From 6436be4cd91bdb8db4389cea4c27c1779619942a Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Wed, 2 Apr 2025 14:40:34 -0600 Subject: [PATCH 01/55] Fix logger for persister and unifier --- backend/persister.py | 2 +- backend/unifier.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/persister.py b/backend/persister.py index 38e8493..a89572c 100644 --- a/backend/persister.py +++ b/backend/persister.py @@ -21,7 +21,7 @@ import pandas as pd import geopandas as gpd -from backend.logging import Loggable +from backend.logger import Loggable try: from google.cloud import storage diff --git a/backend/unifier.py b/backend/unifier.py index 524f695..32042f2 100644 --- a/backend/unifier.py +++ b/backend/unifier.py @@ -17,7 +17,7 @@ from backend.config import Config, get_source from backend.constants import WATERLEVELS -from backend.logging import setup_logging +from backend.logger import setup_logging from backend.persister import CSVPersister, GeoJSONPersister, CloudStoragePersister from backend.source import BaseSiteSource From 3e71a593a6e0c5f06f2ba6b751614fd4bf700bda Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Wed, 2 Apr 2025 16:32:07 -0600 Subject: [PATCH 02/55] Remove USGS records where value is "-999999" --- backend/connectors/usgs/source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/connectors/usgs/source.py b/backend/connectors/usgs/source.py index e60029b..b6eff31 100644 --- a/backend/connectors/usgs/source.py +++ b/backend/connectors/usgs/source.py @@ -181,7 +181,7 @@ def _extract_site_records(self, records, site_record): return [ri for ri in records if ri["site_code"] == site_record.id] def _clean_records(self, records): - return [r for r in records if r["value"] is not None and r["value"].strip()] + return [r for r in records if r["value"] is not None and r["value"].strip() and r["value"] != "-999999"] def _extract_source_parameter_results(self, records): return [float(r["value"]) for r in records] From 5a6681470c2fef875825f33ef1ab8e84557f2c98 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Wed, 2 Apr 2025 17:38:11 -0600 Subject: [PATCH 03/55] Update change log --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 74b032f..80bf398 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), - This naming schema also enables the development of datetime filters as the descriptor will apply to the latest datetime within the provided time frame filter, whereas most recent indicates np filters. - removed sites that are not in New Mexico +### Fixed +- removed records from USGS where the value is "-999999" + ## 0.7.0 From 40f3b9b50b2df90383afe132e5ec443a48d15a1c Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Wed, 2 Apr 2025 17:38:20 -0600 Subject: [PATCH 04/55] Work on updating and maintaining tests --- tests/__init__.py | 126 ++++++++++++++++++++++++++++++ tests/source_tests/__init__.py | 0 tests/source_tests/test_nmbgmr.py | 40 ++++++++++ 3 files changed, 166 insertions(+) create mode 100644 tests/source_tests/__init__.py create mode 100644 tests/source_tests/test_nmbgmr.py diff --git a/tests/__init__.py b/tests/__init__.py index e69de29..1eb65b5 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -0,0 +1,126 @@ +from pathlib import Path +import pytest + +from backend.config import Config, SOURCE_KEYS, get_source +from backend.constants import WATERLEVELS +from backend.unifier import unify_analytes, unify_waterlevels + +class BaseTestClass: + + parameter = None + units = None + agency = None + + dirs_and_files_to_delete = [] + + # restrict results to 10 for testing + site_limit = 39 + + @pytest.fixture(autouse=True) + def setup(self): + # Setup code + self.config = Config() + + for agency in SOURCE_KEYS: + setattr(self.config, f"use_source_{agency}", False) + + setattr(self.config, "site_limit", self.site_limit) + setattr(self.config, "parameter", self.parameter) + setattr(self.config, "units", self.units) + setattr(self.config, f"use_source_{self.agency}", True) + + self.config.finalize() + + # run test + yield + + # Teardown code + self.config = None + self.unifier = None + for p in self.dirs_and_files_to_delete: + if p.is_file(): + p.unlink() + elif p.is_dir(): + for f in p.iterdir(): + f.unlink() + p.rmdir() + self.dirs_and_files_to_delete = [] + + def _unify(self): + self.unifier(self.config) + + def _test_health(self): + # do a health check for the agency + source = self.config.all_site_sources()[0][0] + assert source.health() + + def _test_summary(self): + # Arrange + self.config.output_summary = True + self.config.report() + + # Act + if self.parameter == WATERLEVELS: + unify_waterlevels(self.config) + else: + unify_analytes(self.config) + + # Assert + # Check the summary file + summary_file = Path(self.config.output_path) / "summary.csv" + assert summary_file.exists() + + # Check the column headers + with open(summary_file, "r") as f: + headers = f.readline().strip().split(",") + expected_headers = [ + "source", + "id", + "name", + "usgs_site_id", + "alternate_site_id", + "latitude", + "longitude", + "horizontal_datum", + "elevation", + "elevation_units", + "well_depth", + "well_depth_units", + "parameter_name", + "parameter_units", + "nrecords", + "min", + "max", + "mean", + "earliest_date", + "earliest_time", + "earliest_value", + "earliest_units", + "latest_date", + "latest_time", + "latest_value", + "latest_units", + ] + assert headers == expected_headers + self.dirs_and_files_to_delete.append(summary_file) + + def _test_timeseries_unified(self): + pass + + def _test_timeseries_separated(self): + pass + + def _test_date_range(self): + pass + + def _test_wkt(self): + pass + + def _test_county(self): + pass + + def _test_huc(self): + pass + + def _text_bbox(self): + pass \ No newline at end of file diff --git a/tests/source_tests/__init__.py b/tests/source_tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/source_tests/test_nmbgmr.py b/tests/source_tests/test_nmbgmr.py new file mode 100644 index 0000000..17540e7 --- /dev/null +++ b/tests/source_tests/test_nmbgmr.py @@ -0,0 +1,40 @@ +from backend.constants import WATERLEVELS +from tests import BaseTestClass + +import pytest + +class TestNMBGMRWaterlevels(BaseTestClass): + + parameter = WATERLEVELS + units = "ft" + agency = "nmbgmr_amp" + + def test_health(self): + self._test_health() + + def test_summary(self): + self._test_summary() + + @pytest.mark.skip(reason="Not implemented yet") + def test_timeseries_unified(self): + self._test_timeseries_unified() + + @pytest.mark.skip(reason="Not implemented yet") + def test_timeseries_separated(self): + self._test_timeseries_separated() + + @pytest.mark.skip(reason="Not implemented yet") + def test_date_range(self): + self._test_date_range() + + @pytest.mark.skip(reason="Not implemented yet") + def test_wkt(self): + self._test_wkt() + + @pytest.mark.skip(reason="Not implemented yet") + def test_county(self): + self._test_county() + + @pytest.mark.skip(reason="Not implemented yet") + def test_huc(self): + self._test_huc() \ No newline at end of file From 2ad8ecfbc898af528a4f5523a8864baf3bd700e2 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Wed, 2 Apr 2025 17:38:35 -0600 Subject: [PATCH 05/55] Work on fixing chunk sizes --- backend/connectors/nmbgmr/source.py | 25 +++++++++++-------------- backend/unifier.py | 10 +++++++++- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/backend/connectors/nmbgmr/source.py b/backend/connectors/nmbgmr/source.py index 39c1170..2ec5d10 100644 --- a/backend/connectors/nmbgmr/source.py +++ b/backend/connectors/nmbgmr/source.py @@ -72,9 +72,6 @@ def get_records(self): if config.has_bounds(): params["wkt"] = config.bounding_wkt() - if config.site_limit: - params["limit"] = config.site_limit - if not config.sites_only: if config.parameter.lower() != "waterlevels": @@ -90,18 +87,18 @@ def get_records(self): ) if not config.sites_only: for site in sites: - print(f"Obtaining well data for {site['properties']['point_id']}") - well_data = self._execute_json_request( - _make_url("wells"), - params={"pointid": site["properties"]["point_id"]}, - tag="", - ) - site["properties"]["formation"] = well_data["formation"] - site["properties"]["well_depth"] = well_data["well_depth_ftbgs"] - site["properties"]["well_depth_units"] = FEET - # site["properties"]["formation"] = None - # site["properties"]["well_depth"] = None + # print(f"Obtaining well data for {site['properties']['point_id']}") + # well_data = self._execute_json_request( + # _make_url("wells"), + # params={"pointid": site["properties"]["point_id"]}, + # tag="", + # ) + # site["properties"]["formation"] = well_data["formation"] + # site["properties"]["well_depth"] = well_data["well_depth_ftbgs"] # site["properties"]["well_depth_units"] = FEET + site["properties"]["formation"] = None + site["properties"]["well_depth"] = None + site["properties"]["well_depth_units"] = FEET return sites diff --git a/backend/unifier.py b/backend/unifier.py index 32042f2..86fd157 100644 --- a/backend/unifier.py +++ b/backend/unifier.py @@ -156,11 +156,18 @@ def _site_wrapper(site_source, parameter_source, persister, config): end_ind += n if use_summarize: + print("summarize") + print(sites_with_records_count, site_limit) summary_records = parameter_source.read( sites, use_summarize, start_ind, end_ind ) if summary_records: + print("here", len(summary_records)) persister.records.extend(summary_records) + sites_with_records_count += len(summary_records) + else: + print("there") + continue else: results = parameter_source.read( sites, use_summarize, start_ind, end_ind @@ -175,7 +182,8 @@ def _site_wrapper(site_source, parameter_source, persister, config): persister.timeseries.append((site, records)) persister.sites.append(site) - sites_with_records_count += 1 + print("incrementing sites_with_records_count") + sites_with_records_count += 1 except BaseException: import traceback From 86d63fa39c6bec6547e67019f20101ce57fd763c Mon Sep 17 00:00:00 2001 From: jacob-a-brown Date: Wed, 2 Apr 2025 23:40:12 +0000 Subject: [PATCH 06/55] Formatting changes --- backend/connectors/usgs/source.py | 6 +++++- tests/__init__.py | 9 +++++---- tests/source_tests/test_nmbgmr.py | 3 ++- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/backend/connectors/usgs/source.py b/backend/connectors/usgs/source.py index b6eff31..25e4e87 100644 --- a/backend/connectors/usgs/source.py +++ b/backend/connectors/usgs/source.py @@ -181,7 +181,11 @@ def _extract_site_records(self, records, site_record): return [ri for ri in records if ri["site_code"] == site_record.id] def _clean_records(self, records): - return [r for r in records if r["value"] is not None and r["value"].strip() and r["value"] != "-999999"] + return [ + r + for r in records + if r["value"] is not None and r["value"].strip() and r["value"] != "-999999" + ] def _extract_source_parameter_results(self, records): return [float(r["value"]) for r in records] diff --git a/tests/__init__.py b/tests/__init__.py index 1eb65b5..d40fd4a 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -5,6 +5,7 @@ from backend.constants import WATERLEVELS from backend.unifier import unify_analytes, unify_waterlevels + class BaseTestClass: parameter = None @@ -23,7 +24,7 @@ def setup(self): for agency in SOURCE_KEYS: setattr(self.config, f"use_source_{agency}", False) - + setattr(self.config, "site_limit", self.site_limit) setattr(self.config, "parameter", self.parameter) setattr(self.config, "units", self.units) @@ -58,13 +59,13 @@ def _test_summary(self): # Arrange self.config.output_summary = True self.config.report() - + # Act if self.parameter == WATERLEVELS: unify_waterlevels(self.config) else: unify_analytes(self.config) - + # Assert # Check the summary file summary_file = Path(self.config.output_path) / "summary.csv" @@ -123,4 +124,4 @@ def _test_huc(self): pass def _text_bbox(self): - pass \ No newline at end of file + pass diff --git a/tests/source_tests/test_nmbgmr.py b/tests/source_tests/test_nmbgmr.py index 17540e7..051c7da 100644 --- a/tests/source_tests/test_nmbgmr.py +++ b/tests/source_tests/test_nmbgmr.py @@ -3,6 +3,7 @@ import pytest + class TestNMBGMRWaterlevels(BaseTestClass): parameter = WATERLEVELS @@ -37,4 +38,4 @@ def test_county(self): @pytest.mark.skip(reason="Not implemented yet") def test_huc(self): - self._test_huc() \ No newline at end of file + self._test_huc() From 11c8a1a151505aafc94ac3182486101d59d16006 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 3 Apr 2025 08:28:57 -0600 Subject: [PATCH 07/55] Remove duplicative variable for clarity --- backend/unifier.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/backend/unifier.py b/backend/unifier.py index 86fd157..c2c9bfb 100644 --- a/backend/unifier.py +++ b/backend/unifier.py @@ -129,9 +129,7 @@ def _site_wrapper(site_source, parameter_source, persister, config): use_summarize = config.output_summary site_limit = config.site_limit - sites = site_source.read() - - if not sites: + if not site_source.read(): return sites_with_records_count = 0 @@ -140,7 +138,7 @@ def _site_wrapper(site_source, parameter_source, persister, config): first_flag = True if config.sites_only: - persister.sites.extend(sites) + persister.sites.extend(site_source.read()) else: for sites in site_source.chunks(sites): if site_limit and sites_with_records_count == site_limit: From 4f57d46d35058a8becb60fbbffcfda4533d39a97 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 3 Apr 2025 16:49:28 -0600 Subject: [PATCH 08/55] Intermediate step for this debug --- backend/source.py | 27 ++++++++++++++++++++++++++- backend/unifier.py | 36 +++++++++++++++++++++++++++--------- 2 files changed, 53 insertions(+), 10 deletions(-) diff --git a/backend/source.py b/backend/source.py index b631185..746fb28 100644 --- a/backend/source.py +++ b/backend/source.py @@ -256,7 +256,8 @@ def _execute_json_request( dict the json response """ - # print(url) + print(url) + print(params) resp = httpx.get(url, params=params, **kw) if tag is None: tag = "data" @@ -508,6 +509,30 @@ def chunks(self, records: list, chunk_size: int = None) -> list: ] else: return records + + def get_chunk_size(self): + """ + Returns the chunk size for the source. This is used to determine how many records + to process at once. + + Returns + ------- + int + the chunk size for the source + """ + return self.chunk_size + + def set_chunk_size(self, chunk_size: int): + """ + Sets the chunk size for the source. This is used to determine how many records + to process at once. + + Parameters + ---------- + chunk_size : int + the chunk size for the source + """ + self.chunk_size = chunk_size class BaseParameterSource(BaseSource): diff --git a/backend/unifier.py b/backend/unifier.py index c2c9bfb..ecb377d 100644 --- a/backend/unifier.py +++ b/backend/unifier.py @@ -129,23 +129,35 @@ def _site_wrapper(site_source, parameter_source, persister, config): use_summarize = config.output_summary site_limit = config.site_limit - if not site_source.read(): + sites = site_source.read() + + if not sites: return sites_with_records_count = 0 - start_ind = 1 + start_ind = 0 end_ind = 0 first_flag = True + """ + If site_source.chunk_size is greater than site_limit, set it to site_limit + so that we don't get too many sites at once. This will need to be repeated + within the for loop in conjunction with sites_with_records_count so that + the site_limit is not surpassed + """ + if site_limit > 0 and site_source.get_chunk_size() > site_limit: + site_source.set_chunk_size(site_limit) + if config.sites_only: - persister.sites.extend(site_source.read()) + persister.sites.extend(sites) else: - for sites in site_source.chunks(sites): + for site_records in site_source.chunks(sites): if site_limit and sites_with_records_count == site_limit: break + # elif - if type(sites) == list: - n = len(sites) + if type(site_records) == list: + n = len(site_records) if first_flag: first_flag = False else: @@ -155,9 +167,10 @@ def _site_wrapper(site_source, parameter_source, persister, config): if use_summarize: print("summarize") - print(sites_with_records_count, site_limit) + print("sites_with_records_count:", sites_with_records_count, "site_limit:", site_limit, "chunk_size:", site_source.get_chunk_size()) + print("start_ind:", start_ind, "end_ind:", end_ind) summary_records = parameter_source.read( - sites, use_summarize, start_ind, end_ind + site_records, use_summarize, start_ind, end_ind ) if summary_records: print("here", len(summary_records)) @@ -168,7 +181,7 @@ def _site_wrapper(site_source, parameter_source, persister, config): continue else: results = parameter_source.read( - sites, use_summarize, start_ind, end_ind + site_records, use_summarize, start_ind, end_ind ) # no records are returned if there is no site record for parameter # or if the record isn't clean (doesn't have the correct fields) @@ -183,6 +196,11 @@ def _site_wrapper(site_source, parameter_source, persister, config): print("incrementing sites_with_records_count") sites_with_records_count += 1 + if site_limit > 0 and site_limit < sites_with_records_count + site_source.get_chunk_size(): + new_chunk_size = site_limit - sites_with_records_count + site_source.set_chunk_size(new_chunk_size) + print("new_chunk_size:", new_chunk_size) + except BaseException: import traceback From 2e19085b83d598ac2d2aa1303e87130a88c50c39 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 3 Apr 2025 17:16:38 -0600 Subject: [PATCH 09/55] Ensure that no more sites than site_limit are returned --- backend/source.py | 26 -------------------------- backend/unifier.py | 34 +++++++++++++++++----------------- 2 files changed, 17 insertions(+), 43 deletions(-) diff --git a/backend/source.py b/backend/source.py index 746fb28..acf47d9 100644 --- a/backend/source.py +++ b/backend/source.py @@ -256,8 +256,6 @@ def _execute_json_request( dict the json response """ - print(url) - print(params) resp = httpx.get(url, params=params, **kw) if tag is None: tag = "data" @@ -509,30 +507,6 @@ def chunks(self, records: list, chunk_size: int = None) -> list: ] else: return records - - def get_chunk_size(self): - """ - Returns the chunk size for the source. This is used to determine how many records - to process at once. - - Returns - ------- - int - the chunk size for the source - """ - return self.chunk_size - - def set_chunk_size(self, chunk_size: int): - """ - Sets the chunk size for the source. This is used to determine how many records - to process at once. - - Parameters - ---------- - chunk_size : int - the chunk size for the source - """ - self.chunk_size = chunk_size class BaseParameterSource(BaseSource): diff --git a/backend/unifier.py b/backend/unifier.py index ecb377d..ec27a1d 100644 --- a/backend/unifier.py +++ b/backend/unifier.py @@ -145,16 +145,27 @@ def _site_wrapper(site_source, parameter_source, persister, config): within the for loop in conjunction with sites_with_records_count so that the site_limit is not surpassed """ - if site_limit > 0 and site_source.get_chunk_size() > site_limit: - site_source.set_chunk_size(site_limit) + if site_limit > 0 and site_source. chunk_size > site_limit: + site_source.chunk_size = site_limit if config.sites_only: persister.sites.extend(sites) else: for site_records in site_source.chunks(sites): + print("sites_with_records_count:", sites_with_records_count, "|", "site_limit:", site_limit, "|", "chunk_size:", site_source.chunk_size) if site_limit and sites_with_records_count == site_limit: break - # elif + elif site_limit and sites_with_records_count > site_limit: + # remove any extra sites that were gathered + num_sites_to_remove = sites_with_records_count - site_limit + print("removing", num_sites_to_remove) + + if use_summarize: + persister.records = persister.records[:-num_sites_to_remove] + else: + persister.timeseries = persister.timeseries[:-num_sites_to_remove] + persister.sites = persister.sites[:-num_sites_to_remove] + break if type(site_records) == list: n = len(site_records) @@ -165,19 +176,14 @@ def _site_wrapper(site_source, parameter_source, persister, config): end_ind += n - if use_summarize: - print("summarize") - print("sites_with_records_count:", sites_with_records_count, "site_limit:", site_limit, "chunk_size:", site_source.get_chunk_size()) - print("start_ind:", start_ind, "end_ind:", end_ind) + if use_summarize: summary_records = parameter_source.read( site_records, use_summarize, start_ind, end_ind ) if summary_records: - print("here", len(summary_records)) persister.records.extend(summary_records) sites_with_records_count += len(summary_records) else: - print("there") continue else: results = parameter_source.read( @@ -188,19 +194,13 @@ def _site_wrapper(site_source, parameter_source, persister, config): # don't count these sites to apply to site_limit if results is None or len(results) == 0: continue + else: + sites_with_records_count += len(results) for site, records in results: persister.timeseries.append((site, records)) persister.sites.append(site) - print("incrementing sites_with_records_count") - sites_with_records_count += 1 - - if site_limit > 0 and site_limit < sites_with_records_count + site_source.get_chunk_size(): - new_chunk_size = site_limit - sites_with_records_count - site_source.set_chunk_size(new_chunk_size) - print("new_chunk_size:", new_chunk_size) - except BaseException: import traceback From 15803436a41bd6f8fea835330cb2a5074511cf5a Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 3 Apr 2025 17:34:02 -0600 Subject: [PATCH 10/55] Make tests more efficient and clearer --- backend/unifier.py | 11 +---------- tests/__init__.py | 23 +++++++++-------------- 2 files changed, 10 insertions(+), 24 deletions(-) diff --git a/backend/unifier.py b/backend/unifier.py index ec27a1d..74b57e6 100644 --- a/backend/unifier.py +++ b/backend/unifier.py @@ -139,15 +139,6 @@ def _site_wrapper(site_source, parameter_source, persister, config): end_ind = 0 first_flag = True - """ - If site_source.chunk_size is greater than site_limit, set it to site_limit - so that we don't get too many sites at once. This will need to be repeated - within the for loop in conjunction with sites_with_records_count so that - the site_limit is not surpassed - """ - if site_limit > 0 and site_source. chunk_size > site_limit: - site_source.chunk_size = site_limit - if config.sites_only: persister.sites.extend(sites) else: @@ -158,7 +149,7 @@ def _site_wrapper(site_source, parameter_source, persister, config): elif site_limit and sites_with_records_count > site_limit: # remove any extra sites that were gathered num_sites_to_remove = sites_with_records_count - site_limit - print("removing", num_sites_to_remove) + print(f"removing {num_sites_to_remove} to avoid exceeding the site limit") if use_summarize: persister.records = persister.records[:-num_sites_to_remove] diff --git a/tests/__init__.py b/tests/__init__.py index d40fd4a..b0c0bdd 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -12,10 +12,8 @@ class BaseTestClass: units = None agency = None - dirs_and_files_to_delete = [] - - # restrict results to 10 for testing - site_limit = 39 + # set set_limit for tests + site_limit = 8 @pytest.fixture(autouse=True) def setup(self): @@ -35,17 +33,15 @@ def setup(self): # run test yield - # Teardown code + # Teardown code + path_to_clean = Path(self.config.output_path) + print(f"Cleaning and removing {path_to_clean}") + for f in Path(path_to_clean).iterdir(): + f.unlink() + path_to_clean.rmdir() + self.dirs_to_delete = [] self.config = None self.unifier = None - for p in self.dirs_and_files_to_delete: - if p.is_file(): - p.unlink() - elif p.is_dir(): - for f in p.iterdir(): - f.unlink() - p.rmdir() - self.dirs_and_files_to_delete = [] def _unify(self): self.unifier(self.config) @@ -103,7 +99,6 @@ def _test_summary(self): "latest_units", ] assert headers == expected_headers - self.dirs_and_files_to_delete.append(summary_file) def _test_timeseries_unified(self): pass From d4c0155ba421f246893140f70498d1299048b664 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 3 Apr 2025 17:40:51 -0600 Subject: [PATCH 11/55] site_limit exceedance calculation simplification --- backend/unifier.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/backend/unifier.py b/backend/unifier.py index 74b57e6..7fb388e 100644 --- a/backend/unifier.py +++ b/backend/unifier.py @@ -144,19 +144,18 @@ def _site_wrapper(site_source, parameter_source, persister, config): else: for site_records in site_source.chunks(sites): print("sites_with_records_count:", sites_with_records_count, "|", "site_limit:", site_limit, "|", "chunk_size:", site_source.chunk_size) - if site_limit and sites_with_records_count == site_limit: - break - elif site_limit and sites_with_records_count > site_limit: - # remove any extra sites that were gathered - num_sites_to_remove = sites_with_records_count - site_limit - print(f"removing {num_sites_to_remove} to avoid exceeding the site limit") - - if use_summarize: - persister.records = persister.records[:-num_sites_to_remove] - else: - persister.timeseries = persister.timeseries[:-num_sites_to_remove] - persister.sites = persister.sites[:-num_sites_to_remove] - break + if site_limit: + if sites_with_records_count >= site_limit: + # remove any extra sites that were gathered. removes 0 if site_limit is not exceeded + num_sites_to_remove = sites_with_records_count - site_limit + print(f"removing {num_sites_to_remove} to avoid exceeding the site limit") + + if use_summarize: + persister.records = persister.records[:-num_sites_to_remove] + else: + persister.timeseries = persister.timeseries[:-num_sites_to_remove] + persister.sites = persister.sites[:-num_sites_to_remove] + break if type(site_records) == list: n = len(site_records) From 656596b48236ae84cd15a22f70b26764fd547e3c Mon Sep 17 00:00:00 2001 From: jacob-a-brown Date: Thu, 3 Apr 2025 23:42:37 +0000 Subject: [PATCH 12/55] Formatting changes --- backend/unifier.py | 23 ++++++++++++++++++----- tests/__init__.py | 2 +- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/backend/unifier.py b/backend/unifier.py index 7fb388e..82cd27f 100644 --- a/backend/unifier.py +++ b/backend/unifier.py @@ -143,17 +143,30 @@ def _site_wrapper(site_source, parameter_source, persister, config): persister.sites.extend(sites) else: for site_records in site_source.chunks(sites): - print("sites_with_records_count:", sites_with_records_count, "|", "site_limit:", site_limit, "|", "chunk_size:", site_source.chunk_size) + print( + "sites_with_records_count:", + sites_with_records_count, + "|", + "site_limit:", + site_limit, + "|", + "chunk_size:", + site_source.chunk_size, + ) if site_limit: if sites_with_records_count >= site_limit: # remove any extra sites that were gathered. removes 0 if site_limit is not exceeded num_sites_to_remove = sites_with_records_count - site_limit - print(f"removing {num_sites_to_remove} to avoid exceeding the site limit") - + print( + f"removing {num_sites_to_remove} to avoid exceeding the site limit" + ) + if use_summarize: persister.records = persister.records[:-num_sites_to_remove] else: - persister.timeseries = persister.timeseries[:-num_sites_to_remove] + persister.timeseries = persister.timeseries[ + :-num_sites_to_remove + ] persister.sites = persister.sites[:-num_sites_to_remove] break @@ -166,7 +179,7 @@ def _site_wrapper(site_source, parameter_source, persister, config): end_ind += n - if use_summarize: + if use_summarize: summary_records = parameter_source.read( site_records, use_summarize, start_ind, end_ind ) diff --git a/tests/__init__.py b/tests/__init__.py index b0c0bdd..466ba46 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -33,7 +33,7 @@ def setup(self): # run test yield - # Teardown code + # Teardown code path_to_clean = Path(self.config.output_path) print(f"Cleaning and removing {path_to_clean}") for f in Path(path_to_clean).iterdir(): From cf350b4e3e4181dd5c3e484ddf4a886e8d6dee35 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Fri, 4 Apr 2025 08:54:48 -0600 Subject: [PATCH 13/55] Clearer site limit communication --- backend/unifier.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/unifier.py b/backend/unifier.py index 7fb388e..859f86b 100644 --- a/backend/unifier.py +++ b/backend/unifier.py @@ -148,7 +148,7 @@ def _site_wrapper(site_source, parameter_source, persister, config): if sites_with_records_count >= site_limit: # remove any extra sites that were gathered. removes 0 if site_limit is not exceeded num_sites_to_remove = sites_with_records_count - site_limit - print(f"removing {num_sites_to_remove} to avoid exceeding the site limit") + print(f"removing {num_sites_to_remove} records to avoid exceeding the site limit") if use_summarize: persister.records = persister.records[:-num_sites_to_remove] From 7b12e8ea5a0b31e2315abfe72df8bcfdd9b16063 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Fri, 4 Apr 2025 11:32:50 -0600 Subject: [PATCH 14/55] Archive old/outdated tests but keep for records --- pytest.ini | 3 + tests/archived/__init__.py | 0 tests/archived/test_cli.py | 408 ++++++++++++++++++++++++++++ tests/archived/test_unifier.py | 473 +++++++++++++++++++++++++++++++++ 4 files changed, 884 insertions(+) create mode 100644 pytest.ini create mode 100644 tests/archived/__init__.py create mode 100644 tests/archived/test_cli.py create mode 100644 tests/archived/test_unifier.py diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..8ea4712 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +; skip archived tests but keep for reference +norecursedirs = tests/archived \ No newline at end of file diff --git a/tests/archived/__init__.py b/tests/archived/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/archived/test_cli.py b/tests/archived/test_cli.py new file mode 100644 index 0000000..3d65365 --- /dev/null +++ b/tests/archived/test_cli.py @@ -0,0 +1,408 @@ +# =============================================================================== +# Copyright 2024 Jake Ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +import os + +from click.testing import CliRunner +from frontend.cli import analytes, waterlevels + + +def _tester(function, args, fail=False): + runner = CliRunner() + print(f"invoked with {args}") + result = runner.invoke(function, args) + print(f"result.exit_code={result.exit_code}") + print(f"result.output=\n{result.output}") + + if fail: + assert result.exit_code != 0 + else: + assert result.exit_code == 0 + + +def _make_args(source): + args = [] + if source: + nosources = [ + f + for f in ( + "--no-amp", + "--no-nwis", + "--no-pvacd", + "--no-bor", + "--no-dwb", + "--no-wqp", + "--no-isc-seven-rivers", + "--no-ckan", + ) + if f != f"--no-{source}" + ] + args += nosources + + args += ["--site-limit", 10, "--dry"] + + return args + + +def _make_tds_args(source): + return ["TDS"] + _make_args(source) + + +def _make_wl_args(source=None): + return _make_args(source) + + +def test_waterlevels_nwis(): + args = _make_wl_args("nwis") + _tester(waterlevels, args) + + +def test_waterlevels_pvacd(): + args = _make_wl_args("pvacd") + _tester(waterlevels, args) + + +def test_waterlevels_nmbgmr(): + args = _make_wl_args("nmbgmr") + _tester(waterlevels, args) + + +def test_waterlevels_isc_seven_rivers(): + args = _make_wl_args("iscsevenrivers") + _tester(waterlevels, args) + + +def test_waterlevels_invalid_source(): + args = _make_wl_args() + args.append("--no-foo") + _tester(waterlevels, args, fail=True) + + +def test_waterlevels_invalid_bbox(): + args = _make_wl_args() + args.append("--bbox") + _tester(waterlevels, args, fail=True) + + +def test_waterlevels_invalid_bbox_format(): + args = _make_wl_args() + args.extend(["--bbox", "1 2 3"]) + _tester(waterlevels, args, fail=True) + + +def test_waterlevels_valid_bbox_format(): + args = _make_wl_args() + args.extend(["--bbox", "1 2,3 4"]) + _tester(waterlevels, args) + + +def test_waterlevels_invalid_county(): + args = _make_wl_args() + args.append("--county") + _tester(waterlevels, args, fail=True) + + +def test_waterlevels_invalid_county_name(): + args = _make_wl_args() + args.extend(["--county", "foo"]) + _tester(waterlevels, args, fail=True) + + +# Analyte Tests ======================================================= +def test_analytes_wqp(): + args = _make_tds_args("wqp") + _tester(analytes, args) + + +def test_analytes_bor(): + args = _make_tds_args("bor") + _tester(analytes, args) + + +def test_analytes_amp(): + args = _make_tds_args("amp") + _tester(analytes, args) + + +def test_analytes_dwb(): + args = _make_tds_args("dwb") + _tester(analytes, args) + + +def test_analytes_isc_seven_rivers(): + args = _make_tds_args("isc-seven-rivers") + _tester(analytes, args) + + +def test_analytes_invalid_analyte(): + args = _make_args("wqp") + args[0] = "Foo" + _tester(analytes, args, fail=True) + + +def test_analytes_invalid_source(): + args = _make_tds_args("wqp") + args.append("--no-foo") + _tester(analytes, args, fail=True) + + +def test_analytes_invalid_bbox(): + args = _make_tds_args("wqp") + args.append("--bbox") + _tester(analytes, args, fail=True) + + +def test_analytes_invalid_bbox_format(): + args = _make_tds_args("wqp") + args.extend(["--bbox", "1 2 3"]) + _tester(analytes, args, fail=True) + + +def test_analytes_valid_bbox_format(): + args = _make_tds_args("wqp") + args.extend(["--bbox", "1 2,3 4"]) + _tester(analytes, args) + + +def test_analytes_invalid_county(): + args = _make_tds_args("wqp") + args.append("--county") + _tester(analytes, args, fail=True) + + +def test_analytes_invalid_county_name(): + args = _make_tds_args("wqp") + args.extend(["--county", "foo"]) + _tester(analytes, args, fail=True) + + +def test_waterlevels_date_range_YMD(): + args = _make_wl_args() + args.extend(["--start-date", "2020-01-01", "--end-date", "2020-05-01"]) + _tester(waterlevels, args) + + +def test_waterlevels_date_range_YM(): + args = _make_wl_args() + args.extend(["--start-date", "2020-01", "--end-date", "2020-05"]) + _tester(waterlevels, args) + + +def test_waterlevels_date_range_Y(): + args = _make_wl_args() + args.extend(["--start-date", "2020", "--end-date", "2021"]) + _tester(waterlevels, args) + + +def test_waterlevels_invalid_start(): + args = _make_wl_args() + args.extend(["--start-date", "x-01-01", "--end-date", "2019-05-01"]) + _tester(waterlevels, args, fail=True) + + +def test_waterlevels_invalid_end(): + args = _make_wl_args() + args.extend(["--start-date", "2020-01-01", "--end-date", "x-05-01"]) + _tester(waterlevels, args, fail=True) + + +# +# def _tester(source, func, county, bbox, args=None): +# runner = CliRunner() +# +# nosources = [ +# f +# for f in ( +# "--no-amp", +# "--no-nwis", +# "--no-st2", +# "--no-bor", +# "--no-dwb", +# "--no-wqp", +# "--no-isc-seven-rivers", +# "--no-ckan", +# ) +# if f != f"--no-{source}" +# ] +# +# dargs = nosources + ["--site-limit", 10] +# +# if args: +# args += dargs +# else: +# args = dargs +# +# if county: +# args.extend(("--county", county)) +# elif bbox: +# args.extend(("--bbox", bbox)) +# +# print(" ".join([str(f) for f in args])) +# result = runner.invoke(func, args) +# +# return result + + +# def _summary_tester(source, func, county=None, bbox=None, args=None): +# if not (county or bbox): +# county = "eddy" +# +# runner = CliRunner() +# # with runner.isolated_filesystem(): +# # result = _tester(source, func, county, bbox, args) +# # assert result.exit_code == 0 +# # assert os.path.isfile("output.csv") +# +# +# def _timeseries_tester( +# source, +# func, +# combined_flag=True, +# timeseries_flag=True, +# county=None, +# bbox=None, +# args=None, +# ): +# if args is None: +# args = [] +# # runner = CliRunner() +# # with runner.isolated_filesystem(): +# # result = _tester(source, func, county, bbox, args=args + ["--timeseries"]) +# # assert result.exit_code == 0 +# # print("combined", os.path.isfile("output.combined.csv"), combined_flag) +# # assert os.path.isfile("output.combined.csv") == combined_flag +# # print("timeseries", os.path.isdir("output_timeseries"), timeseries_flag) +# # assert os.path.isdir("output_timeseries") == timeseries_flag +# +# +# # ====== Analyte Tests ======================================================= +# def _analyte_summary_tester(key): +# _summary_tester(key, analytes, args=["TDS"]) +# +# +# def _analyte_county_tester(source, **kw): +# _timeseries_tester(source, analytes, args=["TDS"], county="eddy", **kw) +# +# +# def test_unify_analytes_amp(): +# _analyte_county_tester("amp", timeseries_flag=False) +# +# +# def test_unify_analytes_wqp(): +# _analyte_county_tester("wqp") +# +# +# def test_unify_analytes_bor(): +# _analyte_county_tester("bor", combined_flag=False) +# +# +# def test_unify_analytes_isc_seven_rivers(): +# _analyte_county_tester("isc-seven-rivers") +# +# +# def test_unify_analytes_dwb(): +# _analyte_county_tester("dwb", timeseries_flag=False) +# +# +# def test_unify_analytes_wqp_summary(): +# _analyte_summary_tester("wqp") +# +# +# def test_unify_analytes_bor_summary(): +# _analyte_summary_tester("bor") +# +# +# def test_unify_analytes_amp_summary(): +# _analyte_summary_tester("amp") +# +# +# def test_unify_analytes_dwb_summary(): +# _analyte_summary_tester("dwb") +# +# +# def test_unify_analytes_isc_seven_rivers_summary(): +# _analyte_summary_tester("isc-seven-rivers") + + +# ====== End Analyte Tests ======================================================= + + +# ====== Water Level Tests ======================================================= +# def _waterlevel_county_tester(source, **kw): +# _timeseries_tester(source, waterlevels, county="eddy", **kw) +# +# +# def _waterlevel_bbox_tester(source, **kw): +# _timeseries_tester(source, waterlevels, bbox="-104.5 32.5,-104 33", **kw) + +# +# def test_unify_waterlevels_nwis(): +# _waterlevel_county_tester("nwis", timeseries_flag=False) +# +# +# def test_unify_waterlevels_amp(): +# _waterlevel_county_tester("amp", timeseries_flag=False) +# +# +# def test_unify_waterlevels_st2(): +# _waterlevel_county_tester("st2", combined_flag=False) +# +# +# def test_unify_waterlevels_isc_seven_rivers(): +# _waterlevel_county_tester("isc-seven-rivers") +# +# +# def test_unify_waterlevels_ckan(): +# _waterlevel_county_tester("ckan") +# +# +# def test_unify_waterlevels_nwis_summary(): +# _summary_tester("nwis", waterlevels) +# +# +# def test_unify_waterlevels_amp_summary(): +# _summary_tester("amp", waterlevels) +# +# +# def test_unify_waterlevels_st2_summary(): +# _summary_tester("st2", waterlevels) +# +# +# def test_unify_waterlevels_isc_seven_rivers_summary(): +# _summary_tester("isc-seven-rivers", waterlevels) +# +# +# def test_unify_waterlevels_nwis_bbox(): +# _waterlevel_bbox_tester("nwis", timeseries_flag=False) +# +# +# def test_unify_waterlevels_amp_bbox(): +# _waterlevel_bbox_tester("amp") +# +# +# def test_unify_waterlevels_st2_bbox(): +# _waterlevel_bbox_tester("st2", combined_flag=False) +# +# +# def test_unify_waterlevels_isc_seven_rivers_bbox(): +# _waterlevel_bbox_tester("isc-seven-rivers", combined_flag=False) +# +# +# def test_unify_waterlevels_ckan_bbox(): +# _waterlevel_bbox_tester("ckan") + + +# ====== End Water Level Tests ======================================================= +# ============= EOF ============================================= diff --git a/tests/archived/test_unifier.py b/tests/archived/test_unifier.py new file mode 100644 index 0000000..3947ef6 --- /dev/null +++ b/tests/archived/test_unifier.py @@ -0,0 +1,473 @@ +# =============================================================================== +# Copyright 2024 Jake Ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +import datetime +import os + +import pytest +import shapely.wkt + +from backend.config import Config +from backend.connectors.ckan import HONDO_RESOURCE_ID +from backend.unifier import unify_analytes, unify_waterlevels + + +def config_factory(): + cfg = Config() + cfg.county = "eddy" + cfg.bbox = "-104.5 32.5,-104 33" + cfg.start_date = "2020-01-01" + cfg.end_date = "2024-5-01" + cfg.output_summary = False + + cfg.use_source_nmbgmr = False + cfg.use_source_wqp = False + cfg.use_source_iscsevenrivers = False + cfg.use_source_nwis = False + cfg.use_source_oseroswell = False + cfg.use_source_pvacd = False + cfg.use_source_bor = False + cfg.use_source_dwb = False + cfg.use_source_bernco = False + + cfg.site_limit = 10 + return cfg + + +@pytest.fixture +def waterlevel_summary_cfg(): + cfg = config_factory() + cfg.output_summary = True + return cfg + + +@pytest.fixture +def waterlevel_timeseries_cfg(): + cfg = config_factory() + cfg.output_summary = False + return cfg + + +@pytest.fixture +def analyte_summary_cfg(): + cfg = config_factory() + cfg.output_summary = True + cfg.analyte = "TDS" + return cfg + + +# def test_unify_analytes(cfg): +# unify_analytes(cfg) + + +def _setup(tmp_path, cfg, source, tag): + d = tmp_path / tag + d.mkdir() + cfg.output_dir = str(d) + for stag in ( + "nmbgmr", + "nwis", + "pvacd", + "bor", + "dwb", + "wqp", + "iscsevenrivers", + "oseroswell", + "bernco", + ): + if stag == source: + setattr(cfg, f"use_source_{stag}", True) + return d + + +def _setup_waterlevels(tmp_path, cfg, source): + d = _setup(tmp_path, cfg, source, "waterlevels") + unify_waterlevels(cfg) + return d + + +def _setup_analytes(tmp_path, cfg, source): + d = _setup(tmp_path, cfg, source, "analyte") + unify_analytes(cfg) + return d + + +def _test_analytes_summary(tmp_path, cfg, source): + d = _setup_analytes(tmp_path, cfg, source) + assert (d / "output.csv").is_file() + + +def _test_waterlevels_summary(tmp_path, cfg, source): + d = _setup_waterlevels(tmp_path, cfg, source) + assert (d / "output.csv").is_file() + + +def _test_waterlevels_timeseries( + tmp_path, cfg, source, combined_flag=True, timeseries_flag=False +): + d = _setup_waterlevels(tmp_path, cfg, source) + combined = d / "output.combined.csv" + timeseries = d / "output_timeseries" + print(combined_flag) + + print("combined", combined.is_file(), combined_flag) + assert combined.is_file() == combined_flag + print("timeseries", timeseries.is_dir(), timeseries_flag) + assert timeseries.is_dir() == timeseries_flag + + return combined, timeseries + + +def _test_waterelevels_timeseries_date_range( + tmp_path, cfg, source, timeseries_flag=True, combined_flag=False +): + combined, timeseries = _test_waterlevels_timeseries( + tmp_path, + cfg, + source, + timeseries_flag=timeseries_flag, + combined_flag=combined_flag, + ) + + for p in timeseries.iterdir(): + if os.path.basename(p) == "sites.csv": + continue + + with open(p, "r") as rfile: + lines = rfile.readlines() + for l in lines[1:]: + vs = l.split(",") + dd = vs[3] + dd = datetime.datetime.strptime(dd, "%Y-%m-%d") + assert dd.year >= 2020 and dd.year <= 2024 + + +def test_nwis_site_health_check(): + from backend.connectors.usgs.source import NWISSiteSource + + n = NWISSiteSource() + assert n.health() + + +def test_nmbgmr_site_health_check(): + from backend.connectors.nmbgmr.source import NMBGMRSiteSource + + n = NMBGMRSiteSource() + assert n.health() + + +def test_wqp_site_health_check(): + from backend.connectors.wqp.source import WQPSiteSource + + n = WQPSiteSource() + assert n.health() + + +def test_bor_site_health_check(): + from backend.connectors.bor.source import BORSiteSource + + n = BORSiteSource() + assert n.health() + + +def test_dwb_site_health_check(): + from backend.connectors.nmenv.source import DWBSiteSource + + n = DWBSiteSource() + assert n.health() + + +def test_isc_seven_rivers_site_health_check(): + from backend.connectors.isc_seven_rivers.source import ISCSevenRiversSiteSource + + n = ISCSevenRiversSiteSource() + assert n.health() + + +def test_ckan_site_health_check(): + from backend.connectors.ckan.source import OSERoswellSiteSource + + n = OSERoswellSiteSource(HONDO_RESOURCE_ID) + assert n.health() + + +def test_pvacd_site_health_check(): + from backend.connectors.st2.source import PVACDSiteSource + + n = PVACDSiteSource() + assert n.health() + + +def test_bernco_site_health_check(): + from backend.connectors.st2.source import BernCoSiteSource + + n = BernCoSiteSource() + assert n.health() + + +# def test_ose_roswell_site_health_check(): +# from backend.connectors.ose_roswell.source import OSESiteSource +# n = OSESiteSource() +# assert n.health() + + +# Source tests ======================================================================================================== +def test_source_bounds_nmbgmr(): + from backend.unifier import get_source_bounds + from backend.connectors import NM_STATE_BOUNDING_POLYGON + + sourcekey = "nmbgmr" + bounds = get_source_bounds(sourcekey) + assert bounds + assert bounds.is_valid + assert bounds.geom_type == "Polygon" + assert bounds == NM_STATE_BOUNDING_POLYGON + + +def test_source_bounds_is_seven_rivers(): + from backend.unifier import get_source_bounds + from backend.connectors import ISC_SEVEN_RIVERS_BOUNDING_POLYGON + + sourcekey = "iscsevenrivers" + bounds = get_source_bounds(sourcekey) + assert bounds + assert bounds.is_valid + assert bounds.geom_type == "Polygon" + assert bounds == ISC_SEVEN_RIVERS_BOUNDING_POLYGON + + +def test_source_bounds_oser(): + from backend.unifier import get_source_bounds + from backend.connectors import ( + OSE_ROSWELL_HONDO_BOUNDING_POLYGON, + OSE_ROSWELL_ROSWELL_BOUNDING_POLYGON, + OSE_ROSWELL_FORT_SUMNER_BOUNDING_POLYGON, + ) + + sourcekey = "oseroswell" + bounds = get_source_bounds(sourcekey) + assert bounds + assert bounds.is_valid + assert bounds.geom_type == "GeometryCollection" + assert bounds == shapely.GeometryCollection( + [ + OSE_ROSWELL_HONDO_BOUNDING_POLYGON, + OSE_ROSWELL_FORT_SUMNER_BOUNDING_POLYGON, + OSE_ROSWELL_ROSWELL_BOUNDING_POLYGON, + ] + ) + + +def test_sources_socorro(tmp_path): + cfg = Config() + cfg.county = "socorro" + + from backend.unifier import get_sources + + sources = get_sources(cfg) + assert sources + assert len(sources) == 2 + assert sorted([s.__class__.__name__ for s in sources]) == sorted( + ["NMBGMRSiteSource", "NWISSiteSource"] + ) + + +def test_sources_eddy_dtw(tmp_path): + cfg = Config() + cfg.county = "eddy" + + from backend.unifier import get_sources + + sources = get_sources(cfg) + assert sources + assert len(sources) == 5 + assert sorted([s.__class__.__name__ for s in sources]) == sorted( + [ + "ISCSevenRiversSiteSource", + "NMBGMRSiteSource", + "OSERoswellSiteSource", + "PVACDSiteSource", + "NWISSiteSource", + ] + ) + + +def test_sources_eddy_tds(tmp_path): + cfg = Config() + cfg.county = "eddy" + cfg.analyte = "TDS" + + from backend.unifier import get_sources + + sources = get_sources(cfg) + assert sources + assert len(sources) == 5 + assert sorted([s.__class__.__name__ for s in sources]) == sorted( + [ + "BORSiteSource", + "DWBSiteSource", + "ISCSevenRiversSiteSource", + "NMBGMRSiteSource", + "WQPSiteSource", + ] + ) + + +# Waterlevel Summary tests =========================================================================================== +def test_unify_waterlevels_bernco_summary(tmp_path, waterlevel_summary_cfg): + waterlevel_summary_cfg.county = "bernalillo" + waterlevel_summary_cfg.bbox = None + _test_waterlevels_summary(tmp_path, waterlevel_summary_cfg, "bernco") + + +def test_unify_waterlevels_nwis_summary(tmp_path, waterlevel_summary_cfg): + _test_waterlevels_summary(tmp_path, waterlevel_summary_cfg, "nwis") + + +def test_unify_waterlevels_amp_summary(tmp_path, waterlevel_summary_cfg): + _test_waterlevels_summary(tmp_path, waterlevel_summary_cfg, "nmbgmr") + + +def test_unify_waterlevels_pvacd_summary(tmp_path, waterlevel_summary_cfg): + _test_waterlevels_summary(tmp_path, waterlevel_summary_cfg, "pvacd") + + +def test_unify_waterlevels_isc_seven_rivers_summary(tmp_path, waterlevel_summary_cfg): + _test_waterlevels_summary(tmp_path, waterlevel_summary_cfg, "iscsevenrivers") + + +def test_unify_waterlevels_ose_roswell_summary(tmp_path, waterlevel_summary_cfg): + _test_waterlevels_summary(tmp_path, waterlevel_summary_cfg, "oseroswell") + + +# Waterlevel timeseries tests ========================================================================================= +def test_unify_waterlevels_nwis_timeseries(tmp_path, waterlevel_timeseries_cfg): + # there are one or more locations within the bounding box that have only + # one record, so there is a combined file + _test_waterlevels_timeseries( + tmp_path, + waterlevel_timeseries_cfg, + "nwis", + combined_flag=True, + timeseries_flag=True, + ) + + +def test_unify_waterlevels_amp_timeseries(tmp_path, waterlevel_timeseries_cfg): + _test_waterlevels_timeseries(tmp_path, waterlevel_timeseries_cfg, "nmbgmr") + + +def test_unify_waterlevels_pvacd_timeseries(tmp_path, waterlevel_timeseries_cfg): + # all locations within the bounding box have more than one record + # so there is no combined file + _test_waterlevels_timeseries( + tmp_path, + waterlevel_timeseries_cfg, + "pvacd", + combined_flag=False, + timeseries_flag=True, + ) + + +def test_unify_waterlevels_isc_seven_rivers_timeseries( + tmp_path, waterlevel_timeseries_cfg +): + # all locations within the bounding box have more than one record + # so there is no combined file + _test_waterlevels_timeseries( + tmp_path, + waterlevel_timeseries_cfg, + "iscsevenrivers", + combined_flag=False, + timeseries_flag=True, + ) + + +def test_unify_waterlevels_ose_roswell_timeseries(tmp_path, waterlevel_timeseries_cfg): + _test_waterlevels_timeseries( + tmp_path, waterlevel_timeseries_cfg, "oseroswell", timeseries_flag=True + ) + + +# Waterlevel summary date range tests ================================================================================= +def test_waterlevels_nwis_summary_date_range(tmp_path, waterlevel_summary_cfg): + d = _setup_waterlevels(tmp_path, waterlevel_summary_cfg, "nwis") + assert (d / "output.csv").is_file() + + +# Waterlevel timeseries date range ==================================================================================== +def test_waterlevels_nwis_timeseries_date_range(tmp_path, waterlevel_timeseries_cfg): + # there are one or more locations within the bounding box and date range + # that have only one record, so there is a combined file + _test_waterelevels_timeseries_date_range( + tmp_path, + waterlevel_timeseries_cfg, + "nwis", + timeseries_flag=True, + combined_flag=True, + ) + + +def test_waterlevels_isc_seven_rivers_timeseries_date_range( + tmp_path, waterlevel_timeseries_cfg +): + # all locations within the bounding box and date rangehave more than one + # record so there is no combined file + _test_waterelevels_timeseries_date_range( + tmp_path, + waterlevel_timeseries_cfg, + "iscsevenrivers", + timeseries_flag=True, + combined_flag=False, + ) + + +def test_waterlevels_pvacd_timeseries_date_range(tmp_path, waterlevel_timeseries_cfg): + # all locations within the bounding box and date rangehave more than one + # record so there is no combined file + _test_waterelevels_timeseries_date_range( + tmp_path, + waterlevel_timeseries_cfg, + "pvacd", + timeseries_flag=True, + combined_flag=False, + ) + + +# Analyte summary tests =============================================================================================== +def test_unify_analytes_wqp_summary(tmp_path, analyte_summary_cfg): + _test_analytes_summary(tmp_path, analyte_summary_cfg, "wqp") + + +def test_unify_analytes_amp_summary(tmp_path, analyte_summary_cfg): + _test_analytes_summary(tmp_path, analyte_summary_cfg, "nmbgmr") + + +def test_unify_analytes_bor_summary(tmp_path, analyte_summary_cfg): + # BOR locations are found within Otero County + analyte_summary_cfg.county = "otero" + analyte_summary_cfg.bbox = None + _test_analytes_summary(tmp_path, analyte_summary_cfg, "bor") + + +def test_unify_analytes_isc_seven_rivers_summary(tmp_path, analyte_summary_cfg): + _test_analytes_summary(tmp_path, analyte_summary_cfg, "iscsevenrivers") + + +def test_unify_analytes_dwb_summary(tmp_path, analyte_summary_cfg): + _test_analytes_summary(tmp_path, analyte_summary_cfg, "dwb") + + +# ============= EOF ============================================= From 8045586b332ff06e79494cdabedb02cb15babce4 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Fri, 4 Apr 2025 11:33:43 -0600 Subject: [PATCH 15/55] Archive - but keep - old/outdated tests for reference --- tests/test_cli.py | 408 ------------------------------------ tests/test_unifier.py | 473 ------------------------------------------ 2 files changed, 881 deletions(-) delete mode 100644 tests/test_cli.py delete mode 100644 tests/test_unifier.py diff --git a/tests/test_cli.py b/tests/test_cli.py deleted file mode 100644 index 3d65365..0000000 --- a/tests/test_cli.py +++ /dev/null @@ -1,408 +0,0 @@ -# =============================================================================== -# Copyright 2024 Jake Ross -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# =============================================================================== -import os - -from click.testing import CliRunner -from frontend.cli import analytes, waterlevels - - -def _tester(function, args, fail=False): - runner = CliRunner() - print(f"invoked with {args}") - result = runner.invoke(function, args) - print(f"result.exit_code={result.exit_code}") - print(f"result.output=\n{result.output}") - - if fail: - assert result.exit_code != 0 - else: - assert result.exit_code == 0 - - -def _make_args(source): - args = [] - if source: - nosources = [ - f - for f in ( - "--no-amp", - "--no-nwis", - "--no-pvacd", - "--no-bor", - "--no-dwb", - "--no-wqp", - "--no-isc-seven-rivers", - "--no-ckan", - ) - if f != f"--no-{source}" - ] - args += nosources - - args += ["--site-limit", 10, "--dry"] - - return args - - -def _make_tds_args(source): - return ["TDS"] + _make_args(source) - - -def _make_wl_args(source=None): - return _make_args(source) - - -def test_waterlevels_nwis(): - args = _make_wl_args("nwis") - _tester(waterlevels, args) - - -def test_waterlevels_pvacd(): - args = _make_wl_args("pvacd") - _tester(waterlevels, args) - - -def test_waterlevels_nmbgmr(): - args = _make_wl_args("nmbgmr") - _tester(waterlevels, args) - - -def test_waterlevels_isc_seven_rivers(): - args = _make_wl_args("iscsevenrivers") - _tester(waterlevels, args) - - -def test_waterlevels_invalid_source(): - args = _make_wl_args() - args.append("--no-foo") - _tester(waterlevels, args, fail=True) - - -def test_waterlevels_invalid_bbox(): - args = _make_wl_args() - args.append("--bbox") - _tester(waterlevels, args, fail=True) - - -def test_waterlevels_invalid_bbox_format(): - args = _make_wl_args() - args.extend(["--bbox", "1 2 3"]) - _tester(waterlevels, args, fail=True) - - -def test_waterlevels_valid_bbox_format(): - args = _make_wl_args() - args.extend(["--bbox", "1 2,3 4"]) - _tester(waterlevels, args) - - -def test_waterlevels_invalid_county(): - args = _make_wl_args() - args.append("--county") - _tester(waterlevels, args, fail=True) - - -def test_waterlevels_invalid_county_name(): - args = _make_wl_args() - args.extend(["--county", "foo"]) - _tester(waterlevels, args, fail=True) - - -# Analyte Tests ======================================================= -def test_analytes_wqp(): - args = _make_tds_args("wqp") - _tester(analytes, args) - - -def test_analytes_bor(): - args = _make_tds_args("bor") - _tester(analytes, args) - - -def test_analytes_amp(): - args = _make_tds_args("amp") - _tester(analytes, args) - - -def test_analytes_dwb(): - args = _make_tds_args("dwb") - _tester(analytes, args) - - -def test_analytes_isc_seven_rivers(): - args = _make_tds_args("isc-seven-rivers") - _tester(analytes, args) - - -def test_analytes_invalid_analyte(): - args = _make_args("wqp") - args[0] = "Foo" - _tester(analytes, args, fail=True) - - -def test_analytes_invalid_source(): - args = _make_tds_args("wqp") - args.append("--no-foo") - _tester(analytes, args, fail=True) - - -def test_analytes_invalid_bbox(): - args = _make_tds_args("wqp") - args.append("--bbox") - _tester(analytes, args, fail=True) - - -def test_analytes_invalid_bbox_format(): - args = _make_tds_args("wqp") - args.extend(["--bbox", "1 2 3"]) - _tester(analytes, args, fail=True) - - -def test_analytes_valid_bbox_format(): - args = _make_tds_args("wqp") - args.extend(["--bbox", "1 2,3 4"]) - _tester(analytes, args) - - -def test_analytes_invalid_county(): - args = _make_tds_args("wqp") - args.append("--county") - _tester(analytes, args, fail=True) - - -def test_analytes_invalid_county_name(): - args = _make_tds_args("wqp") - args.extend(["--county", "foo"]) - _tester(analytes, args, fail=True) - - -def test_waterlevels_date_range_YMD(): - args = _make_wl_args() - args.extend(["--start-date", "2020-01-01", "--end-date", "2020-05-01"]) - _tester(waterlevels, args) - - -def test_waterlevels_date_range_YM(): - args = _make_wl_args() - args.extend(["--start-date", "2020-01", "--end-date", "2020-05"]) - _tester(waterlevels, args) - - -def test_waterlevels_date_range_Y(): - args = _make_wl_args() - args.extend(["--start-date", "2020", "--end-date", "2021"]) - _tester(waterlevels, args) - - -def test_waterlevels_invalid_start(): - args = _make_wl_args() - args.extend(["--start-date", "x-01-01", "--end-date", "2019-05-01"]) - _tester(waterlevels, args, fail=True) - - -def test_waterlevels_invalid_end(): - args = _make_wl_args() - args.extend(["--start-date", "2020-01-01", "--end-date", "x-05-01"]) - _tester(waterlevels, args, fail=True) - - -# -# def _tester(source, func, county, bbox, args=None): -# runner = CliRunner() -# -# nosources = [ -# f -# for f in ( -# "--no-amp", -# "--no-nwis", -# "--no-st2", -# "--no-bor", -# "--no-dwb", -# "--no-wqp", -# "--no-isc-seven-rivers", -# "--no-ckan", -# ) -# if f != f"--no-{source}" -# ] -# -# dargs = nosources + ["--site-limit", 10] -# -# if args: -# args += dargs -# else: -# args = dargs -# -# if county: -# args.extend(("--county", county)) -# elif bbox: -# args.extend(("--bbox", bbox)) -# -# print(" ".join([str(f) for f in args])) -# result = runner.invoke(func, args) -# -# return result - - -# def _summary_tester(source, func, county=None, bbox=None, args=None): -# if not (county or bbox): -# county = "eddy" -# -# runner = CliRunner() -# # with runner.isolated_filesystem(): -# # result = _tester(source, func, county, bbox, args) -# # assert result.exit_code == 0 -# # assert os.path.isfile("output.csv") -# -# -# def _timeseries_tester( -# source, -# func, -# combined_flag=True, -# timeseries_flag=True, -# county=None, -# bbox=None, -# args=None, -# ): -# if args is None: -# args = [] -# # runner = CliRunner() -# # with runner.isolated_filesystem(): -# # result = _tester(source, func, county, bbox, args=args + ["--timeseries"]) -# # assert result.exit_code == 0 -# # print("combined", os.path.isfile("output.combined.csv"), combined_flag) -# # assert os.path.isfile("output.combined.csv") == combined_flag -# # print("timeseries", os.path.isdir("output_timeseries"), timeseries_flag) -# # assert os.path.isdir("output_timeseries") == timeseries_flag -# -# -# # ====== Analyte Tests ======================================================= -# def _analyte_summary_tester(key): -# _summary_tester(key, analytes, args=["TDS"]) -# -# -# def _analyte_county_tester(source, **kw): -# _timeseries_tester(source, analytes, args=["TDS"], county="eddy", **kw) -# -# -# def test_unify_analytes_amp(): -# _analyte_county_tester("amp", timeseries_flag=False) -# -# -# def test_unify_analytes_wqp(): -# _analyte_county_tester("wqp") -# -# -# def test_unify_analytes_bor(): -# _analyte_county_tester("bor", combined_flag=False) -# -# -# def test_unify_analytes_isc_seven_rivers(): -# _analyte_county_tester("isc-seven-rivers") -# -# -# def test_unify_analytes_dwb(): -# _analyte_county_tester("dwb", timeseries_flag=False) -# -# -# def test_unify_analytes_wqp_summary(): -# _analyte_summary_tester("wqp") -# -# -# def test_unify_analytes_bor_summary(): -# _analyte_summary_tester("bor") -# -# -# def test_unify_analytes_amp_summary(): -# _analyte_summary_tester("amp") -# -# -# def test_unify_analytes_dwb_summary(): -# _analyte_summary_tester("dwb") -# -# -# def test_unify_analytes_isc_seven_rivers_summary(): -# _analyte_summary_tester("isc-seven-rivers") - - -# ====== End Analyte Tests ======================================================= - - -# ====== Water Level Tests ======================================================= -# def _waterlevel_county_tester(source, **kw): -# _timeseries_tester(source, waterlevels, county="eddy", **kw) -# -# -# def _waterlevel_bbox_tester(source, **kw): -# _timeseries_tester(source, waterlevels, bbox="-104.5 32.5,-104 33", **kw) - -# -# def test_unify_waterlevels_nwis(): -# _waterlevel_county_tester("nwis", timeseries_flag=False) -# -# -# def test_unify_waterlevels_amp(): -# _waterlevel_county_tester("amp", timeseries_flag=False) -# -# -# def test_unify_waterlevels_st2(): -# _waterlevel_county_tester("st2", combined_flag=False) -# -# -# def test_unify_waterlevels_isc_seven_rivers(): -# _waterlevel_county_tester("isc-seven-rivers") -# -# -# def test_unify_waterlevels_ckan(): -# _waterlevel_county_tester("ckan") -# -# -# def test_unify_waterlevels_nwis_summary(): -# _summary_tester("nwis", waterlevels) -# -# -# def test_unify_waterlevels_amp_summary(): -# _summary_tester("amp", waterlevels) -# -# -# def test_unify_waterlevels_st2_summary(): -# _summary_tester("st2", waterlevels) -# -# -# def test_unify_waterlevels_isc_seven_rivers_summary(): -# _summary_tester("isc-seven-rivers", waterlevels) -# -# -# def test_unify_waterlevels_nwis_bbox(): -# _waterlevel_bbox_tester("nwis", timeseries_flag=False) -# -# -# def test_unify_waterlevels_amp_bbox(): -# _waterlevel_bbox_tester("amp") -# -# -# def test_unify_waterlevels_st2_bbox(): -# _waterlevel_bbox_tester("st2", combined_flag=False) -# -# -# def test_unify_waterlevels_isc_seven_rivers_bbox(): -# _waterlevel_bbox_tester("isc-seven-rivers", combined_flag=False) -# -# -# def test_unify_waterlevels_ckan_bbox(): -# _waterlevel_bbox_tester("ckan") - - -# ====== End Water Level Tests ======================================================= -# ============= EOF ============================================= diff --git a/tests/test_unifier.py b/tests/test_unifier.py deleted file mode 100644 index 3947ef6..0000000 --- a/tests/test_unifier.py +++ /dev/null @@ -1,473 +0,0 @@ -# =============================================================================== -# Copyright 2024 Jake Ross -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# =============================================================================== -import datetime -import os - -import pytest -import shapely.wkt - -from backend.config import Config -from backend.connectors.ckan import HONDO_RESOURCE_ID -from backend.unifier import unify_analytes, unify_waterlevels - - -def config_factory(): - cfg = Config() - cfg.county = "eddy" - cfg.bbox = "-104.5 32.5,-104 33" - cfg.start_date = "2020-01-01" - cfg.end_date = "2024-5-01" - cfg.output_summary = False - - cfg.use_source_nmbgmr = False - cfg.use_source_wqp = False - cfg.use_source_iscsevenrivers = False - cfg.use_source_nwis = False - cfg.use_source_oseroswell = False - cfg.use_source_pvacd = False - cfg.use_source_bor = False - cfg.use_source_dwb = False - cfg.use_source_bernco = False - - cfg.site_limit = 10 - return cfg - - -@pytest.fixture -def waterlevel_summary_cfg(): - cfg = config_factory() - cfg.output_summary = True - return cfg - - -@pytest.fixture -def waterlevel_timeseries_cfg(): - cfg = config_factory() - cfg.output_summary = False - return cfg - - -@pytest.fixture -def analyte_summary_cfg(): - cfg = config_factory() - cfg.output_summary = True - cfg.analyte = "TDS" - return cfg - - -# def test_unify_analytes(cfg): -# unify_analytes(cfg) - - -def _setup(tmp_path, cfg, source, tag): - d = tmp_path / tag - d.mkdir() - cfg.output_dir = str(d) - for stag in ( - "nmbgmr", - "nwis", - "pvacd", - "bor", - "dwb", - "wqp", - "iscsevenrivers", - "oseroswell", - "bernco", - ): - if stag == source: - setattr(cfg, f"use_source_{stag}", True) - return d - - -def _setup_waterlevels(tmp_path, cfg, source): - d = _setup(tmp_path, cfg, source, "waterlevels") - unify_waterlevels(cfg) - return d - - -def _setup_analytes(tmp_path, cfg, source): - d = _setup(tmp_path, cfg, source, "analyte") - unify_analytes(cfg) - return d - - -def _test_analytes_summary(tmp_path, cfg, source): - d = _setup_analytes(tmp_path, cfg, source) - assert (d / "output.csv").is_file() - - -def _test_waterlevels_summary(tmp_path, cfg, source): - d = _setup_waterlevels(tmp_path, cfg, source) - assert (d / "output.csv").is_file() - - -def _test_waterlevels_timeseries( - tmp_path, cfg, source, combined_flag=True, timeseries_flag=False -): - d = _setup_waterlevels(tmp_path, cfg, source) - combined = d / "output.combined.csv" - timeseries = d / "output_timeseries" - print(combined_flag) - - print("combined", combined.is_file(), combined_flag) - assert combined.is_file() == combined_flag - print("timeseries", timeseries.is_dir(), timeseries_flag) - assert timeseries.is_dir() == timeseries_flag - - return combined, timeseries - - -def _test_waterelevels_timeseries_date_range( - tmp_path, cfg, source, timeseries_flag=True, combined_flag=False -): - combined, timeseries = _test_waterlevels_timeseries( - tmp_path, - cfg, - source, - timeseries_flag=timeseries_flag, - combined_flag=combined_flag, - ) - - for p in timeseries.iterdir(): - if os.path.basename(p) == "sites.csv": - continue - - with open(p, "r") as rfile: - lines = rfile.readlines() - for l in lines[1:]: - vs = l.split(",") - dd = vs[3] - dd = datetime.datetime.strptime(dd, "%Y-%m-%d") - assert dd.year >= 2020 and dd.year <= 2024 - - -def test_nwis_site_health_check(): - from backend.connectors.usgs.source import NWISSiteSource - - n = NWISSiteSource() - assert n.health() - - -def test_nmbgmr_site_health_check(): - from backend.connectors.nmbgmr.source import NMBGMRSiteSource - - n = NMBGMRSiteSource() - assert n.health() - - -def test_wqp_site_health_check(): - from backend.connectors.wqp.source import WQPSiteSource - - n = WQPSiteSource() - assert n.health() - - -def test_bor_site_health_check(): - from backend.connectors.bor.source import BORSiteSource - - n = BORSiteSource() - assert n.health() - - -def test_dwb_site_health_check(): - from backend.connectors.nmenv.source import DWBSiteSource - - n = DWBSiteSource() - assert n.health() - - -def test_isc_seven_rivers_site_health_check(): - from backend.connectors.isc_seven_rivers.source import ISCSevenRiversSiteSource - - n = ISCSevenRiversSiteSource() - assert n.health() - - -def test_ckan_site_health_check(): - from backend.connectors.ckan.source import OSERoswellSiteSource - - n = OSERoswellSiteSource(HONDO_RESOURCE_ID) - assert n.health() - - -def test_pvacd_site_health_check(): - from backend.connectors.st2.source import PVACDSiteSource - - n = PVACDSiteSource() - assert n.health() - - -def test_bernco_site_health_check(): - from backend.connectors.st2.source import BernCoSiteSource - - n = BernCoSiteSource() - assert n.health() - - -# def test_ose_roswell_site_health_check(): -# from backend.connectors.ose_roswell.source import OSESiteSource -# n = OSESiteSource() -# assert n.health() - - -# Source tests ======================================================================================================== -def test_source_bounds_nmbgmr(): - from backend.unifier import get_source_bounds - from backend.connectors import NM_STATE_BOUNDING_POLYGON - - sourcekey = "nmbgmr" - bounds = get_source_bounds(sourcekey) - assert bounds - assert bounds.is_valid - assert bounds.geom_type == "Polygon" - assert bounds == NM_STATE_BOUNDING_POLYGON - - -def test_source_bounds_is_seven_rivers(): - from backend.unifier import get_source_bounds - from backend.connectors import ISC_SEVEN_RIVERS_BOUNDING_POLYGON - - sourcekey = "iscsevenrivers" - bounds = get_source_bounds(sourcekey) - assert bounds - assert bounds.is_valid - assert bounds.geom_type == "Polygon" - assert bounds == ISC_SEVEN_RIVERS_BOUNDING_POLYGON - - -def test_source_bounds_oser(): - from backend.unifier import get_source_bounds - from backend.connectors import ( - OSE_ROSWELL_HONDO_BOUNDING_POLYGON, - OSE_ROSWELL_ROSWELL_BOUNDING_POLYGON, - OSE_ROSWELL_FORT_SUMNER_BOUNDING_POLYGON, - ) - - sourcekey = "oseroswell" - bounds = get_source_bounds(sourcekey) - assert bounds - assert bounds.is_valid - assert bounds.geom_type == "GeometryCollection" - assert bounds == shapely.GeometryCollection( - [ - OSE_ROSWELL_HONDO_BOUNDING_POLYGON, - OSE_ROSWELL_FORT_SUMNER_BOUNDING_POLYGON, - OSE_ROSWELL_ROSWELL_BOUNDING_POLYGON, - ] - ) - - -def test_sources_socorro(tmp_path): - cfg = Config() - cfg.county = "socorro" - - from backend.unifier import get_sources - - sources = get_sources(cfg) - assert sources - assert len(sources) == 2 - assert sorted([s.__class__.__name__ for s in sources]) == sorted( - ["NMBGMRSiteSource", "NWISSiteSource"] - ) - - -def test_sources_eddy_dtw(tmp_path): - cfg = Config() - cfg.county = "eddy" - - from backend.unifier import get_sources - - sources = get_sources(cfg) - assert sources - assert len(sources) == 5 - assert sorted([s.__class__.__name__ for s in sources]) == sorted( - [ - "ISCSevenRiversSiteSource", - "NMBGMRSiteSource", - "OSERoswellSiteSource", - "PVACDSiteSource", - "NWISSiteSource", - ] - ) - - -def test_sources_eddy_tds(tmp_path): - cfg = Config() - cfg.county = "eddy" - cfg.analyte = "TDS" - - from backend.unifier import get_sources - - sources = get_sources(cfg) - assert sources - assert len(sources) == 5 - assert sorted([s.__class__.__name__ for s in sources]) == sorted( - [ - "BORSiteSource", - "DWBSiteSource", - "ISCSevenRiversSiteSource", - "NMBGMRSiteSource", - "WQPSiteSource", - ] - ) - - -# Waterlevel Summary tests =========================================================================================== -def test_unify_waterlevels_bernco_summary(tmp_path, waterlevel_summary_cfg): - waterlevel_summary_cfg.county = "bernalillo" - waterlevel_summary_cfg.bbox = None - _test_waterlevels_summary(tmp_path, waterlevel_summary_cfg, "bernco") - - -def test_unify_waterlevels_nwis_summary(tmp_path, waterlevel_summary_cfg): - _test_waterlevels_summary(tmp_path, waterlevel_summary_cfg, "nwis") - - -def test_unify_waterlevels_amp_summary(tmp_path, waterlevel_summary_cfg): - _test_waterlevels_summary(tmp_path, waterlevel_summary_cfg, "nmbgmr") - - -def test_unify_waterlevels_pvacd_summary(tmp_path, waterlevel_summary_cfg): - _test_waterlevels_summary(tmp_path, waterlevel_summary_cfg, "pvacd") - - -def test_unify_waterlevels_isc_seven_rivers_summary(tmp_path, waterlevel_summary_cfg): - _test_waterlevels_summary(tmp_path, waterlevel_summary_cfg, "iscsevenrivers") - - -def test_unify_waterlevels_ose_roswell_summary(tmp_path, waterlevel_summary_cfg): - _test_waterlevels_summary(tmp_path, waterlevel_summary_cfg, "oseroswell") - - -# Waterlevel timeseries tests ========================================================================================= -def test_unify_waterlevels_nwis_timeseries(tmp_path, waterlevel_timeseries_cfg): - # there are one or more locations within the bounding box that have only - # one record, so there is a combined file - _test_waterlevels_timeseries( - tmp_path, - waterlevel_timeseries_cfg, - "nwis", - combined_flag=True, - timeseries_flag=True, - ) - - -def test_unify_waterlevels_amp_timeseries(tmp_path, waterlevel_timeseries_cfg): - _test_waterlevels_timeseries(tmp_path, waterlevel_timeseries_cfg, "nmbgmr") - - -def test_unify_waterlevels_pvacd_timeseries(tmp_path, waterlevel_timeseries_cfg): - # all locations within the bounding box have more than one record - # so there is no combined file - _test_waterlevels_timeseries( - tmp_path, - waterlevel_timeseries_cfg, - "pvacd", - combined_flag=False, - timeseries_flag=True, - ) - - -def test_unify_waterlevels_isc_seven_rivers_timeseries( - tmp_path, waterlevel_timeseries_cfg -): - # all locations within the bounding box have more than one record - # so there is no combined file - _test_waterlevels_timeseries( - tmp_path, - waterlevel_timeseries_cfg, - "iscsevenrivers", - combined_flag=False, - timeseries_flag=True, - ) - - -def test_unify_waterlevels_ose_roswell_timeseries(tmp_path, waterlevel_timeseries_cfg): - _test_waterlevels_timeseries( - tmp_path, waterlevel_timeseries_cfg, "oseroswell", timeseries_flag=True - ) - - -# Waterlevel summary date range tests ================================================================================= -def test_waterlevels_nwis_summary_date_range(tmp_path, waterlevel_summary_cfg): - d = _setup_waterlevels(tmp_path, waterlevel_summary_cfg, "nwis") - assert (d / "output.csv").is_file() - - -# Waterlevel timeseries date range ==================================================================================== -def test_waterlevels_nwis_timeseries_date_range(tmp_path, waterlevel_timeseries_cfg): - # there are one or more locations within the bounding box and date range - # that have only one record, so there is a combined file - _test_waterelevels_timeseries_date_range( - tmp_path, - waterlevel_timeseries_cfg, - "nwis", - timeseries_flag=True, - combined_flag=True, - ) - - -def test_waterlevels_isc_seven_rivers_timeseries_date_range( - tmp_path, waterlevel_timeseries_cfg -): - # all locations within the bounding box and date rangehave more than one - # record so there is no combined file - _test_waterelevels_timeseries_date_range( - tmp_path, - waterlevel_timeseries_cfg, - "iscsevenrivers", - timeseries_flag=True, - combined_flag=False, - ) - - -def test_waterlevels_pvacd_timeseries_date_range(tmp_path, waterlevel_timeseries_cfg): - # all locations within the bounding box and date rangehave more than one - # record so there is no combined file - _test_waterelevels_timeseries_date_range( - tmp_path, - waterlevel_timeseries_cfg, - "pvacd", - timeseries_flag=True, - combined_flag=False, - ) - - -# Analyte summary tests =============================================================================================== -def test_unify_analytes_wqp_summary(tmp_path, analyte_summary_cfg): - _test_analytes_summary(tmp_path, analyte_summary_cfg, "wqp") - - -def test_unify_analytes_amp_summary(tmp_path, analyte_summary_cfg): - _test_analytes_summary(tmp_path, analyte_summary_cfg, "nmbgmr") - - -def test_unify_analytes_bor_summary(tmp_path, analyte_summary_cfg): - # BOR locations are found within Otero County - analyte_summary_cfg.county = "otero" - analyte_summary_cfg.bbox = None - _test_analytes_summary(tmp_path, analyte_summary_cfg, "bor") - - -def test_unify_analytes_isc_seven_rivers_summary(tmp_path, analyte_summary_cfg): - _test_analytes_summary(tmp_path, analyte_summary_cfg, "iscsevenrivers") - - -def test_unify_analytes_dwb_summary(tmp_path, analyte_summary_cfg): - _test_analytes_summary(tmp_path, analyte_summary_cfg, "dwb") - - -# ============= EOF ============================================= From 4f894a01007f4d0f01fe500129a457e72c86a336 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Fri, 4 Apr 2025 11:34:07 -0600 Subject: [PATCH 16/55] Setup BaseTestClass for reusability for all sources and tests --- tests/__init__.py | 184 +++++++++++++++++++++++++++++++--------------- 1 file changed, 123 insertions(+), 61 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index b0c0bdd..b1868e7 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,10 +1,27 @@ +from logging import shutdown as logger_shutdown from pathlib import Path import pytest -from backend.config import Config, SOURCE_KEYS, get_source +from backend.config import Config, SOURCE_KEYS from backend.constants import WATERLEVELS +from backend.logger import setup_logging +from backend.record import SummaryRecord, SiteRecord, ParameterRecord from backend.unifier import unify_analytes, unify_waterlevels +SUMMARY_RECORD_HEADERS = list(SummaryRecord.keys) +SITE_RECORD_HEADERS = list(SiteRecord.keys) +PARAMETER_RECORD_HEADERS = list(ParameterRecord.keys) + + +def recursively_clean_directory(path): + """Recursively delete all files and directories in the given path.""" + for item in path.iterdir(): + if item.is_dir(): + recursively_clean_directory(item) + else: + item.unlink() + path.rmdir() + class BaseTestClass: @@ -13,56 +30,88 @@ class BaseTestClass: agency = None # set set_limit for tests - site_limit = 8 + site_limit = 6 @pytest.fixture(autouse=True) def setup(self): - # Setup code + # SETUP CODE ---------------------------------------------------------- + # 1: setup test/config attributes self.config = Config() - for agency in SOURCE_KEYS: setattr(self.config, f"use_source_{agency}", False) - setattr(self.config, "site_limit", self.site_limit) setattr(self.config, "parameter", self.parameter) setattr(self.config, "units", self.units) setattr(self.config, f"use_source_{self.agency}", True) - self.config.finalize() - # run test + # 2: initiate logger + setup_logging(path=self.config.output_path) + + # RUN TESTS ------------------------------------------------------------ yield - # Teardown code + # UNIVERSAL ASSERTIONS ------------------------------------------------- + # 1: log file exists + log_path = Path(self.config.output_path) / "die.log" + assert log_path.exists() + + + # TEARDOWN CODE -------------------------------------------------------- + # 1: close logger to delete log file + logger_shutdown() + + # 2: delete newly created dirs and files path_to_clean = Path(self.config.output_path) print(f"Cleaning and removing {path_to_clean}") - for f in Path(path_to_clean).iterdir(): - f.unlink() - path_to_clean.rmdir() + recursively_clean_directory(path_to_clean) + + # reset test attributes self.dirs_to_delete = [] self.config = None self.unifier = None - def _unify(self): - self.unifier(self.config) + def _run_unifier(self): + if self.parameter == WATERLEVELS: + unify_waterlevels(self.config) + else: + unify_analytes(self.config) + + def _check_sites_file(self): + sites_file = Path(self.config.output_path) / "sites.csv" + assert sites_file.exists() + + with open(sites_file, "r") as f: + headers = f.readline().strip().split(",") + assert headers == SITE_RECORD_HEADERS + + # +1 for the header + with open(sites_file, "r") as f: + lines = f.readlines() + assert len(lines) == self.site_limit + 1 - def _test_health(self): + def _check_timeseries_file(self, timeseries_dir, timeseries_file_name): + timeseries_file = Path(timeseries_dir) / timeseries_file_name + assert timeseries_file.exists() + + with open(timeseries_file, "r") as f: + headers = f.readline().strip().split(",") + assert headers == PARAMETER_RECORD_HEADERS + + def test_health(self): # do a health check for the agency source = self.config.all_site_sources()[0][0] assert source.health() - def _test_summary(self): - # Arrange + def test_summary(self): + # Arrange -------------------------------------------------------------- self.config.output_summary = True self.config.report() - # Act - if self.parameter == WATERLEVELS: - unify_waterlevels(self.config) - else: - unify_analytes(self.config) + # Act ------------------------------------------------------------------ + self._run_unifier() - # Assert + # Assert --------------------------------------------------------------- # Check the summary file summary_file = Path(self.config.output_path) / "summary.csv" assert summary_file.exists() @@ -70,53 +119,66 @@ def _test_summary(self): # Check the column headers with open(summary_file, "r") as f: headers = f.readline().strip().split(",") - expected_headers = [ - "source", - "id", - "name", - "usgs_site_id", - "alternate_site_id", - "latitude", - "longitude", - "horizontal_datum", - "elevation", - "elevation_units", - "well_depth", - "well_depth_units", - "parameter_name", - "parameter_units", - "nrecords", - "min", - "max", - "mean", - "earliest_date", - "earliest_time", - "earliest_value", - "earliest_units", - "latest_date", - "latest_time", - "latest_value", - "latest_units", - ] - assert headers == expected_headers - - def _test_timeseries_unified(self): - pass + assert headers == SUMMARY_RECORD_HEADERS - def _test_timeseries_separated(self): - pass + # +1 for the header + with open(summary_file, "r") as f: + lines = f.readlines() + assert len(lines) == self.site_limit + 1 + + + def test_timeseries_unified(self): + # Arrange -------------------------------------------------------------- + self.config.output_timeseries_unified = True + self.config.report() + + # Act ------------------------------------------------------------------ + self._run_unifier() + + # Assert --------------------------------------------------------------- + # Check the sites file + self._check_sites_file() + + # Check the timeseries file + timeseries_dir = Path(self.config.output_path) + timeseries_file_name = "timeseries_unified.csv" + self._check_timeseries_file(timeseries_dir, timeseries_file_name) + + def test_timeseries_separated(self): + # Arrange -------------------------------------------------------------- + self.config.output_timeseries_separated = True + self.config.report() + + # Act ------------------------------------------------------------------ + self._run_unifier() + + # Assert --------------------------------------------------------------- + # Check the sites file + self._check_sites_file() + + # Check the timeseries files + timeseries_dir = Path(self.config.output_path) / "timeseries" + assert len([f for f in timeseries_dir.iterdir()]) == self.site_limit + + for timeseries_file in timeseries_dir.iterdir(): + self._check_timeseries_file(timeseries_dir, timeseries_file.name) - def _test_date_range(self): + @pytest.mark.skip(reason="Not implemented yet") + def test_date_range(self): pass - def _test_wkt(self): + @pytest.mark.skip(reason="Not implemented yet") + def test_wkt(self): pass - def _test_county(self): + @pytest.mark.skip(reason="Not implemented yet") + def test_county(self): pass - def _test_huc(self): + @pytest.mark.skip(reason="Not implemented yet") + def test_huc(self): pass - def _text_bbox(self): + @pytest.mark.skip(reason="Not implemented yet") + def text_bbox(self): pass From 49b5e9cf11462b88bd5ab8dbb041ef40561497c0 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Fri, 4 Apr 2025 11:34:26 -0600 Subject: [PATCH 17/55] test NMBGMR --- tests/source_tests/test_nmbgmr.py | 34 +++++-------------------------- 1 file changed, 5 insertions(+), 29 deletions(-) diff --git a/tests/source_tests/test_nmbgmr.py b/tests/source_tests/test_nmbgmr.py index 051c7da..b16c54f 100644 --- a/tests/source_tests/test_nmbgmr.py +++ b/tests/source_tests/test_nmbgmr.py @@ -1,4 +1,4 @@ -from backend.constants import WATERLEVELS +from backend.constants import WATERLEVELS, CALCIUM from tests import BaseTestClass import pytest @@ -10,32 +10,8 @@ class TestNMBGMRWaterlevels(BaseTestClass): units = "ft" agency = "nmbgmr_amp" - def test_health(self): - self._test_health() +class TestNMBGMRAnalyte(BaseTestClass): - def test_summary(self): - self._test_summary() - - @pytest.mark.skip(reason="Not implemented yet") - def test_timeseries_unified(self): - self._test_timeseries_unified() - - @pytest.mark.skip(reason="Not implemented yet") - def test_timeseries_separated(self): - self._test_timeseries_separated() - - @pytest.mark.skip(reason="Not implemented yet") - def test_date_range(self): - self._test_date_range() - - @pytest.mark.skip(reason="Not implemented yet") - def test_wkt(self): - self._test_wkt() - - @pytest.mark.skip(reason="Not implemented yet") - def test_county(self): - self._test_county() - - @pytest.mark.skip(reason="Not implemented yet") - def test_huc(self): - self._test_huc() + parameter = CALCIUM + units = "mg/l" + agency = "nmbgmr_amp" \ No newline at end of file From a94c102988674d12d7be020cd6f30fc8829c82da Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Fri, 4 Apr 2025 12:03:19 -0600 Subject: [PATCH 18/55] Fix NMED DWB health check --- backend/connectors/nmenv/source.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/backend/connectors/nmenv/source.py b/backend/connectors/nmenv/source.py index ae62c6b..2ba9694 100644 --- a/backend/connectors/nmenv/source.py +++ b/backend/connectors/nmenv/source.py @@ -27,8 +27,7 @@ DT_MEASURED, SOURCE_PARAMETER_NAME, SOURCE_PARAMETER_UNITS, - EARLIEST, - LATEST, + TDS ) from backend.source import get_analyte_search_param, get_terminal_record @@ -46,7 +45,7 @@ def __repr__(self): return "DWBSiteSource" def health(self): - return self.get_records(top=10, analyte="TDS") + return self.get_records(top=10, analyte=tds) def get_records(self, *args, **kw): From 0451649d3ee739621c035fb79ce6d8a9949817b3 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Fri, 4 Apr 2025 12:03:48 -0600 Subject: [PATCH 19/55] Fix my fix --- backend/connectors/nmenv/source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/connectors/nmenv/source.py b/backend/connectors/nmenv/source.py index 2ba9694..4d99d7a 100644 --- a/backend/connectors/nmenv/source.py +++ b/backend/connectors/nmenv/source.py @@ -45,7 +45,7 @@ def __repr__(self): return "DWBSiteSource" def health(self): - return self.get_records(top=10, analyte=tds) + return self.get_records(top=10, analyte=TDS) def get_records(self, *args, **kw): From 17812616e8b2158e12788e860b87bf9bd8d639f0 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Fri, 4 Apr 2025 12:03:57 -0600 Subject: [PATCH 20/55] Initial setup for source tests --- tests/source_tests/test_bernco.py | 9 +++++++++ tests/source_tests/test_bor.py | 8 ++++++++ tests/source_tests/test_cabq.py | 9 +++++++++ tests/source_tests/test_ebid.py | 9 +++++++++ tests/source_tests/test_nmbgmr.py | 8 +++----- tests/source_tests/test_nmed_dwb.py | 9 +++++++++ tests/source_tests/test_nmose_isc_seven_rivers.py | 15 +++++++++++++++ tests/source_tests/test_nmose_roswell.py | 9 +++++++++ tests/source_tests/test_nwis.py | 9 +++++++++ tests/source_tests/test_pvacd.py | 9 +++++++++ tests/source_tests/test_wqp.py | 15 +++++++++++++++ 11 files changed, 104 insertions(+), 5 deletions(-) create mode 100644 tests/source_tests/test_bernco.py create mode 100644 tests/source_tests/test_bor.py create mode 100644 tests/source_tests/test_cabq.py create mode 100644 tests/source_tests/test_ebid.py create mode 100644 tests/source_tests/test_nmed_dwb.py create mode 100644 tests/source_tests/test_nmose_isc_seven_rivers.py create mode 100644 tests/source_tests/test_nmose_roswell.py create mode 100644 tests/source_tests/test_nwis.py create mode 100644 tests/source_tests/test_pvacd.py create mode 100644 tests/source_tests/test_wqp.py diff --git a/tests/source_tests/test_bernco.py b/tests/source_tests/test_bernco.py new file mode 100644 index 0000000..f9306a4 --- /dev/null +++ b/tests/source_tests/test_bernco.py @@ -0,0 +1,9 @@ +from backend.constants import WATERLEVELS, FEET +from tests import BaseTestClass + + +class TestBernCoWaterlevels(BaseTestClass): + + parameter = WATERLEVELS + units = FEET + agency = "bernco" \ No newline at end of file diff --git a/tests/source_tests/test_bor.py b/tests/source_tests/test_bor.py new file mode 100644 index 0000000..089e831 --- /dev/null +++ b/tests/source_tests/test_bor.py @@ -0,0 +1,8 @@ +from backend.constants import CALCIUM, MILLIGRAMS_PER_LITER +from tests import BaseTestClass + +class TestBoRAnalyte(BaseTestClass): + + parameter = CALCIUM + units = MILLIGRAMS_PER_LITER + agency = "bor" \ No newline at end of file diff --git a/tests/source_tests/test_cabq.py b/tests/source_tests/test_cabq.py new file mode 100644 index 0000000..ae16ad0 --- /dev/null +++ b/tests/source_tests/test_cabq.py @@ -0,0 +1,9 @@ +from backend.constants import WATERLEVELS, FEET +from tests import BaseTestClass + + +class TestCABQWaterlevels(BaseTestClass): + + parameter = WATERLEVELS + units = FEET + agency = "cabq" \ No newline at end of file diff --git a/tests/source_tests/test_ebid.py b/tests/source_tests/test_ebid.py new file mode 100644 index 0000000..6a8bdd5 --- /dev/null +++ b/tests/source_tests/test_ebid.py @@ -0,0 +1,9 @@ +from backend.constants import WATERLEVELS, FEET +from tests import BaseTestClass + + +class TestEBIDWaterlevels(BaseTestClass): + + parameter = WATERLEVELS + units = FEET + agency = "ebid" \ No newline at end of file diff --git a/tests/source_tests/test_nmbgmr.py b/tests/source_tests/test_nmbgmr.py index b16c54f..b8a2cfb 100644 --- a/tests/source_tests/test_nmbgmr.py +++ b/tests/source_tests/test_nmbgmr.py @@ -1,17 +1,15 @@ -from backend.constants import WATERLEVELS, CALCIUM +from backend.constants import WATERLEVELS, CALCIUM, MILLIGRAMS_PER_LITER, FEET from tests import BaseTestClass -import pytest - class TestNMBGMRWaterlevels(BaseTestClass): parameter = WATERLEVELS - units = "ft" + units = FEET agency = "nmbgmr_amp" class TestNMBGMRAnalyte(BaseTestClass): parameter = CALCIUM - units = "mg/l" + units = MILLIGRAMS_PER_LITER agency = "nmbgmr_amp" \ No newline at end of file diff --git a/tests/source_tests/test_nmed_dwb.py b/tests/source_tests/test_nmed_dwb.py new file mode 100644 index 0000000..f50bef0 --- /dev/null +++ b/tests/source_tests/test_nmed_dwb.py @@ -0,0 +1,9 @@ +from backend.constants import CALCIUM, MILLIGRAMS_PER_LITER +from tests import BaseTestClass + + +class TestNMEDDWBAnalyte(BaseTestClass): + + parameter = CALCIUM + units = MILLIGRAMS_PER_LITER + agency = "nmed_dwb" \ No newline at end of file diff --git a/tests/source_tests/test_nmose_isc_seven_rivers.py b/tests/source_tests/test_nmose_isc_seven_rivers.py new file mode 100644 index 0000000..738d901 --- /dev/null +++ b/tests/source_tests/test_nmose_isc_seven_rivers.py @@ -0,0 +1,15 @@ +from backend.constants import WATERLEVELS, CALCIUM, FEET, MILLIGRAMS_PER_LITER +from tests import BaseTestClass + + +class TestNMOSEISCSevenRiversWaterlevels(BaseTestClass): + + parameter = WATERLEVELS + units = FEET + agency = "nmose_isc_seven_rivers" + +class TestNMOSEISCSevenRiversAnalyte(BaseTestClass): + + parameter = CALCIUM + units = MILLIGRAMS_PER_LITER + agency = "nmose_isc_seven_rivers" \ No newline at end of file diff --git a/tests/source_tests/test_nmose_roswell.py b/tests/source_tests/test_nmose_roswell.py new file mode 100644 index 0000000..ee82b4f --- /dev/null +++ b/tests/source_tests/test_nmose_roswell.py @@ -0,0 +1,9 @@ +from backend.constants import WATERLEVELS, FEET +from tests import BaseTestClass + + +class TestNMOSERoswellWaterlevels(BaseTestClass): + + parameter = WATERLEVELS + units = FEET + agency = "nmose_roswell" \ No newline at end of file diff --git a/tests/source_tests/test_nwis.py b/tests/source_tests/test_nwis.py new file mode 100644 index 0000000..f8c0903 --- /dev/null +++ b/tests/source_tests/test_nwis.py @@ -0,0 +1,9 @@ +from backend.constants import WATERLEVELS, FEET +from tests import BaseTestClass + + +class TestNWISWaterlevels(BaseTestClass): + + parameter = WATERLEVELS + units = FEET + agency = "nwis" \ No newline at end of file diff --git a/tests/source_tests/test_pvacd.py b/tests/source_tests/test_pvacd.py new file mode 100644 index 0000000..d9fc1da --- /dev/null +++ b/tests/source_tests/test_pvacd.py @@ -0,0 +1,9 @@ +from backend.constants import WATERLEVELS, FEET +from tests import BaseTestClass + + +class TestPVACDWaterlevels(BaseTestClass): + + parameter = WATERLEVELS + units = FEET + agency = "pvacd" \ No newline at end of file diff --git a/tests/source_tests/test_wqp.py b/tests/source_tests/test_wqp.py new file mode 100644 index 0000000..bcf5695 --- /dev/null +++ b/tests/source_tests/test_wqp.py @@ -0,0 +1,15 @@ +from backend.constants import WATERLEVELS, CALCIUM, MILLIGRAMS_PER_LITER, FEET +from tests import BaseTestClass + + +class TestWQPWaterlevels(BaseTestClass): + + parameter = WATERLEVELS + units = FEET + agency = "wqp" + +class TestWQPAnalyte(BaseTestClass): + + parameter = CALCIUM + units = MILLIGRAMS_PER_LITER + agency = "wqp" \ No newline at end of file From ef9de1ac658675bbabe0a938a7d02452f695c0d9 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Fri, 4 Apr 2025 15:15:21 -0600 Subject: [PATCH 21/55] Fix site_limit trimming --- backend/unifier.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/backend/unifier.py b/backend/unifier.py index 82cd27f..e9e6468 100644 --- a/backend/unifier.py +++ b/backend/unifier.py @@ -161,13 +161,17 @@ def _site_wrapper(site_source, parameter_source, persister, config): f"removing {num_sites_to_remove} to avoid exceeding the site limit" ) + # if sites_with_records_count == sit_limit then num_sites_to_remove = 0 + # and calling list[:0] will retur an empty list, so subtract + # num_sites_to_remove from the length of the list + # to remove the last num_sites_to_remove sites if use_summarize: - persister.records = persister.records[:-num_sites_to_remove] + persister.records = persister.records[:len(persister.records)-num_sites_to_remove] else: persister.timeseries = persister.timeseries[ - :-num_sites_to_remove + :len(persister.timeseries)-num_sites_to_remove ] - persister.sites = persister.sites[:-num_sites_to_remove] + persister.sites = persister.sites[:len(persister.sites)-num_sites_to_remove] break if type(site_records) == list: From 4ad367e913ded4486c9d0911d1a23d526fdcd98d Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Fri, 4 Apr 2025 15:29:15 -0600 Subject: [PATCH 22/55] Trim for site_limit at end of for loop for when site_limit < chunk_size This needs to be done at the end of the for loop to catche situations where site_simit < chunk_size, otherwise the trimming won't occur --- backend/unifier.py | 63 +++++++++++++++++++++++----------------------- tests/__init__.py | 2 +- 2 files changed, 33 insertions(+), 32 deletions(-) diff --git a/backend/unifier.py b/backend/unifier.py index e9e6468..33f3773 100644 --- a/backend/unifier.py +++ b/backend/unifier.py @@ -143,37 +143,6 @@ def _site_wrapper(site_source, parameter_source, persister, config): persister.sites.extend(sites) else: for site_records in site_source.chunks(sites): - print( - "sites_with_records_count:", - sites_with_records_count, - "|", - "site_limit:", - site_limit, - "|", - "chunk_size:", - site_source.chunk_size, - ) - if site_limit: - if sites_with_records_count >= site_limit: - # remove any extra sites that were gathered. removes 0 if site_limit is not exceeded - num_sites_to_remove = sites_with_records_count - site_limit - print( - f"removing {num_sites_to_remove} to avoid exceeding the site limit" - ) - - # if sites_with_records_count == sit_limit then num_sites_to_remove = 0 - # and calling list[:0] will retur an empty list, so subtract - # num_sites_to_remove from the length of the list - # to remove the last num_sites_to_remove sites - if use_summarize: - persister.records = persister.records[:len(persister.records)-num_sites_to_remove] - else: - persister.timeseries = persister.timeseries[ - :len(persister.timeseries)-num_sites_to_remove - ] - persister.sites = persister.sites[:len(persister.sites)-num_sites_to_remove] - break - if type(site_records) == list: n = len(site_records) if first_flag: @@ -208,6 +177,38 @@ def _site_wrapper(site_source, parameter_source, persister, config): persister.timeseries.append((site, records)) persister.sites.append(site) + if site_limit: + print( + "sites_with_records_count:", + sites_with_records_count, + "|", + "site_limit:", + site_limit, + "|", + "chunk_size:", + site_source.chunk_size, + ) + + if sites_with_records_count >= site_limit: + # remove any extra sites that were gathered. removes 0 if site_limit is not exceeded + num_sites_to_remove = sites_with_records_count - site_limit + print( + f"removing {num_sites_to_remove} to avoid exceeding the site limit" + ) + + # if sites_with_records_count == sit_limit then num_sites_to_remove = 0 + # and calling list[:0] will retur an empty list, so subtract + # num_sites_to_remove from the length of the list + # to remove the last num_sites_to_remove sites + if use_summarize: + persister.records = persister.records[:len(persister.records)-num_sites_to_remove] + else: + persister.timeseries = persister.timeseries[ + :len(persister.timeseries)-num_sites_to_remove + ] + persister.sites = persister.sites[:len(persister.sites)-num_sites_to_remove] + break + except BaseException: import traceback diff --git a/tests/__init__.py b/tests/__init__.py index b1868e7..1861aef 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -30,7 +30,7 @@ class BaseTestClass: agency = None # set set_limit for tests - site_limit = 6 + site_limit = 3 @pytest.fixture(autouse=True) def setup(self): From 29b58630ad479829a17e3d83bc8245dcbd911d73 Mon Sep 17 00:00:00 2001 From: jacob-a-brown Date: Fri, 4 Apr 2025 21:31:37 +0000 Subject: [PATCH 23/55] Formatting changes --- backend/connectors/nmenv/source.py | 2 +- backend/unifier.py | 28 +++++++++++-------- tests/__init__.py | 10 +++---- tests/source_tests/test_bernco.py | 2 +- tests/source_tests/test_bor.py | 3 +- tests/source_tests/test_cabq.py | 2 +- tests/source_tests/test_ebid.py | 2 +- tests/source_tests/test_nmbgmr.py | 3 +- tests/source_tests/test_nmed_dwb.py | 2 +- .../test_nmose_isc_seven_rivers.py | 3 +- tests/source_tests/test_nmose_roswell.py | 2 +- tests/source_tests/test_nwis.py | 2 +- tests/source_tests/test_pvacd.py | 2 +- tests/source_tests/test_wqp.py | 3 +- 14 files changed, 36 insertions(+), 30 deletions(-) diff --git a/backend/connectors/nmenv/source.py b/backend/connectors/nmenv/source.py index 4d99d7a..08b1d68 100644 --- a/backend/connectors/nmenv/source.py +++ b/backend/connectors/nmenv/source.py @@ -27,7 +27,7 @@ DT_MEASURED, SOURCE_PARAMETER_NAME, SOURCE_PARAMETER_UNITS, - TDS + TDS, ) from backend.source import get_analyte_search_param, get_terminal_record diff --git a/backend/unifier.py b/backend/unifier.py index 33f3773..c6a8311 100644 --- a/backend/unifier.py +++ b/backend/unifier.py @@ -179,15 +179,15 @@ def _site_wrapper(site_source, parameter_source, persister, config): if site_limit: print( - "sites_with_records_count:", - sites_with_records_count, - "|", - "site_limit:", - site_limit, - "|", - "chunk_size:", - site_source.chunk_size, - ) + "sites_with_records_count:", + sites_with_records_count, + "|", + "site_limit:", + site_limit, + "|", + "chunk_size:", + site_source.chunk_size, + ) if sites_with_records_count >= site_limit: # remove any extra sites that were gathered. removes 0 if site_limit is not exceeded @@ -201,12 +201,16 @@ def _site_wrapper(site_source, parameter_source, persister, config): # num_sites_to_remove from the length of the list # to remove the last num_sites_to_remove sites if use_summarize: - persister.records = persister.records[:len(persister.records)-num_sites_to_remove] + persister.records = persister.records[ + : len(persister.records) - num_sites_to_remove + ] else: persister.timeseries = persister.timeseries[ - :len(persister.timeseries)-num_sites_to_remove + : len(persister.timeseries) - num_sites_to_remove + ] + persister.sites = persister.sites[ + : len(persister.sites) - num_sites_to_remove ] - persister.sites = persister.sites[:len(persister.sites)-num_sites_to_remove] break except BaseException: diff --git a/tests/__init__.py b/tests/__init__.py index 1861aef..52c810e 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -55,13 +55,12 @@ def setup(self): # 1: log file exists log_path = Path(self.config.output_path) / "die.log" assert log_path.exists() - - # TEARDOWN CODE -------------------------------------------------------- - # 1: close logger to delete log file + # TEARDOWN CODE -------------------------------------------------------- + # 1: close logger to delete log file logger_shutdown() - # 2: delete newly created dirs and files + # 2: delete newly created dirs and files path_to_clean = Path(self.config.output_path) print(f"Cleaning and removing {path_to_clean}") recursively_clean_directory(path_to_clean) @@ -125,7 +124,6 @@ def test_summary(self): with open(summary_file, "r") as f: lines = f.readlines() assert len(lines) == self.site_limit + 1 - def test_timeseries_unified(self): # Arrange -------------------------------------------------------------- @@ -138,7 +136,7 @@ def test_timeseries_unified(self): # Assert --------------------------------------------------------------- # Check the sites file self._check_sites_file() - + # Check the timeseries file timeseries_dir = Path(self.config.output_path) timeseries_file_name = "timeseries_unified.csv" diff --git a/tests/source_tests/test_bernco.py b/tests/source_tests/test_bernco.py index f9306a4..b100481 100644 --- a/tests/source_tests/test_bernco.py +++ b/tests/source_tests/test_bernco.py @@ -6,4 +6,4 @@ class TestBernCoWaterlevels(BaseTestClass): parameter = WATERLEVELS units = FEET - agency = "bernco" \ No newline at end of file + agency = "bernco" diff --git a/tests/source_tests/test_bor.py b/tests/source_tests/test_bor.py index 089e831..77bf325 100644 --- a/tests/source_tests/test_bor.py +++ b/tests/source_tests/test_bor.py @@ -1,8 +1,9 @@ from backend.constants import CALCIUM, MILLIGRAMS_PER_LITER from tests import BaseTestClass + class TestBoRAnalyte(BaseTestClass): parameter = CALCIUM units = MILLIGRAMS_PER_LITER - agency = "bor" \ No newline at end of file + agency = "bor" diff --git a/tests/source_tests/test_cabq.py b/tests/source_tests/test_cabq.py index ae16ad0..b430d3a 100644 --- a/tests/source_tests/test_cabq.py +++ b/tests/source_tests/test_cabq.py @@ -6,4 +6,4 @@ class TestCABQWaterlevels(BaseTestClass): parameter = WATERLEVELS units = FEET - agency = "cabq" \ No newline at end of file + agency = "cabq" diff --git a/tests/source_tests/test_ebid.py b/tests/source_tests/test_ebid.py index 6a8bdd5..fa69e00 100644 --- a/tests/source_tests/test_ebid.py +++ b/tests/source_tests/test_ebid.py @@ -6,4 +6,4 @@ class TestEBIDWaterlevels(BaseTestClass): parameter = WATERLEVELS units = FEET - agency = "ebid" \ No newline at end of file + agency = "ebid" diff --git a/tests/source_tests/test_nmbgmr.py b/tests/source_tests/test_nmbgmr.py index b8a2cfb..4643bf0 100644 --- a/tests/source_tests/test_nmbgmr.py +++ b/tests/source_tests/test_nmbgmr.py @@ -8,8 +8,9 @@ class TestNMBGMRWaterlevels(BaseTestClass): units = FEET agency = "nmbgmr_amp" + class TestNMBGMRAnalyte(BaseTestClass): parameter = CALCIUM units = MILLIGRAMS_PER_LITER - agency = "nmbgmr_amp" \ No newline at end of file + agency = "nmbgmr_amp" diff --git a/tests/source_tests/test_nmed_dwb.py b/tests/source_tests/test_nmed_dwb.py index f50bef0..fff8a6e 100644 --- a/tests/source_tests/test_nmed_dwb.py +++ b/tests/source_tests/test_nmed_dwb.py @@ -6,4 +6,4 @@ class TestNMEDDWBAnalyte(BaseTestClass): parameter = CALCIUM units = MILLIGRAMS_PER_LITER - agency = "nmed_dwb" \ No newline at end of file + agency = "nmed_dwb" diff --git a/tests/source_tests/test_nmose_isc_seven_rivers.py b/tests/source_tests/test_nmose_isc_seven_rivers.py index 738d901..a0a5d28 100644 --- a/tests/source_tests/test_nmose_isc_seven_rivers.py +++ b/tests/source_tests/test_nmose_isc_seven_rivers.py @@ -8,8 +8,9 @@ class TestNMOSEISCSevenRiversWaterlevels(BaseTestClass): units = FEET agency = "nmose_isc_seven_rivers" + class TestNMOSEISCSevenRiversAnalyte(BaseTestClass): parameter = CALCIUM units = MILLIGRAMS_PER_LITER - agency = "nmose_isc_seven_rivers" \ No newline at end of file + agency = "nmose_isc_seven_rivers" diff --git a/tests/source_tests/test_nmose_roswell.py b/tests/source_tests/test_nmose_roswell.py index ee82b4f..4c1bd6b 100644 --- a/tests/source_tests/test_nmose_roswell.py +++ b/tests/source_tests/test_nmose_roswell.py @@ -6,4 +6,4 @@ class TestNMOSERoswellWaterlevels(BaseTestClass): parameter = WATERLEVELS units = FEET - agency = "nmose_roswell" \ No newline at end of file + agency = "nmose_roswell" diff --git a/tests/source_tests/test_nwis.py b/tests/source_tests/test_nwis.py index f8c0903..493b801 100644 --- a/tests/source_tests/test_nwis.py +++ b/tests/source_tests/test_nwis.py @@ -6,4 +6,4 @@ class TestNWISWaterlevels(BaseTestClass): parameter = WATERLEVELS units = FEET - agency = "nwis" \ No newline at end of file + agency = "nwis" diff --git a/tests/source_tests/test_pvacd.py b/tests/source_tests/test_pvacd.py index d9fc1da..edf5d48 100644 --- a/tests/source_tests/test_pvacd.py +++ b/tests/source_tests/test_pvacd.py @@ -6,4 +6,4 @@ class TestPVACDWaterlevels(BaseTestClass): parameter = WATERLEVELS units = FEET - agency = "pvacd" \ No newline at end of file + agency = "pvacd" diff --git a/tests/source_tests/test_wqp.py b/tests/source_tests/test_wqp.py index bcf5695..49e61d9 100644 --- a/tests/source_tests/test_wqp.py +++ b/tests/source_tests/test_wqp.py @@ -8,8 +8,9 @@ class TestWQPWaterlevels(BaseTestClass): units = FEET agency = "wqp" + class TestWQPAnalyte(BaseTestClass): parameter = CALCIUM units = MILLIGRAMS_PER_LITER - agency = "wqp" \ No newline at end of file + agency = "wqp" From 1c9f86432a44eb3744f935a7429ebefbd245ca56 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Fri, 4 Apr 2025 15:45:55 -0600 Subject: [PATCH 24/55] Work on mypy linting from CICD --- backend/bounding_polygons.py | 2 +- backend/transformer.py | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/backend/bounding_polygons.py b/backend/bounding_polygons.py index 9ac50aa..8445077 100644 --- a/backend/bounding_polygons.py +++ b/backend/bounding_polygons.py @@ -162,7 +162,7 @@ def get_county_polygon(name, as_wkt=True): _warning(f"Invalid state. {state}") -def get_state_polygon(state, buffer): +def get_state_polygon(state: str, buffer: int | None =None): statefp = _statelookup(state) if statefp: obj = _get_cached_object( diff --git a/backend/transformer.py b/backend/transformer.py index 229fc0c..b993578 100644 --- a/backend/transformer.py +++ b/backend/transformer.py @@ -131,7 +131,7 @@ def convert_units( output_units: str, source_parameter_name: str, die_parameter_name: str, - dt: str = None, + dt: str | None = None, ) -> tuple[float, float, str]: """ Converts the following units for any parameter value: @@ -198,7 +198,7 @@ def convert_units( the source_parameter_name (e.g. nitrate as n). """ if die_parameter_name == "ph": - conversion_factor = 1 + conversion_factor = 1.0 elif output_units == mgl: if input_units in ["mg/l caco3", "mg/l caco3**"]: if die_parameter_name == "bicarbonate": @@ -210,7 +210,7 @@ def convert_units( elif input_units == "mg/l as n": conversion_factor = 4.427 elif input_units in ["mg/l asno3", "mg/l as no3"]: - conversion_factor = 1 + conversion_factor = 1.0 elif input_units == "ug/l as n": conversion_factor = 0.004427 elif input_units == "pci/l": @@ -220,22 +220,22 @@ def convert_units( elif input_units == tpaf: conversion_factor = 735.47 elif input_units == ppm: - conversion_factor = 1 + conversion_factor = 1.0 elif input_units == output_units: if source_parameter_name in ["nitrate as n", "nitrate (as n)"]: conversion_factor = 4.427 else: - conversion_factor = 1 + conversion_factor = 1.0 elif output_units == ft: if input_units in [m, "meters"]: conversion_factor = 3.28084 elif input_units in [ft, "feet"]: - conversion_factor = 1 + conversion_factor = 1.0 elif output_units == m: if input_units in [ft, "feet"]: conversion_factor = 0.3048 elif input_units in [m, "meters"]: - conversion_factor = 1 + conversion_factor = 1.0 if conversion_factor: return input_value * conversion_factor, conversion_factor, warning @@ -395,7 +395,7 @@ def do_transform( # _transform is already implemented in each ParameterTransformer record = self._transform(inrecord, *args, **kw) if not record: - return + return None # ensure that a site or summary record is contained within the boundaing polygon if "longitude" in record and "latitude" in record: @@ -403,7 +403,7 @@ def do_transform( self.warn( f"Skipping site {record['id']}. It is not within the defined geographic bounds" ) - return + return None self._post_transform(record, *args, **kw) From 2f4432caecc872d8e912410d66fe9f3b8e8d7967 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Fri, 4 Apr 2025 15:54:34 -0600 Subject: [PATCH 25/55] Work on mypy changes --- backend/source.py | 6 +- tests/archived/test_cli.py | 274 ++++++------ tests/archived/test_unifier.py | 758 ++++++++++++++++----------------- 3 files changed, 519 insertions(+), 519 deletions(-) diff --git a/backend/source.py b/backend/source.py index acf47d9..5ca8c9b 100644 --- a/backend/source.py +++ b/backend/source.py @@ -19,7 +19,7 @@ import httpx import shapely.wkt from shapely import MultiPoint -from typing import Union, List +from typing import Union, List, Callable from backend.constants import ( MILLIGRAMS_PER_LITER, @@ -47,7 +47,7 @@ from backend.transformer import BaseTransformer, convert_units -def make_site_list(site_record: list | dict) -> list | str: +def make_site_list(site_record: list[SiteRecord] | SiteRecord) -> list | str: """ Returns a list of site ids, as defined by site_record @@ -67,7 +67,7 @@ def make_site_list(site_record: list | dict) -> list | str: return sites -def get_terminal_record(records: list, tag: Union[str, callable], bookend: str) -> dict: +def get_terminal_record(records: list, tag: Union[str, Callable], bookend: str) -> dict: """ Returns the most recent record based on the tag diff --git a/tests/archived/test_cli.py b/tests/archived/test_cli.py index 3d65365..ba8240e 100644 --- a/tests/archived/test_cli.py +++ b/tests/archived/test_cli.py @@ -13,209 +13,209 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== -import os +# import os -from click.testing import CliRunner -from frontend.cli import analytes, waterlevels +# from click.testing import CliRunner +# from frontend.cli import analytes, waterlevels -def _tester(function, args, fail=False): - runner = CliRunner() - print(f"invoked with {args}") - result = runner.invoke(function, args) - print(f"result.exit_code={result.exit_code}") - print(f"result.output=\n{result.output}") +# def _tester(function, args, fail=False): +# runner = CliRunner() +# print(f"invoked with {args}") +# result = runner.invoke(function, args) +# print(f"result.exit_code={result.exit_code}") +# print(f"result.output=\n{result.output}") - if fail: - assert result.exit_code != 0 - else: - assert result.exit_code == 0 +# if fail: +# assert result.exit_code != 0 +# else: +# assert result.exit_code == 0 -def _make_args(source): - args = [] - if source: - nosources = [ - f - for f in ( - "--no-amp", - "--no-nwis", - "--no-pvacd", - "--no-bor", - "--no-dwb", - "--no-wqp", - "--no-isc-seven-rivers", - "--no-ckan", - ) - if f != f"--no-{source}" - ] - args += nosources +# def _make_args(source): +# args = [] +# if source: +# nosources = [ +# f +# for f in ( +# "--no-amp", +# "--no-nwis", +# "--no-pvacd", +# "--no-bor", +# "--no-dwb", +# "--no-wqp", +# "--no-isc-seven-rivers", +# "--no-ckan", +# ) +# if f != f"--no-{source}" +# ] +# args += nosources - args += ["--site-limit", 10, "--dry"] +# args += ["--site-limit", 10, "--dry"] - return args +# return args -def _make_tds_args(source): - return ["TDS"] + _make_args(source) +# def _make_tds_args(source): +# return ["TDS"] + _make_args(source) -def _make_wl_args(source=None): - return _make_args(source) +# def _make_wl_args(source=None): +# return _make_args(source) -def test_waterlevels_nwis(): - args = _make_wl_args("nwis") - _tester(waterlevels, args) +# def test_waterlevels_nwis(): +# args = _make_wl_args("nwis") +# _tester(waterlevels, args) -def test_waterlevels_pvacd(): - args = _make_wl_args("pvacd") - _tester(waterlevels, args) +# def test_waterlevels_pvacd(): +# args = _make_wl_args("pvacd") +# _tester(waterlevels, args) -def test_waterlevels_nmbgmr(): - args = _make_wl_args("nmbgmr") - _tester(waterlevels, args) +# def test_waterlevels_nmbgmr(): +# args = _make_wl_args("nmbgmr") +# _tester(waterlevels, args) -def test_waterlevels_isc_seven_rivers(): - args = _make_wl_args("iscsevenrivers") - _tester(waterlevels, args) +# def test_waterlevels_isc_seven_rivers(): +# args = _make_wl_args("iscsevenrivers") +# _tester(waterlevels, args) -def test_waterlevels_invalid_source(): - args = _make_wl_args() - args.append("--no-foo") - _tester(waterlevels, args, fail=True) +# def test_waterlevels_invalid_source(): +# args = _make_wl_args() +# args.append("--no-foo") +# _tester(waterlevels, args, fail=True) -def test_waterlevels_invalid_bbox(): - args = _make_wl_args() - args.append("--bbox") - _tester(waterlevels, args, fail=True) +# def test_waterlevels_invalid_bbox(): +# args = _make_wl_args() +# args.append("--bbox") +# _tester(waterlevels, args, fail=True) -def test_waterlevels_invalid_bbox_format(): - args = _make_wl_args() - args.extend(["--bbox", "1 2 3"]) - _tester(waterlevels, args, fail=True) +# def test_waterlevels_invalid_bbox_format(): +# args = _make_wl_args() +# args.extend(["--bbox", "1 2 3"]) +# _tester(waterlevels, args, fail=True) -def test_waterlevels_valid_bbox_format(): - args = _make_wl_args() - args.extend(["--bbox", "1 2,3 4"]) - _tester(waterlevels, args) +# def test_waterlevels_valid_bbox_format(): +# args = _make_wl_args() +# args.extend(["--bbox", "1 2,3 4"]) +# _tester(waterlevels, args) -def test_waterlevels_invalid_county(): - args = _make_wl_args() - args.append("--county") - _tester(waterlevels, args, fail=True) +# def test_waterlevels_invalid_county(): +# args = _make_wl_args() +# args.append("--county") +# _tester(waterlevels, args, fail=True) -def test_waterlevels_invalid_county_name(): - args = _make_wl_args() - args.extend(["--county", "foo"]) - _tester(waterlevels, args, fail=True) +# def test_waterlevels_invalid_county_name(): +# args = _make_wl_args() +# args.extend(["--county", "foo"]) +# _tester(waterlevels, args, fail=True) -# Analyte Tests ======================================================= -def test_analytes_wqp(): - args = _make_tds_args("wqp") - _tester(analytes, args) +# # Analyte Tests ======================================================= +# def test_analytes_wqp(): +# args = _make_tds_args("wqp") +# _tester(analytes, args) -def test_analytes_bor(): - args = _make_tds_args("bor") - _tester(analytes, args) +# def test_analytes_bor(): +# args = _make_tds_args("bor") +# _tester(analytes, args) -def test_analytes_amp(): - args = _make_tds_args("amp") - _tester(analytes, args) +# def test_analytes_amp(): +# args = _make_tds_args("amp") +# _tester(analytes, args) -def test_analytes_dwb(): - args = _make_tds_args("dwb") - _tester(analytes, args) +# def test_analytes_dwb(): +# args = _make_tds_args("dwb") +# _tester(analytes, args) -def test_analytes_isc_seven_rivers(): - args = _make_tds_args("isc-seven-rivers") - _tester(analytes, args) +# def test_analytes_isc_seven_rivers(): +# args = _make_tds_args("isc-seven-rivers") +# _tester(analytes, args) -def test_analytes_invalid_analyte(): - args = _make_args("wqp") - args[0] = "Foo" - _tester(analytes, args, fail=True) +# def test_analytes_invalid_analyte(): +# args = _make_args("wqp") +# args[0] = "Foo" +# _tester(analytes, args, fail=True) -def test_analytes_invalid_source(): - args = _make_tds_args("wqp") - args.append("--no-foo") - _tester(analytes, args, fail=True) +# def test_analytes_invalid_source(): +# args = _make_tds_args("wqp") +# args.append("--no-foo") +# _tester(analytes, args, fail=True) -def test_analytes_invalid_bbox(): - args = _make_tds_args("wqp") - args.append("--bbox") - _tester(analytes, args, fail=True) +# def test_analytes_invalid_bbox(): +# args = _make_tds_args("wqp") +# args.append("--bbox") +# _tester(analytes, args, fail=True) -def test_analytes_invalid_bbox_format(): - args = _make_tds_args("wqp") - args.extend(["--bbox", "1 2 3"]) - _tester(analytes, args, fail=True) +# def test_analytes_invalid_bbox_format(): +# args = _make_tds_args("wqp") +# args.extend(["--bbox", "1 2 3"]) +# _tester(analytes, args, fail=True) -def test_analytes_valid_bbox_format(): - args = _make_tds_args("wqp") - args.extend(["--bbox", "1 2,3 4"]) - _tester(analytes, args) +# def test_analytes_valid_bbox_format(): +# args = _make_tds_args("wqp") +# args.extend(["--bbox", "1 2,3 4"]) +# _tester(analytes, args) -def test_analytes_invalid_county(): - args = _make_tds_args("wqp") - args.append("--county") - _tester(analytes, args, fail=True) +# def test_analytes_invalid_county(): +# args = _make_tds_args("wqp") +# args.append("--county") +# _tester(analytes, args, fail=True) -def test_analytes_invalid_county_name(): - args = _make_tds_args("wqp") - args.extend(["--county", "foo"]) - _tester(analytes, args, fail=True) +# def test_analytes_invalid_county_name(): +# args = _make_tds_args("wqp") +# args.extend(["--county", "foo"]) +# _tester(analytes, args, fail=True) -def test_waterlevels_date_range_YMD(): - args = _make_wl_args() - args.extend(["--start-date", "2020-01-01", "--end-date", "2020-05-01"]) - _tester(waterlevels, args) +# def test_waterlevels_date_range_YMD(): +# args = _make_wl_args() +# args.extend(["--start-date", "2020-01-01", "--end-date", "2020-05-01"]) +# _tester(waterlevels, args) -def test_waterlevels_date_range_YM(): - args = _make_wl_args() - args.extend(["--start-date", "2020-01", "--end-date", "2020-05"]) - _tester(waterlevels, args) +# def test_waterlevels_date_range_YM(): +# args = _make_wl_args() +# args.extend(["--start-date", "2020-01", "--end-date", "2020-05"]) +# _tester(waterlevels, args) -def test_waterlevels_date_range_Y(): - args = _make_wl_args() - args.extend(["--start-date", "2020", "--end-date", "2021"]) - _tester(waterlevels, args) +# def test_waterlevels_date_range_Y(): +# args = _make_wl_args() +# args.extend(["--start-date", "2020", "--end-date", "2021"]) +# _tester(waterlevels, args) -def test_waterlevels_invalid_start(): - args = _make_wl_args() - args.extend(["--start-date", "x-01-01", "--end-date", "2019-05-01"]) - _tester(waterlevels, args, fail=True) +# def test_waterlevels_invalid_start(): +# args = _make_wl_args() +# args.extend(["--start-date", "x-01-01", "--end-date", "2019-05-01"]) +# _tester(waterlevels, args, fail=True) -def test_waterlevels_invalid_end(): - args = _make_wl_args() - args.extend(["--start-date", "2020-01-01", "--end-date", "x-05-01"]) - _tester(waterlevels, args, fail=True) +# def test_waterlevels_invalid_end(): +# args = _make_wl_args() +# args.extend(["--start-date", "2020-01-01", "--end-date", "x-05-01"]) +# _tester(waterlevels, args, fail=True) # diff --git a/tests/archived/test_unifier.py b/tests/archived/test_unifier.py index 3947ef6..319c762 100644 --- a/tests/archived/test_unifier.py +++ b/tests/archived/test_unifier.py @@ -13,461 +13,461 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== -import datetime -import os - -import pytest -import shapely.wkt - -from backend.config import Config -from backend.connectors.ckan import HONDO_RESOURCE_ID -from backend.unifier import unify_analytes, unify_waterlevels - - -def config_factory(): - cfg = Config() - cfg.county = "eddy" - cfg.bbox = "-104.5 32.5,-104 33" - cfg.start_date = "2020-01-01" - cfg.end_date = "2024-5-01" - cfg.output_summary = False - - cfg.use_source_nmbgmr = False - cfg.use_source_wqp = False - cfg.use_source_iscsevenrivers = False - cfg.use_source_nwis = False - cfg.use_source_oseroswell = False - cfg.use_source_pvacd = False - cfg.use_source_bor = False - cfg.use_source_dwb = False - cfg.use_source_bernco = False - - cfg.site_limit = 10 - return cfg - - -@pytest.fixture -def waterlevel_summary_cfg(): - cfg = config_factory() - cfg.output_summary = True - return cfg - - -@pytest.fixture -def waterlevel_timeseries_cfg(): - cfg = config_factory() - cfg.output_summary = False - return cfg - - -@pytest.fixture -def analyte_summary_cfg(): - cfg = config_factory() - cfg.output_summary = True - cfg.analyte = "TDS" - return cfg - - -# def test_unify_analytes(cfg): +# import datetime +# import os + +# import pytest +# import shapely.wkt + +# from backend.config import Config +# from backend.connectors.ckan import HONDO_RESOURCE_ID +# from backend.unifier import unify_analytes, unify_waterlevels + + +# def config_factory(): +# cfg = Config() +# cfg.county = "eddy" +# cfg.bbox = "-104.5 32.5,-104 33" +# cfg.start_date = "2020-01-01" +# cfg.end_date = "2024-5-01" +# cfg.output_summary = False + +# cfg.use_source_nmbgmr = False +# cfg.use_source_wqp = False +# cfg.use_source_iscsevenrivers = False +# cfg.use_source_nwis = False +# cfg.use_source_oseroswell = False +# cfg.use_source_pvacd = False +# cfg.use_source_bor = False +# cfg.use_source_dwb = False +# cfg.use_source_bernco = False + +# cfg.site_limit = 10 +# return cfg + + +# @pytest.fixture +# def waterlevel_summary_cfg(): +# cfg = config_factory() +# cfg.output_summary = True +# return cfg + + +# @pytest.fixture +# def waterlevel_timeseries_cfg(): +# cfg = config_factory() +# cfg.output_summary = False +# return cfg + + +# @pytest.fixture +# def analyte_summary_cfg(): +# cfg = config_factory() +# cfg.output_summary = True +# cfg.analyte = "TDS" +# return cfg + + +# # def test_unify_analytes(cfg): +# # unify_analytes(cfg) + + +# def _setup(tmp_path, cfg, source, tag): +# d = tmp_path / tag +# d.mkdir() +# cfg.output_dir = str(d) +# for stag in ( +# "nmbgmr", +# "nwis", +# "pvacd", +# "bor", +# "dwb", +# "wqp", +# "iscsevenrivers", +# "oseroswell", +# "bernco", +# ): +# if stag == source: +# setattr(cfg, f"use_source_{stag}", True) +# return d + + +# def _setup_waterlevels(tmp_path, cfg, source): +# d = _setup(tmp_path, cfg, source, "waterlevels") +# unify_waterlevels(cfg) +# return d + + +# def _setup_analytes(tmp_path, cfg, source): +# d = _setup(tmp_path, cfg, source, "analyte") # unify_analytes(cfg) +# return d -def _setup(tmp_path, cfg, source, tag): - d = tmp_path / tag - d.mkdir() - cfg.output_dir = str(d) - for stag in ( - "nmbgmr", - "nwis", - "pvacd", - "bor", - "dwb", - "wqp", - "iscsevenrivers", - "oseroswell", - "bernco", - ): - if stag == source: - setattr(cfg, f"use_source_{stag}", True) - return d - - -def _setup_waterlevels(tmp_path, cfg, source): - d = _setup(tmp_path, cfg, source, "waterlevels") - unify_waterlevels(cfg) - return d +# def _test_analytes_summary(tmp_path, cfg, source): +# d = _setup_analytes(tmp_path, cfg, source) +# assert (d / "output.csv").is_file() -def _setup_analytes(tmp_path, cfg, source): - d = _setup(tmp_path, cfg, source, "analyte") - unify_analytes(cfg) - return d +# def _test_waterlevels_summary(tmp_path, cfg, source): +# d = _setup_waterlevels(tmp_path, cfg, source) +# assert (d / "output.csv").is_file() -def _test_analytes_summary(tmp_path, cfg, source): - d = _setup_analytes(tmp_path, cfg, source) - assert (d / "output.csv").is_file() +# def _test_waterlevels_timeseries( +# tmp_path, cfg, source, combined_flag=True, timeseries_flag=False +# ): +# d = _setup_waterlevels(tmp_path, cfg, source) +# combined = d / "output.combined.csv" +# timeseries = d / "output_timeseries" +# print(combined_flag) +# print("combined", combined.is_file(), combined_flag) +# assert combined.is_file() == combined_flag +# print("timeseries", timeseries.is_dir(), timeseries_flag) +# assert timeseries.is_dir() == timeseries_flag -def _test_waterlevels_summary(tmp_path, cfg, source): - d = _setup_waterlevels(tmp_path, cfg, source) - assert (d / "output.csv").is_file() +# return combined, timeseries -def _test_waterlevels_timeseries( - tmp_path, cfg, source, combined_flag=True, timeseries_flag=False -): - d = _setup_waterlevels(tmp_path, cfg, source) - combined = d / "output.combined.csv" - timeseries = d / "output_timeseries" - print(combined_flag) +# def _test_waterelevels_timeseries_date_range( +# tmp_path, cfg, source, timeseries_flag=True, combined_flag=False +# ): +# combined, timeseries = _test_waterlevels_timeseries( +# tmp_path, +# cfg, +# source, +# timeseries_flag=timeseries_flag, +# combined_flag=combined_flag, +# ) - print("combined", combined.is_file(), combined_flag) - assert combined.is_file() == combined_flag - print("timeseries", timeseries.is_dir(), timeseries_flag) - assert timeseries.is_dir() == timeseries_flag +# for p in timeseries.iterdir(): +# if os.path.basename(p) == "sites.csv": +# continue - return combined, timeseries +# with open(p, "r") as rfile: +# lines = rfile.readlines() +# for l in lines[1:]: +# vs = l.split(",") +# dd = vs[3] +# dd = datetime.datetime.strptime(dd, "%Y-%m-%d") +# assert dd.year >= 2020 and dd.year <= 2024 -def _test_waterelevels_timeseries_date_range( - tmp_path, cfg, source, timeseries_flag=True, combined_flag=False -): - combined, timeseries = _test_waterlevels_timeseries( - tmp_path, - cfg, - source, - timeseries_flag=timeseries_flag, - combined_flag=combined_flag, - ) +# def test_nwis_site_health_check(): +# from backend.connectors.usgs.source import NWISSiteSource - for p in timeseries.iterdir(): - if os.path.basename(p) == "sites.csv": - continue - - with open(p, "r") as rfile: - lines = rfile.readlines() - for l in lines[1:]: - vs = l.split(",") - dd = vs[3] - dd = datetime.datetime.strptime(dd, "%Y-%m-%d") - assert dd.year >= 2020 and dd.year <= 2024 - - -def test_nwis_site_health_check(): - from backend.connectors.usgs.source import NWISSiteSource - - n = NWISSiteSource() - assert n.health() - - -def test_nmbgmr_site_health_check(): - from backend.connectors.nmbgmr.source import NMBGMRSiteSource +# n = NWISSiteSource() +# assert n.health() - n = NMBGMRSiteSource() - assert n.health() +# def test_nmbgmr_site_health_check(): +# from backend.connectors.nmbgmr.source import NMBGMRSiteSource -def test_wqp_site_health_check(): - from backend.connectors.wqp.source import WQPSiteSource +# n = NMBGMRSiteSource() +# assert n.health() - n = WQPSiteSource() - assert n.health() +# def test_wqp_site_health_check(): +# from backend.connectors.wqp.source import WQPSiteSource -def test_bor_site_health_check(): - from backend.connectors.bor.source import BORSiteSource +# n = WQPSiteSource() +# assert n.health() - n = BORSiteSource() - assert n.health() +# def test_bor_site_health_check(): +# from backend.connectors.bor.source import BORSiteSource -def test_dwb_site_health_check(): - from backend.connectors.nmenv.source import DWBSiteSource +# n = BORSiteSource() +# assert n.health() - n = DWBSiteSource() - assert n.health() +# def test_dwb_site_health_check(): +# from backend.connectors.nmenv.source import DWBSiteSource -def test_isc_seven_rivers_site_health_check(): - from backend.connectors.isc_seven_rivers.source import ISCSevenRiversSiteSource +# n = DWBSiteSource() +# assert n.health() - n = ISCSevenRiversSiteSource() - assert n.health() +# def test_isc_seven_rivers_site_health_check(): +# from backend.connectors.isc_seven_rivers.source import ISCSevenRiversSiteSource -def test_ckan_site_health_check(): - from backend.connectors.ckan.source import OSERoswellSiteSource +# n = ISCSevenRiversSiteSource() +# assert n.health() - n = OSERoswellSiteSource(HONDO_RESOURCE_ID) - assert n.health() +# def test_ckan_site_health_check(): +# from backend.connectors.ckan.source import OSERoswellSiteSource -def test_pvacd_site_health_check(): - from backend.connectors.st2.source import PVACDSiteSource +# n = OSERoswellSiteSource(HONDO_RESOURCE_ID) +# assert n.health() - n = PVACDSiteSource() - assert n.health() +# def test_pvacd_site_health_check(): +# from backend.connectors.st2.source import PVACDSiteSource -def test_bernco_site_health_check(): - from backend.connectors.st2.source import BernCoSiteSource +# n = PVACDSiteSource() +# assert n.health() - n = BernCoSiteSource() - assert n.health() +# def test_bernco_site_health_check(): +# from backend.connectors.st2.source import BernCoSiteSource -# def test_ose_roswell_site_health_check(): -# from backend.connectors.ose_roswell.source import OSESiteSource -# n = OSESiteSource() +# n = BernCoSiteSource() # assert n.health() -# Source tests ======================================================================================================== -def test_source_bounds_nmbgmr(): - from backend.unifier import get_source_bounds - from backend.connectors import NM_STATE_BOUNDING_POLYGON - - sourcekey = "nmbgmr" - bounds = get_source_bounds(sourcekey) - assert bounds - assert bounds.is_valid - assert bounds.geom_type == "Polygon" - assert bounds == NM_STATE_BOUNDING_POLYGON - - -def test_source_bounds_is_seven_rivers(): - from backend.unifier import get_source_bounds - from backend.connectors import ISC_SEVEN_RIVERS_BOUNDING_POLYGON +# # def test_ose_roswell_site_health_check(): +# # from backend.connectors.ose_roswell.source import OSESiteSource +# # n = OSESiteSource() +# # assert n.health() + + +# # Source tests ======================================================================================================== +# def test_source_bounds_nmbgmr(): +# from backend.unifier import get_source_bounds +# from backend.connectors import NM_STATE_BOUNDING_POLYGON + +# sourcekey = "nmbgmr" +# bounds = get_source_bounds(sourcekey) +# assert bounds +# assert bounds.is_valid +# assert bounds.geom_type == "Polygon" +# assert bounds == NM_STATE_BOUNDING_POLYGON + + +# def test_source_bounds_is_seven_rivers(): +# from backend.unifier import get_source_bounds +# from backend.connectors import ISC_SEVEN_RIVERS_BOUNDING_POLYGON + +# sourcekey = "iscsevenrivers" +# bounds = get_source_bounds(sourcekey) +# assert bounds +# assert bounds.is_valid +# assert bounds.geom_type == "Polygon" +# assert bounds == ISC_SEVEN_RIVERS_BOUNDING_POLYGON - sourcekey = "iscsevenrivers" - bounds = get_source_bounds(sourcekey) - assert bounds - assert bounds.is_valid - assert bounds.geom_type == "Polygon" - assert bounds == ISC_SEVEN_RIVERS_BOUNDING_POLYGON +# def test_source_bounds_oser(): +# from backend.unifier import get_source_bounds +# from backend.connectors import ( +# OSE_ROSWELL_HONDO_BOUNDING_POLYGON, +# OSE_ROSWELL_ROSWELL_BOUNDING_POLYGON, +# OSE_ROSWELL_FORT_SUMNER_BOUNDING_POLYGON, +# ) -def test_source_bounds_oser(): - from backend.unifier import get_source_bounds - from backend.connectors import ( - OSE_ROSWELL_HONDO_BOUNDING_POLYGON, - OSE_ROSWELL_ROSWELL_BOUNDING_POLYGON, - OSE_ROSWELL_FORT_SUMNER_BOUNDING_POLYGON, - ) +# sourcekey = "oseroswell" +# bounds = get_source_bounds(sourcekey) +# assert bounds +# assert bounds.is_valid +# assert bounds.geom_type == "GeometryCollection" +# assert bounds == shapely.GeometryCollection( +# [ +# OSE_ROSWELL_HONDO_BOUNDING_POLYGON, +# OSE_ROSWELL_FORT_SUMNER_BOUNDING_POLYGON, +# OSE_ROSWELL_ROSWELL_BOUNDING_POLYGON, +# ] +# ) - sourcekey = "oseroswell" - bounds = get_source_bounds(sourcekey) - assert bounds - assert bounds.is_valid - assert bounds.geom_type == "GeometryCollection" - assert bounds == shapely.GeometryCollection( - [ - OSE_ROSWELL_HONDO_BOUNDING_POLYGON, - OSE_ROSWELL_FORT_SUMNER_BOUNDING_POLYGON, - OSE_ROSWELL_ROSWELL_BOUNDING_POLYGON, - ] - ) +# def test_sources_socorro(tmp_path): +# cfg = Config() +# cfg.county = "socorro" -def test_sources_socorro(tmp_path): - cfg = Config() - cfg.county = "socorro" +# from backend.unifier import get_sources - from backend.unifier import get_sources +# sources = get_sources(cfg) +# assert sources +# assert len(sources) == 2 +# assert sorted([s.__class__.__name__ for s in sources]) == sorted( +# ["NMBGMRSiteSource", "NWISSiteSource"] +# ) - sources = get_sources(cfg) - assert sources - assert len(sources) == 2 - assert sorted([s.__class__.__name__ for s in sources]) == sorted( - ["NMBGMRSiteSource", "NWISSiteSource"] - ) +# def test_sources_eddy_dtw(tmp_path): +# cfg = Config() +# cfg.county = "eddy" -def test_sources_eddy_dtw(tmp_path): - cfg = Config() - cfg.county = "eddy" +# from backend.unifier import get_sources - from backend.unifier import get_sources - - sources = get_sources(cfg) - assert sources - assert len(sources) == 5 - assert sorted([s.__class__.__name__ for s in sources]) == sorted( - [ - "ISCSevenRiversSiteSource", - "NMBGMRSiteSource", - "OSERoswellSiteSource", - "PVACDSiteSource", - "NWISSiteSource", - ] - ) +# sources = get_sources(cfg) +# assert sources +# assert len(sources) == 5 +# assert sorted([s.__class__.__name__ for s in sources]) == sorted( +# [ +# "ISCSevenRiversSiteSource", +# "NMBGMRSiteSource", +# "OSERoswellSiteSource", +# "PVACDSiteSource", +# "NWISSiteSource", +# ] +# ) -def test_sources_eddy_tds(tmp_path): - cfg = Config() - cfg.county = "eddy" - cfg.analyte = "TDS" +# def test_sources_eddy_tds(tmp_path): +# cfg = Config() +# cfg.county = "eddy" +# cfg.analyte = "TDS" - from backend.unifier import get_sources +# from backend.unifier import get_sources - sources = get_sources(cfg) - assert sources - assert len(sources) == 5 - assert sorted([s.__class__.__name__ for s in sources]) == sorted( - [ - "BORSiteSource", - "DWBSiteSource", - "ISCSevenRiversSiteSource", - "NMBGMRSiteSource", - "WQPSiteSource", - ] - ) - - -# Waterlevel Summary tests =========================================================================================== -def test_unify_waterlevels_bernco_summary(tmp_path, waterlevel_summary_cfg): - waterlevel_summary_cfg.county = "bernalillo" - waterlevel_summary_cfg.bbox = None - _test_waterlevels_summary(tmp_path, waterlevel_summary_cfg, "bernco") - - -def test_unify_waterlevels_nwis_summary(tmp_path, waterlevel_summary_cfg): - _test_waterlevels_summary(tmp_path, waterlevel_summary_cfg, "nwis") - - -def test_unify_waterlevels_amp_summary(tmp_path, waterlevel_summary_cfg): - _test_waterlevels_summary(tmp_path, waterlevel_summary_cfg, "nmbgmr") - - -def test_unify_waterlevels_pvacd_summary(tmp_path, waterlevel_summary_cfg): - _test_waterlevels_summary(tmp_path, waterlevel_summary_cfg, "pvacd") - - -def test_unify_waterlevels_isc_seven_rivers_summary(tmp_path, waterlevel_summary_cfg): - _test_waterlevels_summary(tmp_path, waterlevel_summary_cfg, "iscsevenrivers") - - -def test_unify_waterlevels_ose_roswell_summary(tmp_path, waterlevel_summary_cfg): - _test_waterlevels_summary(tmp_path, waterlevel_summary_cfg, "oseroswell") - - -# Waterlevel timeseries tests ========================================================================================= -def test_unify_waterlevels_nwis_timeseries(tmp_path, waterlevel_timeseries_cfg): - # there are one or more locations within the bounding box that have only - # one record, so there is a combined file - _test_waterlevels_timeseries( - tmp_path, - waterlevel_timeseries_cfg, - "nwis", - combined_flag=True, - timeseries_flag=True, - ) +# sources = get_sources(cfg) +# assert sources +# assert len(sources) == 5 +# assert sorted([s.__class__.__name__ for s in sources]) == sorted( +# [ +# "BORSiteSource", +# "DWBSiteSource", +# "ISCSevenRiversSiteSource", +# "NMBGMRSiteSource", +# "WQPSiteSource", +# ] +# ) + + +# # Waterlevel Summary tests =========================================================================================== +# def test_unify_waterlevels_bernco_summary(tmp_path, waterlevel_summary_cfg): +# waterlevel_summary_cfg.county = "bernalillo" +# waterlevel_summary_cfg.bbox = None +# _test_waterlevels_summary(tmp_path, waterlevel_summary_cfg, "bernco") + + +# def test_unify_waterlevels_nwis_summary(tmp_path, waterlevel_summary_cfg): +# _test_waterlevels_summary(tmp_path, waterlevel_summary_cfg, "nwis") + + +# def test_unify_waterlevels_amp_summary(tmp_path, waterlevel_summary_cfg): +# _test_waterlevels_summary(tmp_path, waterlevel_summary_cfg, "nmbgmr") + + +# def test_unify_waterlevels_pvacd_summary(tmp_path, waterlevel_summary_cfg): +# _test_waterlevels_summary(tmp_path, waterlevel_summary_cfg, "pvacd") + + +# def test_unify_waterlevels_isc_seven_rivers_summary(tmp_path, waterlevel_summary_cfg): +# _test_waterlevels_summary(tmp_path, waterlevel_summary_cfg, "iscsevenrivers") + + +# def test_unify_waterlevels_ose_roswell_summary(tmp_path, waterlevel_summary_cfg): +# _test_waterlevels_summary(tmp_path, waterlevel_summary_cfg, "oseroswell") + + +# # Waterlevel timeseries tests ========================================================================================= +# def test_unify_waterlevels_nwis_timeseries(tmp_path, waterlevel_timeseries_cfg): +# # there are one or more locations within the bounding box that have only +# # one record, so there is a combined file +# _test_waterlevels_timeseries( +# tmp_path, +# waterlevel_timeseries_cfg, +# "nwis", +# combined_flag=True, +# timeseries_flag=True, +# ) -def test_unify_waterlevels_amp_timeseries(tmp_path, waterlevel_timeseries_cfg): - _test_waterlevels_timeseries(tmp_path, waterlevel_timeseries_cfg, "nmbgmr") +# def test_unify_waterlevels_amp_timeseries(tmp_path, waterlevel_timeseries_cfg): +# _test_waterlevels_timeseries(tmp_path, waterlevel_timeseries_cfg, "nmbgmr") -def test_unify_waterlevels_pvacd_timeseries(tmp_path, waterlevel_timeseries_cfg): - # all locations within the bounding box have more than one record - # so there is no combined file - _test_waterlevels_timeseries( - tmp_path, - waterlevel_timeseries_cfg, - "pvacd", - combined_flag=False, - timeseries_flag=True, - ) +# def test_unify_waterlevels_pvacd_timeseries(tmp_path, waterlevel_timeseries_cfg): +# # all locations within the bounding box have more than one record +# # so there is no combined file +# _test_waterlevels_timeseries( +# tmp_path, +# waterlevel_timeseries_cfg, +# "pvacd", +# combined_flag=False, +# timeseries_flag=True, +# ) -def test_unify_waterlevels_isc_seven_rivers_timeseries( - tmp_path, waterlevel_timeseries_cfg -): - # all locations within the bounding box have more than one record - # so there is no combined file - _test_waterlevels_timeseries( - tmp_path, - waterlevel_timeseries_cfg, - "iscsevenrivers", - combined_flag=False, - timeseries_flag=True, - ) +# def test_unify_waterlevels_isc_seven_rivers_timeseries( +# tmp_path, waterlevel_timeseries_cfg +# ): +# # all locations within the bounding box have more than one record +# # so there is no combined file +# _test_waterlevels_timeseries( +# tmp_path, +# waterlevel_timeseries_cfg, +# "iscsevenrivers", +# combined_flag=False, +# timeseries_flag=True, +# ) -def test_unify_waterlevels_ose_roswell_timeseries(tmp_path, waterlevel_timeseries_cfg): - _test_waterlevels_timeseries( - tmp_path, waterlevel_timeseries_cfg, "oseroswell", timeseries_flag=True - ) +# def test_unify_waterlevels_ose_roswell_timeseries(tmp_path, waterlevel_timeseries_cfg): +# _test_waterlevels_timeseries( +# tmp_path, waterlevel_timeseries_cfg, "oseroswell", timeseries_flag=True +# ) -# Waterlevel summary date range tests ================================================================================= -def test_waterlevels_nwis_summary_date_range(tmp_path, waterlevel_summary_cfg): - d = _setup_waterlevels(tmp_path, waterlevel_summary_cfg, "nwis") - assert (d / "output.csv").is_file() +# # Waterlevel summary date range tests ================================================================================= +# def test_waterlevels_nwis_summary_date_range(tmp_path, waterlevel_summary_cfg): +# d = _setup_waterlevels(tmp_path, waterlevel_summary_cfg, "nwis") +# assert (d / "output.csv").is_file() -# Waterlevel timeseries date range ==================================================================================== -def test_waterlevels_nwis_timeseries_date_range(tmp_path, waterlevel_timeseries_cfg): - # there are one or more locations within the bounding box and date range - # that have only one record, so there is a combined file - _test_waterelevels_timeseries_date_range( - tmp_path, - waterlevel_timeseries_cfg, - "nwis", - timeseries_flag=True, - combined_flag=True, - ) +# # Waterlevel timeseries date range ==================================================================================== +# def test_waterlevels_nwis_timeseries_date_range(tmp_path, waterlevel_timeseries_cfg): +# # there are one or more locations within the bounding box and date range +# # that have only one record, so there is a combined file +# _test_waterelevels_timeseries_date_range( +# tmp_path, +# waterlevel_timeseries_cfg, +# "nwis", +# timeseries_flag=True, +# combined_flag=True, +# ) -def test_waterlevels_isc_seven_rivers_timeseries_date_range( - tmp_path, waterlevel_timeseries_cfg -): - # all locations within the bounding box and date rangehave more than one - # record so there is no combined file - _test_waterelevels_timeseries_date_range( - tmp_path, - waterlevel_timeseries_cfg, - "iscsevenrivers", - timeseries_flag=True, - combined_flag=False, - ) +# def test_waterlevels_isc_seven_rivers_timeseries_date_range( +# tmp_path, waterlevel_timeseries_cfg +# ): +# # all locations within the bounding box and date rangehave more than one +# # record so there is no combined file +# _test_waterelevels_timeseries_date_range( +# tmp_path, +# waterlevel_timeseries_cfg, +# "iscsevenrivers", +# timeseries_flag=True, +# combined_flag=False, +# ) -def test_waterlevels_pvacd_timeseries_date_range(tmp_path, waterlevel_timeseries_cfg): - # all locations within the bounding box and date rangehave more than one - # record so there is no combined file - _test_waterelevels_timeseries_date_range( - tmp_path, - waterlevel_timeseries_cfg, - "pvacd", - timeseries_flag=True, - combined_flag=False, - ) - - -# Analyte summary tests =============================================================================================== -def test_unify_analytes_wqp_summary(tmp_path, analyte_summary_cfg): - _test_analytes_summary(tmp_path, analyte_summary_cfg, "wqp") - - -def test_unify_analytes_amp_summary(tmp_path, analyte_summary_cfg): - _test_analytes_summary(tmp_path, analyte_summary_cfg, "nmbgmr") - - -def test_unify_analytes_bor_summary(tmp_path, analyte_summary_cfg): - # BOR locations are found within Otero County - analyte_summary_cfg.county = "otero" - analyte_summary_cfg.bbox = None - _test_analytes_summary(tmp_path, analyte_summary_cfg, "bor") +# def test_waterlevels_pvacd_timeseries_date_range(tmp_path, waterlevel_timeseries_cfg): +# # all locations within the bounding box and date rangehave more than one +# # record so there is no combined file +# _test_waterelevels_timeseries_date_range( +# tmp_path, +# waterlevel_timeseries_cfg, +# "pvacd", +# timeseries_flag=True, +# combined_flag=False, +# ) + + +# # Analyte summary tests =============================================================================================== +# def test_unify_analytes_wqp_summary(tmp_path, analyte_summary_cfg): +# _test_analytes_summary(tmp_path, analyte_summary_cfg, "wqp") + + +# def test_unify_analytes_amp_summary(tmp_path, analyte_summary_cfg): +# _test_analytes_summary(tmp_path, analyte_summary_cfg, "nmbgmr") + + +# def test_unify_analytes_bor_summary(tmp_path, analyte_summary_cfg): +# # BOR locations are found within Otero County +# analyte_summary_cfg.county = "otero" +# analyte_summary_cfg.bbox = None +# _test_analytes_summary(tmp_path, analyte_summary_cfg, "bor") -def test_unify_analytes_isc_seven_rivers_summary(tmp_path, analyte_summary_cfg): - _test_analytes_summary(tmp_path, analyte_summary_cfg, "iscsevenrivers") - - -def test_unify_analytes_dwb_summary(tmp_path, analyte_summary_cfg): - _test_analytes_summary(tmp_path, analyte_summary_cfg, "dwb") +# def test_unify_analytes_isc_seven_rivers_summary(tmp_path, analyte_summary_cfg): +# _test_analytes_summary(tmp_path, analyte_summary_cfg, "iscsevenrivers") + + +# def test_unify_analytes_dwb_summary(tmp_path, analyte_summary_cfg): +# _test_analytes_summary(tmp_path, analyte_summary_cfg, "dwb") # ============= EOF ============================================= From 2155fbd3438f9120f8129da5f5b2212dd0de8014 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 7 Apr 2025 10:19:04 -0600 Subject: [PATCH 26/55] type hint BaseTestClass class attributes --- tests/__init__.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index 52c810e..f650d55 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,6 +1,7 @@ from logging import shutdown as logger_shutdown from pathlib import Path import pytest +from typing import Optional from backend.config import Config, SOURCE_KEYS from backend.constants import WATERLEVELS @@ -24,13 +25,12 @@ def recursively_clean_directory(path): class BaseTestClass: + parameter: str + units: str + agency: str - parameter = None - units = None - agency = None - - # set set_limit for tests - site_limit = 3 + # set site_limit for tests + site_limit: int = 3 @pytest.fixture(autouse=True) def setup(self): From f90a1c4cf1b5d0769d1165d193f5f9954bc2afa1 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 7 Apr 2025 10:44:13 -0600 Subject: [PATCH 27/55] mypy typing --- backend/connectors/nmose/source.py | 22 +++++++++++----------- backend/unifier.py | 2 ++ 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/backend/connectors/nmose/source.py b/backend/connectors/nmose/source.py index 92e6fb3..0f1ded0 100644 --- a/backend/connectors/nmose/source.py +++ b/backend/connectors/nmose/source.py @@ -1,4 +1,4 @@ -import os +from typing import List,Dict, Tuple from shapely import wkt from backend.connectors import NM_STATE_BOUNDING_POLYGON @@ -20,10 +20,10 @@ class NMOSEPODSiteSource(BaseSiteSource): """ transformer_klass = NMOSEPODSiteTransformer - chunk_size = 5000 + chunk_size: int = 5000 bounding_polygon = NM_STATE_BOUNDING_POLYGON - def get_records(self, *args, **kw) -> dict: + def get_records(self, *args, **kw) -> List[Dict]: config = self.config params = {} # if config.has_bounds(): @@ -37,25 +37,25 @@ def get_records(self, *args, **kw) -> dict: # if config.end_date: # params["endDt"] = config.end_dt.date().isoformat() - url = "https://services2.arcgis.com/qXZbWTdPDbTjl7Dy/arcgis/rest/services/OSE_PODs/FeatureServer/0/query" + url: str = "https://services2.arcgis.com/qXZbWTdPDbTjl7Dy/arcgis/rest/services/OSE_PODs/FeatureServer/0/query" - params["where"] = ( + params["where"]: Tuple = ( "pod_status = 'ACT' AND pod_basin IN ('A','B','C','CC','CD','CL','CP','CR','CT','E','FS','G','GSF','H', 'HA','HC','HS','HU','J','L','LA','LRG','LV','M','MR','NH','P','PL','PN','RA','RG','S','SB','SJ','SS','T','TU','UP','VV')" ) - params["outFields"] = ( + params["outFields"]: Tuple = ( "OBJECTID,pod_basin,pod_status,easting,northing,datum,utm_accura,status,county,pod_name,pod_nbr,pod_suffix,pod_file" ) - params["outSR"] = 4326 - params["f"] = "json" - params["resultRecordCount"] = self.chunk_size - params["resultOffset"] = 0 + params["outSR"]: int = 4326 + params["f"]: str = "json" + params["resultRecordCount"]: int = self.chunk_size + params["resultOffset"]: int = 0 if config.has_bounds(): wkt = config.bounding_wkt() params["geometry"] = wkt_to_arcgis_json(wkt) params["geometryType"] = "esriGeometryPolygon" - records = [] + records: List = [] i = 1 while 1: rs = self._execute_json_request(url, params, tag="features") diff --git a/backend/unifier.py b/backend/unifier.py index c6a8311..5628746 100644 --- a/backend/unifier.py +++ b/backend/unifier.py @@ -39,6 +39,8 @@ def health_check(source: BaseSiteSource) -> bool: source = get_source(source) if source: return bool(source.health()) + else: + return None def unify_analytes(config): From 91af8ca39f82115ed6dec3711530ce666dfba542 Mon Sep 17 00:00:00 2001 From: jacob-a-brown Date: Mon, 7 Apr 2025 16:45:40 +0000 Subject: [PATCH 28/55] Formatting changes --- backend/bounding_polygons.py | 2 +- backend/connectors/nmose/source.py | 22 ++++++++++++++-------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/backend/bounding_polygons.py b/backend/bounding_polygons.py index 8445077..8a9ccd6 100644 --- a/backend/bounding_polygons.py +++ b/backend/bounding_polygons.py @@ -162,7 +162,7 @@ def get_county_polygon(name, as_wkt=True): _warning(f"Invalid state. {state}") -def get_state_polygon(state: str, buffer: int | None =None): +def get_state_polygon(state: str, buffer: int | None = None): statefp = _statelookup(state) if statefp: obj = _get_cached_object( diff --git a/backend/connectors/nmose/source.py b/backend/connectors/nmose/source.py index 0f1ded0..005d5ca 100644 --- a/backend/connectors/nmose/source.py +++ b/backend/connectors/nmose/source.py @@ -1,4 +1,4 @@ -from typing import List,Dict, Tuple +from typing import List, Dict, Tuple from shapely import wkt from backend.connectors import NM_STATE_BOUNDING_POLYGON @@ -37,14 +37,20 @@ def get_records(self, *args, **kw) -> List[Dict]: # if config.end_date: # params["endDt"] = config.end_dt.date().isoformat() - url: str = "https://services2.arcgis.com/qXZbWTdPDbTjl7Dy/arcgis/rest/services/OSE_PODs/FeatureServer/0/query" - - params["where"]: Tuple = ( - "pod_status = 'ACT' AND pod_basin IN ('A','B','C','CC','CD','CL','CP','CR','CT','E','FS','G','GSF','H', 'HA','HC','HS','HU','J','L','LA','LRG','LV','M','MR','NH','P','PL','PN','RA','RG','S','SB','SJ','SS','T','TU','UP','VV')" - ) - params["outFields"]: Tuple = ( - "OBJECTID,pod_basin,pod_status,easting,northing,datum,utm_accura,status,county,pod_name,pod_nbr,pod_suffix,pod_file" + url: str = ( + "https://services2.arcgis.com/qXZbWTdPDbTjl7Dy/arcgis/rest/services/OSE_PODs/FeatureServer/0/query" ) + + params[ + "where" + ]: ( + Tuple + ) = "pod_status = 'ACT' AND pod_basin IN ('A','B','C','CC','CD','CL','CP','CR','CT','E','FS','G','GSF','H', 'HA','HC','HS','HU','J','L','LA','LRG','LV','M','MR','NH','P','PL','PN','RA','RG','S','SB','SJ','SS','T','TU','UP','VV')" + params[ + "outFields" + ]: ( + Tuple + ) = "OBJECTID,pod_basin,pod_status,easting,northing,datum,utm_accura,status,county,pod_name,pod_nbr,pod_suffix,pod_file" params["outSR"]: int = 4326 params["f"]: str = "json" params["resultRecordCount"]: int = self.chunk_size From 53e29a12baf6588bcd7904bf16374192969b3c51 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Wed, 9 Apr 2025 16:07:54 -0600 Subject: [PATCH 29/55] Work on mypy type checking --- backend/connectors/nmose/source.py | 12 ++++++------ backend/source.py | 7 ++----- backend/unifier.py | 2 +- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/backend/connectors/nmose/source.py b/backend/connectors/nmose/source.py index 0f1ded0..a78d8d6 100644 --- a/backend/connectors/nmose/source.py +++ b/backend/connectors/nmose/source.py @@ -39,16 +39,16 @@ def get_records(self, *args, **kw) -> List[Dict]: url: str = "https://services2.arcgis.com/qXZbWTdPDbTjl7Dy/arcgis/rest/services/OSE_PODs/FeatureServer/0/query" - params["where"]: Tuple = ( + params["where"] = ( "pod_status = 'ACT' AND pod_basin IN ('A','B','C','CC','CD','CL','CP','CR','CT','E','FS','G','GSF','H', 'HA','HC','HS','HU','J','L','LA','LRG','LV','M','MR','NH','P','PL','PN','RA','RG','S','SB','SJ','SS','T','TU','UP','VV')" ) - params["outFields"]: Tuple = ( + params["outFields"] = ( "OBJECTID,pod_basin,pod_status,easting,northing,datum,utm_accura,status,county,pod_name,pod_nbr,pod_suffix,pod_file" ) - params["outSR"]: int = 4326 - params["f"]: str = "json" - params["resultRecordCount"]: int = self.chunk_size - params["resultOffset"]: int = 0 + params["outSR"] = 4326 + params["f"] = "json" + params["resultRecordCount"] = self.chunk_size + params["resultOffset"] = 0 if config.has_bounds(): wkt = config.bounding_wkt() diff --git a/backend/source.py b/backend/source.py index 5ca8c9b..5884861 100644 --- a/backend/source.py +++ b/backend/source.py @@ -19,7 +19,7 @@ import httpx import shapely.wkt from shapely import MultiPoint -from typing import Union, List, Callable +from typing import Union, List, Callable, Dict from backend.constants import ( MILLIGRAMS_PER_LITER, @@ -289,7 +289,7 @@ def read(self, *args, **kw) -> list: # Methods That Need to be Implemented For Each Source # ========================================================================== - def get_records(self, *args, **kw) -> dict: + def get_records(self, *args, **kw) -> List[Dict]: """ Returns records as a dictionary, where the keys are site ids and the values are site or parameter records. @@ -837,9 +837,6 @@ def _extract_site_records(self, records: dict, site_record: dict) -> list: list a list of records for the site """ - if site_record.chunk_size == 1: - return records - raise NotImplementedError( f"{self.__class__.__name__} Must implement _extract_site_records" ) diff --git a/backend/unifier.py b/backend/unifier.py index 5628746..880369f 100644 --- a/backend/unifier.py +++ b/backend/unifier.py @@ -22,7 +22,7 @@ from backend.source import BaseSiteSource -def health_check(source: BaseSiteSource) -> bool: +def health_check(source: BaseSiteSource) -> bool | None: """ Determines if data can be returned from the source (if it is healthy) From 82eaf5342fc5212f7b7deeefc6a9a6133364d4ad Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Wed, 9 Apr 2025 16:36:31 -0600 Subject: [PATCH 30/55] Work on mypy type checking --- backend/transformer.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/backend/transformer.py b/backend/transformer.py index b993578..701866a 100644 --- a/backend/transformer.py +++ b/backend/transformer.py @@ -34,6 +34,7 @@ EARLIEST, LATEST, ) +from backend.config import Config from backend.geo_utils import datum_transform, ALLOWED_DATUMS from backend.logger import Loggable from backend.record import ( @@ -132,7 +133,7 @@ def convert_units( source_parameter_name: str, die_parameter_name: str, dt: str | None = None, -) -> tuple[float, float, str]: +) -> tuple[float, float | None, str]: """ Converts the following units for any parameter value: @@ -331,7 +332,7 @@ class BaseTransformer(Loggable): """ _cached_polygon = None - config = None + config: Config = None check_contained = True # ========================================================================== @@ -347,6 +348,7 @@ def do_transform( | AnalyteSummaryRecord | WaterLevelSummaryRecord | SummaryRecord + | None ): """ Transforms a record, site or parameter, into a standardized format. @@ -667,7 +669,7 @@ def _get_record_klass(self): class SiteTransformer(BaseTransformer): - def _get_record_klass(self) -> SiteRecord: + def _get_record_klass(self) -> type[SiteRecord]: """ Returns the SiteRecord class to use for the transformer for all site records @@ -786,7 +788,7 @@ def _transform_latest_record(self, record, site_id): class WaterLevelTransformer(ParameterTransformer): - def _get_record_klass(self) -> WaterLevelRecord | WaterLevelSummaryRecord: + def _get_record_klass(self) -> type[WaterLevelRecord] | type[WaterLevelSummaryRecord]: """ Returns the WaterLevelRecord class to use for the transformer for water level records if config.output_summary is False, otherwise From 761e4edca248ac8ed7852bf2f23d2c51cef847ff Mon Sep 17 00:00:00 2001 From: jacob-a-brown Date: Wed, 9 Apr 2025 22:37:41 +0000 Subject: [PATCH 31/55] Formatting changes --- backend/transformer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/transformer.py b/backend/transformer.py index 701866a..4d9a1c4 100644 --- a/backend/transformer.py +++ b/backend/transformer.py @@ -788,7 +788,9 @@ def _transform_latest_record(self, record, site_id): class WaterLevelTransformer(ParameterTransformer): - def _get_record_klass(self) -> type[WaterLevelRecord] | type[WaterLevelSummaryRecord]: + def _get_record_klass( + self, + ) -> type[WaterLevelRecord] | type[WaterLevelSummaryRecord]: """ Returns the WaterLevelRecord class to use for the transformer for water level records if config.output_summary is False, otherwise From d90479b5d886f4eb887edc6923927dc36ff90719 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 10 Apr 2025 09:05:29 -0600 Subject: [PATCH 32/55] mypy.ini configuration file --- backend/transformer.py | 4 +- mypy.ini | 3 + tests/archived/test_cli.py | 624 +++++++++++++-------------- tests/archived/test_unifier.py | 758 ++++++++++++++++----------------- 4 files changed, 696 insertions(+), 693 deletions(-) create mode 100644 mypy.ini diff --git a/backend/transformer.py b/backend/transformer.py index 4d9a1c4..0c3796d 100644 --- a/backend/transformer.py +++ b/backend/transformer.py @@ -34,7 +34,6 @@ EARLIEST, LATEST, ) -from backend.config import Config from backend.geo_utils import datum_transform, ALLOWED_DATUMS from backend.logger import Loggable from backend.record import ( @@ -332,7 +331,7 @@ class BaseTransformer(Loggable): """ _cached_polygon = None - config: Config = None + config = None check_contained = True # ========================================================================== @@ -396,6 +395,7 @@ def do_transform( # _transform needs to be implemented by each SiteTransformer # _transform is already implemented in each ParameterTransformer record = self._transform(inrecord, *args, **kw) + print(type(record)) if not record: return None diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 0000000..380b366 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,3 @@ +[mypy] +ignore_missing_imports = True +exclude = ^(venv|.github|.mypy_cache|.pytest_cache|nmuwd.egg-info|__pycache__|build|tests/archived) \ No newline at end of file diff --git a/tests/archived/test_cli.py b/tests/archived/test_cli.py index ba8240e..6fa2baa 100644 --- a/tests/archived/test_cli.py +++ b/tests/archived/test_cli.py @@ -13,395 +13,395 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== -# import os +import os -# from click.testing import CliRunner -# from frontend.cli import analytes, waterlevels +from click.testing import CliRunner +from frontend.cli import analytes, waterlevels -# def _tester(function, args, fail=False): -# runner = CliRunner() -# print(f"invoked with {args}") -# result = runner.invoke(function, args) -# print(f"result.exit_code={result.exit_code}") -# print(f"result.output=\n{result.output}") +def _tester(function, args, fail=False): + runner = CliRunner() + print(f"invoked with {args}") + result = runner.invoke(function, args) + print(f"result.exit_code={result.exit_code}") + print(f"result.output=\n{result.output}") -# if fail: -# assert result.exit_code != 0 -# else: -# assert result.exit_code == 0 + if fail: + assert result.exit_code != 0 + else: + assert result.exit_code == 0 -# def _make_args(source): -# args = [] -# if source: -# nosources = [ -# f -# for f in ( -# "--no-amp", -# "--no-nwis", -# "--no-pvacd", -# "--no-bor", -# "--no-dwb", -# "--no-wqp", -# "--no-isc-seven-rivers", -# "--no-ckan", -# ) -# if f != f"--no-{source}" -# ] -# args += nosources +def _make_args(source): + args = [] + if source: + nosources = [ + f + for f in ( + "--no-amp", + "--no-nwis", + "--no-pvacd", + "--no-bor", + "--no-dwb", + "--no-wqp", + "--no-isc-seven-rivers", + "--no-ckan", + ) + if f != f"--no-{source}" + ] + args += nosources -# args += ["--site-limit", 10, "--dry"] + args += ["--site-limit", 10, "--dry"] -# return args + return args -# def _make_tds_args(source): -# return ["TDS"] + _make_args(source) +def _make_tds_args(source): + return ["TDS"] + _make_args(source) -# def _make_wl_args(source=None): -# return _make_args(source) +def _make_wl_args(source=None): + return _make_args(source) -# def test_waterlevels_nwis(): -# args = _make_wl_args("nwis") -# _tester(waterlevels, args) +def test_waterlevels_nwis(): + args = _make_wl_args("nwis") + _tester(waterlevels, args) -# def test_waterlevels_pvacd(): -# args = _make_wl_args("pvacd") -# _tester(waterlevels, args) +def test_waterlevels_pvacd(): + args = _make_wl_args("pvacd") + _tester(waterlevels, args) -# def test_waterlevels_nmbgmr(): -# args = _make_wl_args("nmbgmr") -# _tester(waterlevels, args) +def test_waterlevels_nmbgmr(): + args = _make_wl_args("nmbgmr") + _tester(waterlevels, args) -# def test_waterlevels_isc_seven_rivers(): -# args = _make_wl_args("iscsevenrivers") -# _tester(waterlevels, args) +def test_waterlevels_isc_seven_rivers(): + args = _make_wl_args("iscsevenrivers") + _tester(waterlevels, args) -# def test_waterlevels_invalid_source(): -# args = _make_wl_args() -# args.append("--no-foo") -# _tester(waterlevels, args, fail=True) +def test_waterlevels_invalid_source(): + args = _make_wl_args() + args.append("--no-foo") + _tester(waterlevels, args, fail=True) -# def test_waterlevels_invalid_bbox(): -# args = _make_wl_args() -# args.append("--bbox") -# _tester(waterlevels, args, fail=True) +def test_waterlevels_invalid_bbox(): + args = _make_wl_args() + args.append("--bbox") + _tester(waterlevels, args, fail=True) -# def test_waterlevels_invalid_bbox_format(): -# args = _make_wl_args() -# args.extend(["--bbox", "1 2 3"]) -# _tester(waterlevels, args, fail=True) +def test_waterlevels_invalid_bbox_format(): + args = _make_wl_args() + args.extend(["--bbox", "1 2 3"]) + _tester(waterlevels, args, fail=True) -# def test_waterlevels_valid_bbox_format(): -# args = _make_wl_args() -# args.extend(["--bbox", "1 2,3 4"]) -# _tester(waterlevels, args) +def test_waterlevels_valid_bbox_format(): + args = _make_wl_args() + args.extend(["--bbox", "1 2,3 4"]) + _tester(waterlevels, args) -# def test_waterlevels_invalid_county(): -# args = _make_wl_args() -# args.append("--county") -# _tester(waterlevels, args, fail=True) +def test_waterlevels_invalid_county(): + args = _make_wl_args() + args.append("--county") + _tester(waterlevels, args, fail=True) -# def test_waterlevels_invalid_county_name(): -# args = _make_wl_args() -# args.extend(["--county", "foo"]) -# _tester(waterlevels, args, fail=True) +def test_waterlevels_invalid_county_name(): + args = _make_wl_args() + args.extend(["--county", "foo"]) + _tester(waterlevels, args, fail=True) -# # Analyte Tests ======================================================= -# def test_analytes_wqp(): -# args = _make_tds_args("wqp") -# _tester(analytes, args) +# Analyte Tests ======================================================= +def test_analytes_wqp(): + args = _make_tds_args("wqp") + _tester(analytes, args) -# def test_analytes_bor(): -# args = _make_tds_args("bor") -# _tester(analytes, args) +def test_analytes_bor(): + args = _make_tds_args("bor") + _tester(analytes, args) -# def test_analytes_amp(): -# args = _make_tds_args("amp") -# _tester(analytes, args) +def test_analytes_amp(): + args = _make_tds_args("amp") + _tester(analytes, args) -# def test_analytes_dwb(): -# args = _make_tds_args("dwb") -# _tester(analytes, args) +def test_analytes_dwb(): + args = _make_tds_args("dwb") + _tester(analytes, args) -# def test_analytes_isc_seven_rivers(): -# args = _make_tds_args("isc-seven-rivers") -# _tester(analytes, args) +def test_analytes_isc_seven_rivers(): + args = _make_tds_args("isc-seven-rivers") + _tester(analytes, args) -# def test_analytes_invalid_analyte(): -# args = _make_args("wqp") -# args[0] = "Foo" -# _tester(analytes, args, fail=True) +def test_analytes_invalid_analyte(): + args = _make_args("wqp") + args[0] = "Foo" + _tester(analytes, args, fail=True) -# def test_analytes_invalid_source(): -# args = _make_tds_args("wqp") -# args.append("--no-foo") -# _tester(analytes, args, fail=True) +def test_analytes_invalid_source(): + args = _make_tds_args("wqp") + args.append("--no-foo") + _tester(analytes, args, fail=True) -# def test_analytes_invalid_bbox(): -# args = _make_tds_args("wqp") -# args.append("--bbox") -# _tester(analytes, args, fail=True) +def test_analytes_invalid_bbox(): + args = _make_tds_args("wqp") + args.append("--bbox") + _tester(analytes, args, fail=True) -# def test_analytes_invalid_bbox_format(): -# args = _make_tds_args("wqp") -# args.extend(["--bbox", "1 2 3"]) -# _tester(analytes, args, fail=True) +def test_analytes_invalid_bbox_format(): + args = _make_tds_args("wqp") + args.extend(["--bbox", "1 2 3"]) + _tester(analytes, args, fail=True) -# def test_analytes_valid_bbox_format(): -# args = _make_tds_args("wqp") -# args.extend(["--bbox", "1 2,3 4"]) -# _tester(analytes, args) +def test_analytes_valid_bbox_format(): + args = _make_tds_args("wqp") + args.extend(["--bbox", "1 2,3 4"]) + _tester(analytes, args) -# def test_analytes_invalid_county(): -# args = _make_tds_args("wqp") -# args.append("--county") -# _tester(analytes, args, fail=True) +def test_analytes_invalid_county(): + args = _make_tds_args("wqp") + args.append("--county") + _tester(analytes, args, fail=True) -# def test_analytes_invalid_county_name(): -# args = _make_tds_args("wqp") -# args.extend(["--county", "foo"]) -# _tester(analytes, args, fail=True) +def test_analytes_invalid_county_name(): + args = _make_tds_args("wqp") + args.extend(["--county", "foo"]) + _tester(analytes, args, fail=True) -# def test_waterlevels_date_range_YMD(): -# args = _make_wl_args() -# args.extend(["--start-date", "2020-01-01", "--end-date", "2020-05-01"]) -# _tester(waterlevels, args) +def test_waterlevels_date_range_YMD(): + args = _make_wl_args() + args.extend(["--start-date", "2020-01-01", "--end-date", "2020-05-01"]) + _tester(waterlevels, args) -# def test_waterlevels_date_range_YM(): -# args = _make_wl_args() -# args.extend(["--start-date", "2020-01", "--end-date", "2020-05"]) -# _tester(waterlevels, args) +def test_waterlevels_date_range_YM(): + args = _make_wl_args() + args.extend(["--start-date", "2020-01", "--end-date", "2020-05"]) + _tester(waterlevels, args) -# def test_waterlevels_date_range_Y(): -# args = _make_wl_args() -# args.extend(["--start-date", "2020", "--end-date", "2021"]) -# _tester(waterlevels, args) +def test_waterlevels_date_range_Y(): + args = _make_wl_args() + args.extend(["--start-date", "2020", "--end-date", "2021"]) + _tester(waterlevels, args) -# def test_waterlevels_invalid_start(): -# args = _make_wl_args() -# args.extend(["--start-date", "x-01-01", "--end-date", "2019-05-01"]) -# _tester(waterlevels, args, fail=True) +def test_waterlevels_invalid_start(): + args = _make_wl_args() + args.extend(["--start-date", "x-01-01", "--end-date", "2019-05-01"]) + _tester(waterlevels, args, fail=True) -# def test_waterlevels_invalid_end(): -# args = _make_wl_args() -# args.extend(["--start-date", "2020-01-01", "--end-date", "x-05-01"]) -# _tester(waterlevels, args, fail=True) +def test_waterlevels_invalid_end(): + args = _make_wl_args() + args.extend(["--start-date", "2020-01-01", "--end-date", "x-05-01"]) + _tester(waterlevels, args, fail=True) -# -# def _tester(source, func, county, bbox, args=None): -# runner = CliRunner() -# -# nosources = [ -# f -# for f in ( -# "--no-amp", -# "--no-nwis", -# "--no-st2", -# "--no-bor", -# "--no-dwb", -# "--no-wqp", -# "--no-isc-seven-rivers", -# "--no-ckan", -# ) -# if f != f"--no-{source}" -# ] -# -# dargs = nosources + ["--site-limit", 10] -# -# if args: -# args += dargs -# else: -# args = dargs -# -# if county: -# args.extend(("--county", county)) -# elif bbox: -# args.extend(("--bbox", bbox)) -# -# print(" ".join([str(f) for f in args])) -# result = runner.invoke(func, args) -# -# return result +def _tester(source, func, county, bbox, args=None): + runner = CliRunner() + + nosources = [ + f + for f in ( + "--no-amp", + "--no-nwis", + "--no-st2", + "--no-bor", + "--no-dwb", + "--no-wqp", + "--no-isc-seven-rivers", + "--no-ckan", + ) + if f != f"--no-{source}" + ] + + dargs = nosources + ["--site-limit", 10] + + if args: + args += dargs + else: + args = dargs + + if county: + args.extend(("--county", county)) + elif bbox: + args.extend(("--bbox", bbox)) + + print(" ".join([str(f) for f in args])) + result = runner.invoke(func, args) + + return result + + +def _summary_tester(source, func, county=None, bbox=None, args=None): + if not (county or bbox): + county = "eddy" + + runner = CliRunner() + # with runner.isolated_filesystem(): + # result = _tester(source, func, county, bbox, args) + # assert result.exit_code == 0 + # assert os.path.isfile("output.csv") + + +def _timeseries_tester( + source, + func, + combined_flag=True, + timeseries_flag=True, + county=None, + bbox=None, + args=None, +): + if args is None: + args = [] + # runner = CliRunner() + # with runner.isolated_filesystem(): + # result = _tester(source, func, county, bbox, args=args + ["--timeseries"]) + # assert result.exit_code == 0 + # print("combined", os.path.isfile("output.combined.csv"), combined_flag) + # assert os.path.isfile("output.combined.csv") == combined_flag + # print("timeseries", os.path.isdir("output_timeseries"), timeseries_flag) + # assert os.path.isdir("output_timeseries") == timeseries_flag + + +# ====== Analyte Tests ======================================================= +def _analyte_summary_tester(key): + _summary_tester(key, analytes, args=["TDS"]) + + +def _analyte_county_tester(source, **kw): + _timeseries_tester(source, analytes, args=["TDS"], county="eddy", **kw) + + +def test_unify_analytes_amp(): + _analyte_county_tester("amp", timeseries_flag=False) + + +def test_unify_analytes_wqp(): + _analyte_county_tester("wqp") + + +def test_unify_analytes_bor(): + _analyte_county_tester("bor", combined_flag=False) + + +def test_unify_analytes_isc_seven_rivers(): + _analyte_county_tester("isc-seven-rivers") + + +def test_unify_analytes_dwb(): + _analyte_county_tester("dwb", timeseries_flag=False) + + +def test_unify_analytes_wqp_summary(): + _analyte_summary_tester("wqp") + + +def test_unify_analytes_bor_summary(): + _analyte_summary_tester("bor") -# def _summary_tester(source, func, county=None, bbox=None, args=None): -# if not (county or bbox): -# county = "eddy" -# -# runner = CliRunner() -# # with runner.isolated_filesystem(): -# # result = _tester(source, func, county, bbox, args) -# # assert result.exit_code == 0 -# # assert os.path.isfile("output.csv") -# -# -# def _timeseries_tester( -# source, -# func, -# combined_flag=True, -# timeseries_flag=True, -# county=None, -# bbox=None, -# args=None, -# ): -# if args is None: -# args = [] -# # runner = CliRunner() -# # with runner.isolated_filesystem(): -# # result = _tester(source, func, county, bbox, args=args + ["--timeseries"]) -# # assert result.exit_code == 0 -# # print("combined", os.path.isfile("output.combined.csv"), combined_flag) -# # assert os.path.isfile("output.combined.csv") == combined_flag -# # print("timeseries", os.path.isdir("output_timeseries"), timeseries_flag) -# # assert os.path.isdir("output_timeseries") == timeseries_flag -# -# -# # ====== Analyte Tests ======================================================= -# def _analyte_summary_tester(key): -# _summary_tester(key, analytes, args=["TDS"]) -# -# -# def _analyte_county_tester(source, **kw): -# _timeseries_tester(source, analytes, args=["TDS"], county="eddy", **kw) -# -# -# def test_unify_analytes_amp(): -# _analyte_county_tester("amp", timeseries_flag=False) -# -# -# def test_unify_analytes_wqp(): -# _analyte_county_tester("wqp") -# -# -# def test_unify_analytes_bor(): -# _analyte_county_tester("bor", combined_flag=False) -# -# -# def test_unify_analytes_isc_seven_rivers(): -# _analyte_county_tester("isc-seven-rivers") -# -# -# def test_unify_analytes_dwb(): -# _analyte_county_tester("dwb", timeseries_flag=False) -# -# -# def test_unify_analytes_wqp_summary(): -# _analyte_summary_tester("wqp") -# -# -# def test_unify_analytes_bor_summary(): -# _analyte_summary_tester("bor") -# -# -# def test_unify_analytes_amp_summary(): -# _analyte_summary_tester("amp") -# -# -# def test_unify_analytes_dwb_summary(): -# _analyte_summary_tester("dwb") -# -# -# def test_unify_analytes_isc_seven_rivers_summary(): -# _analyte_summary_tester("isc-seven-rivers") + +def test_unify_analytes_amp_summary(): + _analyte_summary_tester("amp") + + +def test_unify_analytes_dwb_summary(): + _analyte_summary_tester("dwb") + + +def test_unify_analytes_isc_seven_rivers_summary(): + _analyte_summary_tester("isc-seven-rivers") # ====== End Analyte Tests ======================================================= # ====== Water Level Tests ======================================================= -# def _waterlevel_county_tester(source, **kw): -# _timeseries_tester(source, waterlevels, county="eddy", **kw) -# -# -# def _waterlevel_bbox_tester(source, **kw): -# _timeseries_tester(source, waterlevels, bbox="-104.5 32.5,-104 33", **kw) +def _waterlevel_county_tester(source, **kw): + _timeseries_tester(source, waterlevels, county="eddy", **kw) -# -# def test_unify_waterlevels_nwis(): -# _waterlevel_county_tester("nwis", timeseries_flag=False) -# -# -# def test_unify_waterlevels_amp(): -# _waterlevel_county_tester("amp", timeseries_flag=False) -# -# -# def test_unify_waterlevels_st2(): -# _waterlevel_county_tester("st2", combined_flag=False) -# -# -# def test_unify_waterlevels_isc_seven_rivers(): -# _waterlevel_county_tester("isc-seven-rivers") -# -# -# def test_unify_waterlevels_ckan(): -# _waterlevel_county_tester("ckan") -# -# -# def test_unify_waterlevels_nwis_summary(): -# _summary_tester("nwis", waterlevels) -# -# -# def test_unify_waterlevels_amp_summary(): -# _summary_tester("amp", waterlevels) -# -# -# def test_unify_waterlevels_st2_summary(): -# _summary_tester("st2", waterlevels) -# -# -# def test_unify_waterlevels_isc_seven_rivers_summary(): -# _summary_tester("isc-seven-rivers", waterlevels) -# -# -# def test_unify_waterlevels_nwis_bbox(): -# _waterlevel_bbox_tester("nwis", timeseries_flag=False) -# -# -# def test_unify_waterlevels_amp_bbox(): -# _waterlevel_bbox_tester("amp") -# -# -# def test_unify_waterlevels_st2_bbox(): -# _waterlevel_bbox_tester("st2", combined_flag=False) -# -# -# def test_unify_waterlevels_isc_seven_rivers_bbox(): -# _waterlevel_bbox_tester("isc-seven-rivers", combined_flag=False) -# -# -# def test_unify_waterlevels_ckan_bbox(): -# _waterlevel_bbox_tester("ckan") + +def _waterlevel_bbox_tester(source, **kw): + _timeseries_tester(source, waterlevels, bbox="-104.5 32.5,-104 33", **kw) + + +def test_unify_waterlevels_nwis(): + _waterlevel_county_tester("nwis", timeseries_flag=False) + + +def test_unify_waterlevels_amp(): + _waterlevel_county_tester("amp", timeseries_flag=False) + + +def test_unify_waterlevels_st2(): + _waterlevel_county_tester("st2", combined_flag=False) + + +def test_unify_waterlevels_isc_seven_rivers(): + _waterlevel_county_tester("isc-seven-rivers") + + +def test_unify_waterlevels_ckan(): + _waterlevel_county_tester("ckan") + + +def test_unify_waterlevels_nwis_summary(): + _summary_tester("nwis", waterlevels) + + +def test_unify_waterlevels_amp_summary(): + _summary_tester("amp", waterlevels) + + +def test_unify_waterlevels_st2_summary(): + _summary_tester("st2", waterlevels) + + +def test_unify_waterlevels_isc_seven_rivers_summary(): + _summary_tester("isc-seven-rivers", waterlevels) + + +def test_unify_waterlevels_nwis_bbox(): + _waterlevel_bbox_tester("nwis", timeseries_flag=False) + + +def test_unify_waterlevels_amp_bbox(): + _waterlevel_bbox_tester("amp") + + +def test_unify_waterlevels_st2_bbox(): + _waterlevel_bbox_tester("st2", combined_flag=False) + + +def test_unify_waterlevels_isc_seven_rivers_bbox(): + _waterlevel_bbox_tester("isc-seven-rivers", combined_flag=False) + + +def test_unify_waterlevels_ckan_bbox(): + _waterlevel_bbox_tester("ckan") # ====== End Water Level Tests ======================================================= diff --git a/tests/archived/test_unifier.py b/tests/archived/test_unifier.py index 319c762..3947ef6 100644 --- a/tests/archived/test_unifier.py +++ b/tests/archived/test_unifier.py @@ -13,461 +13,461 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== -# import datetime -# import os - -# import pytest -# import shapely.wkt - -# from backend.config import Config -# from backend.connectors.ckan import HONDO_RESOURCE_ID -# from backend.unifier import unify_analytes, unify_waterlevels - - -# def config_factory(): -# cfg = Config() -# cfg.county = "eddy" -# cfg.bbox = "-104.5 32.5,-104 33" -# cfg.start_date = "2020-01-01" -# cfg.end_date = "2024-5-01" -# cfg.output_summary = False - -# cfg.use_source_nmbgmr = False -# cfg.use_source_wqp = False -# cfg.use_source_iscsevenrivers = False -# cfg.use_source_nwis = False -# cfg.use_source_oseroswell = False -# cfg.use_source_pvacd = False -# cfg.use_source_bor = False -# cfg.use_source_dwb = False -# cfg.use_source_bernco = False - -# cfg.site_limit = 10 -# return cfg - - -# @pytest.fixture -# def waterlevel_summary_cfg(): -# cfg = config_factory() -# cfg.output_summary = True -# return cfg - - -# @pytest.fixture -# def waterlevel_timeseries_cfg(): -# cfg = config_factory() -# cfg.output_summary = False -# return cfg - - -# @pytest.fixture -# def analyte_summary_cfg(): -# cfg = config_factory() -# cfg.output_summary = True -# cfg.analyte = "TDS" -# return cfg - - -# # def test_unify_analytes(cfg): -# # unify_analytes(cfg) - - -# def _setup(tmp_path, cfg, source, tag): -# d = tmp_path / tag -# d.mkdir() -# cfg.output_dir = str(d) -# for stag in ( -# "nmbgmr", -# "nwis", -# "pvacd", -# "bor", -# "dwb", -# "wqp", -# "iscsevenrivers", -# "oseroswell", -# "bernco", -# ): -# if stag == source: -# setattr(cfg, f"use_source_{stag}", True) -# return d - - -# def _setup_waterlevels(tmp_path, cfg, source): -# d = _setup(tmp_path, cfg, source, "waterlevels") -# unify_waterlevels(cfg) -# return d - - -# def _setup_analytes(tmp_path, cfg, source): -# d = _setup(tmp_path, cfg, source, "analyte") +import datetime +import os + +import pytest +import shapely.wkt + +from backend.config import Config +from backend.connectors.ckan import HONDO_RESOURCE_ID +from backend.unifier import unify_analytes, unify_waterlevels + + +def config_factory(): + cfg = Config() + cfg.county = "eddy" + cfg.bbox = "-104.5 32.5,-104 33" + cfg.start_date = "2020-01-01" + cfg.end_date = "2024-5-01" + cfg.output_summary = False + + cfg.use_source_nmbgmr = False + cfg.use_source_wqp = False + cfg.use_source_iscsevenrivers = False + cfg.use_source_nwis = False + cfg.use_source_oseroswell = False + cfg.use_source_pvacd = False + cfg.use_source_bor = False + cfg.use_source_dwb = False + cfg.use_source_bernco = False + + cfg.site_limit = 10 + return cfg + + +@pytest.fixture +def waterlevel_summary_cfg(): + cfg = config_factory() + cfg.output_summary = True + return cfg + + +@pytest.fixture +def waterlevel_timeseries_cfg(): + cfg = config_factory() + cfg.output_summary = False + return cfg + + +@pytest.fixture +def analyte_summary_cfg(): + cfg = config_factory() + cfg.output_summary = True + cfg.analyte = "TDS" + return cfg + + +# def test_unify_analytes(cfg): # unify_analytes(cfg) -# return d -# def _test_analytes_summary(tmp_path, cfg, source): -# d = _setup_analytes(tmp_path, cfg, source) -# assert (d / "output.csv").is_file() +def _setup(tmp_path, cfg, source, tag): + d = tmp_path / tag + d.mkdir() + cfg.output_dir = str(d) + for stag in ( + "nmbgmr", + "nwis", + "pvacd", + "bor", + "dwb", + "wqp", + "iscsevenrivers", + "oseroswell", + "bernco", + ): + if stag == source: + setattr(cfg, f"use_source_{stag}", True) + return d -# def _test_waterlevels_summary(tmp_path, cfg, source): -# d = _setup_waterlevels(tmp_path, cfg, source) -# assert (d / "output.csv").is_file() +def _setup_waterlevels(tmp_path, cfg, source): + d = _setup(tmp_path, cfg, source, "waterlevels") + unify_waterlevels(cfg) + return d -# def _test_waterlevels_timeseries( -# tmp_path, cfg, source, combined_flag=True, timeseries_flag=False -# ): -# d = _setup_waterlevels(tmp_path, cfg, source) -# combined = d / "output.combined.csv" -# timeseries = d / "output_timeseries" -# print(combined_flag) +def _setup_analytes(tmp_path, cfg, source): + d = _setup(tmp_path, cfg, source, "analyte") + unify_analytes(cfg) + return d -# print("combined", combined.is_file(), combined_flag) -# assert combined.is_file() == combined_flag -# print("timeseries", timeseries.is_dir(), timeseries_flag) -# assert timeseries.is_dir() == timeseries_flag -# return combined, timeseries +def _test_analytes_summary(tmp_path, cfg, source): + d = _setup_analytes(tmp_path, cfg, source) + assert (d / "output.csv").is_file() -# def _test_waterelevels_timeseries_date_range( -# tmp_path, cfg, source, timeseries_flag=True, combined_flag=False -# ): -# combined, timeseries = _test_waterlevels_timeseries( -# tmp_path, -# cfg, -# source, -# timeseries_flag=timeseries_flag, -# combined_flag=combined_flag, -# ) +def _test_waterlevels_summary(tmp_path, cfg, source): + d = _setup_waterlevels(tmp_path, cfg, source) + assert (d / "output.csv").is_file() -# for p in timeseries.iterdir(): -# if os.path.basename(p) == "sites.csv": -# continue -# with open(p, "r") as rfile: -# lines = rfile.readlines() -# for l in lines[1:]: -# vs = l.split(",") -# dd = vs[3] -# dd = datetime.datetime.strptime(dd, "%Y-%m-%d") -# assert dd.year >= 2020 and dd.year <= 2024 +def _test_waterlevels_timeseries( + tmp_path, cfg, source, combined_flag=True, timeseries_flag=False +): + d = _setup_waterlevels(tmp_path, cfg, source) + combined = d / "output.combined.csv" + timeseries = d / "output_timeseries" + print(combined_flag) + print("combined", combined.is_file(), combined_flag) + assert combined.is_file() == combined_flag + print("timeseries", timeseries.is_dir(), timeseries_flag) + assert timeseries.is_dir() == timeseries_flag -# def test_nwis_site_health_check(): -# from backend.connectors.usgs.source import NWISSiteSource + return combined, timeseries -# n = NWISSiteSource() -# assert n.health() +def _test_waterelevels_timeseries_date_range( + tmp_path, cfg, source, timeseries_flag=True, combined_flag=False +): + combined, timeseries = _test_waterlevels_timeseries( + tmp_path, + cfg, + source, + timeseries_flag=timeseries_flag, + combined_flag=combined_flag, + ) -# def test_nmbgmr_site_health_check(): -# from backend.connectors.nmbgmr.source import NMBGMRSiteSource + for p in timeseries.iterdir(): + if os.path.basename(p) == "sites.csv": + continue -# n = NMBGMRSiteSource() -# assert n.health() + with open(p, "r") as rfile: + lines = rfile.readlines() + for l in lines[1:]: + vs = l.split(",") + dd = vs[3] + dd = datetime.datetime.strptime(dd, "%Y-%m-%d") + assert dd.year >= 2020 and dd.year <= 2024 -# def test_wqp_site_health_check(): -# from backend.connectors.wqp.source import WQPSiteSource +def test_nwis_site_health_check(): + from backend.connectors.usgs.source import NWISSiteSource -# n = WQPSiteSource() -# assert n.health() + n = NWISSiteSource() + assert n.health() -# def test_bor_site_health_check(): -# from backend.connectors.bor.source import BORSiteSource +def test_nmbgmr_site_health_check(): + from backend.connectors.nmbgmr.source import NMBGMRSiteSource -# n = BORSiteSource() -# assert n.health() + n = NMBGMRSiteSource() + assert n.health() -# def test_dwb_site_health_check(): -# from backend.connectors.nmenv.source import DWBSiteSource +def test_wqp_site_health_check(): + from backend.connectors.wqp.source import WQPSiteSource -# n = DWBSiteSource() -# assert n.health() + n = WQPSiteSource() + assert n.health() -# def test_isc_seven_rivers_site_health_check(): -# from backend.connectors.isc_seven_rivers.source import ISCSevenRiversSiteSource +def test_bor_site_health_check(): + from backend.connectors.bor.source import BORSiteSource -# n = ISCSevenRiversSiteSource() -# assert n.health() + n = BORSiteSource() + assert n.health() -# def test_ckan_site_health_check(): -# from backend.connectors.ckan.source import OSERoswellSiteSource +def test_dwb_site_health_check(): + from backend.connectors.nmenv.source import DWBSiteSource -# n = OSERoswellSiteSource(HONDO_RESOURCE_ID) -# assert n.health() + n = DWBSiteSource() + assert n.health() -# def test_pvacd_site_health_check(): -# from backend.connectors.st2.source import PVACDSiteSource +def test_isc_seven_rivers_site_health_check(): + from backend.connectors.isc_seven_rivers.source import ISCSevenRiversSiteSource -# n = PVACDSiteSource() -# assert n.health() + n = ISCSevenRiversSiteSource() + assert n.health() -# def test_bernco_site_health_check(): -# from backend.connectors.st2.source import BernCoSiteSource +def test_ckan_site_health_check(): + from backend.connectors.ckan.source import OSERoswellSiteSource + + n = OSERoswellSiteSource(HONDO_RESOURCE_ID) + assert n.health() -# n = BernCoSiteSource() -# assert n.health() +def test_pvacd_site_health_check(): + from backend.connectors.st2.source import PVACDSiteSource + + n = PVACDSiteSource() + assert n.health() + + +def test_bernco_site_health_check(): + from backend.connectors.st2.source import BernCoSiteSource + + n = BernCoSiteSource() + assert n.health() + + +# def test_ose_roswell_site_health_check(): +# from backend.connectors.ose_roswell.source import OSESiteSource +# n = OSESiteSource() +# assert n.health() -# # def test_ose_roswell_site_health_check(): -# # from backend.connectors.ose_roswell.source import OSESiteSource -# # n = OSESiteSource() -# # assert n.health() - - -# # Source tests ======================================================================================================== -# def test_source_bounds_nmbgmr(): -# from backend.unifier import get_source_bounds -# from backend.connectors import NM_STATE_BOUNDING_POLYGON - -# sourcekey = "nmbgmr" -# bounds = get_source_bounds(sourcekey) -# assert bounds -# assert bounds.is_valid -# assert bounds.geom_type == "Polygon" -# assert bounds == NM_STATE_BOUNDING_POLYGON - - -# def test_source_bounds_is_seven_rivers(): -# from backend.unifier import get_source_bounds -# from backend.connectors import ISC_SEVEN_RIVERS_BOUNDING_POLYGON -# sourcekey = "iscsevenrivers" -# bounds = get_source_bounds(sourcekey) -# assert bounds -# assert bounds.is_valid -# assert bounds.geom_type == "Polygon" -# assert bounds == ISC_SEVEN_RIVERS_BOUNDING_POLYGON +# Source tests ======================================================================================================== +def test_source_bounds_nmbgmr(): + from backend.unifier import get_source_bounds + from backend.connectors import NM_STATE_BOUNDING_POLYGON + + sourcekey = "nmbgmr" + bounds = get_source_bounds(sourcekey) + assert bounds + assert bounds.is_valid + assert bounds.geom_type == "Polygon" + assert bounds == NM_STATE_BOUNDING_POLYGON + + +def test_source_bounds_is_seven_rivers(): + from backend.unifier import get_source_bounds + from backend.connectors import ISC_SEVEN_RIVERS_BOUNDING_POLYGON + sourcekey = "iscsevenrivers" + bounds = get_source_bounds(sourcekey) + assert bounds + assert bounds.is_valid + assert bounds.geom_type == "Polygon" + assert bounds == ISC_SEVEN_RIVERS_BOUNDING_POLYGON -# def test_source_bounds_oser(): -# from backend.unifier import get_source_bounds -# from backend.connectors import ( -# OSE_ROSWELL_HONDO_BOUNDING_POLYGON, -# OSE_ROSWELL_ROSWELL_BOUNDING_POLYGON, -# OSE_ROSWELL_FORT_SUMNER_BOUNDING_POLYGON, -# ) -# sourcekey = "oseroswell" -# bounds = get_source_bounds(sourcekey) -# assert bounds -# assert bounds.is_valid -# assert bounds.geom_type == "GeometryCollection" -# assert bounds == shapely.GeometryCollection( -# [ -# OSE_ROSWELL_HONDO_BOUNDING_POLYGON, -# OSE_ROSWELL_FORT_SUMNER_BOUNDING_POLYGON, -# OSE_ROSWELL_ROSWELL_BOUNDING_POLYGON, -# ] -# ) +def test_source_bounds_oser(): + from backend.unifier import get_source_bounds + from backend.connectors import ( + OSE_ROSWELL_HONDO_BOUNDING_POLYGON, + OSE_ROSWELL_ROSWELL_BOUNDING_POLYGON, + OSE_ROSWELL_FORT_SUMNER_BOUNDING_POLYGON, + ) + sourcekey = "oseroswell" + bounds = get_source_bounds(sourcekey) + assert bounds + assert bounds.is_valid + assert bounds.geom_type == "GeometryCollection" + assert bounds == shapely.GeometryCollection( + [ + OSE_ROSWELL_HONDO_BOUNDING_POLYGON, + OSE_ROSWELL_FORT_SUMNER_BOUNDING_POLYGON, + OSE_ROSWELL_ROSWELL_BOUNDING_POLYGON, + ] + ) -# def test_sources_socorro(tmp_path): -# cfg = Config() -# cfg.county = "socorro" -# from backend.unifier import get_sources +def test_sources_socorro(tmp_path): + cfg = Config() + cfg.county = "socorro" -# sources = get_sources(cfg) -# assert sources -# assert len(sources) == 2 -# assert sorted([s.__class__.__name__ for s in sources]) == sorted( -# ["NMBGMRSiteSource", "NWISSiteSource"] -# ) + from backend.unifier import get_sources + sources = get_sources(cfg) + assert sources + assert len(sources) == 2 + assert sorted([s.__class__.__name__ for s in sources]) == sorted( + ["NMBGMRSiteSource", "NWISSiteSource"] + ) -# def test_sources_eddy_dtw(tmp_path): -# cfg = Config() -# cfg.county = "eddy" -# from backend.unifier import get_sources +def test_sources_eddy_dtw(tmp_path): + cfg = Config() + cfg.county = "eddy" -# sources = get_sources(cfg) -# assert sources -# assert len(sources) == 5 -# assert sorted([s.__class__.__name__ for s in sources]) == sorted( -# [ -# "ISCSevenRiversSiteSource", -# "NMBGMRSiteSource", -# "OSERoswellSiteSource", -# "PVACDSiteSource", -# "NWISSiteSource", -# ] -# ) + from backend.unifier import get_sources + + sources = get_sources(cfg) + assert sources + assert len(sources) == 5 + assert sorted([s.__class__.__name__ for s in sources]) == sorted( + [ + "ISCSevenRiversSiteSource", + "NMBGMRSiteSource", + "OSERoswellSiteSource", + "PVACDSiteSource", + "NWISSiteSource", + ] + ) -# def test_sources_eddy_tds(tmp_path): -# cfg = Config() -# cfg.county = "eddy" -# cfg.analyte = "TDS" +def test_sources_eddy_tds(tmp_path): + cfg = Config() + cfg.county = "eddy" + cfg.analyte = "TDS" -# from backend.unifier import get_sources + from backend.unifier import get_sources -# sources = get_sources(cfg) -# assert sources -# assert len(sources) == 5 -# assert sorted([s.__class__.__name__ for s in sources]) == sorted( -# [ -# "BORSiteSource", -# "DWBSiteSource", -# "ISCSevenRiversSiteSource", -# "NMBGMRSiteSource", -# "WQPSiteSource", -# ] -# ) - - -# # Waterlevel Summary tests =========================================================================================== -# def test_unify_waterlevels_bernco_summary(tmp_path, waterlevel_summary_cfg): -# waterlevel_summary_cfg.county = "bernalillo" -# waterlevel_summary_cfg.bbox = None -# _test_waterlevels_summary(tmp_path, waterlevel_summary_cfg, "bernco") - - -# def test_unify_waterlevels_nwis_summary(tmp_path, waterlevel_summary_cfg): -# _test_waterlevels_summary(tmp_path, waterlevel_summary_cfg, "nwis") - - -# def test_unify_waterlevels_amp_summary(tmp_path, waterlevel_summary_cfg): -# _test_waterlevels_summary(tmp_path, waterlevel_summary_cfg, "nmbgmr") - - -# def test_unify_waterlevels_pvacd_summary(tmp_path, waterlevel_summary_cfg): -# _test_waterlevels_summary(tmp_path, waterlevel_summary_cfg, "pvacd") - - -# def test_unify_waterlevels_isc_seven_rivers_summary(tmp_path, waterlevel_summary_cfg): -# _test_waterlevels_summary(tmp_path, waterlevel_summary_cfg, "iscsevenrivers") - - -# def test_unify_waterlevels_ose_roswell_summary(tmp_path, waterlevel_summary_cfg): -# _test_waterlevels_summary(tmp_path, waterlevel_summary_cfg, "oseroswell") - - -# # Waterlevel timeseries tests ========================================================================================= -# def test_unify_waterlevels_nwis_timeseries(tmp_path, waterlevel_timeseries_cfg): -# # there are one or more locations within the bounding box that have only -# # one record, so there is a combined file -# _test_waterlevels_timeseries( -# tmp_path, -# waterlevel_timeseries_cfg, -# "nwis", -# combined_flag=True, -# timeseries_flag=True, -# ) + sources = get_sources(cfg) + assert sources + assert len(sources) == 5 + assert sorted([s.__class__.__name__ for s in sources]) == sorted( + [ + "BORSiteSource", + "DWBSiteSource", + "ISCSevenRiversSiteSource", + "NMBGMRSiteSource", + "WQPSiteSource", + ] + ) + + +# Waterlevel Summary tests =========================================================================================== +def test_unify_waterlevels_bernco_summary(tmp_path, waterlevel_summary_cfg): + waterlevel_summary_cfg.county = "bernalillo" + waterlevel_summary_cfg.bbox = None + _test_waterlevels_summary(tmp_path, waterlevel_summary_cfg, "bernco") + + +def test_unify_waterlevels_nwis_summary(tmp_path, waterlevel_summary_cfg): + _test_waterlevels_summary(tmp_path, waterlevel_summary_cfg, "nwis") + + +def test_unify_waterlevels_amp_summary(tmp_path, waterlevel_summary_cfg): + _test_waterlevels_summary(tmp_path, waterlevel_summary_cfg, "nmbgmr") + + +def test_unify_waterlevels_pvacd_summary(tmp_path, waterlevel_summary_cfg): + _test_waterlevels_summary(tmp_path, waterlevel_summary_cfg, "pvacd") + + +def test_unify_waterlevels_isc_seven_rivers_summary(tmp_path, waterlevel_summary_cfg): + _test_waterlevels_summary(tmp_path, waterlevel_summary_cfg, "iscsevenrivers") + + +def test_unify_waterlevels_ose_roswell_summary(tmp_path, waterlevel_summary_cfg): + _test_waterlevels_summary(tmp_path, waterlevel_summary_cfg, "oseroswell") + + +# Waterlevel timeseries tests ========================================================================================= +def test_unify_waterlevels_nwis_timeseries(tmp_path, waterlevel_timeseries_cfg): + # there are one or more locations within the bounding box that have only + # one record, so there is a combined file + _test_waterlevels_timeseries( + tmp_path, + waterlevel_timeseries_cfg, + "nwis", + combined_flag=True, + timeseries_flag=True, + ) -# def test_unify_waterlevels_amp_timeseries(tmp_path, waterlevel_timeseries_cfg): -# _test_waterlevels_timeseries(tmp_path, waterlevel_timeseries_cfg, "nmbgmr") +def test_unify_waterlevels_amp_timeseries(tmp_path, waterlevel_timeseries_cfg): + _test_waterlevels_timeseries(tmp_path, waterlevel_timeseries_cfg, "nmbgmr") -# def test_unify_waterlevels_pvacd_timeseries(tmp_path, waterlevel_timeseries_cfg): -# # all locations within the bounding box have more than one record -# # so there is no combined file -# _test_waterlevels_timeseries( -# tmp_path, -# waterlevel_timeseries_cfg, -# "pvacd", -# combined_flag=False, -# timeseries_flag=True, -# ) +def test_unify_waterlevels_pvacd_timeseries(tmp_path, waterlevel_timeseries_cfg): + # all locations within the bounding box have more than one record + # so there is no combined file + _test_waterlevels_timeseries( + tmp_path, + waterlevel_timeseries_cfg, + "pvacd", + combined_flag=False, + timeseries_flag=True, + ) -# def test_unify_waterlevels_isc_seven_rivers_timeseries( -# tmp_path, waterlevel_timeseries_cfg -# ): -# # all locations within the bounding box have more than one record -# # so there is no combined file -# _test_waterlevels_timeseries( -# tmp_path, -# waterlevel_timeseries_cfg, -# "iscsevenrivers", -# combined_flag=False, -# timeseries_flag=True, -# ) +def test_unify_waterlevels_isc_seven_rivers_timeseries( + tmp_path, waterlevel_timeseries_cfg +): + # all locations within the bounding box have more than one record + # so there is no combined file + _test_waterlevels_timeseries( + tmp_path, + waterlevel_timeseries_cfg, + "iscsevenrivers", + combined_flag=False, + timeseries_flag=True, + ) -# def test_unify_waterlevels_ose_roswell_timeseries(tmp_path, waterlevel_timeseries_cfg): -# _test_waterlevels_timeseries( -# tmp_path, waterlevel_timeseries_cfg, "oseroswell", timeseries_flag=True -# ) +def test_unify_waterlevels_ose_roswell_timeseries(tmp_path, waterlevel_timeseries_cfg): + _test_waterlevels_timeseries( + tmp_path, waterlevel_timeseries_cfg, "oseroswell", timeseries_flag=True + ) -# # Waterlevel summary date range tests ================================================================================= -# def test_waterlevels_nwis_summary_date_range(tmp_path, waterlevel_summary_cfg): -# d = _setup_waterlevels(tmp_path, waterlevel_summary_cfg, "nwis") -# assert (d / "output.csv").is_file() +# Waterlevel summary date range tests ================================================================================= +def test_waterlevels_nwis_summary_date_range(tmp_path, waterlevel_summary_cfg): + d = _setup_waterlevels(tmp_path, waterlevel_summary_cfg, "nwis") + assert (d / "output.csv").is_file() -# # Waterlevel timeseries date range ==================================================================================== -# def test_waterlevels_nwis_timeseries_date_range(tmp_path, waterlevel_timeseries_cfg): -# # there are one or more locations within the bounding box and date range -# # that have only one record, so there is a combined file -# _test_waterelevels_timeseries_date_range( -# tmp_path, -# waterlevel_timeseries_cfg, -# "nwis", -# timeseries_flag=True, -# combined_flag=True, -# ) +# Waterlevel timeseries date range ==================================================================================== +def test_waterlevels_nwis_timeseries_date_range(tmp_path, waterlevel_timeseries_cfg): + # there are one or more locations within the bounding box and date range + # that have only one record, so there is a combined file + _test_waterelevels_timeseries_date_range( + tmp_path, + waterlevel_timeseries_cfg, + "nwis", + timeseries_flag=True, + combined_flag=True, + ) -# def test_waterlevels_isc_seven_rivers_timeseries_date_range( -# tmp_path, waterlevel_timeseries_cfg -# ): -# # all locations within the bounding box and date rangehave more than one -# # record so there is no combined file -# _test_waterelevels_timeseries_date_range( -# tmp_path, -# waterlevel_timeseries_cfg, -# "iscsevenrivers", -# timeseries_flag=True, -# combined_flag=False, -# ) +def test_waterlevels_isc_seven_rivers_timeseries_date_range( + tmp_path, waterlevel_timeseries_cfg +): + # all locations within the bounding box and date rangehave more than one + # record so there is no combined file + _test_waterelevels_timeseries_date_range( + tmp_path, + waterlevel_timeseries_cfg, + "iscsevenrivers", + timeseries_flag=True, + combined_flag=False, + ) -# def test_waterlevels_pvacd_timeseries_date_range(tmp_path, waterlevel_timeseries_cfg): -# # all locations within the bounding box and date rangehave more than one -# # record so there is no combined file -# _test_waterelevels_timeseries_date_range( -# tmp_path, -# waterlevel_timeseries_cfg, -# "pvacd", -# timeseries_flag=True, -# combined_flag=False, -# ) - - -# # Analyte summary tests =============================================================================================== -# def test_unify_analytes_wqp_summary(tmp_path, analyte_summary_cfg): -# _test_analytes_summary(tmp_path, analyte_summary_cfg, "wqp") - - -# def test_unify_analytes_amp_summary(tmp_path, analyte_summary_cfg): -# _test_analytes_summary(tmp_path, analyte_summary_cfg, "nmbgmr") - - -# def test_unify_analytes_bor_summary(tmp_path, analyte_summary_cfg): -# # BOR locations are found within Otero County -# analyte_summary_cfg.county = "otero" -# analyte_summary_cfg.bbox = None -# _test_analytes_summary(tmp_path, analyte_summary_cfg, "bor") +def test_waterlevels_pvacd_timeseries_date_range(tmp_path, waterlevel_timeseries_cfg): + # all locations within the bounding box and date rangehave more than one + # record so there is no combined file + _test_waterelevels_timeseries_date_range( + tmp_path, + waterlevel_timeseries_cfg, + "pvacd", + timeseries_flag=True, + combined_flag=False, + ) + + +# Analyte summary tests =============================================================================================== +def test_unify_analytes_wqp_summary(tmp_path, analyte_summary_cfg): + _test_analytes_summary(tmp_path, analyte_summary_cfg, "wqp") + + +def test_unify_analytes_amp_summary(tmp_path, analyte_summary_cfg): + _test_analytes_summary(tmp_path, analyte_summary_cfg, "nmbgmr") + + +def test_unify_analytes_bor_summary(tmp_path, analyte_summary_cfg): + # BOR locations are found within Otero County + analyte_summary_cfg.county = "otero" + analyte_summary_cfg.bbox = None + _test_analytes_summary(tmp_path, analyte_summary_cfg, "bor") -# def test_unify_analytes_isc_seven_rivers_summary(tmp_path, analyte_summary_cfg): -# _test_analytes_summary(tmp_path, analyte_summary_cfg, "iscsevenrivers") - - -# def test_unify_analytes_dwb_summary(tmp_path, analyte_summary_cfg): -# _test_analytes_summary(tmp_path, analyte_summary_cfg, "dwb") +def test_unify_analytes_isc_seven_rivers_summary(tmp_path, analyte_summary_cfg): + _test_analytes_summary(tmp_path, analyte_summary_cfg, "iscsevenrivers") + + +def test_unify_analytes_dwb_summary(tmp_path, analyte_summary_cfg): + _test_analytes_summary(tmp_path, analyte_summary_cfg, "dwb") # ============= EOF ============================================= From ccc5cad52869f8facc0e46f7745ee60c9e61eceb Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 10 Apr 2025 16:56:32 -0600 Subject: [PATCH 33/55] Comment out debugging print statements --- backend/unifier.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/unifier.py b/backend/unifier.py index 880369f..40a6296 100644 --- a/backend/unifier.py +++ b/backend/unifier.py @@ -194,9 +194,9 @@ def _site_wrapper(site_source, parameter_source, persister, config): if sites_with_records_count >= site_limit: # remove any extra sites that were gathered. removes 0 if site_limit is not exceeded num_sites_to_remove = sites_with_records_count - site_limit - print( - f"removing {num_sites_to_remove} to avoid exceeding the site limit" - ) + # print( + # f"removing {num_sites_to_remove} to avoid exceeding the site limit" + # ) # if sites_with_records_count == sit_limit then num_sites_to_remove = 0 # and calling list[:0] will retur an empty list, so subtract From 64928747efe867cbc3c46409f2622e332a78d2e4 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 10 Apr 2025 16:57:13 -0600 Subject: [PATCH 34/55] remove debugging print statements --- backend/unifier.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/backend/unifier.py b/backend/unifier.py index 40a6296..ce03a3d 100644 --- a/backend/unifier.py +++ b/backend/unifier.py @@ -180,16 +180,16 @@ def _site_wrapper(site_source, parameter_source, persister, config): persister.sites.append(site) if site_limit: - print( - "sites_with_records_count:", - sites_with_records_count, - "|", - "site_limit:", - site_limit, - "|", - "chunk_size:", - site_source.chunk_size, - ) + # print( + # "sites_with_records_count:", + # sites_with_records_count, + # "|", + # "site_limit:", + # site_limit, + # "|", + # "chunk_size:", + # site_source.chunk_size, + # ) if sites_with_records_count >= site_limit: # remove any extra sites that were gathered. removes 0 if site_limit is not exceeded From f359dc2544eefc007384b6e5d7788ace050058af Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 10 Apr 2025 17:10:02 -0600 Subject: [PATCH 35/55] mypy type checking --- backend/config.py | 11 ----- backend/source.py | 40 ++++++++---------- backend/transformer.py | 93 +++++++++++++++++++++--------------------- 3 files changed, 64 insertions(+), 80 deletions(-) diff --git a/backend/config.py b/backend/config.py index 9f0f2d7..25f61fd 100644 --- a/backend/config.py +++ b/backend/config.py @@ -15,7 +15,6 @@ # =============================================================================== import os import sys -import time from datetime import datetime, timedelta import shapely.wkt @@ -29,15 +28,6 @@ NMBGMRAnalyteSource, ) from .connectors.bor.source import BORSiteSource, BORAnalyteSource -from .connectors.ckan import ( - HONDO_RESOURCE_ID, - FORT_SUMNER_RESOURCE_ID, - ROSWELL_RESOURCE_ID, -) -from .connectors.ckan.source import ( - OSERoswellSiteSource, - OSERoswellWaterLevelSource, -) from .connectors.nmenv.source import DWBSiteSource, DWBAnalyteSource from .connectors.nmose.source import NMOSEPODSiteSource from .constants import ( @@ -67,7 +57,6 @@ ISCSevenRiversAnalyteSource, ) from .connectors.st2.source import ( - ST2SiteSource, PVACDSiteSource, PVACDWaterLevelSource, EBIDSiteSource, diff --git a/backend/source.py b/backend/source.py index 5884861..5d8196a 100644 --- a/backend/source.py +++ b/backend/source.py @@ -15,19 +15,13 @@ # =============================================================================== from json import JSONDecodeError -import click import httpx import shapely.wkt from shapely import MultiPoint from typing import Union, List, Callable, Dict from backend.constants import ( - MILLIGRAMS_PER_LITER, FEET, - METERS, - PARTS_PER_MILLION, - DTW, - DTW_UNITS, DT_MEASURED, PARAMETER_NAME, PARAMETER_UNITS, @@ -36,7 +30,6 @@ LATEST, ) from backend.logger import Loggable -from backend.persister import BasePersister, CSVPersister from backend.record import ( AnalyteRecord, AnalyteSummaryRecord, @@ -106,7 +99,10 @@ def func(x): return sorted(records, key=func)[0] elif bookend == LATEST: return sorted(records, key=func)[-1] - + else: + raise ValueError( + f"Invalid bookend {bookend}. Must be either {EARLIEST} or {LATEST}" + ) def get_analyte_search_param(parameter: str, mapping: dict) -> str: """ @@ -178,11 +174,9 @@ class BaseSource(Loggable): """ transformer_klass = BaseTransformer - config = None - def __init__(self, config=None): + def __init__(self): self.transformer = self.transformer_klass() - self.set_config(config) super().__init__() @property @@ -205,7 +199,7 @@ def discover(self, *args, **kw): # Methods Already Implemented # ========================================================================== - def _execute_text_request(self, url: str, params=None, **kw) -> str: + def _execute_text_request(self, url: str, params: dict | None = None, **kw) -> str: """ Executes a get request to the provided url and returns the text response. @@ -235,8 +229,8 @@ def _execute_text_request(self, url: str, params=None, **kw) -> str: return "" def _execute_json_request( - self, url: str, params: dict = None, tag: str = None, **kw - ) -> dict: + self, url: str, params: dict | None = None, tag: str | None = None, **kw + ) -> dict | None: """ Executes a get request to the provided url and returns the json response. @@ -268,18 +262,18 @@ def _execute_json_request( return obj except JSONDecodeError: self.warn(f"service responded but with no data. \n{resp.text}") - return [] + return None else: self.warn(f"service responded with status {resp.status_code}") self.warn(f"service responded with text {resp.text}") self.warn(f"service at url: {resp.url}") - return [] + return None # ========================================================================== # Methods Implemented in BaseSiteSource and BaseParameterSource # ========================================================================== - def read(self, *args, **kw) -> list: + def read(self, *args, **kw) -> list | None: """ Returns the records. Implemented in BaseSiteSource and BaseAnalyteSource """ @@ -437,7 +431,7 @@ def intersects(self, wkt: str) -> bool: return True - def read(self, *args, **kw) -> List[SiteRecord]: + def read(self, *args, **kw) -> List[SiteRecord] | None: """ Returns a list of transformed site records. Calls self.get_records, which needs to be implemented for each source @@ -454,6 +448,7 @@ def read(self, *args, **kw) -> List[SiteRecord]: return self._transform_sites(records) else: self.warn("No site records returned") + return None def _transform_sites(self, records: list) -> List[SiteRecord]: """ @@ -479,7 +474,7 @@ def _transform_sites(self, records: list) -> List[SiteRecord]: self.log(f"processed nrecords={len(transformed_records)}") return transformed_records - def chunks(self, records: list, chunk_size: int = None) -> list: + def chunks(self, records: list, chunk_size: int | None = None) -> list: """ Returns a list of records split into lists of size chunk_size. If chunk_size less than 1 then the records are not split @@ -613,13 +608,13 @@ def _extract_latest_record(self, records: list) -> dict: return self._extract_terminal_record(records, bookend=LATEST) def read( - self, site_record: SiteRecord, use_summarize: bool, start_ind: int, end_ind: int + self, site_record: SiteRecord | list, use_summarize: bool, start_ind: int, end_ind: int ) -> List[ AnalyteRecord | AnalyteSummaryRecord | WaterLevelRecord | WaterLevelSummaryRecord - ]: + ] | None: """ Returns a list of transformed parameter records. Transformed parameter records are standardized so that all of the records have the same format. They are @@ -772,6 +767,7 @@ def read( name = ",".join(names) self.warn(f"{name}: No records found") + return None # ========================================================================== # Methods Implemented in BaseAnalyteSource and BaseWaterLevelSource @@ -820,7 +816,7 @@ def _get_output_units(self) -> str: # Methods That Need to be Implemented For Each Source # ========================================================================== - def _extract_site_records(self, records: dict, site_record: dict) -> list: + def _extract_site_records(self, records: list[dict], site_record: dict) -> list: """ Returns all records for a single site as a list of records (which are dictionaries). diff --git a/backend/transformer.py b/backend/transformer.py index 0c3796d..904e895 100644 --- a/backend/transformer.py +++ b/backend/transformer.py @@ -394,56 +394,55 @@ def do_transform( """ # _transform needs to be implemented by each SiteTransformer # _transform is already implemented in each ParameterTransformer - record = self._transform(inrecord, *args, **kw) - print(type(record)) - if not record: + transformed_record = self._transform(inrecord, *args, **kw) + if not transformed_record: return None # ensure that a site or summary record is contained within the boundaing polygon - if "longitude" in record and "latitude" in record: - if not self.contained(record["longitude"], record["latitude"]): + if "longitude" in transformed_record and "latitude" in transformed_record: + if not self.contained(transformed_record["longitude"], transformed_record["latitude"]): self.warn( - f"Skipping site {record['id']}. It is not within the defined geographic bounds" + f"Skipping site {transformed_record['id']}. It is not within the defined geographic bounds" ) return None - self._post_transform(record, *args, **kw) + self._post_transform(transformed_record, *args, **kw) # standardize datetime - dt = record.get(DT_MEASURED) + dt = transformed_record.get(DT_MEASURED) if dt: - d, t = standardize_datetime(dt, record["id"]) - record["date_measured"] = d - record["time_measured"] = t + d, t = standardize_datetime(dt, transformed_record["id"]) + transformed_record["date_measured"] = d + transformed_record["time_measured"] = t else: - mrd = record.get("latest_datetime") + mrd = transformed_record.get("latest_datetime") if mrd: - d, t = standardize_datetime(mrd, record["id"]) - record["date_measured"] = d - record["time_measured"] = t + d, t = standardize_datetime(mrd, transformed_record["id"]) + transformed_record["date_measured"] = d + transformed_record["time_measured"] = t # convert to proper record type # a record klass holds the original record's data as a dictionary, and has methods to update the record's data and get the record's data klass = self._get_record_klass() - record = klass(record) + klassed_record = klass(transformed_record) # update the record's geographic information and well data if it is a SiteRecord or SummaryRecord # transforms the horizontal datum and lon/lat coordinates to WGS84 # transforms the elevation and well depth units to the output unit specified in the config # transforms the well depth and well depth units to the output unit specified in the config - if isinstance(record, (SiteRecord, SummaryRecord)): - y = float(record.latitude) - x = float(record.longitude) + if isinstance(klassed_record, (SiteRecord, SummaryRecord)): + y = float(klassed_record.latitude) + x = float(klassed_record.longitude) if x == 0 or y == 0: - self.warn(f"Skipping site {record.id}. Latitude or Longitude is 0") + self.warn(f"Skipping site {klassed_record.id}. Latitude or Longitude is 0") return None - input_horizontal_datum = record.horizontal_datum + input_horizontal_datum = klassed_record.horizontal_datum if input_horizontal_datum not in ALLOWED_DATUMS: self.warn( - f"Skipping site {record.id}. Datum {input_horizontal_datum} cannot be processed" + f"Skipping site {klassed_record.id}. Datum {input_horizontal_datum} cannot be processed" ) return None @@ -464,43 +463,43 @@ def do_transform( if not self.in_nm(lng, lat): self.warn( - f"Skipping site {record.id}. Coordinates {x}, {y} with datum {input_horizontal_datum} are not within 25km of New Mexico" + f"Skipping site {klassed_record.id}. Coordinates {x}, {y} with datum {input_horizontal_datum} are not within 25km of New Mexico" ) return None - record.update(latitude=lat) - record.update(longitude=lng) - record.update(horizontal_datum=datum) + klassed_record.update(latitude=lat) + klassed_record.update(longitude=lng) + klassed_record.update(horizontal_datum=datum) elevation, elevation_unit = transform_length_units( - record.elevation, - record.elevation_units, + klassed_record.elevation, + klassed_record.elevation_units, output_elevation_units, ) - record.update(elevation=elevation) - record.update(elevation_units=elevation_unit) + klassed_record.update(elevation=elevation) + klassed_record.update(elevation_units=elevation_unit) well_depth, well_depth_unit = transform_length_units( - record.well_depth, - record.well_depth_units, + klassed_record.well_depth, + klassed_record.well_depth_units, well_depth_units, ) - record.update(well_depth=well_depth) - record.update(well_depth_units=well_depth_unit) + klassed_record.update(well_depth=well_depth) + klassed_record.update(well_depth_units=well_depth_unit) # update the units to the output unit for analyte records # this is done after converting the units to the output unit for the analyte records # convert the parameter value to the output unit specified in the config - elif isinstance(record, (AnalyteRecord, WaterLevelRecord)): - if isinstance(record, AnalyteRecord): + elif isinstance(klassed_record, (AnalyteRecord, WaterLevelRecord)): + if isinstance(klassed_record, AnalyteRecord): output_units = self.config.analyte_output_units else: output_units = self.config.waterlevel_output_units - source_result = record.parameter_value - source_unit = record.source_parameter_units - dt = record.date_measured - source_name = record.source_parameter_name + source_result = klassed_record.parameter_value + source_unit = klassed_record.source_parameter_units + dt = klassed_record.date_measured + source_name = klassed_record.source_parameter_name conversion_factor = None # conversion factor will remain None if record is kept for time series and cannot be converted, such as non-detects warning_msg = "" try: @@ -516,21 +515,21 @@ def do_transform( msg = f"{warning_msg} for {record.id}" self.warn(msg) except TypeError: - msg = f"Keeping {source_result} for {record.id} on {record.date_measured} for time series data" + msg = f"Keeping {source_result} for {record.id} on {klassed_record.date_measured} for time series data" self.warn(msg) converted_result = source_result except ValueError: - msg = f"Keeping {source_result} for {record.id} on {record.date_measured} for time series data" + msg = f"Keeping {source_result} for {record.id} on {klassed_record.date_measured} for time series data" self.warn(msg) converted_result = source_result if warning_msg == "": - record.update(conversion_factor=conversion_factor) - record.update(parameter_value=converted_result) + klassed_record.update(conversion_factor=conversion_factor) + klassed_record.update(parameter_value=converted_result) else: - record = None + klassed_record = None - return record + return klassed_record def in_nm(self, lng: float | int | str, lat: float | int | str) -> bool: """ @@ -819,7 +818,7 @@ def _get_parameter_name_and_units(self) -> tuple: class AnalyteTransformer(ParameterTransformer): - def _get_record_klass(self) -> AnalyteRecord | AnalyteSummaryRecord: + def _get_record_klass(self) -> type[AnalyteRecord] | type[AnalyteSummaryRecord]: """ Returns the AnalyteRecord class to use for the transformer for water level records if config.output_summary is False, otherwise From 1ff9d6e3ff62346bbb88d7b737bc8ddd76933136 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 10 Apr 2025 17:10:58 -0600 Subject: [PATCH 36/55] mypy type hinting --- backend/transformer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/transformer.py b/backend/transformer.py index 904e895..c9eafd8 100644 --- a/backend/transformer.py +++ b/backend/transformer.py @@ -512,14 +512,14 @@ def do_transform( dt, ) if warning_msg != "": - msg = f"{warning_msg} for {record.id}" + msg = f"{warning_msg} for {klassed_record.id}" self.warn(msg) except TypeError: - msg = f"Keeping {source_result} for {record.id} on {klassed_record.date_measured} for time series data" + msg = f"Keeping {source_result} for {klassed_record.id} on {klassed_record.date_measured} for time series data" self.warn(msg) converted_result = source_result except ValueError: - msg = f"Keeping {source_result} for {record.id} on {klassed_record.date_measured} for time series data" + msg = f"Keeping {source_result} for {klassed_record.id} on {klassed_record.date_measured} for time series data" self.warn(msg) converted_result = source_result From dad5e50dd9bdc049b401c0c18d76cea63de43d78 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Fri, 11 Apr 2025 10:05:52 -0600 Subject: [PATCH 37/55] Finished mypy type checking for tests in GitHub Actions --- backend/connectors/nmose/source.py | 9 ++++++--- backend/source.py | 2 +- backend/transformer.py | 14 +++++++++++++- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/backend/connectors/nmose/source.py b/backend/connectors/nmose/source.py index df80f7c..249c7db 100644 --- a/backend/connectors/nmose/source.py +++ b/backend/connectors/nmose/source.py @@ -1,4 +1,4 @@ -from typing import List, Dict, Tuple +from typing import List, Dict, Any from shapely import wkt from backend.connectors import NM_STATE_BOUNDING_POLYGON @@ -25,7 +25,7 @@ class NMOSEPODSiteSource(BaseSiteSource): def get_records(self, *args, **kw) -> List[Dict]: config = self.config - params = {} + params: Dict[str, Any] = {} # if config.has_bounds(): # bbox = config.bbox_bounding_points() # params["bBox"] = ",".join([str(b) for b in bbox]) @@ -61,7 +61,10 @@ def get_records(self, *args, **kw) -> List[Dict]: i = 1 while 1: rs = self._execute_json_request(url, params, tag="features") - records.extend(rs) + if rs is None: + continue + else: + records.extend(rs) params["resultOffset"] += self.chunk_size if len(rs) < self.chunk_size: break diff --git a/backend/source.py b/backend/source.py index 5d8196a..c10aaf6 100644 --- a/backend/source.py +++ b/backend/source.py @@ -185,7 +185,7 @@ def tag(self): def set_config(self, config): self.config = config - self.transformer.config = config + self.transformer.set_config(config) def check(self, *args, **kw): return True diff --git a/backend/transformer.py b/backend/transformer.py index c9eafd8..80e8721 100644 --- a/backend/transformer.py +++ b/backend/transformer.py @@ -331,13 +331,25 @@ class BaseTransformer(Loggable): """ _cached_polygon = None - config = None + # config = None check_contained = True # ========================================================================== # Methods Already Implemented # ========================================================================== + def set_config(self, config): + """ + Sets the config for the transformer. Called in BaseSource.set_config() + to set the config for both the source and the transformer. + + Parameters + -------- + config: Config + The config to set for the transformer + """ + self.config = config + def do_transform( self, inrecord: dict, *args, **kw ) -> ( From fbd9b7595d9b059962e1cd4a2fc2485b70e5cf5e Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Fri, 11 Apr 2025 10:31:14 -0600 Subject: [PATCH 38/55] mypy fix --- backend/source.py | 5 ++++- tests/{source_tests => test_sources}/__init__.py | 0 tests/{source_tests => test_sources}/test_bernco.py | 0 tests/{source_tests => test_sources}/test_bor.py | 0 tests/{source_tests => test_sources}/test_cabq.py | 0 tests/{source_tests => test_sources}/test_ebid.py | 0 tests/{source_tests => test_sources}/test_nmbgmr.py | 0 tests/{source_tests => test_sources}/test_nmed_dwb.py | 0 .../test_nmose_isc_seven_rivers.py | 0 tests/{source_tests => test_sources}/test_nmose_roswell.py | 0 tests/{source_tests => test_sources}/test_nwis.py | 0 tests/{source_tests => test_sources}/test_pvacd.py | 0 tests/{source_tests => test_sources}/test_wqp.py | 0 13 files changed, 4 insertions(+), 1 deletion(-) rename tests/{source_tests => test_sources}/__init__.py (100%) rename tests/{source_tests => test_sources}/test_bernco.py (100%) rename tests/{source_tests => test_sources}/test_bor.py (100%) rename tests/{source_tests => test_sources}/test_cabq.py (100%) rename tests/{source_tests => test_sources}/test_ebid.py (100%) rename tests/{source_tests => test_sources}/test_nmbgmr.py (100%) rename tests/{source_tests => test_sources}/test_nmed_dwb.py (100%) rename tests/{source_tests => test_sources}/test_nmose_isc_seven_rivers.py (100%) rename tests/{source_tests => test_sources}/test_nmose_roswell.py (100%) rename tests/{source_tests => test_sources}/test_nwis.py (100%) rename tests/{source_tests => test_sources}/test_pvacd.py (100%) rename tests/{source_tests => test_sources}/test_wqp.py (100%) diff --git a/backend/source.py b/backend/source.py index c10aaf6..f81b26f 100644 --- a/backend/source.py +++ b/backend/source.py @@ -816,7 +816,7 @@ def _get_output_units(self) -> str: # Methods That Need to be Implemented For Each Source # ========================================================================== - def _extract_site_records(self, records: list[dict], site_record: dict) -> list: + def _extract_site_records(self, records: list[dict], site_record) -> list: """ Returns all records for a single site as a list of records (which are dictionaries). @@ -833,6 +833,9 @@ def _extract_site_records(self, records: list[dict], site_record: dict) -> list: list a list of records for the site """ + if site_record.chunk_size == 1: + return records + raise NotImplementedError( f"{self.__class__.__name__} Must implement _extract_site_records" ) diff --git a/tests/source_tests/__init__.py b/tests/test_sources/__init__.py similarity index 100% rename from tests/source_tests/__init__.py rename to tests/test_sources/__init__.py diff --git a/tests/source_tests/test_bernco.py b/tests/test_sources/test_bernco.py similarity index 100% rename from tests/source_tests/test_bernco.py rename to tests/test_sources/test_bernco.py diff --git a/tests/source_tests/test_bor.py b/tests/test_sources/test_bor.py similarity index 100% rename from tests/source_tests/test_bor.py rename to tests/test_sources/test_bor.py diff --git a/tests/source_tests/test_cabq.py b/tests/test_sources/test_cabq.py similarity index 100% rename from tests/source_tests/test_cabq.py rename to tests/test_sources/test_cabq.py diff --git a/tests/source_tests/test_ebid.py b/tests/test_sources/test_ebid.py similarity index 100% rename from tests/source_tests/test_ebid.py rename to tests/test_sources/test_ebid.py diff --git a/tests/source_tests/test_nmbgmr.py b/tests/test_sources/test_nmbgmr.py similarity index 100% rename from tests/source_tests/test_nmbgmr.py rename to tests/test_sources/test_nmbgmr.py diff --git a/tests/source_tests/test_nmed_dwb.py b/tests/test_sources/test_nmed_dwb.py similarity index 100% rename from tests/source_tests/test_nmed_dwb.py rename to tests/test_sources/test_nmed_dwb.py diff --git a/tests/source_tests/test_nmose_isc_seven_rivers.py b/tests/test_sources/test_nmose_isc_seven_rivers.py similarity index 100% rename from tests/source_tests/test_nmose_isc_seven_rivers.py rename to tests/test_sources/test_nmose_isc_seven_rivers.py diff --git a/tests/source_tests/test_nmose_roswell.py b/tests/test_sources/test_nmose_roswell.py similarity index 100% rename from tests/source_tests/test_nmose_roswell.py rename to tests/test_sources/test_nmose_roswell.py diff --git a/tests/source_tests/test_nwis.py b/tests/test_sources/test_nwis.py similarity index 100% rename from tests/source_tests/test_nwis.py rename to tests/test_sources/test_nwis.py diff --git a/tests/source_tests/test_pvacd.py b/tests/test_sources/test_pvacd.py similarity index 100% rename from tests/source_tests/test_pvacd.py rename to tests/test_sources/test_pvacd.py diff --git a/tests/source_tests/test_wqp.py b/tests/test_sources/test_wqp.py similarity index 100% rename from tests/source_tests/test_wqp.py rename to tests/test_sources/test_wqp.py From b6b368b07c38478104284755682bcbfc5b98baeb Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Fri, 11 Apr 2025 14:03:24 -0600 Subject: [PATCH 39/55] rename BaseTestClass to BaseSourceTestClass for clarity --- tests/__init__.py | 6 +++++- tests/test_sources/test_bernco.py | 4 ++-- tests/test_sources/test_bor.py | 4 ++-- tests/test_sources/test_cabq.py | 4 ++-- tests/test_sources/test_ebid.py | 4 ++-- tests/test_sources/test_nmbgmr.py | 6 +++--- tests/test_sources/test_nmed_dwb.py | 4 ++-- tests/test_sources/test_nmose_isc_seven_rivers.py | 6 +++--- tests/test_sources/test_nmose_roswell.py | 4 ++-- tests/test_sources/test_nwis.py | 4 ++-- tests/test_sources/test_pvacd.py | 4 ++-- tests/test_sources/test_wqp.py | 6 +++--- 12 files changed, 30 insertions(+), 26 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index f650d55..7a32897 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -24,7 +24,7 @@ def recursively_clean_directory(path): path.rmdir() -class BaseTestClass: +class BaseSourceTestClass: parameter: str units: str agency: str @@ -97,6 +97,10 @@ def _check_timeseries_file(self, timeseries_dir, timeseries_file_name): headers = f.readline().strip().split(",") assert headers == PARAMETER_RECORD_HEADERS + @pytest.mark.skip(reason="Not implemented yet") + def test_bounds(self): + pass + def test_health(self): # do a health check for the agency source = self.config.all_site_sources()[0][0] diff --git a/tests/test_sources/test_bernco.py b/tests/test_sources/test_bernco.py index b100481..16290c8 100644 --- a/tests/test_sources/test_bernco.py +++ b/tests/test_sources/test_bernco.py @@ -1,8 +1,8 @@ from backend.constants import WATERLEVELS, FEET -from tests import BaseTestClass +from tests import BaseSourceTestClass -class TestBernCoWaterlevels(BaseTestClass): +class TestBernCoWaterlevels(BaseSourceTestClass): parameter = WATERLEVELS units = FEET diff --git a/tests/test_sources/test_bor.py b/tests/test_sources/test_bor.py index 77bf325..623491b 100644 --- a/tests/test_sources/test_bor.py +++ b/tests/test_sources/test_bor.py @@ -1,8 +1,8 @@ from backend.constants import CALCIUM, MILLIGRAMS_PER_LITER -from tests import BaseTestClass +from tests import BaseSourceTestClass -class TestBoRAnalyte(BaseTestClass): +class TestBoRAnalyte(BaseSourceTestClass): parameter = CALCIUM units = MILLIGRAMS_PER_LITER diff --git a/tests/test_sources/test_cabq.py b/tests/test_sources/test_cabq.py index b430d3a..da0324b 100644 --- a/tests/test_sources/test_cabq.py +++ b/tests/test_sources/test_cabq.py @@ -1,8 +1,8 @@ from backend.constants import WATERLEVELS, FEET -from tests import BaseTestClass +from tests import BaseSourceTestClass -class TestCABQWaterlevels(BaseTestClass): +class TestCABQWaterlevels(BaseSourceTestClass): parameter = WATERLEVELS units = FEET diff --git a/tests/test_sources/test_ebid.py b/tests/test_sources/test_ebid.py index fa69e00..1ddbeda 100644 --- a/tests/test_sources/test_ebid.py +++ b/tests/test_sources/test_ebid.py @@ -1,8 +1,8 @@ from backend.constants import WATERLEVELS, FEET -from tests import BaseTestClass +from tests import BaseSourceTestClass -class TestEBIDWaterlevels(BaseTestClass): +class TestEBIDWaterlevels(BaseSourceTestClass): parameter = WATERLEVELS units = FEET diff --git a/tests/test_sources/test_nmbgmr.py b/tests/test_sources/test_nmbgmr.py index 4643bf0..1d86731 100644 --- a/tests/test_sources/test_nmbgmr.py +++ b/tests/test_sources/test_nmbgmr.py @@ -1,15 +1,15 @@ from backend.constants import WATERLEVELS, CALCIUM, MILLIGRAMS_PER_LITER, FEET -from tests import BaseTestClass +from tests import BaseSourceTestClass -class TestNMBGMRWaterlevels(BaseTestClass): +class TestNMBGMRWaterlevels(BaseSourceTestClass): parameter = WATERLEVELS units = FEET agency = "nmbgmr_amp" -class TestNMBGMRAnalyte(BaseTestClass): +class TestNMBGMRAnalyte(BaseSourceTestClass): parameter = CALCIUM units = MILLIGRAMS_PER_LITER diff --git a/tests/test_sources/test_nmed_dwb.py b/tests/test_sources/test_nmed_dwb.py index fff8a6e..5b35718 100644 --- a/tests/test_sources/test_nmed_dwb.py +++ b/tests/test_sources/test_nmed_dwb.py @@ -1,8 +1,8 @@ from backend.constants import CALCIUM, MILLIGRAMS_PER_LITER -from tests import BaseTestClass +from tests import BaseSourceTestClass -class TestNMEDDWBAnalyte(BaseTestClass): +class TestNMEDDWBAnalyte(BaseSourceTestClass): parameter = CALCIUM units = MILLIGRAMS_PER_LITER diff --git a/tests/test_sources/test_nmose_isc_seven_rivers.py b/tests/test_sources/test_nmose_isc_seven_rivers.py index a0a5d28..6a4b021 100644 --- a/tests/test_sources/test_nmose_isc_seven_rivers.py +++ b/tests/test_sources/test_nmose_isc_seven_rivers.py @@ -1,15 +1,15 @@ from backend.constants import WATERLEVELS, CALCIUM, FEET, MILLIGRAMS_PER_LITER -from tests import BaseTestClass +from tests import BaseSourceTestClass -class TestNMOSEISCSevenRiversWaterlevels(BaseTestClass): +class TestNMOSEISCSevenRiversWaterlevels(BaseSourceTestClass): parameter = WATERLEVELS units = FEET agency = "nmose_isc_seven_rivers" -class TestNMOSEISCSevenRiversAnalyte(BaseTestClass): +class TestNMOSEISCSevenRiversAnalyte(BaseSourceTestClass): parameter = CALCIUM units = MILLIGRAMS_PER_LITER diff --git a/tests/test_sources/test_nmose_roswell.py b/tests/test_sources/test_nmose_roswell.py index 4c1bd6b..3c21f4d 100644 --- a/tests/test_sources/test_nmose_roswell.py +++ b/tests/test_sources/test_nmose_roswell.py @@ -1,8 +1,8 @@ from backend.constants import WATERLEVELS, FEET -from tests import BaseTestClass +from tests import BaseSourceTestClass -class TestNMOSERoswellWaterlevels(BaseTestClass): +class TestNMOSERoswellWaterlevels(BaseSourceTestClass): parameter = WATERLEVELS units = FEET diff --git a/tests/test_sources/test_nwis.py b/tests/test_sources/test_nwis.py index 493b801..ff74edc 100644 --- a/tests/test_sources/test_nwis.py +++ b/tests/test_sources/test_nwis.py @@ -1,8 +1,8 @@ from backend.constants import WATERLEVELS, FEET -from tests import BaseTestClass +from tests import BaseSourceTestClass -class TestNWISWaterlevels(BaseTestClass): +class TestNWISWaterlevels(BaseSourceTestClass): parameter = WATERLEVELS units = FEET diff --git a/tests/test_sources/test_pvacd.py b/tests/test_sources/test_pvacd.py index edf5d48..6f5e551 100644 --- a/tests/test_sources/test_pvacd.py +++ b/tests/test_sources/test_pvacd.py @@ -1,8 +1,8 @@ from backend.constants import WATERLEVELS, FEET -from tests import BaseTestClass +from tests import BaseSourceTestClass -class TestPVACDWaterlevels(BaseTestClass): +class TestPVACDWaterlevels(BaseSourceTestClass): parameter = WATERLEVELS units = FEET diff --git a/tests/test_sources/test_wqp.py b/tests/test_sources/test_wqp.py index 49e61d9..10f9ea2 100644 --- a/tests/test_sources/test_wqp.py +++ b/tests/test_sources/test_wqp.py @@ -1,15 +1,15 @@ from backend.constants import WATERLEVELS, CALCIUM, MILLIGRAMS_PER_LITER, FEET -from tests import BaseTestClass +from tests import BaseSourceTestClass -class TestWQPWaterlevels(BaseTestClass): +class TestWQPWaterlevels(BaseSourceTestClass): parameter = WATERLEVELS units = FEET agency = "wqp" -class TestWQPAnalyte(BaseTestClass): +class TestWQPAnalyte(BaseSourceTestClass): parameter = CALCIUM units = MILLIGRAMS_PER_LITER From 76f77dc45328e241b064bc1947c2f131ad5f1fa7 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Fri, 11 Apr 2025 14:17:47 -0600 Subject: [PATCH 40/55] Update pytest skip messages for clarity --- tests/__init__.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index 7a32897..9e23564 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,7 +1,7 @@ from logging import shutdown as logger_shutdown from pathlib import Path import pytest -from typing import Optional +from shapely import Geometry from backend.config import Config, SOURCE_KEYS from backend.constants import WATERLEVELS @@ -28,6 +28,7 @@ class BaseSourceTestClass: parameter: str units: str agency: str + bounds: Geometry # set site_limit for tests site_limit: int = 3 @@ -97,10 +98,6 @@ def _check_timeseries_file(self, timeseries_dir, timeseries_file_name): headers = f.readline().strip().split(",") assert headers == PARAMETER_RECORD_HEADERS - @pytest.mark.skip(reason="Not implemented yet") - def test_bounds(self): - pass - def test_health(self): # do a health check for the agency source = self.config.all_site_sources()[0][0] @@ -165,22 +162,26 @@ def test_timeseries_separated(self): for timeseries_file in timeseries_dir.iterdir(): self._check_timeseries_file(timeseries_dir, timeseries_file.name) - @pytest.mark.skip(reason="Not implemented yet") + @pytest.mark.skip(reason="test_date_range not implemented yet") def test_date_range(self): pass - @pytest.mark.skip(reason="Not implemented yet") + @pytest.mark.skip(reason="test_bounds not implemented yet") + def test_bounds(self): + pass + + @pytest.mark.skip(reason="test_wkt not implemented yet") def test_wkt(self): pass - @pytest.mark.skip(reason="Not implemented yet") + @pytest.mark.skip(reason="test_county not implemented yet") def test_county(self): pass - @pytest.mark.skip(reason="Not implemented yet") + @pytest.mark.skip(reason="test_huc not implemented yet") def test_huc(self): pass - @pytest.mark.skip(reason="Not implemented yet") + @pytest.mark.skip(reason="test_bbox not implemented yet") def text_bbox(self): pass From d4cd969c032c45c65b0011bfa714caa08c794665 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Fri, 11 Apr 2025 14:30:48 -0600 Subject: [PATCH 41/55] reararnge testing folder --- tests/__init__.py | 187 ------------------ tests/test_sources/__init__.py | 187 ++++++++++++++++++ tests/test_sources/test_bernco.py | 2 +- tests/test_sources/test_bor.py | 2 +- tests/test_sources/test_cabq.py | 2 +- tests/test_sources/test_ebid.py | 2 +- tests/test_sources/test_nmbgmr.py | 2 +- tests/test_sources/test_nmed_dwb.py | 2 +- .../test_nmose_isc_seven_rivers.py | 2 +- tests/test_sources/test_nmose_roswell.py | 2 +- tests/test_sources/test_nwis.py | 2 +- tests/test_sources/test_pvacd.py | 2 +- tests/test_sources/test_wqp.py | 2 +- 13 files changed, 198 insertions(+), 198 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index 9e23564..e69de29 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,187 +0,0 @@ -from logging import shutdown as logger_shutdown -from pathlib import Path -import pytest -from shapely import Geometry - -from backend.config import Config, SOURCE_KEYS -from backend.constants import WATERLEVELS -from backend.logger import setup_logging -from backend.record import SummaryRecord, SiteRecord, ParameterRecord -from backend.unifier import unify_analytes, unify_waterlevels - -SUMMARY_RECORD_HEADERS = list(SummaryRecord.keys) -SITE_RECORD_HEADERS = list(SiteRecord.keys) -PARAMETER_RECORD_HEADERS = list(ParameterRecord.keys) - - -def recursively_clean_directory(path): - """Recursively delete all files and directories in the given path.""" - for item in path.iterdir(): - if item.is_dir(): - recursively_clean_directory(item) - else: - item.unlink() - path.rmdir() - - -class BaseSourceTestClass: - parameter: str - units: str - agency: str - bounds: Geometry - - # set site_limit for tests - site_limit: int = 3 - - @pytest.fixture(autouse=True) - def setup(self): - # SETUP CODE ---------------------------------------------------------- - # 1: setup test/config attributes - self.config = Config() - for agency in SOURCE_KEYS: - setattr(self.config, f"use_source_{agency}", False) - setattr(self.config, "site_limit", self.site_limit) - setattr(self.config, "parameter", self.parameter) - setattr(self.config, "units", self.units) - setattr(self.config, f"use_source_{self.agency}", True) - self.config.finalize() - - # 2: initiate logger - setup_logging(path=self.config.output_path) - - # RUN TESTS ------------------------------------------------------------ - yield - - # UNIVERSAL ASSERTIONS ------------------------------------------------- - # 1: log file exists - log_path = Path(self.config.output_path) / "die.log" - assert log_path.exists() - - # TEARDOWN CODE -------------------------------------------------------- - # 1: close logger to delete log file - logger_shutdown() - - # 2: delete newly created dirs and files - path_to_clean = Path(self.config.output_path) - print(f"Cleaning and removing {path_to_clean}") - recursively_clean_directory(path_to_clean) - - # reset test attributes - self.dirs_to_delete = [] - self.config = None - self.unifier = None - - def _run_unifier(self): - if self.parameter == WATERLEVELS: - unify_waterlevels(self.config) - else: - unify_analytes(self.config) - - def _check_sites_file(self): - sites_file = Path(self.config.output_path) / "sites.csv" - assert sites_file.exists() - - with open(sites_file, "r") as f: - headers = f.readline().strip().split(",") - assert headers == SITE_RECORD_HEADERS - - # +1 for the header - with open(sites_file, "r") as f: - lines = f.readlines() - assert len(lines) == self.site_limit + 1 - - def _check_timeseries_file(self, timeseries_dir, timeseries_file_name): - timeseries_file = Path(timeseries_dir) / timeseries_file_name - assert timeseries_file.exists() - - with open(timeseries_file, "r") as f: - headers = f.readline().strip().split(",") - assert headers == PARAMETER_RECORD_HEADERS - - def test_health(self): - # do a health check for the agency - source = self.config.all_site_sources()[0][0] - assert source.health() - - def test_summary(self): - # Arrange -------------------------------------------------------------- - self.config.output_summary = True - self.config.report() - - # Act ------------------------------------------------------------------ - self._run_unifier() - - # Assert --------------------------------------------------------------- - # Check the summary file - summary_file = Path(self.config.output_path) / "summary.csv" - assert summary_file.exists() - - # Check the column headers - with open(summary_file, "r") as f: - headers = f.readline().strip().split(",") - assert headers == SUMMARY_RECORD_HEADERS - - # +1 for the header - with open(summary_file, "r") as f: - lines = f.readlines() - assert len(lines) == self.site_limit + 1 - - def test_timeseries_unified(self): - # Arrange -------------------------------------------------------------- - self.config.output_timeseries_unified = True - self.config.report() - - # Act ------------------------------------------------------------------ - self._run_unifier() - - # Assert --------------------------------------------------------------- - # Check the sites file - self._check_sites_file() - - # Check the timeseries file - timeseries_dir = Path(self.config.output_path) - timeseries_file_name = "timeseries_unified.csv" - self._check_timeseries_file(timeseries_dir, timeseries_file_name) - - def test_timeseries_separated(self): - # Arrange -------------------------------------------------------------- - self.config.output_timeseries_separated = True - self.config.report() - - # Act ------------------------------------------------------------------ - self._run_unifier() - - # Assert --------------------------------------------------------------- - # Check the sites file - self._check_sites_file() - - # Check the timeseries files - timeseries_dir = Path(self.config.output_path) / "timeseries" - assert len([f for f in timeseries_dir.iterdir()]) == self.site_limit - - for timeseries_file in timeseries_dir.iterdir(): - self._check_timeseries_file(timeseries_dir, timeseries_file.name) - - @pytest.mark.skip(reason="test_date_range not implemented yet") - def test_date_range(self): - pass - - @pytest.mark.skip(reason="test_bounds not implemented yet") - def test_bounds(self): - pass - - @pytest.mark.skip(reason="test_wkt not implemented yet") - def test_wkt(self): - pass - - @pytest.mark.skip(reason="test_county not implemented yet") - def test_county(self): - pass - - @pytest.mark.skip(reason="test_huc not implemented yet") - def test_huc(self): - pass - - @pytest.mark.skip(reason="test_bbox not implemented yet") - def text_bbox(self): - pass diff --git a/tests/test_sources/__init__.py b/tests/test_sources/__init__.py index e69de29..9e23564 100644 --- a/tests/test_sources/__init__.py +++ b/tests/test_sources/__init__.py @@ -0,0 +1,187 @@ +from logging import shutdown as logger_shutdown +from pathlib import Path +import pytest +from shapely import Geometry + +from backend.config import Config, SOURCE_KEYS +from backend.constants import WATERLEVELS +from backend.logger import setup_logging +from backend.record import SummaryRecord, SiteRecord, ParameterRecord +from backend.unifier import unify_analytes, unify_waterlevels + +SUMMARY_RECORD_HEADERS = list(SummaryRecord.keys) +SITE_RECORD_HEADERS = list(SiteRecord.keys) +PARAMETER_RECORD_HEADERS = list(ParameterRecord.keys) + + +def recursively_clean_directory(path): + """Recursively delete all files and directories in the given path.""" + for item in path.iterdir(): + if item.is_dir(): + recursively_clean_directory(item) + else: + item.unlink() + path.rmdir() + + +class BaseSourceTestClass: + parameter: str + units: str + agency: str + bounds: Geometry + + # set site_limit for tests + site_limit: int = 3 + + @pytest.fixture(autouse=True) + def setup(self): + # SETUP CODE ---------------------------------------------------------- + # 1: setup test/config attributes + self.config = Config() + for agency in SOURCE_KEYS: + setattr(self.config, f"use_source_{agency}", False) + setattr(self.config, "site_limit", self.site_limit) + setattr(self.config, "parameter", self.parameter) + setattr(self.config, "units", self.units) + setattr(self.config, f"use_source_{self.agency}", True) + self.config.finalize() + + # 2: initiate logger + setup_logging(path=self.config.output_path) + + # RUN TESTS ------------------------------------------------------------ + yield + + # UNIVERSAL ASSERTIONS ------------------------------------------------- + # 1: log file exists + log_path = Path(self.config.output_path) / "die.log" + assert log_path.exists() + + # TEARDOWN CODE -------------------------------------------------------- + # 1: close logger to delete log file + logger_shutdown() + + # 2: delete newly created dirs and files + path_to_clean = Path(self.config.output_path) + print(f"Cleaning and removing {path_to_clean}") + recursively_clean_directory(path_to_clean) + + # reset test attributes + self.dirs_to_delete = [] + self.config = None + self.unifier = None + + def _run_unifier(self): + if self.parameter == WATERLEVELS: + unify_waterlevels(self.config) + else: + unify_analytes(self.config) + + def _check_sites_file(self): + sites_file = Path(self.config.output_path) / "sites.csv" + assert sites_file.exists() + + with open(sites_file, "r") as f: + headers = f.readline().strip().split(",") + assert headers == SITE_RECORD_HEADERS + + # +1 for the header + with open(sites_file, "r") as f: + lines = f.readlines() + assert len(lines) == self.site_limit + 1 + + def _check_timeseries_file(self, timeseries_dir, timeseries_file_name): + timeseries_file = Path(timeseries_dir) / timeseries_file_name + assert timeseries_file.exists() + + with open(timeseries_file, "r") as f: + headers = f.readline().strip().split(",") + assert headers == PARAMETER_RECORD_HEADERS + + def test_health(self): + # do a health check for the agency + source = self.config.all_site_sources()[0][0] + assert source.health() + + def test_summary(self): + # Arrange -------------------------------------------------------------- + self.config.output_summary = True + self.config.report() + + # Act ------------------------------------------------------------------ + self._run_unifier() + + # Assert --------------------------------------------------------------- + # Check the summary file + summary_file = Path(self.config.output_path) / "summary.csv" + assert summary_file.exists() + + # Check the column headers + with open(summary_file, "r") as f: + headers = f.readline().strip().split(",") + assert headers == SUMMARY_RECORD_HEADERS + + # +1 for the header + with open(summary_file, "r") as f: + lines = f.readlines() + assert len(lines) == self.site_limit + 1 + + def test_timeseries_unified(self): + # Arrange -------------------------------------------------------------- + self.config.output_timeseries_unified = True + self.config.report() + + # Act ------------------------------------------------------------------ + self._run_unifier() + + # Assert --------------------------------------------------------------- + # Check the sites file + self._check_sites_file() + + # Check the timeseries file + timeseries_dir = Path(self.config.output_path) + timeseries_file_name = "timeseries_unified.csv" + self._check_timeseries_file(timeseries_dir, timeseries_file_name) + + def test_timeseries_separated(self): + # Arrange -------------------------------------------------------------- + self.config.output_timeseries_separated = True + self.config.report() + + # Act ------------------------------------------------------------------ + self._run_unifier() + + # Assert --------------------------------------------------------------- + # Check the sites file + self._check_sites_file() + + # Check the timeseries files + timeseries_dir = Path(self.config.output_path) / "timeseries" + assert len([f for f in timeseries_dir.iterdir()]) == self.site_limit + + for timeseries_file in timeseries_dir.iterdir(): + self._check_timeseries_file(timeseries_dir, timeseries_file.name) + + @pytest.mark.skip(reason="test_date_range not implemented yet") + def test_date_range(self): + pass + + @pytest.mark.skip(reason="test_bounds not implemented yet") + def test_bounds(self): + pass + + @pytest.mark.skip(reason="test_wkt not implemented yet") + def test_wkt(self): + pass + + @pytest.mark.skip(reason="test_county not implemented yet") + def test_county(self): + pass + + @pytest.mark.skip(reason="test_huc not implemented yet") + def test_huc(self): + pass + + @pytest.mark.skip(reason="test_bbox not implemented yet") + def text_bbox(self): + pass diff --git a/tests/test_sources/test_bernco.py b/tests/test_sources/test_bernco.py index 16290c8..48004a9 100644 --- a/tests/test_sources/test_bernco.py +++ b/tests/test_sources/test_bernco.py @@ -1,5 +1,5 @@ from backend.constants import WATERLEVELS, FEET -from tests import BaseSourceTestClass +from tests.test_sources import BaseSourceTestClass class TestBernCoWaterlevels(BaseSourceTestClass): diff --git a/tests/test_sources/test_bor.py b/tests/test_sources/test_bor.py index 623491b..003391d 100644 --- a/tests/test_sources/test_bor.py +++ b/tests/test_sources/test_bor.py @@ -1,5 +1,5 @@ from backend.constants import CALCIUM, MILLIGRAMS_PER_LITER -from tests import BaseSourceTestClass +from tests.test_sources import BaseSourceTestClass class TestBoRAnalyte(BaseSourceTestClass): diff --git a/tests/test_sources/test_cabq.py b/tests/test_sources/test_cabq.py index da0324b..9f3ff3c 100644 --- a/tests/test_sources/test_cabq.py +++ b/tests/test_sources/test_cabq.py @@ -1,5 +1,5 @@ from backend.constants import WATERLEVELS, FEET -from tests import BaseSourceTestClass +from tests.test_sources import BaseSourceTestClass class TestCABQWaterlevels(BaseSourceTestClass): diff --git a/tests/test_sources/test_ebid.py b/tests/test_sources/test_ebid.py index 1ddbeda..aa38894 100644 --- a/tests/test_sources/test_ebid.py +++ b/tests/test_sources/test_ebid.py @@ -1,5 +1,5 @@ from backend.constants import WATERLEVELS, FEET -from tests import BaseSourceTestClass +from teststest_sources import BaseSourceTestClass class TestEBIDWaterlevels(BaseSourceTestClass): diff --git a/tests/test_sources/test_nmbgmr.py b/tests/test_sources/test_nmbgmr.py index 1d86731..90bba2c 100644 --- a/tests/test_sources/test_nmbgmr.py +++ b/tests/test_sources/test_nmbgmr.py @@ -1,5 +1,5 @@ from backend.constants import WATERLEVELS, CALCIUM, MILLIGRAMS_PER_LITER, FEET -from tests import BaseSourceTestClass +from tests.test_sources import BaseSourceTestClass class TestNMBGMRWaterlevels(BaseSourceTestClass): diff --git a/tests/test_sources/test_nmed_dwb.py b/tests/test_sources/test_nmed_dwb.py index 5b35718..2a27be3 100644 --- a/tests/test_sources/test_nmed_dwb.py +++ b/tests/test_sources/test_nmed_dwb.py @@ -1,5 +1,5 @@ from backend.constants import CALCIUM, MILLIGRAMS_PER_LITER -from tests import BaseSourceTestClass +from tests.test_sources import BaseSourceTestClass class TestNMEDDWBAnalyte(BaseSourceTestClass): diff --git a/tests/test_sources/test_nmose_isc_seven_rivers.py b/tests/test_sources/test_nmose_isc_seven_rivers.py index 6a4b021..55b345e 100644 --- a/tests/test_sources/test_nmose_isc_seven_rivers.py +++ b/tests/test_sources/test_nmose_isc_seven_rivers.py @@ -1,5 +1,5 @@ from backend.constants import WATERLEVELS, CALCIUM, FEET, MILLIGRAMS_PER_LITER -from tests import BaseSourceTestClass +from tests.test_sources import BaseSourceTestClass class TestNMOSEISCSevenRiversWaterlevels(BaseSourceTestClass): diff --git a/tests/test_sources/test_nmose_roswell.py b/tests/test_sources/test_nmose_roswell.py index 3c21f4d..585090f 100644 --- a/tests/test_sources/test_nmose_roswell.py +++ b/tests/test_sources/test_nmose_roswell.py @@ -1,5 +1,5 @@ from backend.constants import WATERLEVELS, FEET -from tests import BaseSourceTestClass +from tests.test_sources import BaseSourceTestClass class TestNMOSERoswellWaterlevels(BaseSourceTestClass): diff --git a/tests/test_sources/test_nwis.py b/tests/test_sources/test_nwis.py index ff74edc..b7bf272 100644 --- a/tests/test_sources/test_nwis.py +++ b/tests/test_sources/test_nwis.py @@ -1,5 +1,5 @@ from backend.constants import WATERLEVELS, FEET -from tests import BaseSourceTestClass +from tests.test_sources import BaseSourceTestClass class TestNWISWaterlevels(BaseSourceTestClass): diff --git a/tests/test_sources/test_pvacd.py b/tests/test_sources/test_pvacd.py index 6f5e551..715acf7 100644 --- a/tests/test_sources/test_pvacd.py +++ b/tests/test_sources/test_pvacd.py @@ -1,5 +1,5 @@ from backend.constants import WATERLEVELS, FEET -from tests import BaseSourceTestClass +from tests.test_sources import BaseSourceTestClass class TestPVACDWaterlevels(BaseSourceTestClass): diff --git a/tests/test_sources/test_wqp.py b/tests/test_sources/test_wqp.py index 10f9ea2..4f8437e 100644 --- a/tests/test_sources/test_wqp.py +++ b/tests/test_sources/test_wqp.py @@ -1,5 +1,5 @@ from backend.constants import WATERLEVELS, CALCIUM, MILLIGRAMS_PER_LITER, FEET -from tests import BaseSourceTestClass +from tests.test_sources import BaseSourceTestClass class TestWQPWaterlevels(BaseSourceTestClass): From 6cef8c456b081210c1bdacbf57ed05d4892ecdf7 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Fri, 11 Apr 2025 16:30:15 -0600 Subject: [PATCH 42/55] Work on CLI tests --- tests/test_cli/__init__.py | 70 +++++++++++++++++++++++++++++++++++ tests/test_cli/test_nmbgmr.py | 13 +++++++ 2 files changed, 83 insertions(+) create mode 100644 tests/test_cli/__init__.py create mode 100644 tests/test_cli/test_nmbgmr.py diff --git a/tests/test_cli/__init__.py b/tests/test_cli/__init__.py new file mode 100644 index 0000000..7a8da3e --- /dev/null +++ b/tests/test_cli/__init__.py @@ -0,0 +1,70 @@ +from click.testing import CliRunner +import pytest +from typing import List, Any + +from backend.config import SOURCE_KEYS +from backend.constants import ( + WATERLEVELS, + ARSENIC, + BICARBONATE, + CALCIUM, + CARBONATE, + CHLORIDE, + FLUORIDE, + MAGNESIUM, + NITRATE, + PH, + POTASSIUM, + SILICA, + SODIUM, + SULFATE, + TDS, + URANIUM +) +from frontend.cli import weave + +class BaseCLITestClass(): + + runner: CliRunner + agency: str + no_agencies: List[str] = [] + + @pytest.fixture(autouse=True) + def setup(self): + # STEUP CODE ----------------------------------------------------------- + self.runner = CliRunner() + + # turn off all sources except for the one being tested + for source in SOURCE_KEYS: + if source == self.agency: + continue + else: + source_with_dash = source.replace("_", "-") + self.no_agencies.append(f"--no-{source_with_dash}") + + + # RUN TESTS ------------------------------------------------------------ + yield + + # TEARDOWN CODE --------------------------------------------------------- + self.no_agencies = [] + + def _test_weave(self, parameter, output): + # Arrange + arguments = [ + parameter, + f"--output {output}", + "--dry" + ] + + arguments.extend(self.no_agencies) + + print(arguments) + + # Act + result = self.runner.invoke(weave, arguments) + print(result.output) + print(result.__dict__) + + # Assert + assert result.exit_code == 0 \ No newline at end of file diff --git a/tests/test_cli/test_nmbgmr.py b/tests/test_cli/test_nmbgmr.py new file mode 100644 index 0000000..faa8e47 --- /dev/null +++ b/tests/test_cli/test_nmbgmr.py @@ -0,0 +1,13 @@ +from tests.test_cli import BaseCLITestClass + +class TestNMBGMRCLI(BaseCLITestClass): + """Test the CLI for the NMBGMR source.""" + + agency = "nmbgmr-amp" + + def test_weave(self): + # Test the weave command for NMBGMR + self._test_weave( + parameter="waterlevels", + output="summary" + ) \ No newline at end of file From 4adafb1f51483a0061ca541b71cbebe5eace74f3 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Fri, 11 Apr 2025 16:30:37 -0600 Subject: [PATCH 43/55] Comment out test dev in progress --- tests/test_cli/test_nmbgmr.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_cli/test_nmbgmr.py b/tests/test_cli/test_nmbgmr.py index faa8e47..9a5a1b8 100644 --- a/tests/test_cli/test_nmbgmr.py +++ b/tests/test_cli/test_nmbgmr.py @@ -5,9 +5,9 @@ class TestNMBGMRCLI(BaseCLITestClass): agency = "nmbgmr-amp" - def test_weave(self): - # Test the weave command for NMBGMR - self._test_weave( - parameter="waterlevels", - output="summary" - ) \ No newline at end of file + # def test_weave(self): + # # Test the weave command for NMBGMR + # self._test_weave( + # parameter="waterlevels", + # output="summary" + # ) \ No newline at end of file From d2b6d6c5b327715d7b2c1f79f78b0a3c0b9b412c Mon Sep 17 00:00:00 2001 From: jacob-a-brown Date: Fri, 11 Apr 2025 22:47:46 +0000 Subject: [PATCH 44/55] Formatting changes --- backend/source.py | 24 ++++++++++++++++-------- backend/transformer.py | 8 ++++++-- tests/archived/test_cli.py | 1 - tests/test_cli/__init__.py | 14 +++++--------- tests/test_cli/test_nmbgmr.py | 3 ++- 5 files changed, 29 insertions(+), 21 deletions(-) diff --git a/backend/source.py b/backend/source.py index f81b26f..5189258 100644 --- a/backend/source.py +++ b/backend/source.py @@ -102,7 +102,8 @@ def func(x): else: raise ValueError( f"Invalid bookend {bookend}. Must be either {EARLIEST} or {LATEST}" - ) + ) + def get_analyte_search_param(parameter: str, mapping: dict) -> str: """ @@ -608,13 +609,20 @@ def _extract_latest_record(self, records: list) -> dict: return self._extract_terminal_record(records, bookend=LATEST) def read( - self, site_record: SiteRecord | list, use_summarize: bool, start_ind: int, end_ind: int - ) -> List[ - AnalyteRecord - | AnalyteSummaryRecord - | WaterLevelRecord - | WaterLevelSummaryRecord - ] | None: + self, + site_record: SiteRecord | list, + use_summarize: bool, + start_ind: int, + end_ind: int, + ) -> ( + List[ + AnalyteRecord + | AnalyteSummaryRecord + | WaterLevelRecord + | WaterLevelSummaryRecord + ] + | None + ): """ Returns a list of transformed parameter records. Transformed parameter records are standardized so that all of the records have the same format. They are diff --git a/backend/transformer.py b/backend/transformer.py index 80e8721..cb3afe5 100644 --- a/backend/transformer.py +++ b/backend/transformer.py @@ -412,7 +412,9 @@ def do_transform( # ensure that a site or summary record is contained within the boundaing polygon if "longitude" in transformed_record and "latitude" in transformed_record: - if not self.contained(transformed_record["longitude"], transformed_record["latitude"]): + if not self.contained( + transformed_record["longitude"], transformed_record["latitude"] + ): self.warn( f"Skipping site {transformed_record['id']}. It is not within the defined geographic bounds" ) @@ -447,7 +449,9 @@ def do_transform( x = float(klassed_record.longitude) if x == 0 or y == 0: - self.warn(f"Skipping site {klassed_record.id}. Latitude or Longitude is 0") + self.warn( + f"Skipping site {klassed_record.id}. Latitude or Longitude is 0" + ) return None input_horizontal_datum = klassed_record.horizontal_datum diff --git a/tests/archived/test_cli.py b/tests/archived/test_cli.py index 6fa2baa..3e53924 100644 --- a/tests/archived/test_cli.py +++ b/tests/archived/test_cli.py @@ -218,7 +218,6 @@ def test_waterlevels_invalid_end(): _tester(waterlevels, args, fail=True) - def _tester(source, func, county, bbox, args=None): runner = CliRunner() diff --git a/tests/test_cli/__init__.py b/tests/test_cli/__init__.py index 7a8da3e..e9ae2f1 100644 --- a/tests/test_cli/__init__.py +++ b/tests/test_cli/__init__.py @@ -19,11 +19,12 @@ SODIUM, SULFATE, TDS, - URANIUM + URANIUM, ) from frontend.cli import weave -class BaseCLITestClass(): + +class BaseCLITestClass: runner: CliRunner agency: str @@ -41,7 +42,6 @@ def setup(self): else: source_with_dash = source.replace("_", "-") self.no_agencies.append(f"--no-{source_with_dash}") - # RUN TESTS ------------------------------------------------------------ yield @@ -51,11 +51,7 @@ def setup(self): def _test_weave(self, parameter, output): # Arrange - arguments = [ - parameter, - f"--output {output}", - "--dry" - ] + arguments = [parameter, f"--output {output}", "--dry"] arguments.extend(self.no_agencies) @@ -67,4 +63,4 @@ def _test_weave(self, parameter, output): print(result.__dict__) # Assert - assert result.exit_code == 0 \ No newline at end of file + assert result.exit_code == 0 diff --git a/tests/test_cli/test_nmbgmr.py b/tests/test_cli/test_nmbgmr.py index 9a5a1b8..6928473 100644 --- a/tests/test_cli/test_nmbgmr.py +++ b/tests/test_cli/test_nmbgmr.py @@ -1,5 +1,6 @@ from tests.test_cli import BaseCLITestClass + class TestNMBGMRCLI(BaseCLITestClass): """Test the CLI for the NMBGMR source.""" @@ -10,4 +11,4 @@ class TestNMBGMRCLI(BaseCLITestClass): # self._test_weave( # parameter="waterlevels", # output="summary" - # ) \ No newline at end of file + # ) From b4f4f2dee2400a4e2c8de561f917c3cd69b0ad8f Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 14 Apr 2025 08:00:26 -0700 Subject: [PATCH 45/55] fix import error --- tests/test_sources/test_ebid.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_sources/test_ebid.py b/tests/test_sources/test_ebid.py index aa38894..6adfd6f 100644 --- a/tests/test_sources/test_ebid.py +++ b/tests/test_sources/test_ebid.py @@ -1,5 +1,5 @@ from backend.constants import WATERLEVELS, FEET -from teststest_sources import BaseSourceTestClass +from tests.test_sources import BaseSourceTestClass class TestEBIDWaterlevels(BaseSourceTestClass): From cae4227730d5af2a90d22d5bf0ac2f5629190941 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 14 Apr 2025 12:12:37 -0700 Subject: [PATCH 46/55] put cleanup function in base dir for reuse --- tests/__init__.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/__init__.py b/tests/__init__.py index e69de29..9316fba 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -0,0 +1,8 @@ +def recursively_clean_directory(path): + """Recursively delete all files and directories in the given path.""" + for item in path.iterdir(): + if item.is_dir(): + recursively_clean_directory(item) + else: + item.unlink() + path.rmdir() \ No newline at end of file From 1b42e2df341004a389d4b87688aa57c7b485b639 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 14 Apr 2025 12:12:53 -0700 Subject: [PATCH 47/55] rename nmbgmr to nmbgmr_amp for clarity/consistency --- tests/test_sources/__init__.py | 11 +---------- .../{test_nmbgmr.py => test_nmbgmr_amp.py} | 0 2 files changed, 1 insertion(+), 10 deletions(-) rename tests/test_sources/{test_nmbgmr.py => test_nmbgmr_amp.py} (100%) diff --git a/tests/test_sources/__init__.py b/tests/test_sources/__init__.py index 9e23564..a18dd94 100644 --- a/tests/test_sources/__init__.py +++ b/tests/test_sources/__init__.py @@ -8,22 +8,13 @@ from backend.logger import setup_logging from backend.record import SummaryRecord, SiteRecord, ParameterRecord from backend.unifier import unify_analytes, unify_waterlevels +from tests import recursively_clean_directory SUMMARY_RECORD_HEADERS = list(SummaryRecord.keys) SITE_RECORD_HEADERS = list(SiteRecord.keys) PARAMETER_RECORD_HEADERS = list(ParameterRecord.keys) -def recursively_clean_directory(path): - """Recursively delete all files and directories in the given path.""" - for item in path.iterdir(): - if item.is_dir(): - recursively_clean_directory(item) - else: - item.unlink() - path.rmdir() - - class BaseSourceTestClass: parameter: str units: str diff --git a/tests/test_sources/test_nmbgmr.py b/tests/test_sources/test_nmbgmr_amp.py similarity index 100% rename from tests/test_sources/test_nmbgmr.py rename to tests/test_sources/test_nmbgmr_amp.py From 685f242490e532904b223108cb8559301a37c80f Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 14 Apr 2025 12:13:00 -0700 Subject: [PATCH 48/55] CLI tests for all sources and ways to set config --- tests/test_cli/__init__.py | 271 ++++++++++++++++-- tests/test_cli/test_bernco.py | 43 +++ tests/test_cli/test_cabq.py | 43 +++ tests/test_cli/test_ebid.py | 43 +++ tests/test_cli/test_nmbgmr.py | 14 - tests/test_cli/test_nmbgmr_amp.py | 44 +++ tests/test_cli/test_nmed_dwb.py | 43 +++ tests/test_cli/test_nmose_isc_seven_rivers.py | 44 +++ tests/test_cli/test_nmose_roswell.py | 43 +++ tests/test_cli/test_nwis.py | 43 +++ tests/test_cli/test_pvacd.py | 43 +++ tests/test_cli/test_wqp.py | 43 +++ 12 files changed, 684 insertions(+), 33 deletions(-) create mode 100644 tests/test_cli/test_bernco.py create mode 100644 tests/test_cli/test_cabq.py create mode 100644 tests/test_cli/test_ebid.py delete mode 100644 tests/test_cli/test_nmbgmr.py create mode 100644 tests/test_cli/test_nmbgmr_amp.py create mode 100644 tests/test_cli/test_nmed_dwb.py create mode 100644 tests/test_cli/test_nmose_isc_seven_rivers.py create mode 100644 tests/test_cli/test_nmose_roswell.py create mode 100644 tests/test_cli/test_nwis.py create mode 100644 tests/test_cli/test_pvacd.py create mode 100644 tests/test_cli/test_wqp.py diff --git a/tests/test_cli/__init__.py b/tests/test_cli/__init__.py index e9ae2f1..0ede68e 100644 --- a/tests/test_cli/__init__.py +++ b/tests/test_cli/__init__.py @@ -1,6 +1,8 @@ from click.testing import CliRunner +from logging import shutdown as logger_shutdown +from pathlib import Path import pytest -from typing import List, Any +from typing import List from backend.config import SOURCE_KEYS from backend.constants import ( @@ -22,45 +24,276 @@ URANIUM, ) from frontend.cli import weave +from tests import recursively_clean_directory class BaseCLITestClass: runner: CliRunner agency: str - no_agencies: List[str] = [] + agency_reports_parameter: dict + output_dir: Path @pytest.fixture(autouse=True) def setup(self): - # STEUP CODE ----------------------------------------------------------- + # SETUP CODE ----------------------------------------------------------- self.runner = CliRunner() - # turn off all sources except for the one being tested - for source in SOURCE_KEYS: - if source == self.agency: - continue - else: - source_with_dash = source.replace("_", "-") - self.no_agencies.append(f"--no-{source_with_dash}") - # RUN TESTS ------------------------------------------------------------ yield # TEARDOWN CODE --------------------------------------------------------- - self.no_agencies = [] + logger_shutdown() + recursively_clean_directory(self.output_dir) - def _test_weave(self, parameter, output): + def _test_weave( + self, + parameter: str, + output: str, + site_limit: int = 4, + start_date: str = "1990-08-10", + end_date: str = "1990-08-11", + bbox: str | None = None, + county: str | None = None, + wkt: str | None = None, + ): # Arrange - arguments = [parameter, f"--output {output}", "--dry"] + # turn off all sources except for the one being tested + no_agencies = [] + for source in SOURCE_KEYS: + source_with_dash = source.replace("_", "-") + if source_with_dash == self.agency: + continue + else: + no_agencies.append(f"--no-{source_with_dash}") - arguments.extend(self.no_agencies) + geographic_filter_name: str | None = None + geographic_filter_value: str | None = None + if bbox: + geographic_filter_name = "bbox" + geographic_filter_value = bbox + elif county: + geographic_filter_name = "county" + geographic_filter_value = county + elif wkt: + geographic_filter_name = "wkt" + geographic_filter_value = wkt + + arguments = [ + parameter, + "--output", + output, + "--dry", + "--site-limit", + site_limit, + "--start-date", + start_date, + "--end-date", + end_date, + ] - print(arguments) + if geographic_filter_name and geographic_filter_value: + arguments.extend([f"--{geographic_filter_name}", geographic_filter_value]) + + arguments.extend(no_agencies) # Act - result = self.runner.invoke(weave, arguments) - print(result.output) - print(result.__dict__) + result = self.runner.invoke(weave, arguments, standalone_mode=False) # Assert assert result.exit_code == 0 + + """ + For the config, check that + + 0. (set output dir to clean up tests results even in event of failure) + 1. The parameter is set correctly + 2. The agencies are set correctly + 3. The output types are set correctly + 4. The site limit is set correctly + 5. The dry is set correctly + 6. The start date is set correctly + 7. The end date is set correctly + 8. The geographic filter is set correctly + """ + config = result.return_value + + # 0 + self.output_dir = Path(config.output_path) + + # 1 + assert getattr(config, "parameter") == parameter + + # 2 + agency_with_underscore = self.agency.replace("-", "_") + if self.agency_reports_parameter[parameter]: + assert getattr(config, f"use_source_{agency_with_underscore}") is True + else: + assert getattr(config, f"use_source_{agency_with_underscore}") is False + + for no_agency in no_agencies: + no_agency_with_underscore = no_agency.replace("--no-", "").replace("-", "_") + assert getattr(config, f"use_source_{no_agency_with_underscore}") is False + + # 3 + output_types = ["summary", "timeseries_unified", "timeseries_separated"] + for output_type in output_types: + if output_type == output: + assert getattr(config, f"output_{output_type}") is True + else: + assert getattr(config, f"output_{output_type}") is False + + # 4 + assert getattr(config, "site_limit") == 4 + + # 5 + assert getattr(config, "dry") is True + + # 6 + assert getattr(config, "start_date") == start_date + + # 7 + assert getattr(config, "end_date") == end_date + + # 8 + if geographic_filter_name and geographic_filter_value: + for _geographic_filter_name in ["bbox", "county", "wkt"]: + if _geographic_filter_name == geographic_filter_name: + assert getattr(config, _geographic_filter_name) == geographic_filter_value + else: + assert getattr(config, _geographic_filter_name) == "" + + def test_weave_summary(self): + self._test_weave( + parameter=WATERLEVELS, + output="summary" + ) + + def test_weave_timeseries_unified(self): + self._test_weave( + parameter=WATERLEVELS, + output="timeseries_unified" + ) + + def test_weave_timeseries_separated(self): + self._test_weave( + parameter=WATERLEVELS, + output="timeseries_separated" + ) + + def test_weave_bbox(self): + self._test_weave( + parameter=WATERLEVELS, + output="summary", + bbox="32.0,-106.0,36.0,-102.0" + ) + + def test_weave_county(self): + self._test_weave( + parameter=WATERLEVELS, + output="summary", + county="Bernalillo" + ) + + def test_weave_wkt(self): + self._test_weave( + parameter=WATERLEVELS, + output="summary", + wkt="POLYGON((-106.0 32.0, -102.0 32.0, -102.0 36.0, -106.0 36.0, -106.0 32.0))" + ) + + def test_weave_waterlevels(self): + self._test_weave( + parameter=WATERLEVELS, + output="summary" + ) + + def test_weave_arsenic(self): + self._test_weave( + parameter=ARSENIC, + output="summary" + ) + + def test_weave_bicarbonate(self): + self._test_weave( + parameter=BICARBONATE, + output="summary" + ) + + def test_weave_calcium(self): + self._test_weave( + parameter=CALCIUM, + output="summary" + ) + + def test_weave_carbonate(self): + self._test_weave( + parameter=CARBONATE, + output="summary" + ) + + def test_weave_chloride(self): + self._test_weave( + parameter=CHLORIDE, + output="summary" + ) + + def test_weave_fluoride(self): + self._test_weave( + parameter=FLUORIDE, + output="summary" + ) + + def test_weave_magnesium(self): + self._test_weave( + parameter=MAGNESIUM, + output="summary" + ) + + def test_weave_nitrate(self): + self._test_weave( + parameter=NITRATE, + output="summary" + ) + + def test_weave_ph(self): + self._test_weave( + parameter=PH, + output="summary" + ) + + def test_weave_potassium(self): + self._test_weave( + parameter=POTASSIUM, + output="summary" + ) + + def test_weave_silica(self): + self._test_weave( + parameter=SILICA, + output="summary" + ) + + def test_weave_sodium(self): + self._test_weave( + parameter=SODIUM, + output="summary" + ) + + def test_weave_sulfate(self): + self._test_weave( + parameter=SULFATE, + output="summary" + ) + + def test_weave_tds(self): + self._test_weave( + parameter=TDS, + output="summary" + ) + + def test_weave_uranium(self): + self._test_weave( + parameter=URANIUM, + output="summary" + ) \ No newline at end of file diff --git a/tests/test_cli/test_bernco.py b/tests/test_cli/test_bernco.py new file mode 100644 index 0000000..c6e4031 --- /dev/null +++ b/tests/test_cli/test_bernco.py @@ -0,0 +1,43 @@ +from backend.constants import ( + WATERLEVELS, + ARSENIC, + BICARBONATE, + CALCIUM, + CARBONATE, + CHLORIDE, + FLUORIDE, + MAGNESIUM, + NITRATE, + PH, + POTASSIUM, + SILICA, + SODIUM, + SULFATE, + TDS, + URANIUM, +) +from tests.test_cli import BaseCLITestClass + +class TestBernCoCLI(BaseCLITestClass): + + agency = "bernco" + agency_reports_parameter = { + WATERLEVELS: True, + ARSENIC: False, + BICARBONATE: False, + CALCIUM: False, + CARBONATE: False, + CHLORIDE: False, + FLUORIDE: False, + MAGNESIUM: False, + NITRATE: False, + PH: False, + POTASSIUM: False, + SILICA: False, + SODIUM: False, + SULFATE: False, + TDS: False, + URANIUM: False, + } + + diff --git a/tests/test_cli/test_cabq.py b/tests/test_cli/test_cabq.py new file mode 100644 index 0000000..5e96a07 --- /dev/null +++ b/tests/test_cli/test_cabq.py @@ -0,0 +1,43 @@ +from backend.constants import ( + WATERLEVELS, + ARSENIC, + BICARBONATE, + CALCIUM, + CARBONATE, + CHLORIDE, + FLUORIDE, + MAGNESIUM, + NITRATE, + PH, + POTASSIUM, + SILICA, + SODIUM, + SULFATE, + TDS, + URANIUM, +) +from tests.test_cli import BaseCLITestClass + +class TestCABQCLI(BaseCLITestClass): + + agency = "cabq" + agency_reports_parameter = { + WATERLEVELS: True, + ARSENIC: False, + BICARBONATE: False, + CALCIUM: False, + CARBONATE: False, + CHLORIDE: False, + FLUORIDE: False, + MAGNESIUM: False, + NITRATE: False, + PH: False, + POTASSIUM: False, + SILICA: False, + SODIUM: False, + SULFATE: False, + TDS: False, + URANIUM: False, + } + + diff --git a/tests/test_cli/test_ebid.py b/tests/test_cli/test_ebid.py new file mode 100644 index 0000000..1f88692 --- /dev/null +++ b/tests/test_cli/test_ebid.py @@ -0,0 +1,43 @@ +from backend.constants import ( + WATERLEVELS, + ARSENIC, + BICARBONATE, + CALCIUM, + CARBONATE, + CHLORIDE, + FLUORIDE, + MAGNESIUM, + NITRATE, + PH, + POTASSIUM, + SILICA, + SODIUM, + SULFATE, + TDS, + URANIUM, +) +from tests.test_cli import BaseCLITestClass + +class TestEBIDCLI(BaseCLITestClass): + + agency = "ebid" + agency_reports_parameter = { + WATERLEVELS: True, + ARSENIC: False, + BICARBONATE: False, + CALCIUM: False, + CARBONATE: False, + CHLORIDE: False, + FLUORIDE: False, + MAGNESIUM: False, + NITRATE: False, + PH: False, + POTASSIUM: False, + SILICA: False, + SODIUM: False, + SULFATE: False, + TDS: False, + URANIUM: False, + } + + diff --git a/tests/test_cli/test_nmbgmr.py b/tests/test_cli/test_nmbgmr.py deleted file mode 100644 index 6928473..0000000 --- a/tests/test_cli/test_nmbgmr.py +++ /dev/null @@ -1,14 +0,0 @@ -from tests.test_cli import BaseCLITestClass - - -class TestNMBGMRCLI(BaseCLITestClass): - """Test the CLI for the NMBGMR source.""" - - agency = "nmbgmr-amp" - - # def test_weave(self): - # # Test the weave command for NMBGMR - # self._test_weave( - # parameter="waterlevels", - # output="summary" - # ) diff --git a/tests/test_cli/test_nmbgmr_amp.py b/tests/test_cli/test_nmbgmr_amp.py new file mode 100644 index 0000000..b582045 --- /dev/null +++ b/tests/test_cli/test_nmbgmr_amp.py @@ -0,0 +1,44 @@ +from backend.constants import ( + WATERLEVELS, + ARSENIC, + BICARBONATE, + CALCIUM, + CARBONATE, + CHLORIDE, + FLUORIDE, + MAGNESIUM, + NITRATE, + PH, + POTASSIUM, + SILICA, + SODIUM, + SULFATE, + TDS, + URANIUM, +) +from tests.test_cli import BaseCLITestClass + + +class TestNMBGMRCLI(BaseCLITestClass): + + agency = "nmbgmr-amp" + agency_reports_parameter = { + WATERLEVELS: True, + ARSENIC: True, + BICARBONATE: True, + CALCIUM: True, + CARBONATE: True, + CHLORIDE: True, + FLUORIDE: True, + MAGNESIUM: True, + NITRATE: True, + PH: True, + POTASSIUM: True, + SILICA: True, + SODIUM: True, + SULFATE: True, + TDS: True, + URANIUM: True, + } + + diff --git a/tests/test_cli/test_nmed_dwb.py b/tests/test_cli/test_nmed_dwb.py new file mode 100644 index 0000000..4e8660c --- /dev/null +++ b/tests/test_cli/test_nmed_dwb.py @@ -0,0 +1,43 @@ +from backend.constants import ( + WATERLEVELS, + ARSENIC, + BICARBONATE, + CALCIUM, + CARBONATE, + CHLORIDE, + FLUORIDE, + MAGNESIUM, + NITRATE, + PH, + POTASSIUM, + SILICA, + SODIUM, + SULFATE, + TDS, + URANIUM, +) +from tests.test_cli import BaseCLITestClass + +class TestNMEDDWBCLI(BaseCLITestClass): + + agency = "nmed-dwb" + agency_reports_parameter = { + WATERLEVELS: False, + ARSENIC: True, + BICARBONATE: True, + CALCIUM: True, + CARBONATE: False, + CHLORIDE: True, + FLUORIDE: True, + MAGNESIUM: True, + NITRATE: True, + PH: True, + POTASSIUM: True, + SILICA: True, + SODIUM: True, + SULFATE: True, + TDS: True, + URANIUM: True, + } + + diff --git a/tests/test_cli/test_nmose_isc_seven_rivers.py b/tests/test_cli/test_nmose_isc_seven_rivers.py new file mode 100644 index 0000000..2ab6bc1 --- /dev/null +++ b/tests/test_cli/test_nmose_isc_seven_rivers.py @@ -0,0 +1,44 @@ +from backend.constants import ( + WATERLEVELS, + ARSENIC, + BICARBONATE, + CALCIUM, + CARBONATE, + CHLORIDE, + FLUORIDE, + MAGNESIUM, + NITRATE, + PH, + POTASSIUM, + SILICA, + SODIUM, + SULFATE, + TDS, + URANIUM, +) +from tests.test_cli import BaseCLITestClass + + +class TestNMOSEISCSevenRiversCLI(BaseCLITestClass): + + agency = "nmose-isc-seven-rivers" + agency_reports_parameter = { + WATERLEVELS: True, + ARSENIC: False, + BICARBONATE: True, + CALCIUM: True, + CARBONATE: False, + CHLORIDE: True, + FLUORIDE: True, + MAGNESIUM: True, + NITRATE: True, + PH: True, + POTASSIUM: True, + SILICA: True, + SODIUM: True, + SULFATE: True, + TDS: True, + URANIUM: False, + } + + diff --git a/tests/test_cli/test_nmose_roswell.py b/tests/test_cli/test_nmose_roswell.py new file mode 100644 index 0000000..1fafcc8 --- /dev/null +++ b/tests/test_cli/test_nmose_roswell.py @@ -0,0 +1,43 @@ +from backend.constants import ( + WATERLEVELS, + ARSENIC, + BICARBONATE, + CALCIUM, + CARBONATE, + CHLORIDE, + FLUORIDE, + MAGNESIUM, + NITRATE, + PH, + POTASSIUM, + SILICA, + SODIUM, + SULFATE, + TDS, + URANIUM, +) +from tests.test_cli import BaseCLITestClass + +class TestNMOSERoswellCLI(BaseCLITestClass): + + agency = "nmose-roswell" + agency_reports_parameter = { + WATERLEVELS: True, + ARSENIC: False, + BICARBONATE: False, + CALCIUM: False, + CARBONATE: False, + CHLORIDE: False, + FLUORIDE: False, + MAGNESIUM: False, + NITRATE: False, + PH: False, + POTASSIUM: False, + SILICA: False, + SODIUM: False, + SULFATE: False, + TDS: False, + URANIUM: False, + } + + diff --git a/tests/test_cli/test_nwis.py b/tests/test_cli/test_nwis.py new file mode 100644 index 0000000..3f5dd55 --- /dev/null +++ b/tests/test_cli/test_nwis.py @@ -0,0 +1,43 @@ +from backend.constants import ( + WATERLEVELS, + ARSENIC, + BICARBONATE, + CALCIUM, + CARBONATE, + CHLORIDE, + FLUORIDE, + MAGNESIUM, + NITRATE, + PH, + POTASSIUM, + SILICA, + SODIUM, + SULFATE, + TDS, + URANIUM, +) +from tests.test_cli import BaseCLITestClass + +class TestNWISCLI(BaseCLITestClass): + + agency = "nwis" + agency_reports_parameter = { + WATERLEVELS: True, + ARSENIC: False, + BICARBONATE: False, + CALCIUM: False, + CARBONATE: False, + CHLORIDE: False, + FLUORIDE: False, + MAGNESIUM: False, + NITRATE: False, + PH: False, + POTASSIUM: False, + SILICA: False, + SODIUM: False, + SULFATE: False, + TDS: False, + URANIUM: False, + } + + diff --git a/tests/test_cli/test_pvacd.py b/tests/test_cli/test_pvacd.py new file mode 100644 index 0000000..167e8d6 --- /dev/null +++ b/tests/test_cli/test_pvacd.py @@ -0,0 +1,43 @@ +from backend.constants import ( + WATERLEVELS, + ARSENIC, + BICARBONATE, + CALCIUM, + CARBONATE, + CHLORIDE, + FLUORIDE, + MAGNESIUM, + NITRATE, + PH, + POTASSIUM, + SILICA, + SODIUM, + SULFATE, + TDS, + URANIUM, +) +from tests.test_cli import BaseCLITestClass + +class TestPVACDCLI(BaseCLITestClass): + + agency = "pvacd" + agency_reports_parameter = { + WATERLEVELS: True, + ARSENIC: False, + BICARBONATE: False, + CALCIUM: False, + CARBONATE: False, + CHLORIDE: False, + FLUORIDE: False, + MAGNESIUM: False, + NITRATE: False, + PH: False, + POTASSIUM: False, + SILICA: False, + SODIUM: False, + SULFATE: False, + TDS: False, + URANIUM: False, + } + + diff --git a/tests/test_cli/test_wqp.py b/tests/test_cli/test_wqp.py new file mode 100644 index 0000000..3ef021c --- /dev/null +++ b/tests/test_cli/test_wqp.py @@ -0,0 +1,43 @@ +from backend.constants import ( + WATERLEVELS, + ARSENIC, + BICARBONATE, + CALCIUM, + CARBONATE, + CHLORIDE, + FLUORIDE, + MAGNESIUM, + NITRATE, + PH, + POTASSIUM, + SILICA, + SODIUM, + SULFATE, + TDS, + URANIUM, +) +from tests.test_cli import BaseCLITestClass + +class TestWQPCLI(BaseCLITestClass): + + agency = "wqp" + agency_reports_parameter = { + WATERLEVELS: True, + ARSENIC: True, + BICARBONATE: True, + CALCIUM: True, + CARBONATE: True, + CHLORIDE: True, + FLUORIDE: True, + MAGNESIUM: True, + NITRATE: True, + PH: True, + POTASSIUM: True, + SILICA: True, + SODIUM: True, + SULFATE: True, + TDS: True, + URANIUM: True, + } + + From 22c3be07fb6fbfcbcfa242ef4914c1a70151a0cd Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 14 Apr 2025 12:13:31 -0700 Subject: [PATCH 49/55] keep bbox str to be consistent with other geographic filters --- backend/config.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/backend/config.py b/backend/config.py index 25f61fd..a1ce0e8 100644 --- a/backend/config.py +++ b/backend/config.py @@ -109,7 +109,7 @@ class Config(Loggable): end_date: str = "" # spatial - bbox: dict # dict or str + bbox: str = "" county: str = "" wkt: str = "" @@ -155,7 +155,6 @@ def __init__(self, model=None, payload=None): # need to initialize logger super().__init__() - self.bbox = {} if model: if model.wkt: self.wkt = model.wkt From c4e0d29d129b5686c74544bfde66032249373f2f Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 14 Apr 2025 12:13:58 -0700 Subject: [PATCH 50/55] add wkt spatial option --- frontend/cli.py | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/frontend/cli.py b/frontend/cli.py index 425fdfa..f724874 100644 --- a/frontend/cli.py +++ b/frontend/cli.py @@ -130,6 +130,11 @@ def cli(): default="", help="New Mexico county name", ), + click.option( + "--wkt", + default="", + help="Well known text (WKT) representation of a geometry. For example, 'POLYGON((x1 y1, x2 y2, x3 y3, x1 y1))'", + ) ] DEBUG_OPTIONS = [ click.option( @@ -227,6 +232,7 @@ def weave( start_date, end_date, bbox, + wkt, county, no_bernco, no_bor, @@ -249,7 +255,7 @@ def weave( """ parameter = weave # instantiate config and set up parameter - config = setup_config(f"{parameter}", bbox, county, site_limit, dry) + config = setup_config(f"{parameter}", bbox, wkt, county, site_limit, dry) config.parameter = parameter # output type @@ -288,18 +294,17 @@ def weave( config.finalize() # setup logging here so that the path can be set to config.output_path setup_logging(path=config.output_path) - + + config.report() if not dry: - config.report() # prompt user to continue if not click.confirm("Do you want to continue?", default=True): return - - if parameter.lower() == "waterlevels": - unify_waterlevels(config) - else: - unify_analytes(config) - + if parameter.lower() == "waterlevels": + unify_waterlevels(config) + else: + unify_analytes(config) + return config @cli.command() @add_options(SPATIAL_OPTIONS) @@ -308,6 +313,7 @@ def weave( @add_options(DEBUG_OPTIONS) def wells( bbox, + wkt, county, output_dir, no_bernco, @@ -330,7 +336,7 @@ def wells( Get locations """ - config = setup_config("sites", bbox, county, site_limit, dry) + config = setup_config("sites", bbox, wkt, county, site_limit, dry) config_agencies = [ "bernco", "bor", @@ -370,7 +376,7 @@ def wells( required=True, ) @add_options(SPATIAL_OPTIONS) -def sources(sources, bbox, county): +def sources(sources, bbox, wkt, county): """ List available sources """ @@ -381,6 +387,8 @@ def sources(sources, bbox, county): config.county = county elif bbox: config.bbox = bbox + elif wkt: + config.wkt = wkt parameter = sources config.parameter = parameter @@ -394,7 +402,7 @@ def sources(sources, bbox, county): click.echo(s) -def setup_config(tag, bbox, county, site_limit, dry): +def setup_config(tag, bbox, wkt, county, site_limit, dry): config = Config() if county: click.echo(f"Getting {tag} for county {county}") @@ -403,6 +411,9 @@ def setup_config(tag, bbox, county, site_limit, dry): click.echo(f"Getting {tag} for bounding box {bbox}") # bbox = -105.396826 36.219290, -106.024162 35.384307 config.bbox = bbox + elif wkt: + click.echo(f"Getting {tag} for WKT {wkt}") + config.wkt = wkt config.site_limit = site_limit config.dry = dry From a1bc4d04a8981004f4e587f5daf3e181811cef11 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 14 Apr 2025 12:14:16 -0700 Subject: [PATCH 51/55] code cleanup --- backend/unifier.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/unifier.py b/backend/unifier.py index ce03a3d..7872743 100644 --- a/backend/unifier.py +++ b/backend/unifier.py @@ -115,7 +115,7 @@ def _perister_factory(config): def _site_wrapper(site_source, parameter_source, persister, config): - + try: # TODO: fully develop checks/discoveries below # if not site_source.check(): From 08525c2108cdd387be28377ca0bde94b850145f6 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 14 Apr 2025 13:32:23 -0700 Subject: [PATCH 52/55] mypy fix --- frontend/cli.py | 2 +- tests/test_cli/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/cli.py b/frontend/cli.py index f724874..abf6ed4 100644 --- a/frontend/cli.py +++ b/frontend/cli.py @@ -415,7 +415,7 @@ def setup_config(tag, bbox, wkt, county, site_limit, dry): click.echo(f"Getting {tag} for WKT {wkt}") config.wkt = wkt - config.site_limit = site_limit + config.site_limit = int(site_limit) config.dry = dry return config diff --git a/tests/test_cli/__init__.py b/tests/test_cli/__init__.py index 0ede68e..6a7eb7c 100644 --- a/tests/test_cli/__init__.py +++ b/tests/test_cli/__init__.py @@ -85,7 +85,7 @@ def _test_weave( output, "--dry", "--site-limit", - site_limit, + str(site_limit), "--start-date", start_date, "--end-date", From ea23c47653dbc50413d96dc9b5f1011d1ea5bd0d Mon Sep 17 00:00:00 2001 From: jacob-a-brown Date: Mon, 14 Apr 2025 20:33:26 +0000 Subject: [PATCH 53/55] Formatting changes --- backend/config.py | 2 +- backend/unifier.py | 2 +- frontend/cli.py | 5 +- tests/__init__.py | 2 +- tests/test_cli/__init__.py | 140 +++++------------- tests/test_cli/test_bernco.py | 3 +- tests/test_cli/test_cabq.py | 3 +- tests/test_cli/test_ebid.py | 3 +- tests/test_cli/test_nmbgmr_amp.py | 4 +- tests/test_cli/test_nmed_dwb.py | 3 +- tests/test_cli/test_nmose_isc_seven_rivers.py | 4 +- tests/test_cli/test_nmose_roswell.py | 3 +- tests/test_cli/test_nwis.py | 3 +- tests/test_cli/test_pvacd.py | 3 +- tests/test_cli/test_wqp.py | 3 +- 15 files changed, 56 insertions(+), 127 deletions(-) diff --git a/backend/config.py b/backend/config.py index a1ce0e8..0ec1dd7 100644 --- a/backend/config.py +++ b/backend/config.py @@ -109,7 +109,7 @@ class Config(Loggable): end_date: str = "" # spatial - bbox: str = "" + bbox: str = "" county: str = "" wkt: str = "" diff --git a/backend/unifier.py b/backend/unifier.py index 7872743..ce03a3d 100644 --- a/backend/unifier.py +++ b/backend/unifier.py @@ -115,7 +115,7 @@ def _perister_factory(config): def _site_wrapper(site_source, parameter_source, persister, config): - + try: # TODO: fully develop checks/discoveries below # if not site_source.check(): diff --git a/frontend/cli.py b/frontend/cli.py index abf6ed4..3dcad53 100644 --- a/frontend/cli.py +++ b/frontend/cli.py @@ -134,7 +134,7 @@ def cli(): "--wkt", default="", help="Well known text (WKT) representation of a geometry. For example, 'POLYGON((x1 y1, x2 y2, x3 y3, x1 y1))'", - ) + ), ] DEBUG_OPTIONS = [ click.option( @@ -294,7 +294,7 @@ def weave( config.finalize() # setup logging here so that the path can be set to config.output_path setup_logging(path=config.output_path) - + config.report() if not dry: # prompt user to continue @@ -306,6 +306,7 @@ def weave( unify_analytes(config) return config + @cli.command() @add_options(SPATIAL_OPTIONS) @add_options(PERSISTER_OPTIONS) diff --git a/tests/__init__.py b/tests/__init__.py index 9316fba..bcf9e80 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -5,4 +5,4 @@ def recursively_clean_directory(path): recursively_clean_directory(item) else: item.unlink() - path.rmdir() \ No newline at end of file + path.rmdir() diff --git a/tests/test_cli/__init__.py b/tests/test_cli/__init__.py index 6a7eb7c..84923b8 100644 --- a/tests/test_cli/__init__.py +++ b/tests/test_cli/__init__.py @@ -47,16 +47,16 @@ def setup(self): recursively_clean_directory(self.output_dir) def _test_weave( - self, - parameter: str, - output: str, - site_limit: int = 4, - start_date: str = "1990-08-10", - end_date: str = "1990-08-11", - bbox: str | None = None, - county: str | None = None, - wkt: str | None = None, - ): + self, + parameter: str, + output: str, + site_limit: int = 4, + start_date: str = "1990-08-10", + end_date: str = "1990-08-11", + bbox: str | None = None, + county: str | None = None, + wkt: str | None = None, + ): # Arrange # turn off all sources except for the one being tested no_agencies = [] @@ -78,7 +78,7 @@ def _test_weave( elif wkt: geographic_filter_name = "wkt" geographic_filter_value = wkt - + arguments = [ parameter, "--output", @@ -119,14 +119,14 @@ def _test_weave( config = result.return_value # 0 - self.output_dir = Path(config.output_path) + self.output_dir = Path(config.output_path) # 1 assert getattr(config, "parameter") == parameter # 2 agency_with_underscore = self.agency.replace("-", "_") - if self.agency_reports_parameter[parameter]: + if self.agency_reports_parameter[parameter]: assert getattr(config, f"use_source_{agency_with_underscore}") is True else: assert getattr(config, f"use_source_{agency_with_underscore}") is False @@ -134,7 +134,7 @@ def _test_weave( for no_agency in no_agencies: no_agency_with_underscore = no_agency.replace("--no-", "").replace("-", "_") assert getattr(config, f"use_source_{no_agency_with_underscore}") is False - + # 3 output_types = ["summary", "timeseries_unified", "timeseries_separated"] for output_type in output_types: @@ -159,141 +159,81 @@ def _test_weave( if geographic_filter_name and geographic_filter_value: for _geographic_filter_name in ["bbox", "county", "wkt"]: if _geographic_filter_name == geographic_filter_name: - assert getattr(config, _geographic_filter_name) == geographic_filter_value + assert ( + getattr(config, _geographic_filter_name) + == geographic_filter_value + ) else: assert getattr(config, _geographic_filter_name) == "" def test_weave_summary(self): - self._test_weave( - parameter=WATERLEVELS, - output="summary" - ) + self._test_weave(parameter=WATERLEVELS, output="summary") def test_weave_timeseries_unified(self): - self._test_weave( - parameter=WATERLEVELS, - output="timeseries_unified" - ) + self._test_weave(parameter=WATERLEVELS, output="timeseries_unified") def test_weave_timeseries_separated(self): - self._test_weave( - parameter=WATERLEVELS, - output="timeseries_separated" - ) + self._test_weave(parameter=WATERLEVELS, output="timeseries_separated") def test_weave_bbox(self): self._test_weave( - parameter=WATERLEVELS, - output="summary", - bbox="32.0,-106.0,36.0,-102.0" + parameter=WATERLEVELS, output="summary", bbox="32.0,-106.0,36.0,-102.0" ) def test_weave_county(self): - self._test_weave( - parameter=WATERLEVELS, - output="summary", - county="Bernalillo" - ) + self._test_weave(parameter=WATERLEVELS, output="summary", county="Bernalillo") def test_weave_wkt(self): self._test_weave( parameter=WATERLEVELS, output="summary", - wkt="POLYGON((-106.0 32.0, -102.0 32.0, -102.0 36.0, -106.0 36.0, -106.0 32.0))" + wkt="POLYGON((-106.0 32.0, -102.0 32.0, -102.0 36.0, -106.0 36.0, -106.0 32.0))", ) def test_weave_waterlevels(self): - self._test_weave( - parameter=WATERLEVELS, - output="summary" - ) + self._test_weave(parameter=WATERLEVELS, output="summary") def test_weave_arsenic(self): - self._test_weave( - parameter=ARSENIC, - output="summary" - ) + self._test_weave(parameter=ARSENIC, output="summary") def test_weave_bicarbonate(self): - self._test_weave( - parameter=BICARBONATE, - output="summary" - ) + self._test_weave(parameter=BICARBONATE, output="summary") def test_weave_calcium(self): - self._test_weave( - parameter=CALCIUM, - output="summary" - ) + self._test_weave(parameter=CALCIUM, output="summary") def test_weave_carbonate(self): - self._test_weave( - parameter=CARBONATE, - output="summary" - ) + self._test_weave(parameter=CARBONATE, output="summary") def test_weave_chloride(self): - self._test_weave( - parameter=CHLORIDE, - output="summary" - ) + self._test_weave(parameter=CHLORIDE, output="summary") def test_weave_fluoride(self): - self._test_weave( - parameter=FLUORIDE, - output="summary" - ) + self._test_weave(parameter=FLUORIDE, output="summary") def test_weave_magnesium(self): - self._test_weave( - parameter=MAGNESIUM, - output="summary" - ) + self._test_weave(parameter=MAGNESIUM, output="summary") def test_weave_nitrate(self): - self._test_weave( - parameter=NITRATE, - output="summary" - ) + self._test_weave(parameter=NITRATE, output="summary") def test_weave_ph(self): - self._test_weave( - parameter=PH, - output="summary" - ) + self._test_weave(parameter=PH, output="summary") def test_weave_potassium(self): - self._test_weave( - parameter=POTASSIUM, - output="summary" - ) + self._test_weave(parameter=POTASSIUM, output="summary") def test_weave_silica(self): - self._test_weave( - parameter=SILICA, - output="summary" - ) + self._test_weave(parameter=SILICA, output="summary") def test_weave_sodium(self): - self._test_weave( - parameter=SODIUM, - output="summary" - ) + self._test_weave(parameter=SODIUM, output="summary") def test_weave_sulfate(self): - self._test_weave( - parameter=SULFATE, - output="summary" - ) + self._test_weave(parameter=SULFATE, output="summary") def test_weave_tds(self): - self._test_weave( - parameter=TDS, - output="summary" - ) + self._test_weave(parameter=TDS, output="summary") def test_weave_uranium(self): - self._test_weave( - parameter=URANIUM, - output="summary" - ) \ No newline at end of file + self._test_weave(parameter=URANIUM, output="summary") diff --git a/tests/test_cli/test_bernco.py b/tests/test_cli/test_bernco.py index c6e4031..331ed26 100644 --- a/tests/test_cli/test_bernco.py +++ b/tests/test_cli/test_bernco.py @@ -18,6 +18,7 @@ ) from tests.test_cli import BaseCLITestClass + class TestBernCoCLI(BaseCLITestClass): agency = "bernco" @@ -39,5 +40,3 @@ class TestBernCoCLI(BaseCLITestClass): TDS: False, URANIUM: False, } - - diff --git a/tests/test_cli/test_cabq.py b/tests/test_cli/test_cabq.py index 5e96a07..1748975 100644 --- a/tests/test_cli/test_cabq.py +++ b/tests/test_cli/test_cabq.py @@ -18,6 +18,7 @@ ) from tests.test_cli import BaseCLITestClass + class TestCABQCLI(BaseCLITestClass): agency = "cabq" @@ -39,5 +40,3 @@ class TestCABQCLI(BaseCLITestClass): TDS: False, URANIUM: False, } - - diff --git a/tests/test_cli/test_ebid.py b/tests/test_cli/test_ebid.py index 1f88692..76429f1 100644 --- a/tests/test_cli/test_ebid.py +++ b/tests/test_cli/test_ebid.py @@ -18,6 +18,7 @@ ) from tests.test_cli import BaseCLITestClass + class TestEBIDCLI(BaseCLITestClass): agency = "ebid" @@ -39,5 +40,3 @@ class TestEBIDCLI(BaseCLITestClass): TDS: False, URANIUM: False, } - - diff --git a/tests/test_cli/test_nmbgmr_amp.py b/tests/test_cli/test_nmbgmr_amp.py index b582045..df4ea49 100644 --- a/tests/test_cli/test_nmbgmr_amp.py +++ b/tests/test_cli/test_nmbgmr_amp.py @@ -39,6 +39,4 @@ class TestNMBGMRCLI(BaseCLITestClass): SULFATE: True, TDS: True, URANIUM: True, - } - - + } diff --git a/tests/test_cli/test_nmed_dwb.py b/tests/test_cli/test_nmed_dwb.py index 4e8660c..edd9d68 100644 --- a/tests/test_cli/test_nmed_dwb.py +++ b/tests/test_cli/test_nmed_dwb.py @@ -18,6 +18,7 @@ ) from tests.test_cli import BaseCLITestClass + class TestNMEDDWBCLI(BaseCLITestClass): agency = "nmed-dwb" @@ -39,5 +40,3 @@ class TestNMEDDWBCLI(BaseCLITestClass): TDS: True, URANIUM: True, } - - diff --git a/tests/test_cli/test_nmose_isc_seven_rivers.py b/tests/test_cli/test_nmose_isc_seven_rivers.py index 2ab6bc1..0f99e70 100644 --- a/tests/test_cli/test_nmose_isc_seven_rivers.py +++ b/tests/test_cli/test_nmose_isc_seven_rivers.py @@ -39,6 +39,4 @@ class TestNMOSEISCSevenRiversCLI(BaseCLITestClass): SULFATE: True, TDS: True, URANIUM: False, - } - - + } diff --git a/tests/test_cli/test_nmose_roswell.py b/tests/test_cli/test_nmose_roswell.py index 1fafcc8..0c2be39 100644 --- a/tests/test_cli/test_nmose_roswell.py +++ b/tests/test_cli/test_nmose_roswell.py @@ -18,6 +18,7 @@ ) from tests.test_cli import BaseCLITestClass + class TestNMOSERoswellCLI(BaseCLITestClass): agency = "nmose-roswell" @@ -39,5 +40,3 @@ class TestNMOSERoswellCLI(BaseCLITestClass): TDS: False, URANIUM: False, } - - diff --git a/tests/test_cli/test_nwis.py b/tests/test_cli/test_nwis.py index 3f5dd55..0fd236a 100644 --- a/tests/test_cli/test_nwis.py +++ b/tests/test_cli/test_nwis.py @@ -18,6 +18,7 @@ ) from tests.test_cli import BaseCLITestClass + class TestNWISCLI(BaseCLITestClass): agency = "nwis" @@ -39,5 +40,3 @@ class TestNWISCLI(BaseCLITestClass): TDS: False, URANIUM: False, } - - diff --git a/tests/test_cli/test_pvacd.py b/tests/test_cli/test_pvacd.py index 167e8d6..041c9a9 100644 --- a/tests/test_cli/test_pvacd.py +++ b/tests/test_cli/test_pvacd.py @@ -18,6 +18,7 @@ ) from tests.test_cli import BaseCLITestClass + class TestPVACDCLI(BaseCLITestClass): agency = "pvacd" @@ -39,5 +40,3 @@ class TestPVACDCLI(BaseCLITestClass): TDS: False, URANIUM: False, } - - diff --git a/tests/test_cli/test_wqp.py b/tests/test_cli/test_wqp.py index 3ef021c..f3beb7b 100644 --- a/tests/test_cli/test_wqp.py +++ b/tests/test_cli/test_wqp.py @@ -18,6 +18,7 @@ ) from tests.test_cli import BaseCLITestClass + class TestWQPCLI(BaseCLITestClass): agency = "wqp" @@ -39,5 +40,3 @@ class TestWQPCLI(BaseCLITestClass): TDS: True, URANIUM: True, } - - From 3620df46ed104b84cbb798f50a840cd6d507700d Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 14 Apr 2025 13:42:48 -0700 Subject: [PATCH 54/55] bump version to 0.8.2 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9f43e06..f06990d 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ setup( name="nmuwd", - version="0.8.1", + version="0.8.2", author="Jake Ross", description="New Mexico Water Data Integration Engine", long_description=long_description, From 0cf98b4ae98e6bb90ae0891d970f4c2553b85f8f Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Mon, 14 Apr 2025 16:13:48 -0700 Subject: [PATCH 55/55] PR 43 review changes --- frontend/cli.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/frontend/cli.py b/frontend/cli.py index 3dcad53..04e2949 100644 --- a/frontend/cli.py +++ b/frontend/cli.py @@ -215,7 +215,7 @@ def _add_options(func): @cli.command() @click.argument( - "weave", + "parameter", type=click.Choice(PARAMETER_OPTIONS, case_sensitive=False), required=True, ) @@ -226,7 +226,7 @@ def _add_options(func): @add_options(ALL_SOURCE_OPTIONS) @add_options(DEBUG_OPTIONS) def weave( - weave, + parameter, output, output_dir, start_date, @@ -253,9 +253,8 @@ def weave( """ Get parameter timeseries or summary data """ - parameter = weave # instantiate config and set up parameter - config = setup_config(f"{parameter}", bbox, wkt, county, site_limit, dry) + config = setup_config(parameter, bbox, wkt, county, site_limit, dry) config.parameter = parameter # output type @@ -416,7 +415,10 @@ def setup_config(tag, bbox, wkt, county, site_limit, dry): click.echo(f"Getting {tag} for WKT {wkt}") config.wkt = wkt - config.site_limit = int(site_limit) + if site_limit: + config.site_limit = int(site_limit) + else: + config.site_limit = None config.dry = dry return config