v. 0.8.2: testing suite | mpyp cleanup | NWIS response cleanup | NMBGMR health check cleanup | site limit fix |

CHANGELOG.md

@@ -20,6 +20,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
   - This naming schema also enables the development of datetime filters as the descriptor will apply to the latest datetime within the provided time frame filter, whereas most recent indicates np filters.
 - removed sites that are not in New Mexico  
 
+### Fixed
+- removed records from USGS where the value is "-999999"
+
 
 ## 0.7.0
 

backend/bounding_polygons.py

@@ -162,7 +162,7 @@ def get_county_polygon(name, as_wkt=True):
         _warning(f"Invalid state. {state}")
 
 
-def get_state_polygon(state, buffer):
+def get_state_polygon(state: str, buffer: int | None = None):
     statefp = _statelookup(state)
     if statefp:
         obj = _get_cached_object(

backend/config.py

@@ -15,7 +15,6 @@
 # ===============================================================================
 import os
 import sys
-import time
 from datetime import datetime, timedelta
 
 import shapely.wkt
@@ -29,15 +28,6 @@
     NMBGMRAnalyteSource,
 )
 from .connectors.bor.source import BORSiteSource, BORAnalyteSource
-from .connectors.ckan import (
-    HONDO_RESOURCE_ID,
-    FORT_SUMNER_RESOURCE_ID,
-    ROSWELL_RESOURCE_ID,
-)
-from .connectors.ckan.source import (
-    OSERoswellSiteSource,
-    OSERoswellWaterLevelSource,
-)
 from .connectors.nmenv.source import DWBSiteSource, DWBAnalyteSource
 from .connectors.nmose.source import NMOSEPODSiteSource
 from .constants import (
@@ -67,7 +57,6 @@
     ISCSevenRiversAnalyteSource,
 )
 from .connectors.st2.source import (
-    ST2SiteSource,
     PVACDSiteSource,
     PVACDWaterLevelSource,
     EBIDSiteSource,
@@ -120,7 +109,7 @@ class Config(Loggable):
     end_date: str = ""
 
     # spatial
-    bbox: dict  # dict or str
+    bbox: str = ""
     county: str = ""
     wkt: str = ""
 
@@ -166,7 +155,6 @@ def __init__(self, model=None, payload=None):
         # need to initialize logger
         super().__init__()
 
-        self.bbox = {}
         if model:
             if model.wkt:
                 self.wkt = model.wkt

backend/connectors/nmbgmr/source.py

@@ -72,9 +72,6 @@ def get_records(self):
         if config.has_bounds():
             params["wkt"] = config.bounding_wkt()
 
-        if config.site_limit:
-            params["limit"] = config.site_limit
-
         if not config.sites_only:
 
             if config.parameter.lower() != "waterlevels":
@@ -90,18 +87,18 @@ def get_records(self):
         )
         if not config.sites_only:
             for site in sites:
-                print(f"Obtaining well data for {site['properties']['point_id']}")
-                well_data = self._execute_json_request(
-                    _make_url("wells"),
-                    params={"pointid": site["properties"]["point_id"]},
-                    tag="",
-                )
-                site["properties"]["formation"] = well_data["formation"]
-                site["properties"]["well_depth"] = well_data["well_depth_ftbgs"]
-                site["properties"]["well_depth_units"] = FEET
-                # site["properties"]["formation"] = None
-                # site["properties"]["well_depth"] = None
+                # print(f"Obtaining well data for {site['properties']['point_id']}")
+                # well_data = self._execute_json_request(
+                #     _make_url("wells"),
+                #     params={"pointid": site["properties"]["point_id"]},
+                #     tag="",
+                # )
+                # site["properties"]["formation"] = well_data["formation"]
+                # site["properties"]["well_depth"] = well_data["well_depth_ftbgs"]
                 # site["properties"]["well_depth_units"] = FEET
+                site["properties"]["formation"] = None
+                site["properties"]["well_depth"] = None
+                site["properties"]["well_depth_units"] = FEET
 
         return sites
 

backend/connectors/nmenv/source.py

@@ -27,8 +27,7 @@
     DT_MEASURED,
     SOURCE_PARAMETER_NAME,
     SOURCE_PARAMETER_UNITS,
-    EARLIEST,
-    LATEST,
+    TDS,
 )
 from backend.source import get_analyte_search_param, get_terminal_record
 
@@ -46,7 +45,7 @@ def __repr__(self):
         return "DWBSiteSource"
 
     def health(self):
-        return self.get_records(top=10, analyte="TDS")
+        return self.get_records(top=10, analyte=TDS)
 
     def get_records(self, *args, **kw):
 

backend/connectors/nmose/source.py

@@ -1,4 +1,4 @@
-import os
+from typing import List, Dict, Any
 
 from shapely import wkt
 from backend.connectors import NM_STATE_BOUNDING_POLYGON
@@ -20,12 +20,12 @@ class NMOSEPODSiteSource(BaseSiteSource):
     """
 
     transformer_klass = NMOSEPODSiteTransformer
-    chunk_size = 5000
+    chunk_size: int = 5000
     bounding_polygon = NM_STATE_BOUNDING_POLYGON
 
-    def get_records(self, *args, **kw) -> dict:
+    def get_records(self, *args, **kw) -> List[Dict]:
         config = self.config
-        params = {}
+        params: Dict[str, Any] = {}
         # if config.has_bounds():
         #     bbox = config.bbox_bounding_points()
         #     params["bBox"] = ",".join([str(b) for b in bbox])
@@ -37,7 +37,9 @@ def get_records(self, *args, **kw) -> dict:
         # if config.end_date:
         #     params["endDt"] = config.end_dt.date().isoformat()
 
-        url = "https://services2.arcgis.com/qXZbWTdPDbTjl7Dy/arcgis/rest/services/OSE_PODs/FeatureServer/0/query"
+        url: str = (
+            "https://services2.arcgis.com/qXZbWTdPDbTjl7Dy/arcgis/rest/services/OSE_PODs/FeatureServer/0/query"
+        )
 
         params["where"] = (
             "pod_status = 'ACT' AND pod_basin IN ('A','B','C','CC','CD','CL','CP','CR','CT','E','FS','G','GSF','H', 'HA','HC','HS','HU','J','L','LA','LRG','LV','M','MR','NH','P','PL','PN','RA','RG','S','SB','SJ','SS','T','TU','UP','VV')"
@@ -55,11 +57,14 @@ def get_records(self, *args, **kw) -> dict:
             params["geometry"] = wkt_to_arcgis_json(wkt)
             params["geometryType"] = "esriGeometryPolygon"
 
-        records = []
+        records: List = []
         i = 1
         while 1:
             rs = self._execute_json_request(url, params, tag="features")
-            records.extend(rs)
+            if rs is None:
+                continue
+            else:
+                records.extend(rs)
             params["resultOffset"] += self.chunk_size
             if len(rs) < self.chunk_size:
                 break

backend/connectors/usgs/source.py

@@ -181,7 +181,11 @@ def _extract_site_records(self, records, site_record):
         return [ri for ri in records if ri["site_code"] == site_record.id]
 
     def _clean_records(self, records):
-        return [r for r in records if r["value"] is not None and r["value"].strip()]
+        return [
+            r
+            for r in records
+            if r["value"] is not None and r["value"].strip() and r["value"] != "-999999"
+        ]
 
     def _extract_source_parameter_results(self, records):
         return [float(r["value"]) for r in records]

backend/persister.py

@@ -21,7 +21,7 @@
 import pandas as pd
 import geopandas as gpd
 
-from backend.logging import Loggable
+from backend.logger import Loggable
 
 try:
     from google.cloud import storage