feat: add new thing types and transfer functions for rock samples, surface water diversions, lakes, soil gas samples, and outfalls

admin/views/chemistry_sampleinfo.py

@@ -28,6 +28,8 @@
 - thing_id: Integer FK to Thing.id
 """
 
+from starlette.requests import Request
+from starlette_admin import HasOne
 
 from admin.views.base import OcotilloModelView
 

core/lexicon.json

@@ -2007,6 +2007,48 @@
       "term": "meteorological station",
       "definition": "a station that measures the weather conditions at a particular location"
     },
+    {
+      "categories": [
+        "thing_type"
+      ],
+      "term": "rock sample location",
+      "definition": "a location where rock samples are collected"
+    },
+    {
+      "categories": [
+        "thing_type"
+      ],
+      "term": "diversion of surface water, etc.",
+      "definition": "a diversion structure for surface water such as a ditch, canal, or intake"
+    },
+    {
+      "categories": [
+        "thing_type"
+      ],
+      "term": "lake, pond or reservoir",
+      "definition": "a natural or artificial standing body of water"
+    },
+    {
+      "categories": [
+        "thing_type"
+      ],
+      "term": "soil gas sample location",
+      "definition": "a location where soil gas samples are collected"
+    },
+    {
+      "categories": [
+        "thing_type"
+      ],
+      "term": "other",
+      "definition": "a thing type that does not fit other categories"
+    },
+    {
+      "categories": [
+        "thing_type"
+      ],
+      "term": "outfall of wastewater or return flow",
+      "definition": "a discharge point for wastewater or return flows"
+    },
     {
       "categories": [
         "groundwater_level_reason"
@@ -8149,4 +8191,4 @@
       "definition": "Data were not field checked but are considered reliable"
     }
   ]
-}
+}

transfers/thing_transfer.py

@@ -14,6 +14,7 @@
 # limitations under the License.
 # ===============================================================================
 import time
+
 from pandas import isna
 from pydantic import ValidationError
 from sqlalchemy.orm import Session
@@ -93,48 +94,118 @@ def transfer_thing(session: Session, site_type: str, make_payload, limit=None) -
     logger.info("Completed transfer: Things (%s)", site_type)
 
 
+def _release_status(row) -> str:
+    return "public" if row.PublicRelease else "private"
+
+
 def transfer_springs(session, limit=None):
     def make_payload(row):
         return {
             "name": row.PointID,
             "thing_type": "spring",
-            "release_status": "public" if row.PublicRelease else "private",
+            "release_status": _release_status(row),
         }
 
     transfer_thing(session, "SP", make_payload, limit)
 
 
-def transfer_perennial_stream(session, limit=None):
+def transfer_perennial_streams(session, limit=None):
     def make_payload(row):
         return {
             "name": row.PointID,
             "thing_type": "perennial stream",
-            "release_status": "public" if row.PublicRelease else "private",
+            "release_status": _release_status(row),
         }
 
     transfer_thing(session, "PS", make_payload, limit)
 
 
-def transfer_ephemeral_stream(session, limit=None):
+def transfer_ephemeral_streams(session, limit=None):
     def make_payload(row):
         return {
             "name": row.PointID,
             "thing_type": "ephemeral stream",
-            "release_status": "public" if row.PublicRelease else "private",
+            "release_status": _release_status(row),
         }
 
     transfer_thing(session, "ES", make_payload, limit)
 
 
-def transfer_met(session, limit=None):
+def transfer_met_stations(session, limit=None):
     def make_payload(row):
         return {
             "name": row.PointID,
             "thing_type": "meteorological station",
-            "release_status": "public" if row.PublicRelease else "private",
+            "release_status": _release_status(row),
         }
 
     transfer_thing(session, "M", make_payload, limit)
 
 
+def transfer_rock_sample_locations(session, limit=None):
+    def make_payload(row):
+        return {
+            "name": row.PointID,
+            "thing_type": "rock sample location",
+            "release_status": _release_status(row),
+        }
+
+    transfer_thing(session, "R", make_payload, limit)
+
+
+def transfer_diversion_of_surface_water(session, limit=None):
+    def make_payload(row):
+        return {
+            "name": row.PointID,
+            "thing_type": "diversion of surface water, etc.",
+            "release_status": _release_status(row),
+        }
+
+    transfer_thing(session, "D", make_payload, limit)
+
+
+def transfer_lake_pond_reservoir(session, limit=None):
+    def make_payload(row):
+        return {
+            "name": row.PointID,
+            "thing_type": "lake, pond or reservoir",
+            "release_status": _release_status(row),
+        }
+
+    transfer_thing(session, "L", make_payload, limit)
+
+
+def transfer_soil_gas_sample_locations(session, limit=None):
+    def make_payload(row):
+        return {
+            "name": row.PointID,
+            "thing_type": "soil gas sample location",
+            "release_status": _release_status(row),
+        }
+
+    transfer_thing(session, "S", make_payload, limit)
+
+
+def transfer_other_site_types(session, limit=None):
+    def make_payload(row):
+        return {
+            "name": row.PointID,
+            "thing_type": "other",
+            "release_status": _release_status(row),
+        }
+
+    transfer_thing(session, "OT", make_payload, limit)
+
+
+def transfer_outfall_wastewater_return_flow(session, limit=None):
+    def make_payload(row):
+        return {
+            "name": row.PointID,
+            "thing_type": "outfall of wastewater or return flow",
+            "release_status": _release_status(row),
+        }
+
+    transfer_thing(session, "O", make_payload, limit)
+
+
 # ============= EOF =============================================

transfers/transfer.py

@@ -21,6 +21,19 @@
 
 from dotenv import load_dotenv
 
+from transfers.thing_transfer import (
+    transfer_rock_sample_locations,
+    transfer_springs,
+    transfer_perennial_streams,
+    transfer_ephemeral_streams,
+    transfer_met_stations,
+    transfer_diversion_of_surface_water,
+    transfer_lake_pond_reservoir,
+    transfer_soil_gas_sample_locations,
+    transfer_other_site_types,
+    transfer_outfall_wastewater_return_flow,
+)
+
 # Load .env file FIRST, before any database imports, to ensure correct port/database settings
 load_dotenv(override=True)
 
@@ -61,12 +74,6 @@
     WellScreenTransferer,
 )
 from transfers.well_transfer_util import cleanup_locations
-from transfers.thing_transfer import (
-    transfer_springs,
-    transfer_perennial_stream,
-    transfer_ephemeral_stream,
-    transfer_met,
-)
 from transfers.minor_trace_chemistry_transfer import MinorTraceChemistryTransferer
 
 from transfers.asset_transfer import AssetTransferer
@@ -125,6 +132,12 @@ class TransferOptions:
     transfer_perennial_streams: bool
     transfer_ephemeral_streams: bool
     transfer_met_stations: bool
+    transfer_rock_sample_locations: bool
+    transfer_diversion_of_surface_water: bool
+    transfer_lake_pond_reservoir: bool
+    transfer_soil_gas_sample_locations: bool
+    transfer_other_site_types: bool
+    transfer_outfall_wastewater_return_flow: bool
 
 
 def load_transfer_options() -> TransferOptions:
@@ -168,6 +181,20 @@ def load_transfer_options() -> TransferOptions:
         transfer_perennial_streams=get_bool_env("TRANSFER_PERENNIAL_STREAMS", True),
         transfer_ephemeral_streams=get_bool_env("TRANSFER_EPHEMERAL_STREAMS", True),
         transfer_met_stations=get_bool_env("TRANSFER_MET_STATIONS", True),
+        transfer_rock_sample_locations=get_bool_env(
+            "TRANSFER_ROCK_SAMPLE_LOCATIONS", True
+        ),
+        transfer_diversion_of_surface_water=get_bool_env(
+            "TRANSFER_DIVERSION_OF_SURFACE_WATER", True
+        ),
+        transfer_lake_pond_reservoir=get_bool_env("TRANSFER_LAKE_POND_RESERVOIR", True),
+        transfer_soil_gas_sample_locations=get_bool_env(
+            "TRANSFER_SOIL_GAS_SAMPLE_LOCATIONS", True
+        ),
+        transfer_other_site_types=get_bool_env("TRANSFER_OTHER_SITE_TYPES", True),
+        transfer_outfall_wastewater_return_flow=get_bool_env(
+            "TRANSFER_OUTFALL_WASTEWATER_RETURN_FLOW", True
+        ),
     )
 
 
@@ -360,14 +387,37 @@ def transfer_all(metrics: Metrics) -> list[ProfileArtifact]:
         # These create Things and Locations that chemistry/other transfers depend on.
         # =========================================================================
         non_well_tasks = []
-        if transfer_options.transfer_springs:
-            non_well_tasks.append(("Springs", transfer_springs))
-        if transfer_options.transfer_perennial_streams:
-            non_well_tasks.append(("PerennialStreams", transfer_perennial_stream))
-        if transfer_options.transfer_ephemeral_streams:
-            non_well_tasks.append(("EphemeralStreams", transfer_ephemeral_stream))
-        if transfer_options.transfer_met_stations:
-            non_well_tasks.append(("MetStations", transfer_met))
+        transfer_functions = {
+            "transfer_springs": transfer_springs,
+            "transfer_perennial_streams": transfer_perennial_streams,
+            "transfer_ephemeral_streams": transfer_ephemeral_streams,
+            "transfer_met_stations": transfer_met_stations,
+            "transfer_rock_sample_locations": transfer_rock_sample_locations,
+            "transfer_diversion_of_surface_water": transfer_diversion_of_surface_water,
+            "transfer_lake_pond_reservoir": transfer_lake_pond_reservoir,
+            "transfer_soil_gas_sample_locations": transfer_soil_gas_sample_locations,
+            "transfer_other_site_types": transfer_other_site_types,
+            "transfer_outfall_wastewater_return_flow": (
+                transfer_outfall_wastewater_return_flow
+            ),
+        }
+
+        for attr, thing_type in (
+            ("springs", "Springs"),
+            ("perennial_streams", "PerennialStreams"),
+            ("ephemeral_streams", "EphemeralStreams"),
+            ("met_stations", "MetStations"),
+            ("rock_sample_locations", "RockSampleLocations"),
+            ("diversion_of_surface_water", "DiversionOfSurfaceWater"),
+            ("lake_pond_reservoir", "LakePondReservoir"),
+            ("soil_gas_sample_locations", "SoilGasSampleLocations"),
+            ("other_site_types", "OtherSiteTypes"),
+            ("outfall_wastewater_return_flow", "OutfallWastewaterReturnFlow"),
+        ):
+            attr_name = f"transfer_{attr}"
+            if getattr(transfer_options, attr_name):
+                transfer_func = transfer_functions[attr_name]
+                non_well_tasks.append((thing_type, transfer_func))
 
         if non_well_tasks:
             message("PHASE 1.5: NON-WELL LOCATION TYPES (PARALLEL)")

transfers/well_transfer.py

@@ -150,8 +150,6 @@ def process_batch(batch_idx: int, batch_df: pd.DataFrame) -> dict:
                             # Process single well with all dependent objects
                             self._step_parallel_complete(
                                 session,
-                                batch_df,
-                                i,
                                 row,
                                 local_aquifers,
                                 local_formations,
@@ -879,8 +877,6 @@ def _add_histories(self, session: Session, row, well: Thing) -> None:
     def _step_parallel_complete(
         self,
         session: Session,
-        df: pd.DataFrame,
-        i: int,
         row,
         local_aquifers: list,
         local_formations: dict,