From d6bf42cd20dd21a31fe34afe7edada21d59f44db Mon Sep 17 00:00:00 2001 From: jakeross Date: Wed, 4 Feb 2026 20:05:03 +1100 Subject: [PATCH 1/6] feat: add new thing types and transfer functions for rock samples, surface water diversions, lakes, soil gas samples, and outfalls --- core/lexicon.json | 44 ++++++++++++++++++- transfers/thing_transfer.py | 85 ++++++++++++++++++++++++++++++++++--- transfers/transfer.py | 52 +++++++++++++++++------ 3 files changed, 159 insertions(+), 22 deletions(-) diff --git a/core/lexicon.json b/core/lexicon.json index 01539f2d..cf605117 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -2007,6 +2007,48 @@ "term": "meteorological station", "definition": "a station that measures the weather conditions at a particular location" }, + { + "categories": [ + "thing_type" + ], + "term": "Rock sample location", + "definition": "a location where rock samples are collected" + }, + { + "categories": [ + "thing_type" + ], + "term": "Diversion of surface water, etc.", + "definition": "a diversion structure for surface water such as a ditch, canal, or intake" + }, + { + "categories": [ + "thing_type" + ], + "term": "Lake, pond or reservoir", + "definition": "a natural or artificial standing body of water" + }, + { + "categories": [ + "thing_type" + ], + "term": "Soil gas sample location", + "definition": "a location where soil gas samples are collected" + }, + { + "categories": [ + "thing_type" + ], + "term": "Other", + "definition": "a thing type that does not fit other categories" + }, + { + "categories": [ + "thing_type" + ], + "term": "Outfall of wastewater or return flow", + "definition": "a discharge point for wastewater or return flows" + }, { "categories": [ "groundwater_level_reason" @@ -8149,4 +8191,4 @@ "definition": "Data were not field checked but are considered reliable" } ] -} \ No newline at end of file +} diff --git a/transfers/thing_transfer.py b/transfers/thing_transfer.py index 754634b7..dcdeb85e 100644 --- a/transfers/thing_transfer.py +++ b/transfers/thing_transfer.py @@ -14,6 +14,7 @@ # limitations under the License. # =============================================================================== import time + from pandas import isna from pydantic import ValidationError from sqlalchemy.orm import Session @@ -93,48 +94,118 @@ def transfer_thing(session: Session, site_type: str, make_payload, limit=None) - logger.info("Completed transfer: Things (%s)", site_type) +def _release_status(row) -> str: + return "public" if row.PublicRelease else "private" + + def transfer_springs(session, limit=None): def make_payload(row): return { "name": row.PointID, "thing_type": "spring", - "release_status": "public" if row.PublicRelease else "private", + "release_status": _release_status(row), } transfer_thing(session, "SP", make_payload, limit) -def transfer_perennial_stream(session, limit=None): +def transfer_perennial_streams(session, limit=None): def make_payload(row): return { "name": row.PointID, "thing_type": "perennial stream", - "release_status": "public" if row.PublicRelease else "private", + "release_status": _release_status(row), } transfer_thing(session, "PS", make_payload, limit) -def transfer_ephemeral_stream(session, limit=None): +def transfer_ephemeral_streams(session, limit=None): def make_payload(row): return { "name": row.PointID, "thing_type": "ephemeral stream", - "release_status": "public" if row.PublicRelease else "private", + "release_status": _release_status(row), } transfer_thing(session, "ES", make_payload, limit) -def transfer_met(session, limit=None): +def transfer_met_stations(session, limit=None): def make_payload(row): return { "name": row.PointID, "thing_type": "meteorological station", - "release_status": "public" if row.PublicRelease else "private", + "release_status": _release_status(row), } transfer_thing(session, "M", make_payload, limit) +def transfer_rock_sample_locations(session, limit=None): + def make_payload(row): + return { + "name": row.PointID, + "thing_type": "Rock sample location", + "release_status": _release_status(row), + } + + transfer_thing(session, "R", make_payload, limit) + + +def transfer_diversion_of_surface_water(session, limit=None): + def make_payload(row): + return { + "name": row.PointID, + "thing_type": "Diversion of surface water, etc.", + "release_status": _release_status(row), + } + + transfer_thing(session, "D", make_payload, limit) + + +def transfer_lake_pond_reservoir(session, limit=None): + def make_payload(row): + return { + "name": row.PointID, + "thing_type": "Lake, pond or reservoir", + "release_status": _release_status(row), + } + + transfer_thing(session, "L", make_payload, limit) + + +def transfer_soil_gas_sample_locations(session, limit=None): + def make_payload(row): + return { + "name": row.PointID, + "thing_type": "Soil gas sample location", + "release_status": _release_status(row), + } + + transfer_thing(session, "S", make_payload, limit) + + +def transfer_other_site_types(session, limit=None): + def make_payload(row): + return { + "name": row.PointID, + "thing_type": "Other", + "release_status": _release_status(row), + } + + transfer_thing(session, "OT", make_payload, limit) + + +def transfer_outfall_wastewater_return_flow(session, limit=None): + def make_payload(row): + return { + "name": row.PointID, + "thing_type": "Outfall of wastewater or return flow", + "release_status": _release_status(row), + } + + transfer_thing(session, "O", make_payload, limit) + + # ============= EOF ============================================= diff --git a/transfers/transfer.py b/transfers/transfer.py index 73c82a21..45dda85b 100644 --- a/transfers/transfer.py +++ b/transfers/transfer.py @@ -61,12 +61,6 @@ WellScreenTransferer, ) from transfers.well_transfer_util import cleanup_locations -from transfers.thing_transfer import ( - transfer_springs, - transfer_perennial_stream, - transfer_ephemeral_stream, - transfer_met, -) from transfers.minor_trace_chemistry_transfer import MinorTraceChemistryTransferer from transfers.asset_transfer import AssetTransferer @@ -125,6 +119,12 @@ class TransferOptions: transfer_perennial_streams: bool transfer_ephemeral_streams: bool transfer_met_stations: bool + transfer_rock_sample_locations: bool + transfer_diversion_of_surface_water: bool + transfer_lake_pond_reservoir: bool + transfer_soil_gas_sample_locations: bool + transfer_other_site_types: bool + transfer_outfall_wastewater_return_flow: bool def load_transfer_options() -> TransferOptions: @@ -168,6 +168,20 @@ def load_transfer_options() -> TransferOptions: transfer_perennial_streams=get_bool_env("TRANSFER_PERENNIAL_STREAMS", True), transfer_ephemeral_streams=get_bool_env("TRANSFER_EPHEMERAL_STREAMS", True), transfer_met_stations=get_bool_env("TRANSFER_MET_STATIONS", True), + transfer_rock_sample_locations=get_bool_env( + "TRANSFER_ROCK_SAMPLE_LOCATIONS", True + ), + transfer_diversion_of_surface_water=get_bool_env( + "TRANSFER_DIVERSION_OF_SURFACE_WATER", True + ), + transfer_lake_pond_reservoir=get_bool_env("TRANSFER_LAKE_POND_RESERVOIR", True), + transfer_soil_gas_sample_locations=get_bool_env( + "TRANSFER_SOIL_GAS_SAMPLE_LOCATIONS", True + ), + transfer_other_site_types=get_bool_env("TRANSFER_OTHER_SITE_TYPES", True), + transfer_outfall_wastewater_return_flow=get_bool_env( + "TRANSFER_OUTFALL_WASTEWATER_RETURN_FLOW", True + ), ) @@ -360,14 +374,24 @@ def transfer_all(metrics: Metrics) -> list[ProfileArtifact]: # These create Things and Locations that chemistry/other transfers depend on. # ========================================================================= non_well_tasks = [] - if transfer_options.transfer_springs: - non_well_tasks.append(("Springs", transfer_springs)) - if transfer_options.transfer_perennial_streams: - non_well_tasks.append(("PerennialStreams", transfer_perennial_stream)) - if transfer_options.transfer_ephemeral_streams: - non_well_tasks.append(("EphemeralStreams", transfer_ephemeral_stream)) - if transfer_options.transfer_met_stations: - non_well_tasks.append(("MetStations", transfer_met)) + gs = globals() + for attr in ( + "springs", + "perennial_streams", + "ephemeral_streams", + "met_stations", + "rock_sample_locations", + "diversion_of_surface_water", + "lake_pond_reservoir", + "soil_gas_sample_locations", + "other_site_types", + "outfall_wastewater_return_flow", + ): + thing_type = "".join(part.capitalize() for part in attr.split("_")) + attr_name = f"transfer_{attr}" + if getattr(transfer_options, attr_name): + transfer_func = gs[attr_name] + non_well_tasks.append((thing_type, transfer_func)) if non_well_tasks: message("PHASE 1.5: NON-WELL LOCATION TYPES (PARALLEL)") From e436125bbe9025e4139b97d077421baa5cd5afaa Mon Sep 17 00:00:00 2001 From: jakeross Date: Wed, 4 Feb 2026 22:12:11 +1100 Subject: [PATCH 2/6] fix: standardize thing type terminology in lexicon and payloads --- core/lexicon.json | 12 ++++++------ transfers/thing_transfer.py | 12 ++++++------ transfers/transfer.py | 29 +++++++++++++++++++++++++++-- 3 files changed, 39 insertions(+), 14 deletions(-) diff --git a/core/lexicon.json b/core/lexicon.json index cf605117..5b99accb 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -2011,42 +2011,42 @@ "categories": [ "thing_type" ], - "term": "Rock sample location", + "term": "rock sample location", "definition": "a location where rock samples are collected" }, { "categories": [ "thing_type" ], - "term": "Diversion of surface water, etc.", + "term": "diversion of surface water, etc.", "definition": "a diversion structure for surface water such as a ditch, canal, or intake" }, { "categories": [ "thing_type" ], - "term": "Lake, pond or reservoir", + "term": "lake, pond or reservoir", "definition": "a natural or artificial standing body of water" }, { "categories": [ "thing_type" ], - "term": "Soil gas sample location", + "term": "soil gas sample location", "definition": "a location where soil gas samples are collected" }, { "categories": [ "thing_type" ], - "term": "Other", + "term": "other", "definition": "a thing type that does not fit other categories" }, { "categories": [ "thing_type" ], - "term": "Outfall of wastewater or return flow", + "term": "outfall of wastewater or return flow", "definition": "a discharge point for wastewater or return flows" }, { diff --git a/transfers/thing_transfer.py b/transfers/thing_transfer.py index dcdeb85e..5d4456db 100644 --- a/transfers/thing_transfer.py +++ b/transfers/thing_transfer.py @@ -146,7 +146,7 @@ def transfer_rock_sample_locations(session, limit=None): def make_payload(row): return { "name": row.PointID, - "thing_type": "Rock sample location", + "thing_type": "rock sample location", "release_status": _release_status(row), } @@ -157,7 +157,7 @@ def transfer_diversion_of_surface_water(session, limit=None): def make_payload(row): return { "name": row.PointID, - "thing_type": "Diversion of surface water, etc.", + "thing_type": "diversion of surface water, etc.", "release_status": _release_status(row), } @@ -168,7 +168,7 @@ def transfer_lake_pond_reservoir(session, limit=None): def make_payload(row): return { "name": row.PointID, - "thing_type": "Lake, pond or reservoir", + "thing_type": "lake, pond or reservoir", "release_status": _release_status(row), } @@ -179,7 +179,7 @@ def transfer_soil_gas_sample_locations(session, limit=None): def make_payload(row): return { "name": row.PointID, - "thing_type": "Soil gas sample location", + "thing_type": "soil gas sample location", "release_status": _release_status(row), } @@ -190,7 +190,7 @@ def transfer_other_site_types(session, limit=None): def make_payload(row): return { "name": row.PointID, - "thing_type": "Other", + "thing_type": "other", "release_status": _release_status(row), } @@ -201,7 +201,7 @@ def transfer_outfall_wastewater_return_flow(session, limit=None): def make_payload(row): return { "name": row.PointID, - "thing_type": "Outfall of wastewater or return flow", + "thing_type": "outfall of wastewater or return flow", "release_status": _release_status(row), } diff --git a/transfers/transfer.py b/transfers/transfer.py index 45dda85b..357d9342 100644 --- a/transfers/transfer.py +++ b/transfers/transfer.py @@ -21,6 +21,19 @@ from dotenv import load_dotenv +from transfers.thing_transfer import ( + transfer_rock_sample_locations, + transfer_springs, + transfer_perennial_streams, + transfer_ephemeral_streams, + transfer_met_stations, + transfer_diversion_of_surface_water, + transfer_lake_pond_reservoir, + transfer_soil_gas_sample_locations, + transfer_other_site_types, + transfer_outfall_wastewater_return_flow, +) + # Load .env file FIRST, before any database imports, to ensure correct port/database settings load_dotenv(override=True) @@ -374,7 +387,19 @@ def transfer_all(metrics: Metrics) -> list[ProfileArtifact]: # These create Things and Locations that chemistry/other transfers depend on. # ========================================================================= non_well_tasks = [] - gs = globals() + transfer_functions = { + "springs": transfer_springs, + "perennial_streams": transfer_perennial_streams, + "ephemeral_streams": transfer_ephemeral_streams, + "met_stations": transfer_met_stations, + "rock_sample_locations": transfer_rock_sample_locations, + "diversion_of_surface_water": transfer_diversion_of_surface_water, + "lake_pond_reservoir": transfer_lake_pond_reservoir, + "soil_gas_sample_locations": transfer_soil_gas_sample_locations, + "other_site_types": transfer_other_site_types, + "outfall_wastewater_return_flow": transfer_outfall_wastewater_return_flow, + } + for attr in ( "springs", "perennial_streams", @@ -390,7 +415,7 @@ def transfer_all(metrics: Metrics) -> list[ProfileArtifact]: thing_type = "".join(part.capitalize() for part in attr.split("_")) attr_name = f"transfer_{attr}" if getattr(transfer_options, attr_name): - transfer_func = gs[attr_name] + transfer_func = transfer_functions[attr] non_well_tasks.append((thing_type, transfer_func)) if non_well_tasks: From 05811884182be6f2f3cdf997f75f57537b8fd3ba Mon Sep 17 00:00:00 2001 From: jakeross Date: Wed, 4 Feb 2026 22:17:41 +1100 Subject: [PATCH 3/6] fix: import necessary modules for handling requests and relationships in chemistry_sampleinfo --- admin/views/chemistry_sampleinfo.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/admin/views/chemistry_sampleinfo.py b/admin/views/chemistry_sampleinfo.py index 9aa6654e..ac31f618 100644 --- a/admin/views/chemistry_sampleinfo.py +++ b/admin/views/chemistry_sampleinfo.py @@ -27,7 +27,8 @@ FK Change (2026-01): - thing_id: Integer FK to Thing.id """ - +from starlette.requests import Request +from starlette_admin import HasOne from admin.views.base import OcotilloModelView From 7b2f43d776e64100e430a67f9df884d0c42105b0 Mon Sep 17 00:00:00 2001 From: jirhiker Date: Wed, 4 Feb 2026 11:18:03 +0000 Subject: [PATCH 4/6] Formatting changes --- admin/views/chemistry_sampleinfo.py | 1 + 1 file changed, 1 insertion(+) diff --git a/admin/views/chemistry_sampleinfo.py b/admin/views/chemistry_sampleinfo.py index ac31f618..b28bd112 100644 --- a/admin/views/chemistry_sampleinfo.py +++ b/admin/views/chemistry_sampleinfo.py @@ -27,6 +27,7 @@ FK Change (2026-01): - thing_id: Integer FK to Thing.id """ + from starlette.requests import Request from starlette_admin import HasOne From 205927ab34cf1580bc6c07bfd4bff2f265f5bf39 Mon Sep 17 00:00:00 2001 From: jakeross Date: Wed, 4 Feb 2026 23:12:23 +1100 Subject: [PATCH 5/6] fix: import necessary modules for handling requests and relationships in chemistry_sampleinfo --- transfers/transfer.py | 47 ++++++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/transfers/transfer.py b/transfers/transfer.py index 357d9342..5bca4378 100644 --- a/transfers/transfer.py +++ b/transfers/transfer.py @@ -388,34 +388,35 @@ def transfer_all(metrics: Metrics) -> list[ProfileArtifact]: # ========================================================================= non_well_tasks = [] transfer_functions = { - "springs": transfer_springs, - "perennial_streams": transfer_perennial_streams, - "ephemeral_streams": transfer_ephemeral_streams, - "met_stations": transfer_met_stations, - "rock_sample_locations": transfer_rock_sample_locations, - "diversion_of_surface_water": transfer_diversion_of_surface_water, - "lake_pond_reservoir": transfer_lake_pond_reservoir, - "soil_gas_sample_locations": transfer_soil_gas_sample_locations, - "other_site_types": transfer_other_site_types, - "outfall_wastewater_return_flow": transfer_outfall_wastewater_return_flow, + "transfer_springs": transfer_springs, + "transfer_perennial_streams": transfer_perennial_streams, + "transfer_ephemeral_streams": transfer_ephemeral_streams, + "transfer_met_stations": transfer_met_stations, + "transfer_rock_sample_locations": transfer_rock_sample_locations, + "transfer_diversion_of_surface_water": transfer_diversion_of_surface_water, + "transfer_lake_pond_reservoir": transfer_lake_pond_reservoir, + "transfer_soil_gas_sample_locations": transfer_soil_gas_sample_locations, + "transfer_other_site_types": transfer_other_site_types, + "transfer_outfall_wastewater_return_flow": ( + transfer_outfall_wastewater_return_flow + ), } - for attr in ( - "springs", - "perennial_streams", - "ephemeral_streams", - "met_stations", - "rock_sample_locations", - "diversion_of_surface_water", - "lake_pond_reservoir", - "soil_gas_sample_locations", - "other_site_types", - "outfall_wastewater_return_flow", + for attr, thing_type in ( + ("springs", "Springs"), + ("perennial_streams", "PerennialStreams"), + ("ephemeral_streams", "EphemeralStreams"), + ("met_stations", "MetStations"), + ("rock_sample_locations", "RockSampleLocations"), + ("diversion_of_surface_water", "DiversionOfSurfaceWater"), + ("lake_pond_reservoir", "LakePondReservoir"), + ("soil_gas_sample_locations", "SoilGasSampleLocations"), + ("other_site_types", "OtherSiteTypes"), + ("outfall_wastewater_return_flow", "OutfallWastewaterReturnFlow"), ): - thing_type = "".join(part.capitalize() for part in attr.split("_")) attr_name = f"transfer_{attr}" if getattr(transfer_options, attr_name): - transfer_func = transfer_functions[attr] + transfer_func = transfer_functions[attr_name] non_well_tasks.append((thing_type, transfer_func)) if non_well_tasks: From 23c4450d0a38b6a5d9e37267a1bb1ff73303c304 Mon Sep 17 00:00:00 2001 From: jakeross Date: Wed, 4 Feb 2026 23:21:57 +1100 Subject: [PATCH 6/6] fix: remove unused parameters from _step_parallel_complete method in well_transfer.py --- transfers/well_transfer.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/transfers/well_transfer.py b/transfers/well_transfer.py index c57491de..77ab09b2 100644 --- a/transfers/well_transfer.py +++ b/transfers/well_transfer.py @@ -150,8 +150,6 @@ def process_batch(batch_idx: int, batch_df: pd.DataFrame) -> dict: # Process single well with all dependent objects self._step_parallel_complete( session, - batch_df, - i, row, local_aquifers, local_formations, @@ -879,8 +877,6 @@ def _add_histories(self, session: Session, row, well: Thing) -> None: def _step_parallel_complete( self, session: Session, - df: pd.DataFrame, - i: int, row, local_aquifers: list, local_formations: dict,