diff --git a/admin/views/chemistry_sampleinfo.py b/admin/views/chemistry_sampleinfo.py index 9aa6654e..b28bd112 100644 --- a/admin/views/chemistry_sampleinfo.py +++ b/admin/views/chemistry_sampleinfo.py @@ -28,6 +28,8 @@ - thing_id: Integer FK to Thing.id """ +from starlette.requests import Request +from starlette_admin import HasOne from admin.views.base import OcotilloModelView diff --git a/core/lexicon.json b/core/lexicon.json index 01539f2d..5b99accb 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -2007,6 +2007,48 @@ "term": "meteorological station", "definition": "a station that measures the weather conditions at a particular location" }, + { + "categories": [ + "thing_type" + ], + "term": "rock sample location", + "definition": "a location where rock samples are collected" + }, + { + "categories": [ + "thing_type" + ], + "term": "diversion of surface water, etc.", + "definition": "a diversion structure for surface water such as a ditch, canal, or intake" + }, + { + "categories": [ + "thing_type" + ], + "term": "lake, pond or reservoir", + "definition": "a natural or artificial standing body of water" + }, + { + "categories": [ + "thing_type" + ], + "term": "soil gas sample location", + "definition": "a location where soil gas samples are collected" + }, + { + "categories": [ + "thing_type" + ], + "term": "other", + "definition": "a thing type that does not fit other categories" + }, + { + "categories": [ + "thing_type" + ], + "term": "outfall of wastewater or return flow", + "definition": "a discharge point for wastewater or return flows" + }, { "categories": [ "groundwater_level_reason" @@ -8149,4 +8191,4 @@ "definition": "Data were not field checked but are considered reliable" } ] -} \ No newline at end of file +} diff --git a/transfers/thing_transfer.py b/transfers/thing_transfer.py index 754634b7..5d4456db 100644 --- a/transfers/thing_transfer.py +++ b/transfers/thing_transfer.py @@ -14,6 +14,7 @@ # limitations under the License. # =============================================================================== import time + from pandas import isna from pydantic import ValidationError from sqlalchemy.orm import Session @@ -93,48 +94,118 @@ def transfer_thing(session: Session, site_type: str, make_payload, limit=None) - logger.info("Completed transfer: Things (%s)", site_type) +def _release_status(row) -> str: + return "public" if row.PublicRelease else "private" + + def transfer_springs(session, limit=None): def make_payload(row): return { "name": row.PointID, "thing_type": "spring", - "release_status": "public" if row.PublicRelease else "private", + "release_status": _release_status(row), } transfer_thing(session, "SP", make_payload, limit) -def transfer_perennial_stream(session, limit=None): +def transfer_perennial_streams(session, limit=None): def make_payload(row): return { "name": row.PointID, "thing_type": "perennial stream", - "release_status": "public" if row.PublicRelease else "private", + "release_status": _release_status(row), } transfer_thing(session, "PS", make_payload, limit) -def transfer_ephemeral_stream(session, limit=None): +def transfer_ephemeral_streams(session, limit=None): def make_payload(row): return { "name": row.PointID, "thing_type": "ephemeral stream", - "release_status": "public" if row.PublicRelease else "private", + "release_status": _release_status(row), } transfer_thing(session, "ES", make_payload, limit) -def transfer_met(session, limit=None): +def transfer_met_stations(session, limit=None): def make_payload(row): return { "name": row.PointID, "thing_type": "meteorological station", - "release_status": "public" if row.PublicRelease else "private", + "release_status": _release_status(row), } transfer_thing(session, "M", make_payload, limit) +def transfer_rock_sample_locations(session, limit=None): + def make_payload(row): + return { + "name": row.PointID, + "thing_type": "rock sample location", + "release_status": _release_status(row), + } + + transfer_thing(session, "R", make_payload, limit) + + +def transfer_diversion_of_surface_water(session, limit=None): + def make_payload(row): + return { + "name": row.PointID, + "thing_type": "diversion of surface water, etc.", + "release_status": _release_status(row), + } + + transfer_thing(session, "D", make_payload, limit) + + +def transfer_lake_pond_reservoir(session, limit=None): + def make_payload(row): + return { + "name": row.PointID, + "thing_type": "lake, pond or reservoir", + "release_status": _release_status(row), + } + + transfer_thing(session, "L", make_payload, limit) + + +def transfer_soil_gas_sample_locations(session, limit=None): + def make_payload(row): + return { + "name": row.PointID, + "thing_type": "soil gas sample location", + "release_status": _release_status(row), + } + + transfer_thing(session, "S", make_payload, limit) + + +def transfer_other_site_types(session, limit=None): + def make_payload(row): + return { + "name": row.PointID, + "thing_type": "other", + "release_status": _release_status(row), + } + + transfer_thing(session, "OT", make_payload, limit) + + +def transfer_outfall_wastewater_return_flow(session, limit=None): + def make_payload(row): + return { + "name": row.PointID, + "thing_type": "outfall of wastewater or return flow", + "release_status": _release_status(row), + } + + transfer_thing(session, "O", make_payload, limit) + + # ============= EOF ============================================= diff --git a/transfers/transfer.py b/transfers/transfer.py index 73c82a21..5bca4378 100644 --- a/transfers/transfer.py +++ b/transfers/transfer.py @@ -21,6 +21,19 @@ from dotenv import load_dotenv +from transfers.thing_transfer import ( + transfer_rock_sample_locations, + transfer_springs, + transfer_perennial_streams, + transfer_ephemeral_streams, + transfer_met_stations, + transfer_diversion_of_surface_water, + transfer_lake_pond_reservoir, + transfer_soil_gas_sample_locations, + transfer_other_site_types, + transfer_outfall_wastewater_return_flow, +) + # Load .env file FIRST, before any database imports, to ensure correct port/database settings load_dotenv(override=True) @@ -61,12 +74,6 @@ WellScreenTransferer, ) from transfers.well_transfer_util import cleanup_locations -from transfers.thing_transfer import ( - transfer_springs, - transfer_perennial_stream, - transfer_ephemeral_stream, - transfer_met, -) from transfers.minor_trace_chemistry_transfer import MinorTraceChemistryTransferer from transfers.asset_transfer import AssetTransferer @@ -125,6 +132,12 @@ class TransferOptions: transfer_perennial_streams: bool transfer_ephemeral_streams: bool transfer_met_stations: bool + transfer_rock_sample_locations: bool + transfer_diversion_of_surface_water: bool + transfer_lake_pond_reservoir: bool + transfer_soil_gas_sample_locations: bool + transfer_other_site_types: bool + transfer_outfall_wastewater_return_flow: bool def load_transfer_options() -> TransferOptions: @@ -168,6 +181,20 @@ def load_transfer_options() -> TransferOptions: transfer_perennial_streams=get_bool_env("TRANSFER_PERENNIAL_STREAMS", True), transfer_ephemeral_streams=get_bool_env("TRANSFER_EPHEMERAL_STREAMS", True), transfer_met_stations=get_bool_env("TRANSFER_MET_STATIONS", True), + transfer_rock_sample_locations=get_bool_env( + "TRANSFER_ROCK_SAMPLE_LOCATIONS", True + ), + transfer_diversion_of_surface_water=get_bool_env( + "TRANSFER_DIVERSION_OF_SURFACE_WATER", True + ), + transfer_lake_pond_reservoir=get_bool_env("TRANSFER_LAKE_POND_RESERVOIR", True), + transfer_soil_gas_sample_locations=get_bool_env( + "TRANSFER_SOIL_GAS_SAMPLE_LOCATIONS", True + ), + transfer_other_site_types=get_bool_env("TRANSFER_OTHER_SITE_TYPES", True), + transfer_outfall_wastewater_return_flow=get_bool_env( + "TRANSFER_OUTFALL_WASTEWATER_RETURN_FLOW", True + ), ) @@ -360,14 +387,37 @@ def transfer_all(metrics: Metrics) -> list[ProfileArtifact]: # These create Things and Locations that chemistry/other transfers depend on. # ========================================================================= non_well_tasks = [] - if transfer_options.transfer_springs: - non_well_tasks.append(("Springs", transfer_springs)) - if transfer_options.transfer_perennial_streams: - non_well_tasks.append(("PerennialStreams", transfer_perennial_stream)) - if transfer_options.transfer_ephemeral_streams: - non_well_tasks.append(("EphemeralStreams", transfer_ephemeral_stream)) - if transfer_options.transfer_met_stations: - non_well_tasks.append(("MetStations", transfer_met)) + transfer_functions = { + "transfer_springs": transfer_springs, + "transfer_perennial_streams": transfer_perennial_streams, + "transfer_ephemeral_streams": transfer_ephemeral_streams, + "transfer_met_stations": transfer_met_stations, + "transfer_rock_sample_locations": transfer_rock_sample_locations, + "transfer_diversion_of_surface_water": transfer_diversion_of_surface_water, + "transfer_lake_pond_reservoir": transfer_lake_pond_reservoir, + "transfer_soil_gas_sample_locations": transfer_soil_gas_sample_locations, + "transfer_other_site_types": transfer_other_site_types, + "transfer_outfall_wastewater_return_flow": ( + transfer_outfall_wastewater_return_flow + ), + } + + for attr, thing_type in ( + ("springs", "Springs"), + ("perennial_streams", "PerennialStreams"), + ("ephemeral_streams", "EphemeralStreams"), + ("met_stations", "MetStations"), + ("rock_sample_locations", "RockSampleLocations"), + ("diversion_of_surface_water", "DiversionOfSurfaceWater"), + ("lake_pond_reservoir", "LakePondReservoir"), + ("soil_gas_sample_locations", "SoilGasSampleLocations"), + ("other_site_types", "OtherSiteTypes"), + ("outfall_wastewater_return_flow", "OutfallWastewaterReturnFlow"), + ): + attr_name = f"transfer_{attr}" + if getattr(transfer_options, attr_name): + transfer_func = transfer_functions[attr_name] + non_well_tasks.append((thing_type, transfer_func)) if non_well_tasks: message("PHASE 1.5: NON-WELL LOCATION TYPES (PARALLEL)") diff --git a/transfers/well_transfer.py b/transfers/well_transfer.py index c57491de..77ab09b2 100644 --- a/transfers/well_transfer.py +++ b/transfers/well_transfer.py @@ -150,8 +150,6 @@ def process_batch(batch_idx: int, batch_df: pd.DataFrame) -> dict: # Process single well with all dependent objects self._step_parallel_complete( session, - batch_df, - i, row, local_aquifers, local_formations, @@ -879,8 +877,6 @@ def _add_histories(self, session: Session, row, well: Thing) -> None: def _step_parallel_complete( self, session: Session, - df: pd.DataFrame, - i: int, row, local_aquifers: list, local_formations: dict,