From 52f60c742f678e9aaec15dff5e942063ec98f722 Mon Sep 17 00:00:00 2001 From: Kelsey Smuczynski Date: Wed, 28 Jan 2026 15:11:51 -0700 Subject: [PATCH 01/13] refactor: Use UUID types for GlobalID/WellID in NMA_WaterLevelsContinuous_Pressure_Daily Context: updates db/nma_legacy.py to map GlobalID and WellID as UUID(as_uuid=True) and documents the WellID FK to Thing for the continuous pressure daily model. --- db/nma_legacy.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/db/nma_legacy.py b/db/nma_legacy.py index 5ea1337e1..5129a1b59 100644 --- a/db/nma_legacy.py +++ b/db/nma_legacy.py @@ -56,11 +56,14 @@ class NMA_WaterLevelsContinuous_Pressure_Daily(Base): __tablename__ = "NMA_WaterLevelsContinuous_Pressure_Daily" - global_id: Mapped[str] = mapped_column("GlobalID", String(40), primary_key=True) + global_id: Mapped[uuid.UUID] = mapped_column( + "GlobalID", UUID(as_uuid=True), primary_key=True + ) object_id: Mapped[Optional[int]] = mapped_column( "OBJECTID", Integer, autoincrement=True ) - well_id: Mapped[Optional[str]] = mapped_column("WellID", String(40)) + # FK to Thing table (well_id --> Thing.nma_pk_welldata) + well_id: Mapped[Optional[uuid.UUID]] = mapped_column("WellID", UUID(as_uuid=True)) point_id: Mapped[Optional[str]] = mapped_column("PointID", String(50)) date_measured: Mapped[datetime] = mapped_column( "DateMeasured", DateTime, nullable=False From e46ac1580b05a143761ec600af5f2cdcf4e82c29 Mon Sep 17 00:00:00 2001 From: Kelsey Smuczynski Date: Wed, 28 Jan 2026 15:36:50 -0700 Subject: [PATCH 02/13] feat: add admin view for legacy continuous pressure daily water levels - Define read-only Starlette Admin view with full legacy-order fields - Register view in admin/views/__init__.py and admin/config.py --- admin/config.py | 9 ++ admin/views/__init__.py | 4 + .../waterlevelscontinuous_pressure_daily.py | 147 ++++++++++++++++++ 3 files changed, 160 insertions(+) create mode 100644 admin/views/waterlevelscontinuous_pressure_daily.py diff --git a/admin/config.py b/admin/config.py index 5aec1a5f4..1c3bb14f0 100644 --- a/admin/config.py +++ b/admin/config.py @@ -53,6 +53,7 @@ SurfaceWaterPhotosAdmin, ThingAdmin, TransducerObservationAdmin, + WaterLevelsContinuousPressureDailyAdmin, WeatherPhotosAdmin, WeatherDataAdmin, FieldParametersAdmin, @@ -80,6 +81,7 @@ NMA_Soil_Rock_Results, NMA_Stratigraphy, NMA_SurfaceWaterData, + NMA_WaterLevelsContinuous_Pressure_Daily, NMA_WeatherPhotos, NMA_SurfaceWaterPhotos, NMA_WeatherData, @@ -192,6 +194,13 @@ def create_admin(app): # Transducer observations admin.add_view(TransducerObservationAdmin(TransducerObservation)) + # Water Levels - Continuous (legacy) + admin.add_view( + WaterLevelsContinuousPressureDailyAdmin( + NMA_WaterLevelsContinuous_Pressure_Daily + ) + ) + # Weather admin.add_view(WeatherPhotosAdmin(NMA_WeatherPhotos)) diff --git a/admin/views/__init__.py b/admin/views/__init__.py index 33920b856..285d5ef5f 100644 --- a/admin/views/__init__.py +++ b/admin/views/__init__.py @@ -52,6 +52,9 @@ from admin.views.surface_water_photos import SurfaceWaterPhotosAdmin from admin.views.thing import ThingAdmin from admin.views.transducer_observation import TransducerObservationAdmin +from admin.views.waterlevelscontinuous_pressure_daily import ( + WaterLevelsContinuousPressureDailyAdmin, +) from admin.views.weather_photos import WeatherPhotosAdmin from admin.views.weather_data import WeatherDataAdmin @@ -88,6 +91,7 @@ "SurfaceWaterPhotosAdmin", "ThingAdmin", "TransducerObservationAdmin", + "WaterLevelsContinuousPressureDailyAdmin", "WeatherPhotosAdmin", "WeatherDataAdmin", ] diff --git a/admin/views/waterlevelscontinuous_pressure_daily.py b/admin/views/waterlevelscontinuous_pressure_daily.py new file mode 100644 index 000000000..094700f1c --- /dev/null +++ b/admin/views/waterlevelscontinuous_pressure_daily.py @@ -0,0 +1,147 @@ +# =============================================================================== +# Copyright 2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +WaterLevelsContinuousPressureDailyAdmin view for legacy NMA_WaterLevelsContinuous_Pressure_Daily. +""" +from starlette.requests import Request + +from admin.views.base import OcotilloModelView + + +class WaterLevelsContinuousPressureDailyAdmin(OcotilloModelView): + """ + Admin view for NMA_WaterLevelsContinuous_Pressure_Daily model. + """ + + # ========== Basic Configuration ========== + name = "NMA Water Levels Continuous Pressure Daily" + label = "NMA Water Levels Continuous Pressure Daily" + icon = "fa fa-tachometer-alt" + + def can_create(self, request: Request) -> bool: + return False + + def can_edit(self, request: Request) -> bool: + return False + + def can_delete(self, request: Request) -> bool: + return False + + # ========== List View ========== + list_fields = [ + "global_id", + "object_id", + "well_id", + "point_id", + "date_measured", + "temperature_water", + "water_head", + "water_head_adjusted", + "depth_to_water_bgs", + "measurement_method", + "data_source", + "measuring_agency", + "qced", + "notes", + "created", + "updated", + "processed_by", + "checked_by", + "cond_dl_ms_cm", + ] + + sortable_fields = [ + "global_id", + "object_id", + "well_id", + "point_id", + "date_measured", + "water_head", + "depth_to_water_bgs", + "measurement_method", + "data_source", + "measuring_agency", + "qced", + "created", + "updated", + "processed_by", + "checked_by", + "cond_dl_ms_cm", + ] + + fields_default_sort = [("date_measured", True)] + + searchable_fields = [ + "global_id", + "well_id", + "point_id", + "date_measured", + "measurement_method", + "data_source", + "measuring_agency", + "notes", + ] + + page_size = 50 + page_size_options = [25, 50, 100, 200] + + # ========== Detail View ========== + fields = [ + "global_id", + "object_id", + "well_id", + "point_id", + "date_measured", + "temperature_water", + "water_head", + "water_head_adjusted", + "depth_to_water_bgs", + "measurement_method", + "data_source", + "measuring_agency", + "qced", + "notes", + "created", + "updated", + "processed_by", + "checked_by", + "cond_dl_ms_cm", + ] + + field_labels = { + "global_id": "GlobalID", + "object_id": "OBJECTID", + "well_id": "WellID", + "point_id": "PointID", + "date_measured": "Date Measured", + "temperature_water": "Temperature Water", + "water_head": "Water Head", + "water_head_adjusted": "Water Head Adjusted", + "depth_to_water_bgs": "Depth To Water (BGS)", + "measurement_method": "Measurement Method", + "data_source": "Data Source", + "measuring_agency": "Measuring Agency", + "qced": "QCed", + "notes": "Notes", + "created": "Created", + "updated": "Updated", + "processed_by": "Processed By", + "checked_by": "Checked By", + "cond_dl_ms_cm": "CONDDL (mS/cm)", + } + + +# ============= EOF ============================================= From 3cd8c56dab3475aa4c8865a45e204ede031fad41 Mon Sep 17 00:00:00 2001 From: Kelsey Smuczynski Date: Wed, 28 Jan 2026 16:09:14 -0700 Subject: [PATCH 03/13] refactor (test): Update test_waterlevelscontinuous_pressure_daily_legacy.py for UUID IDs - Updated the test helper to return UUIDs for GlobalID and WellID in legacy model tests - Changed well_id to use a UUID so it matches the model mapping. --- tests/test_waterlevelscontinuous_pressure_daily_legacy.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_waterlevelscontinuous_pressure_daily_legacy.py b/tests/test_waterlevelscontinuous_pressure_daily_legacy.py index 7328e4059..d2622c72f 100644 --- a/tests/test_waterlevelscontinuous_pressure_daily_legacy.py +++ b/tests/test_waterlevelscontinuous_pressure_daily_legacy.py @@ -21,14 +21,14 @@ """ from datetime import datetime -from uuid import uuid4 +from uuid import UUID, uuid4 from db.engine import session_ctx from db.nma_legacy import NMA_WaterLevelsContinuous_Pressure_Daily -def _next_global_id() -> str: - return str(uuid4()) +def _next_global_id() -> UUID: + return uuid4() def _next_object_id() -> int: @@ -44,7 +44,7 @@ def test_create_pressure_daily_all_fields(): record = NMA_WaterLevelsContinuous_Pressure_Daily( global_id=_next_global_id(), object_id=_next_object_id(), - well_id="WELL-1", + well_id=uuid4(), point_id="PD-1001", date_measured=now, temperature_water=12.3, From 9b8a398f7077b99b621435185635718af6990f82 Mon Sep 17 00:00:00 2001 From: ksmuczynski Date: Wed, 28 Jan 2026 23:18:24 +0000 Subject: [PATCH 04/13] Formatting changes --- admin/views/waterlevelscontinuous_pressure_daily.py | 1 + 1 file changed, 1 insertion(+) diff --git a/admin/views/waterlevelscontinuous_pressure_daily.py b/admin/views/waterlevelscontinuous_pressure_daily.py index 094700f1c..ac2afb020 100644 --- a/admin/views/waterlevelscontinuous_pressure_daily.py +++ b/admin/views/waterlevelscontinuous_pressure_daily.py @@ -16,6 +16,7 @@ """ WaterLevelsContinuousPressureDailyAdmin view for legacy NMA_WaterLevelsContinuous_Pressure_Daily. """ + from starlette.requests import Request from admin.views.base import OcotilloModelView From 91d3aa5c38dbb6b55af655e75e92e1439092e94b Mon Sep 17 00:00:00 2001 From: Kelsey Smuczynski Date: Thu, 29 Jan 2026 10:02:24 -0700 Subject: [PATCH 05/13] fix: add ForeignKey constraint to well_id in nma_legacy model --- db/nma_legacy.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/db/nma_legacy.py b/db/nma_legacy.py index 5129a1b59..d475e362d 100644 --- a/db/nma_legacy.py +++ b/db/nma_legacy.py @@ -63,7 +63,12 @@ class NMA_WaterLevelsContinuous_Pressure_Daily(Base): "OBJECTID", Integer, autoincrement=True ) # FK to Thing table (well_id --> Thing.nma_pk_welldata) - well_id: Mapped[Optional[uuid.UUID]] = mapped_column("WellID", UUID(as_uuid=True)) + well_id: Mapped[Optional[uuid.UUID]] = mapped_column( + "WellID", + UUID(as_uuid=True), + ForeignKey("Thing.nma_pk_welldata"), + nullable=False, + ) point_id: Mapped[Optional[str]] = mapped_column("PointID", String(50)) date_measured: Mapped[datetime] = mapped_column( "DateMeasured", DateTime, nullable=False From f09a67dc6d21744d82674604b7148894371d6572 Mon Sep 17 00:00:00 2001 From: Kelsey Smuczynski Date: Thu, 29 Jan 2026 10:07:29 -0700 Subject: [PATCH 06/13] fix: update well_id to be non-optional and enforce ForeignKey constraint in nma_legacy model --- db/nma_legacy.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/db/nma_legacy.py b/db/nma_legacy.py index d475e362d..a97efcde1 100644 --- a/db/nma_legacy.py +++ b/db/nma_legacy.py @@ -63,7 +63,7 @@ class NMA_WaterLevelsContinuous_Pressure_Daily(Base): "OBJECTID", Integer, autoincrement=True ) # FK to Thing table (well_id --> Thing.nma_pk_welldata) - well_id: Mapped[Optional[uuid.UUID]] = mapped_column( + well_id: Mapped[uuid.UUID] = mapped_column( "WellID", UUID(as_uuid=True), ForeignKey("Thing.nma_pk_welldata"), @@ -179,7 +179,12 @@ class NMA_HydraulicsData(Base): global_id: Mapped[uuid.UUID] = mapped_column( "GlobalID", UUID(as_uuid=True), primary_key=True ) - well_id: Mapped[Optional[uuid.UUID]] = mapped_column("WellID", UUID(as_uuid=True)) + well_id: Mapped[uuid.UUID] = mapped_column( + "WellID", + UUID(as_uuid=True), + ForeignKey("thing.nma_pk_welldata"), + nullable=False, + ) point_id: Mapped[Optional[str]] = mapped_column("PointID", String(50)) data_source: Mapped[Optional[str]] = mapped_column("Data Source", String(255)) thing_id: Mapped[int] = mapped_column( From 71899fe13b6afad5617de443d227e0643f0a6128 Mon Sep 17 00:00:00 2001 From: Kelsey Smuczynski Date: Thu, 29 Jan 2026 10:14:11 -0700 Subject: [PATCH 07/13] fix: remove well_id field from NMA_HydraulicsData model --- db/nma_legacy.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/db/nma_legacy.py b/db/nma_legacy.py index a97efcde1..64e58c81e 100644 --- a/db/nma_legacy.py +++ b/db/nma_legacy.py @@ -179,12 +179,6 @@ class NMA_HydraulicsData(Base): global_id: Mapped[uuid.UUID] = mapped_column( "GlobalID", UUID(as_uuid=True), primary_key=True ) - well_id: Mapped[uuid.UUID] = mapped_column( - "WellID", - UUID(as_uuid=True), - ForeignKey("thing.nma_pk_welldata"), - nullable=False, - ) point_id: Mapped[Optional[str]] = mapped_column("PointID", String(50)) data_source: Mapped[Optional[str]] = mapped_column("Data Source", String(255)) thing_id: Mapped[int] = mapped_column( From 5c7b730009abb03e6e7fed7a498f754c4a18652e Mon Sep 17 00:00:00 2001 From: Kelsey Smuczynski Date: Thu, 29 Jan 2026 10:22:56 -0700 Subject: [PATCH 08/13] fix: update well_id field to be optional and add thing_id ForeignKey constraint in nma_legacy model --- db/nma_legacy.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/db/nma_legacy.py b/db/nma_legacy.py index 64e58c81e..45f4ce2d4 100644 --- a/db/nma_legacy.py +++ b/db/nma_legacy.py @@ -62,12 +62,10 @@ class NMA_WaterLevelsContinuous_Pressure_Daily(Base): object_id: Mapped[Optional[int]] = mapped_column( "OBJECTID", Integer, autoincrement=True ) - # FK to Thing table (well_id --> Thing.nma_pk_welldata) - well_id: Mapped[uuid.UUID] = mapped_column( - "WellID", - UUID(as_uuid=True), - ForeignKey("Thing.nma_pk_welldata"), - nullable=False, + well_id: Mapped[Optional[uuid.UUID]] = mapped_column("WellID", UUID(as_uuid=True)) + # FK to Thing table - required for all WaterLevelsContinuous_Pressure_Daily records + thing_id: Mapped[int] = mapped_column( + Integer, ForeignKey("thing.id", ondelete="CASCADE"), nullable=False ) point_id: Mapped[Optional[str]] = mapped_column("PointID", String(50)) date_measured: Mapped[datetime] = mapped_column( From 0ff8926945edfe7dc71ea381b07d6d2e2550fa9d Mon Sep 17 00:00:00 2001 From: Kelsey Smuczynski Date: Thu, 29 Jan 2026 10:32:53 -0700 Subject: [PATCH 09/13] fix: make well_id field optional in nma_legacy model --- db/nma_legacy.py | 1 + 1 file changed, 1 insertion(+) diff --git a/db/nma_legacy.py b/db/nma_legacy.py index 45f4ce2d4..f9b55cfe3 100644 --- a/db/nma_legacy.py +++ b/db/nma_legacy.py @@ -177,6 +177,7 @@ class NMA_HydraulicsData(Base): global_id: Mapped[uuid.UUID] = mapped_column( "GlobalID", UUID(as_uuid=True), primary_key=True ) + well_id: Mapped[Optional[uuid.UUID]] = mapped_column("WellID", UUID(as_uuid=True)) point_id: Mapped[Optional[str]] = mapped_column("PointID", String(50)) data_source: Mapped[Optional[str]] = mapped_column("Data Source", String(255)) thing_id: Mapped[int] = mapped_column( From 8d16b93c20929719eca47ee870e8e30426a1bf1d Mon Sep 17 00:00:00 2001 From: Kelsey Smuczynski Date: Thu, 29 Jan 2026 12:21:02 -0700 Subject: [PATCH 10/13] test: add tests for thing_id foreign key integrity in water levels continuous pressure daily records --- ...rlevelscontinuous_pressure_daily_legacy.py | 77 ++++++++++++++++--- 1 file changed, 67 insertions(+), 10 deletions(-) diff --git a/tests/test_waterlevelscontinuous_pressure_daily_legacy.py b/tests/test_waterlevelscontinuous_pressure_daily_legacy.py index d2622c72f..d98b03ab5 100644 --- a/tests/test_waterlevelscontinuous_pressure_daily_legacy.py +++ b/tests/test_waterlevelscontinuous_pressure_daily_legacy.py @@ -23,6 +23,9 @@ from datetime import datetime from uuid import UUID, uuid4 +import pytest +from sqlalchemy.exc import IntegrityError, ProgrammingError + from db.engine import session_ctx from db.nma_legacy import NMA_WaterLevelsContinuous_Pressure_Daily @@ -37,7 +40,7 @@ def _next_object_id() -> int: # ===================== CREATE tests ========================== -def test_create_pressure_daily_all_fields(): +def test_create_pressure_daily_all_fields(water_well_thing): """Test creating a pressure daily record with required fields.""" with session_ctx() as session: now = datetime(2024, 1, 1, 12, 0, 0) @@ -45,7 +48,7 @@ def test_create_pressure_daily_all_fields(): global_id=_next_global_id(), object_id=_next_object_id(), well_id=uuid4(), - point_id="PD-1001", + point_id=water_well_thing.name, date_measured=now, temperature_water=12.3, water_head=4.5, @@ -61,6 +64,7 @@ def test_create_pressure_daily_all_fields(): processed_by="AB", checked_by="CD", cond_dl_ms_cm=0.2, + thing_id=water_well_thing.id, ) session.add(record) session.commit() @@ -74,16 +78,17 @@ def test_create_pressure_daily_all_fields(): session.commit() -def test_create_pressure_daily_minimal(): +def test_create_pressure_daily_minimal(water_well_thing): """Test creating a pressure daily record with minimal fields.""" with session_ctx() as session: now = datetime(2024, 1, 2, 12, 0, 0) record = NMA_WaterLevelsContinuous_Pressure_Daily( global_id=_next_global_id(), - point_id="PD-1002", + point_id=water_well_thing.name, date_measured=now, created=now, updated=now, + thing_id=water_well_thing.id, ) session.add(record) session.commit() @@ -97,16 +102,17 @@ def test_create_pressure_daily_minimal(): # ===================== READ tests ========================== -def test_read_pressure_daily_by_global_id(): +def test_read_pressure_daily_by_global_id(water_well_thing): """Test reading a pressure daily record by GlobalID.""" with session_ctx() as session: now = datetime(2024, 1, 3, 12, 0, 0) record = NMA_WaterLevelsContinuous_Pressure_Daily( global_id=_next_global_id(), - point_id="PD-1003", + point_id=water_well_thing.name, date_measured=now, created=now, updated=now, + thing_id=water_well_thing.id, ) session.add(record) session.commit() @@ -123,16 +129,17 @@ def test_read_pressure_daily_by_global_id(): # ===================== UPDATE tests ========================== -def test_update_pressure_daily(): +def test_update_pressure_daily(water_well_thing): """Test updating a pressure daily record.""" with session_ctx() as session: now = datetime(2024, 1, 4, 12, 0, 0) record = NMA_WaterLevelsContinuous_Pressure_Daily( global_id=_next_global_id(), - point_id="PD-1004", + point_id=water_well_thing.name, date_measured=now, created=now, updated=now, + thing_id=water_well_thing.id, ) session.add(record) session.commit() @@ -150,16 +157,17 @@ def test_update_pressure_daily(): # ===================== DELETE tests ========================== -def test_delete_pressure_daily(): +def test_delete_pressure_daily(water_well_thing): """Test deleting a pressure daily record.""" with session_ctx() as session: now = datetime(2024, 1, 5, 12, 0, 0) record = NMA_WaterLevelsContinuous_Pressure_Daily( global_id=_next_global_id(), - point_id="PD-1005", + point_id=water_well_thing.name, date_measured=now, created=now, updated=now, + thing_id=water_well_thing.id, ) session.add(record) session.commit() @@ -180,6 +188,7 @@ def test_pressure_daily_has_all_migrated_columns(): "global_id", "object_id", "well_id", + "thing_id", "point_id", "date_measured", "temperature_water", @@ -212,4 +221,52 @@ def test_pressure_daily_table_name(): ) +# ===================== Relational Integrity Tests ====================== + + +def test_pressure_daily_thing_id_required(): + """ + VERIFIES: 'thing_id IS NOT NULL' and Foreign Key presence. + Ensures the DB rejects records without a Thing linkage. + """ + with session_ctx() as session: + now = datetime(2024, 1, 6, 12, 0, 0) + record = NMA_WaterLevelsContinuous_Pressure_Daily( + global_id=_next_global_id(), + point_id="PD-1006", + date_measured=now, + created=now, + updated=now, + ) + session.add(record) + + with pytest.raises((IntegrityError, ProgrammingError)): + session.flush() + session.rollback() + + +def test_pressure_daily_invalid_thing_id_rejected(water_well_thing): + """ + VERIFIES: foreign key integrity on thing_id. + Ensures the DB rejects updates to a non-existent Thing. + """ + with session_ctx() as session: + now = datetime(2024, 1, 7, 12, 0, 0) + record = NMA_WaterLevelsContinuous_Pressure_Daily( + global_id=_next_global_id(), + point_id=water_well_thing.name, + date_measured=now, + created=now, + updated=now, + thing_id=water_well_thing.id, + ) + session.add(record) + session.commit() + + with pytest.raises((IntegrityError, ProgrammingError)): + record.thing_id = 999999 + session.flush() + session.rollback() + + # ============= EOF ============================================= From d6fb0fae5cab513e7d56084e14c3349ce55dac45 Mon Sep 17 00:00:00 2001 From: Kelsey Smuczynski Date: Thu, 29 Jan 2026 12:45:50 -0700 Subject: [PATCH 11/13] feat: Add thing_id FK to NMA pressure daily table - Added migrations for thing_id FK and UUID column alignment - Updated pressure daily legacy tests for Thing linkage and FK enforcement --- ...ma_waterlevelscontinuous_pressure_daily.py | 92 +++++++++++++++++++ ...7b6a5_align_pressure_daily_uuid_columns.py | 85 +++++++++++++++++ ...rlevelscontinuous_pressure_daily_legacy.py | 6 +- 3 files changed, 180 insertions(+), 3 deletions(-) create mode 100644 alembic/versions/e8a7c6b5d4f3_add_thing_id_to_nma_waterlevelscontinuous_pressure_daily.py create mode 100644 alembic/versions/f0c9d8e7b6a5_align_pressure_daily_uuid_columns.py diff --git a/alembic/versions/e8a7c6b5d4f3_add_thing_id_to_nma_waterlevelscontinuous_pressure_daily.py b/alembic/versions/e8a7c6b5d4f3_add_thing_id_to_nma_waterlevelscontinuous_pressure_daily.py new file mode 100644 index 000000000..f825e81ae --- /dev/null +++ b/alembic/versions/e8a7c6b5d4f3_add_thing_id_to_nma_waterlevelscontinuous_pressure_daily.py @@ -0,0 +1,92 @@ +"""Add thing_id FK to NMA_WaterLevelsContinuous_Pressure_Daily. + +Revision ID: e8a7c6b5d4f3 +Revises: b12e3919077e +Create Date: 2026-01-29 12:45:00.000000 +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op +from sqlalchemy import inspect + +# revision identifiers, used by Alembic. +revision: str = "e8a7c6b5d4f3" +down_revision: Union[str, Sequence[str], None] = "b12e3919077e" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Add thing_id and FK to legacy pressure daily table.""" + bind = op.get_bind() + inspector = inspect(bind) + if not inspector.has_table("NMA_WaterLevelsContinuous_Pressure_Daily"): + return + + columns = { + col["name"] + for col in inspector.get_columns("NMA_WaterLevelsContinuous_Pressure_Daily") + } + if "thing_id" not in columns: + op.add_column( + "NMA_WaterLevelsContinuous_Pressure_Daily", + sa.Column("thing_id", sa.Integer(), nullable=True), + ) + + existing_fks = { + fk["name"] + for fk in inspector.get_foreign_keys("NMA_WaterLevelsContinuous_Pressure_Daily") + if fk.get("name") + } + if "fk_pressure_daily_thing" not in existing_fks: + op.create_foreign_key( + "fk_pressure_daily_thing", + "NMA_WaterLevelsContinuous_Pressure_Daily", + "thing", + ["thing_id"], + ["id"], + ondelete="CASCADE", + ) + + null_count = bind.execute( + sa.text( + 'SELECT COUNT(*) FROM "NMA_WaterLevelsContinuous_Pressure_Daily" ' + 'WHERE "thing_id" IS NULL' + ) + ).scalar() + if null_count == 0: + op.alter_column( + "NMA_WaterLevelsContinuous_Pressure_Daily", + "thing_id", + existing_type=sa.Integer(), + nullable=False, + ) + + +def downgrade() -> None: + """Remove thing_id FK from legacy pressure daily table.""" + bind = op.get_bind() + inspector = inspect(bind) + if not inspector.has_table("NMA_WaterLevelsContinuous_Pressure_Daily"): + return + + existing_fks = { + fk["name"] + for fk in inspector.get_foreign_keys("NMA_WaterLevelsContinuous_Pressure_Daily") + if fk.get("name") + } + if "fk_pressure_daily_thing" in existing_fks: + op.drop_constraint( + "fk_pressure_daily_thing", + "NMA_WaterLevelsContinuous_Pressure_Daily", + type_="foreignkey", + ) + + columns = { + col["name"] + for col in inspector.get_columns("NMA_WaterLevelsContinuous_Pressure_Daily") + } + if "thing_id" in columns: + op.drop_column("NMA_WaterLevelsContinuous_Pressure_Daily", "thing_id") diff --git a/alembic/versions/f0c9d8e7b6a5_align_pressure_daily_uuid_columns.py b/alembic/versions/f0c9d8e7b6a5_align_pressure_daily_uuid_columns.py new file mode 100644 index 000000000..38d113068 --- /dev/null +++ b/alembic/versions/f0c9d8e7b6a5_align_pressure_daily_uuid_columns.py @@ -0,0 +1,85 @@ +"""Align UUID column types on NMA_WaterLevelsContinuous_Pressure_Daily. + +Revision ID: f0c9d8e7b6a5 +Revises: e8a7c6b5d4f3 +Create Date: 2026-01-29 12:55:00.000000 +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy import inspect +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = "f0c9d8e7b6a5" +down_revision: Union[str, Sequence[str], None] = "e8a7c6b5d4f3" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def _column_is_uuid(col) -> bool: + return isinstance(col.get("type"), postgresql.UUID) + + +def upgrade() -> None: + """Alter UUID columns to proper UUID types.""" + bind = op.get_bind() + inspector = inspect(bind) + if not inspector.has_table("NMA_WaterLevelsContinuous_Pressure_Daily"): + return + + columns = { + col["name"]: col + for col in inspector.get_columns("NMA_WaterLevelsContinuous_Pressure_Daily") + } + + global_id_col = columns.get("GlobalID") + if global_id_col is not None and not _column_is_uuid(global_id_col): + op.alter_column( + "NMA_WaterLevelsContinuous_Pressure_Daily", + "GlobalID", + type_=postgresql.UUID(as_uuid=True), + postgresql_using='"GlobalID"::uuid', + ) + + well_id_col = columns.get("WellID") + if well_id_col is not None and not _column_is_uuid(well_id_col): + op.alter_column( + "NMA_WaterLevelsContinuous_Pressure_Daily", + "WellID", + type_=postgresql.UUID(as_uuid=True), + postgresql_using='"WellID"::uuid', + ) + + +def downgrade() -> None: + """Revert UUID columns back to strings.""" + bind = op.get_bind() + inspector = inspect(bind) + if not inspector.has_table("NMA_WaterLevelsContinuous_Pressure_Daily"): + return + + columns = { + col["name"]: col + for col in inspector.get_columns("NMA_WaterLevelsContinuous_Pressure_Daily") + } + + global_id_col = columns.get("GlobalID") + if global_id_col is not None and _column_is_uuid(global_id_col): + op.alter_column( + "NMA_WaterLevelsContinuous_Pressure_Daily", + "GlobalID", + type_=sa.String(length=40), + postgresql_using='"GlobalID"::text', + ) + + well_id_col = columns.get("WellID") + if well_id_col is not None and _column_is_uuid(well_id_col): + op.alter_column( + "NMA_WaterLevelsContinuous_Pressure_Daily", + "WellID", + type_=sa.String(length=40), + postgresql_using='"WellID"::text', + ) diff --git a/tests/test_waterlevelscontinuous_pressure_daily_legacy.py b/tests/test_waterlevelscontinuous_pressure_daily_legacy.py index d98b03ab5..9b6a55dac 100644 --- a/tests/test_waterlevelscontinuous_pressure_daily_legacy.py +++ b/tests/test_waterlevelscontinuous_pressure_daily_legacy.py @@ -71,7 +71,7 @@ def test_create_pressure_daily_all_fields(water_well_thing): session.refresh(record) assert record.global_id is not None - assert record.point_id == "PD-1001" + assert record.point_id == water_well_thing.name assert record.date_measured == now session.delete(record) @@ -95,7 +95,7 @@ def test_create_pressure_daily_minimal(water_well_thing): session.refresh(record) assert record.global_id is not None - assert record.point_id == "PD-1002" + assert record.point_id == water_well_thing.name session.delete(record) session.commit() @@ -122,7 +122,7 @@ def test_read_pressure_daily_by_global_id(water_well_thing): ) assert fetched is not None assert fetched.global_id == record.global_id - assert fetched.point_id == "PD-1003" + assert fetched.point_id == water_well_thing.name session.delete(record) session.commit() From 8ae5bbbca363ecb0e60937f6ee81c100732bd49e Mon Sep 17 00:00:00 2001 From: Kelsey Smuczynski Date: Thu, 29 Jan 2026 13:11:45 -0700 Subject: [PATCH 12/13] refactor: Enhance transfer of waterlevelscontinuous_pressure_daily - Cache Thing IDs and map PointID to thing_id to satisfy new FK - Filter orphan rows to prevent invalid inserts - Add focused transfer unit test to validate mapping and filtering --- ...evelscontinuous_pressure_daily_transfer.py | 47 +++++++++++++++++++ .../waterlevelscontinuous_pressure_daily.py | 33 +++++++++++-- 2 files changed, 77 insertions(+), 3 deletions(-) create mode 100644 tests/transfers/test_waterlevelscontinuous_pressure_daily_transfer.py diff --git a/tests/transfers/test_waterlevelscontinuous_pressure_daily_transfer.py b/tests/transfers/test_waterlevelscontinuous_pressure_daily_transfer.py new file mode 100644 index 000000000..a5616f81b --- /dev/null +++ b/tests/transfers/test_waterlevelscontinuous_pressure_daily_transfer.py @@ -0,0 +1,47 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== + +import pandas as pd + +from transfers.waterlevelscontinuous_pressure_daily import ( + NMA_WaterLevelsContinuous_Pressure_DailyTransferer, +) + + +def test_pressure_daily_transfer_filters_orphans(water_well_thing): + transferer = NMA_WaterLevelsContinuous_Pressure_DailyTransferer(batch_size=1) + df = pd.DataFrame( + [ + {"PointID": water_well_thing.name, "GlobalID": "gid-1"}, + {"PointID": "MISSING-THING", "GlobalID": "gid-2"}, + ] + ) + + filtered = transferer._filter_to_valid_things(df) + + assert list(filtered["PointID"]) == [water_well_thing.name] + + +def test_pressure_daily_row_dict_sets_thing_id(water_well_thing): + transferer = NMA_WaterLevelsContinuous_Pressure_DailyTransferer(batch_size=1) + row = {"PointID": water_well_thing.name, "GlobalID": "gid-3"} + + mapped = transferer._row_dict(row) + + assert mapped["thing_id"] == water_well_thing.id + + +# ============= EOF ============================================= diff --git a/transfers/waterlevelscontinuous_pressure_daily.py b/transfers/waterlevelscontinuous_pressure_daily.py index c41423f78..6caa348c3 100644 --- a/transfers/waterlevelscontinuous_pressure_daily.py +++ b/transfers/waterlevelscontinuous_pressure_daily.py @@ -22,7 +22,8 @@ from sqlalchemy.dialects.postgresql import insert from sqlalchemy.orm import Session -from db import NMA_WaterLevelsContinuous_Pressure_Daily +from db import NMA_WaterLevelsContinuous_Pressure_Daily, Thing +from db.engine import session_ctx from transfers.logger import logger from transfers.transferer import Transferer from transfers.util import read_csv @@ -41,6 +42,30 @@ class NMA_WaterLevelsContinuous_Pressure_DailyTransferer(Transferer): def __init__(self, *args, batch_size: int = 1000, **kwargs): super().__init__(*args, **kwargs) self.batch_size = batch_size + self._thing_id_cache: dict[str, int] = {} + self._build_thing_id_cache() + + def _build_thing_id_cache(self) -> None: + with session_ctx() as session: + things = session.query(Thing.name, Thing.id).all() + self._thing_id_cache = {name: thing_id for name, thing_id in things} + logger.info(f"Built Thing ID cache with {len(self._thing_id_cache)} entries") + + def _filter_to_valid_things(self, df: pd.DataFrame) -> pd.DataFrame: + valid_point_ids = set(self._thing_id_cache.keys()) + before_count = len(df) + filtered_df = df[df["PointID"].isin(valid_point_ids)].copy() + after_count = len(filtered_df) + if before_count > after_count: + skipped = before_count - after_count + logger.warning( + "Filtered out %s WaterLevelsContinuous_Pressure_Daily records without matching Things " + "(%s valid, %s orphan records prevented)", + skipped, + after_count, + skipped, + ) + return filtered_df def _get_dfs(self) -> tuple[pd.DataFrame, pd.DataFrame]: # Parse key datetime columns eagerly to avoid per-row parsing later. @@ -48,8 +73,8 @@ def _get_dfs(self) -> tuple[pd.DataFrame, pd.DataFrame]: self.source_table, parse_dates=["DateMeasured", "Created", "Updated"], ) - # No special cleaning/validation beyond raw import; keep identical copy. - return input_df, input_df + cleaned_df = self._filter_to_valid_things(input_df) + return input_df, cleaned_df def _transfer_hook(self, session: Session) -> None: rows = self._dedupe_rows( @@ -71,6 +96,7 @@ def _transfer_hook(self, session: Session) -> None: "OBJECTID": excluded.OBJECTID, "WellID": excluded.WellID, "PointID": excluded.PointID, + "thing_id": excluded.thing_id, "DateMeasured": excluded.DateMeasured, "TemperatureWater": excluded.TemperatureWater, "WaterHead": excluded.WaterHead, @@ -104,6 +130,7 @@ def val(key: str) -> Optional[Any]: "OBJECTID": val("OBJECTID"), "WellID": val("WellID"), "PointID": val("PointID"), + "thing_id": self._thing_id_cache.get(val("PointID")), "DateMeasured": val("DateMeasured"), "TemperatureWater": val("TemperatureWater"), "WaterHead": val("WaterHead"), From 4466e5a50ab14b1d6deaa1cc942b8fb446c46b65 Mon Sep 17 00:00:00 2001 From: Kelsey Smuczynski Date: Thu, 29 Jan 2026 15:24:37 -0700 Subject: [PATCH 13/13] feat: Add missing relationship/backref between Thing and NMA_WaterLevelsContinuous_Pressure_Daily --- db/nma_legacy.py | 4 ++++ db/thing.py | 14 +++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/db/nma_legacy.py b/db/nma_legacy.py index 33981bfae..6f1954e72 100644 --- a/db/nma_legacy.py +++ b/db/nma_legacy.py @@ -96,6 +96,10 @@ class NMA_WaterLevelsContinuous_Pressure_Daily(Base): checked_by: Mapped[Optional[str]] = mapped_column("CheckedBy", String(4)) cond_dl_ms_cm: Mapped[Optional[float]] = mapped_column("CONDDL (mS/cm)", Float) + thing: Mapped["Thing"] = relationship( + "Thing", back_populates="pressure_daily_levels" + ) + class NMA_view_NGWMN_WellConstruction(Base): """ diff --git a/db/thing.py b/db/thing.py index 8c3f4d315..66dc55244 100644 --- a/db/thing.py +++ b/db/thing.py @@ -47,7 +47,11 @@ from db.thing_geologic_formation_association import ( ThingGeologicFormationAssociation, ) - from db.nma_legacy import NMA_Chemistry_SampleInfo, NMA_Stratigraphy + from db.nma_legacy import ( + NMA_Chemistry_SampleInfo, + NMA_Stratigraphy, + NMA_WaterLevelsContinuous_Pressure_Daily, + ) class Thing( @@ -318,6 +322,14 @@ class Thing( cascade="all, delete-orphan", passive_deletes=True, ) + pressure_daily_levels: Mapped[List["NMA_WaterLevelsContinuous_Pressure_Daily"]] = ( + relationship( + "NMA_WaterLevelsContinuous_Pressure_Daily", + back_populates="thing", + cascade="all, delete-orphan", + passive_deletes=True, + ) + ) # --- Association Proxies --- assets: AssociationProxy[list["Asset"]] = association_proxy(