diff --git a/.env.example b/.env.example index 84f3fbc90..dffd3dfd8 100644 --- a/.env.example +++ b/.env.example @@ -46,6 +46,9 @@ MODE=development # disable authentication (for development only) AUTHENTIK_DISABLE_AUTHENTICATION=1 +# erase and rebuild the database for step tests +REBUILD_DB=1 + # authentik AUTHENTIK_URL= AUTHENTIK_CLIENT_ID= diff --git a/.gitignore b/.gitignore index f848fa5a2..197d03556 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,9 @@ requirements.txt # VS Code +# macOS +.DS_Store + # local development files development.db .env diff --git a/admin/views/chemistry_sampleinfo.py b/admin/views/chemistry_sampleinfo.py index 9aa6654ea..b588da038 100644 --- a/admin/views/chemistry_sampleinfo.py +++ b/admin/views/chemistry_sampleinfo.py @@ -28,6 +28,8 @@ - thing_id: Integer FK to Thing.id """ +from starlette.requests import Request +from starlette_admin.fields import HasOne from admin.views.base import OcotilloModelView diff --git a/admin/views/thing.py b/admin/views/thing.py index e7c413fbd..d74e0b9df 100644 --- a/admin/views/thing.py +++ b/admin/views/thing.py @@ -88,7 +88,6 @@ class ThingAdmin(OcotilloModelView): "well_pump_type", "well_pump_depth", "formation_completion_code", - "is_suitable_for_datalogger", # Spring-specific "spring_type", # Release Status diff --git a/alembic/versions/7b8c9d0e1f2a_delete_is_suitable_for_datalogger.py b/alembic/versions/7b8c9d0e1f2a_delete_is_suitable_for_datalogger.py new file mode 100644 index 000000000..e2f8b0fcf --- /dev/null +++ b/alembic/versions/7b8c9d0e1f2a_delete_is_suitable_for_datalogger.py @@ -0,0 +1,31 @@ +""" +Revision ID: 7b8c9d0e1f2a +Revises: 71a4c6b3d2e8 +Create Date: 2026-02-02 00:00:00.000000 + +Removes the is_suitable_for_datalogger column from the thing and thing_version tables. +""" + +# revision identifiers, used by Alembic. +revision = "7b8c9d0e1f2a" +down_revision = "71a4c6b3d2e8" +branch_labels = None +depends_on = None + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + op.drop_column("thing", "is_suitable_for_datalogger") + op.drop_column("thing_version", "is_suitable_for_datalogger") + + +def downgrade(): + op.add_column( + "thing", sa.Column("is_suitable_for_datalogger", sa.Boolean(), nullable=True) + ) + op.add_column( + "thing_version", + sa.Column("is_suitable_for_datalogger", sa.Boolean(), nullable=True), + ) diff --git a/api/lexicon.py b/api/lexicon.py index 933fb7a08..e0f08b56e 100644 --- a/api/lexicon.py +++ b/api/lexicon.py @@ -262,6 +262,7 @@ async def get_lexicon_term( async def get_lexicon_categories( session: session_dependency, user: viewer_dependency, + name: str | None = None, sort: str = "name", order: str = "asc", filter_: str = Query(alias="filter", default=None), @@ -269,6 +270,10 @@ async def get_lexicon_categories( """ Endpoint to retrieve lexicon categories. """ + if name: + sql = select(LexiconCategory).where(LexiconCategory.name.ilike(f"%{name}%")) + return paginated_all_getter(session, LexiconCategory, sort, order, filter_, sql) + return paginated_all_getter(session, LexiconCategory, sort, order, filter_) diff --git a/api/ogc/features.py b/api/ogc/features.py index 7fef38e82..47a1024e5 100644 --- a/api/ogc/features.py +++ b/api/ogc/features.py @@ -263,7 +263,6 @@ def _build_feature(row, collection_id: str) -> dict[str, Any]: "well_pump_type": model.well_pump_type, "well_pump_depth": model.well_pump_depth, "formation_completion_code": model.formation_completion_code, - "is_suitable_for_datalogger": model.is_suitable_for_datalogger, } if collection_id == "wells": properties["well_purposes"] = [ @@ -281,6 +280,10 @@ def _build_feature(row, collection_id: str) -> dict[str, Any]: } for screen in (model.screens or []) ] + properties["open_status"] = model.open_status + properties["datalogger_suitability_status"] = ( + model.datalogger_suitability_status + ) if hasattr(model, "nma_formation_zone"): properties["nma_formation_zone"] = model.nma_formation_zone return { @@ -350,7 +353,9 @@ def get_items( "well_pump_type": Thing.well_pump_type, "well_pump_depth": Thing.well_pump_depth, "formation_completion_code": Thing.formation_completion_code, - "is_suitable_for_datalogger": Thing.is_suitable_for_datalogger, + "well_status": Thing.well_status, + "open_status": Thing.open_status, + "datalogger_suitability_status": Thing.datalogger_suitability_status, } if hasattr(Thing, "nma_formation_zone"): column_map["nma_formation_zone"] = Thing.nma_formation_zone diff --git a/api/well_inventory.py b/api/well_inventory.py new file mode 100644 index 000000000..a73c1d11c --- /dev/null +++ b/api/well_inventory.py @@ -0,0 +1,664 @@ +# =============================================================================== +# Copyright 2025 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +import csv +from datetime import date +import logging +import re +from collections import Counter +from io import StringIO +from itertools import groupby +from typing import Set + +from fastapi import APIRouter, UploadFile, File +from fastapi.responses import JSONResponse +from pydantic import ValidationError +from shapely import Point +from sqlalchemy import select, and_ +from sqlalchemy.exc import DatabaseError +from sqlalchemy.orm import Session +from starlette.status import ( + HTTP_201_CREATED, + HTTP_422_UNPROCESSABLE_ENTITY, + HTTP_400_BAD_REQUEST, +) + +from core.constants import SRID_UTM_ZONE_13N, SRID_UTM_ZONE_12N, SRID_WGS84 +from core.dependencies import session_dependency, amp_editor_dependency +from db import ( + Group, + Location, + DataProvenance, + FieldEvent, + FieldEventParticipant, + FieldActivity, + Contact, + PermissionHistory, + Thing, +) +from schemas.thing import CreateWell +from schemas.well_inventory import WellInventoryRow +from services.contact_helper import add_contact +from services.exceptions_helper import PydanticStyleException +from services.thing_helper import add_thing +from services.util import transform_srid, convert_ft_to_m + +router = APIRouter(prefix="/well-inventory-csv") + + +def _make_location(model) -> Location: + point = Point(model.utm_easting, model.utm_northing) + + # TODO: this needs to be more sophisticated in the future. Likely more than 13N and 12N will be used + if model.utm_zone == "13N": + source_srid = SRID_UTM_ZONE_13N + else: + source_srid = SRID_UTM_ZONE_12N + + # Convert the point to a WGS84 coordinate system + transformed_point = transform_srid( + point, source_srid=source_srid, target_srid=SRID_WGS84 + ) + elevation_ft = float(model.elevation_ft) + elevation_m = convert_ft_to_m(elevation_ft) + + loc = Location( + point=transformed_point.wkt, + elevation=elevation_m, + ) + + return loc + + +def _make_contact(model: WellInventoryRow, well: Thing, idx) -> dict: + # add contact + notes = [] + for content, note_type in ( + (model.result_communication_preference, "Communication"), + (model.contact_special_requests_notes, "General"), + ): + if content is not None: + notes.append({"content": content, "note_type": note_type}) + + emails = [] + phones = [] + addresses = [] + name = getattr(model, f"contact_{idx}_name") + if name: + for i in (1, 2): + email = getattr(model, f"contact_{idx}_email_{i}") + etype = getattr(model, f"contact_{idx}_email_{i}_type") + if email and etype: + emails.append({"email": email, "email_type": etype}) + phone = getattr(model, f"contact_{idx}_phone_{i}") + ptype = getattr(model, f"contact_{idx}_phone_{i}_type") + if phone and ptype: + phones.append({"phone_number": phone, "phone_type": ptype}) + + address_line_1 = getattr(model, f"contact_{idx}_address_{i}_line_1") + address_line_2 = getattr(model, f"contact_{idx}_address_{i}_line_2") + city = getattr(model, f"contact_{idx}_address_{i}_city") + state = getattr(model, f"contact_{idx}_address_{i}_state") + postal_code = getattr(model, f"contact_{idx}_address_{i}_postal_code") + address_type = getattr(model, f"contact_{idx}_address_{i}_type") + if address_line_1 and city and state and postal_code and address_type: + addresses.append( + { + "address_line_1": address_line_1, + "address_line_2": address_line_2, + "city": city, + "state": state, + "postal_code": postal_code, + "address_type": address_type, + } + ) + + return { + "thing_id": well.id, + "name": name, + "organization": getattr(model, f"contact_{idx}_organization"), + "role": getattr(model, f"contact_{idx}_role"), + "contact_type": getattr(model, f"contact_{idx}_type"), + "emails": emails, + "phones": phones, + "addresses": addresses, + "notes": notes, + } + + +def _make_well_permission( + well: Thing, + contact: Contact | None, + permission_type: str, + permission_allowed: bool, + start_date: date, +) -> PermissionHistory: + """ + Makes a PermissionHistory record for the given well and contact. + If the contact has not been provided, but a permission is to be created, + no PermissionHistory record is created and a 400 error is raised. + """ + if contact is None: + raise PydanticStyleException( + HTTP_400_BAD_REQUEST, + detail=[ + { + "loc": [], + "msg": "At least one contact required for permission", + "type": "Contact required for permission", + "input": None, + } + ], + ) + + permission = PermissionHistory( + target_table="thing", + target_id=well.id, + contact=contact, + permission_type=permission_type, + permission_allowed=permission_allowed, + start_date=start_date, + end_date=None, + ) + return permission + + +AUTOGEN_REGEX = re.compile(r"^[A-Za-z]{2}-$") + + +def generate_autogen_well_id(session, prefix: str, offset: int = 0) -> str: + # get the latest well_name_point_id that starts with the same prefix + if not offset: + latest_well = session.scalars( + select(Thing) + .where(Thing.name.like(f"{prefix}%")) + .order_by(Thing.name.desc()) + ).first() + + if latest_well: + latest_id = latest_well.name + # extract the numeric part and increment it + number_part = latest_id.replace(prefix, "") + if number_part.isdigit(): + new_number = int(number_part) + 1 + else: + new_number = 1 + else: + new_number = 1 + else: + new_number = offset + 1 + + return f"{prefix}{new_number:04d}", new_number + + +def _make_row_models(rows, session): + models = [] + validation_errors = [] + seen_ids: Set[str] = set() + offset = 0 + for idx, row in enumerate(rows): + try: + if all(key == row.get(key) for key in row.keys()): + raise ValueError("Duplicate header row") + + well_id = row.get("well_name_point_id") + if not well_id: + raise ValueError("Field required") + print(f"Processing well_name_point_id: {well_id}") + if AUTOGEN_REGEX.match(well_id): + well_id, offset = generate_autogen_well_id(session, well_id, offset) + row["well_name_point_id"] = well_id + + if well_id in seen_ids: + print(seen_ids) + raise ValueError("Duplicate value for well_name_point_id") + seen_ids.add(well_id) + + model = WellInventoryRow(**row) + models.append(model) + + except ValidationError as e: + for err in e.errors(): + loc = err["loc"] + + field = loc[0] if loc else "composite field error" + value = row.get(field) if loc else None + validation_errors.append( + { + "row": idx + 1, + "error": err["msg"], + "field": field, + "value": value, + } + ) + except ValueError as e: + field = "well_name_point_id" + # Map specific controlled errors to safe, non-revealing messages + if str(e) == "Field required": + error_msg = "Field required" + elif str(e) == "Duplicate value for well_name_point_id": + error_msg = "Duplicate value for well_name_point_id" + elif str(e) == "Duplicate header row": + error_msg = "Duplicate header row" + field = "header" + else: + error_msg = "Invalid value" + + validation_errors.append( + {"row": idx + 1, "field": field, "error": error_msg} + ) + return models, validation_errors + + +@router.post("") +async def well_inventory_csv( + user: amp_editor_dependency, + session: session_dependency, + file: UploadFile = File(...), +): + if not file.content_type.startswith("text/csv") or not file.filename.endswith( + ".csv" + ): + raise PydanticStyleException( + HTTP_400_BAD_REQUEST, + detail=[ + { + "loc": [], + "msg": "Unsupported file type", + "type": "Unsupported file type", + "input": f"file.content_type {file.content_type} name={file.filename}", + } + ], + ) + + content = await file.read() + if not content: + raise PydanticStyleException( + HTTP_400_BAD_REQUEST, + detail=[ + {"loc": [], "msg": "Empty file", "type": "Empty file", "input": ""} + ], + ) + + try: + text = content.decode("utf-8") + except UnicodeDecodeError: + raise PydanticStyleException( + HTTP_400_BAD_REQUEST, + detail=[ + { + "loc": [], + "msg": "File encoding error", + "type": "File encoding error", + "input": "", + } + ], + ) + + reader = csv.DictReader(StringIO(text)) + rows = list(reader) + + if not rows: + raise PydanticStyleException( + HTTP_400_BAD_REQUEST, + detail=[ + { + "loc": [], + "msg": "No data rows found", + "type": "No data rows found", + "input": str(rows), + } + ], + ) + + if len(rows) > 2000: + raise PydanticStyleException( + HTTP_400_BAD_REQUEST, + detail=[ + { + "loc": [], + "msg": f"Too many rows {len(rows)}>2000", + "type": "Too many rows", + } + ], + ) + + header = text.splitlines()[0] + dialect = csv.Sniffer().sniff(header) + + if dialect.delimiter in (";", "\t"): + raise PydanticStyleException( + HTTP_400_BAD_REQUEST, + detail=[ + { + "loc": [], + "msg": f"Unsupported delimiter '{dialect.delimiter}'", + "type": "Unsupported delimiter", + } + ], + ) + + header = header.split(dialect.delimiter) + counts = Counter(header) + duplicates = [col for col, count in counts.items() if count > 1] + + wells = [] + if duplicates: + validation_errors = [ + { + "row": 0, + "field": f"{duplicates}", + "error": "Duplicate columns found", + } + ] + + else: + models, validation_errors = _make_row_models(rows, session) + if models and not validation_errors: + for project, items in groupby( + sorted(models, key=lambda x: x.project), key=lambda x: x.project + ): + # get project and add if does not exist + # BDMS-221 adds group_type + sql = select(Group).where( + and_(Group.group_type == "Monitoring Plan", Group.name == project) + ) + group = session.scalars(sql).one_or_none() + if not group: + group = Group(name=project, group_type="Monitoring Plan") + session.add(group) + + for model in items: + try: + added = _add_csv_row(session, group, model, user) + if added: + session.commit() + except ValueError as e: + validation_errors.append( + { + "row": model.well_name_point_id, + "field": "Invalid value", + "error": str(e), + } + ) + continue + except DatabaseError as e: + logging.error( + f"Database error while importing row '{model.well_name_point_id}': {e}" + ) + validation_errors.append( + { + "row": model.well_name_point_id, + "field": "Database error", + "error": "A database error occurred while importing this row.", + } + ) + continue + + wells.append(added) + + rows_imported = len(wells) + rows_processed = len(rows) + rows_with_validation_errors_or_warnings = len(validation_errors) + + status_code = HTTP_201_CREATED + if validation_errors: + status_code = HTTP_422_UNPROCESSABLE_ENTITY + + return JSONResponse( + status_code=status_code, + content={ + "validation_errors": validation_errors, + "summary": { + "total_rows_processed": rows_processed, + "total_rows_imported": rows_imported, + "validation_errors_or_warnings": rows_with_validation_errors_or_warnings, + }, + "wells": wells, + }, + ) + + +def _add_field_staff( + session: Session, fs: str, field_event: FieldEvent, role: str, user: str +) -> None: + ct = "Field Event Participant" + org = "NMBGMR" + contact = session.scalars( + select(Contact) + .where(Contact.name == fs) + .where(Contact.organization == org) + .where(Contact.contact_type == ct) + ).first() + + if not contact: + payload = dict(name=fs, role="Technician", organization=org, contact_type=ct) + contact = add_contact(session, payload, user) + + fec = FieldEventParticipant( + field_event=field_event, contact_id=contact.id, participant_role=role + ) + session.add(fec) + + +def _add_csv_row(session: Session, group: Group, model: WellInventoryRow, user) -> str: + name = model.well_name_point_id + date_time = model.date_time + + # -------------------- + # Location and associated tables + # -------------------- + + # add Location + loc = _make_location(model) + session.add(loc) + session.flush() + + # add location notes + if model.directions_to_site: + directions_note = loc.add_note( + content=model.directions_to_site, note_type="Directions" + ) + session.add(directions_note) + + # add data provenance records + dp = DataProvenance( + target_id=loc.id, + target_table="location", + field_name="elevation", + collection_method=model.elevation_method, + ) + session.add(dp) + + # -------------------- + # Thing and associated tables + # -------------------- + + # add Thing + """ + Developer's note + + Laila said that the depth source is almost always the source for the historic depth to water. + She indicated that it would be acceptable to use the depth source for the historic depth to water source. + """ + if model.depth_source: + historic_depth_to_water_source = model.depth_source.lower() + else: + historic_depth_to_water_source = "unknown" + + if model.historic_depth_to_water_ft is not None: + historic_depth_note = f"historic depth to water: {model.historic_depth_to_water_ft} ft - source: {historic_depth_to_water_source}" + else: + historic_depth_note = None + + well_notes = [] + for note_content, note_type in ( + (model.specific_location_of_well, "Access"), + (model.contact_special_requests_notes, "General"), + (model.well_measuring_notes, "Sampling Procedure"), + (model.sampling_scenario_notes, "Sampling Procedure"), + (historic_depth_note, "Historical"), + ): + if note_content is not None: + well_notes.append({"content": note_content, "note_type": note_type}) + + alternate_ids = [] + for alternate_id, alternate_organization in ( + (model.site_name, "NMBGMR"), + (model.ose_well_record_id, "NMOSE"), + ): + if alternate_id is not None: + alternate_ids.append( + { + "alternate_id": alternate_id, + "alternate_organization": alternate_organization, + "relation": "same_as", + } + ) + + well_purposes = [] + if model.well_purpose: + well_purposes.append(model.well_purpose) + if model.well_purpose_2: + well_purposes.append(model.well_purpose_2) + + monitoring_frequencies = [] + if model.monitoring_frequency: + monitoring_frequencies.append( + { + "monitoring_frequency": model.monitoring_frequency, + "start_date": date_time.date(), + } + ) + + data = CreateWell( + location_id=loc.id, + group_id=group.id, + name=name, + first_visit_date=date_time.date(), + well_depth=model.total_well_depth_ft, + well_depth_source=model.depth_source, + well_casing_diameter=model.casing_diameter_ft, + measuring_point_height=model.measuring_point_height_ft, + measuring_point_description=model.measuring_point_description, + well_completion_date=model.date_drilled, + well_completion_date_source=model.completion_source, + well_pump_type=model.well_pump_type, + well_pump_depth=model.well_pump_depth_ft, + is_suitable_for_datalogger=model.datalogger_possible, + is_open=model.is_open, + notes=well_notes, + well_purposes=well_purposes, + monitoring_frequencies=monitoring_frequencies, + ) + well_data = data.model_dump() + + """ + Developer's notes + + the add_thing function also handles: + - MeasuringPointHistory + - GroupThingAssociation + - LocationThingAssociation + - DataProvenance for well_completion_date + - DataProvenance for well_depth + - Notes + - WellPurpose + - MonitoringFrequencyHistory + - StatusHistory for status_type 'Open Status' + - StatusHistory for status_type 'Datalogger Suitability Status' + """ + well = add_thing( + session=session, data=well_data, user=user, thing_type="water well" + ) + session.refresh(well) + + # ------------------ + # Field Events and related tables + # ------------------ + """ + Developer's notes + + These tables are not handled in add_thing because they are only relevant if + the well has been inventoried in the field, not if the well is added from + another source like a report, database, or map. + """ + + # add field event + fe = FieldEvent( + event_date=date_time, + notes="Initial field event from well inventory import", + thing_id=well.id, + ) + session.add(fe) + + # add field staff + for fsi, role in ( + (model.field_staff, "Lead"), + (model.field_staff_2, "Participant"), + (model.field_staff_3, "Participant"), + ): + if not fsi: + continue + + _add_field_staff(session, fsi, fe, role, user) + + # add field activity + fa = FieldActivity( + field_event=fe, + activity_type="well inventory", + notes="Well inventory conducted during field event.", + ) + session.add(fa) + + # ------------------ + # Contacts + # ------------------ + + # add contacts + contact_for_permissions = None + for idx in (1, 2): + contact_dict = _make_contact(model, well, idx) + if contact_dict: + contact = add_contact(session, contact_dict, user=user) + + # Use the first created contact for permissions if available + if contact_for_permissions is None: + contact_for_permissions = contact + + # ------------------ + # Permissions + # ------------------ + + # add permissions + for permission_type, permission_allowed in ( + ("Water Level Sample", model.repeat_measurement_permission), + ("Water Chemistry Sample", model.sampling_permission), + ("Datalogger Installation", model.datalogger_installation_permission), + ): + if permission_allowed is not None: + permission = _make_well_permission( + well=well, + contact=contact_for_permissions, + permission_type=permission_type, + permission_allowed=permission_allowed, + start_date=model.date_time.date(), + ) + session.add(permission) + + return model.well_name_point_id + + +# ============= EOF ============================================= diff --git a/core/constants.py b/core/constants.py index 93179ddb1..5938d0d6a 100644 --- a/core/constants.py +++ b/core/constants.py @@ -16,4 +16,58 @@ SRID_WGS84 = 4326 SRID_UTM_ZONE_13N = 26913 +SRID_UTM_ZONE_12N = 26912 + +STATE_CODES = ( + "AL", + "AK", + "AZ", + "AR", + "CA", + "CO", + "CT", + "DE", + "FL", + "GA", + "HI", + "ID", + "IL", + "IN", + "IA", + "KS", + "KY", + "LA", + "ME", + "MD", + "MA", + "MI", + "MN", + "MS", + "MO", + "MT", + "NE", + "NV", + "NH", + "NJ", + "NM", + "NY", + "NC", + "ND", + "OH", + "OK", + "OR", + "PA", + "RI", + "SC", + "SD", + "TN", + "TX", + "UT", + "VT", + "VA", + "WA", + "WV", + "WI", + "WY", +) # ============= EOF ============================================= diff --git a/core/enums.py b/core/enums.py index 9ed99a82e..43c16c2d3 100644 --- a/core/enums.py +++ b/core/enums.py @@ -51,7 +51,7 @@ MonitoringStatus: type[Enum] = build_enum_from_lexicon_category("monitoring_status") ParameterName: type[Enum] = build_enum_from_lexicon_category("parameter_name") Organization: type[Enum] = build_enum_from_lexicon_category("organization") -OriginSource: type[Enum] = build_enum_from_lexicon_category("origin_source") +OriginType: type[Enum] = build_enum_from_lexicon_category("origin_type") ParameterType: type[Enum] = build_enum_from_lexicon_category("parameter_type") PhoneType: type[Enum] = build_enum_from_lexicon_category("phone_type") PublicationType: type[Enum] = build_enum_from_lexicon_category("publication_type") @@ -81,4 +81,5 @@ GeographicScale: type[Enum] = build_enum_from_lexicon_category("geographic_scale") Lithology: type[Enum] = build_enum_from_lexicon_category("lithology") FormationCode: type[Enum] = build_enum_from_lexicon_category("formation_code") +NoteType: type[Enum] = build_enum_from_lexicon_category("note_type") # ============= EOF ============================================= diff --git a/core/initializers.py b/core/initializers.py index 330ade9fc..7b002ac9b 100644 --- a/core/initializers.py +++ b/core/initializers.py @@ -126,7 +126,9 @@ def register_routes(app): from api.geospatial import router as geospatial_router from api.ngwmn import router as ngwmn_router from api.ogc.router import router as ogc_router + from api.well_inventory import router as well_inventory_router + app.include_router(well_inventory_router) app.include_router(asset_router) app.include_router(admin_auth_router) app.include_router(author_router) diff --git a/core/lexicon.json b/core/lexicon.json index 01539f2d2..f5c2c0a64 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -221,7 +221,7 @@ "description": null }, { - "name": "origin_source", + "name": "origin_type", "description": null }, { @@ -1832,6 +1832,13 @@ "term": "PLSS", "definition": "Public Land Survey System ID" }, + { + "categories": [ + "activity_type" + ], + "term": "well inventory", + "definition": "well inventory" + }, { "categories": [ "activity_type" @@ -2189,6 +2196,40 @@ "term": "Access Status", "definition": "Defines the well's access status for field personnel." }, + { + "categories": [ + "status_type" + ], + "term": "Open Status", + "definition": "Defines if the well is open or closed." + }, + { + "categories": [ + "status_type" + ], + "term": "Datalogger Suitability Status", + "definition": "Defines if a datalogger can or cannot be installed at the well." + }, + { + "categories": ["status_value"], + "term": "Open", + "definition": "The well is open." + }, + { + "categories": ["status_value"], + "term": "Closed", + "definition": "The well is closed." + }, + { + "categories": ["status_value"], + "term": "Datalogger can be installed", + "definition": "A datalogger can be installed at the well" + }, + { + "categories": ["status_value"], + "term": "Datalogger cannot be installed", + "definition": "A datalogger cannot be installed at the well" + }, { "categories": [ "status_value" @@ -7933,77 +7974,77 @@ }, { "categories": [ - "origin_source" + "origin_type" ], "term": "Reported by another agency", "definition": "Reported by another agency" }, { "categories": [ - "origin_source" + "origin_type" ], "term": "From driller's log or well report", "definition": "From driller's log or well report" }, { "categories": [ - "origin_source" + "origin_type" ], "term": "Private geologist, consultant or univ associate", "definition": "Private geologist, consultant or univ associate" }, { "categories": [ - "origin_source" + "origin_type" ], "term": "Interpreted fr geophys logs by source agency", "definition": "Interpreted fr geophys logs by source agency" }, { "categories": [ - "origin_source" + "origin_type" ], "term": "Memory of owner, operator, driller", "definition": "Memory of owner, operator, driller" }, { "categories": [ - "origin_source" + "origin_type" ], "term": "Measured by source agency", "definition": "Measured by source agency" }, { "categories": [ - "origin_source" + "origin_type" ], "term": "Reported by owner of well", "definition": "Reported by owner of well" }, { "categories": [ - "origin_source" + "origin_type" ], "term": "Reported by person other than driller owner agency", "definition": "Reported by person other than driller owner agency" }, { "categories": [ - "origin_source" + "origin_type" ], "term": "Measured by NMBGMR staff", "definition": "Measured by NMBGMR staff" }, { "categories": [ - "origin_source" + "origin_type" ], "term": "Other", "definition": "Other" }, { "categories": [ - "origin_source" + "origin_type" ], "term": "Data Portal", "definition": "Data Portal" @@ -8015,6 +8056,20 @@ "term": "Access", "definition": "Access instructions, gate codes, permission requirements, etc." }, + { + "categories": [ + "note_type" + ], + "term": "Directions", + "definition": "Notes about directions to a location." + }, + { + "categories": [ + "note_type" + ], + "term": "Communication", + "definition": "Notes about communication preferences/requests for a contact." + }, { "categories": [ "note_type" diff --git a/db/__init__.py b/db/__init__.py index 1c5e33896..a376381b1 100644 --- a/db/__init__.py +++ b/db/__init__.py @@ -16,20 +16,35 @@ # import all models from db package so that Alembic can discover them -from db.base import * -from db.base import Base +from sqlalchemy import ( + desc, + cast, +) +from sqlalchemy.dialects.postgresql import REGCONFIG +from sqlalchemy.orm import configure_mappers +from sqlalchemy_searchable import ( + inspect_search_vectors, + search_manager, +) from db.analysis_method import * +from db.aquifer_system import * +from db.aquifer_type import * from db.asset import * +from db.base import * +from db.base import Base from db.collabnet import * from db.contact import * +from db.data_provenance import * from db.deployment import * +from db.field import * from db.geochronology import * +from db.geologic_formation import * from db.geothermal import * -from db.field import * from db.group import * from db.lexicon import * from db.location import * +from db.measuring_point_history import * from db.notes import * from db.observation import * from db.parameter import * @@ -40,15 +55,11 @@ from db.sensor import * from db.status_history import * from db.thing import * -from db.transducer import * -from db.measuring_point_history import * -from db.data_provenance import * -from db.aquifer_system import * -from db.geologic_formation import * from db.thing_aquifer_association import * from db.thing_geologic_formation_association import * from db.aquifer_type import * from db.nma_legacy import * +from db.transducer import * from sqlalchemy import ( func, diff --git a/db/contact.py b/db/contact.py index 656e4d9d4..fa3146df1 100644 --- a/db/contact.py +++ b/db/contact.py @@ -125,6 +125,14 @@ class Contact(Base, AutoBaseMixin, ReleaseMixin, NotesMixin): UniqueConstraint("name", "organization", name="uq_contact_name_organization"), ) + @property + def communication_notes(self): + return self._get_notes("Communication") + + @property + def general_notes(self): + return self._get_notes("General") + class IncompleteNMAPhone(Base, AutoBaseMixin): """ diff --git a/db/data_provenance.py b/db/data_provenance.py index 20505d94c..50d0204cf 100644 --- a/db/data_provenance.py +++ b/db/data_provenance.py @@ -19,9 +19,8 @@ from sqlalchemy import Integer, Index, and_ from sqlalchemy.orm import relationship, Mapped, mapped_column, declared_attr, foreign -from db.base import Base, AutoBaseMixin, ReleaseMixin - from db import lexicon_term +from db.base import Base, AutoBaseMixin, ReleaseMixin if TYPE_CHECKING: from db.thing import Thing diff --git a/db/group.py b/db/group.py index 5be1dedc6..451e32b9d 100644 --- a/db/group.py +++ b/db/group.py @@ -16,7 +16,7 @@ from typing import Optional, List, TYPE_CHECKING from geoalchemy2 import Geometry, WKBElement -from sqlalchemy import String, Integer, ForeignKey +from sqlalchemy import String, Integer, ForeignKey, UniqueConstraint from sqlalchemy.ext.associationproxy import association_proxy, AssociationProxy from sqlalchemy.orm import relationship, Mapped from sqlalchemy.testing.schema import mapped_column @@ -31,7 +31,7 @@ class Group(Base, AutoBaseMixin, ReleaseMixin): # --- Column Definitions --- - name: Mapped[str] = mapped_column(String(100), nullable=False, unique=True) + name: Mapped[str] = mapped_column(String(100), nullable=False) description: Mapped[str] = mapped_column(String(255), nullable=True) project_area: Mapped[Optional[WKBElement]] = mapped_column( Geometry(geometry_type="MULTIPOLYGON", srid=SRID_WGS84, spatial_index=True) @@ -56,6 +56,10 @@ class Group(Base, AutoBaseMixin, ReleaseMixin): "thing_associations", "thing" ) + __table_args__ = ( + UniqueConstraint("name", "group_type", name="uq_group_name_group_type"), + ) + class GroupThingAssociation(Base, AutoBaseMixin): group_id: Mapped[int] = mapped_column( diff --git a/db/permission_history.py b/db/permission_history.py index fbc0007e8..0e1526e2a 100644 --- a/db/permission_history.py +++ b/db/permission_history.py @@ -7,8 +7,9 @@ which entity the permission applies to, and for what period of time. """ -from typing import TYPE_CHECKING from datetime import date +from typing import TYPE_CHECKING + from sqlalchemy import Integer, ForeignKey, String, and_ from sqlalchemy.orm import relationship, Mapped, mapped_column, declared_attr, foreign diff --git a/db/thing.py b/db/thing.py index bdfff8e58..71e131211 100644 --- a/db/thing.py +++ b/db/thing.py @@ -151,11 +151,6 @@ class Thing( nullable=True, comment="Raw FormationZone value from legacy WellData (NM_Aquifer).", ) - # TODO: should this be required for every well in the database? AMMP review - is_suitable_for_datalogger: Mapped[bool] = mapped_column( - nullable=True, - comment="Indicates if the well is suitable for datalogger installation.", - ) # Spring-related columns spring_type: Mapped[str] = lexicon_term( @@ -467,6 +462,32 @@ def monitoring_status(self) -> str | None: ) return latest_status.status_value if latest_status else None + @property + def open_status(self) -> str | None: + """ + Returns the open status from the most recent status history entry + where status_type is "Open Status". + + Since status_history is eagerly loaded, this should not introduce N+1 query issues. + """ + latest_status = retrieve_latest_polymorphic_history_table_record( + self, "status_history", "Open Status" + ) + return latest_status.status_value if latest_status else None + + @property + def datalogger_suitability_status(self) -> str | None: + """ + Returns the datalogger installation status from the most recent status history entry + where status_type is "Datalogger Suitability Status". + + Since status_history is eagerly loaded, this should not introduce N+1 query issues. + """ + latest_status = retrieve_latest_polymorphic_history_table_record( + self, "status_history", "Datalogger Suitability Status" + ) + return latest_status.status_value if latest_status else None + @property def measuring_point_height(self) -> int | None: """ diff --git a/db/thing_aquifer_association.py b/db/thing_aquifer_association.py index d9a3e6ac4..003801243 100644 --- a/db/thing_aquifer_association.py +++ b/db/thing_aquifer_association.py @@ -8,7 +8,6 @@ from typing import TYPE_CHECKING from sqlalchemy import ForeignKey - from sqlalchemy.orm import relationship, Mapped, mapped_column from db.base import Base, AutoBaseMixin, ReleaseMixin diff --git a/pyproject.toml b/pyproject.toml index 22539c00a..fd2cf3d09 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,6 +97,7 @@ dependencies = [ "typing-inspection==0.4.1", "tzdata==2025.2", "urllib3==2.6.0", + "utm>=0.8.1", "uvicorn==0.38.0", "yarl==1.20.1", ] diff --git a/schemas/__init__.py b/schemas/__init__.py index cd8e62d62..5a31f9229 100644 --- a/schemas/__init__.py +++ b/schemas/__init__.py @@ -53,13 +53,19 @@ class BaseUpdateModel(BaseCreateModel): release_status: ReleaseStatus | None = None -def past_or_today_validator(value: date) -> date: - if value > date.today(): +def past_or_today_validator(value: date | datetime) -> date | datetime: + if isinstance(value, datetime): + if value > datetime.now(timezone.utc): + raise ValueError("Datetime must be in the past or present.") + elif value > date.today(): raise ValueError("Date must be today or in the past.") return value -PastOrTodayDate = Annotated[date, AfterValidator(past_or_today_validator)] +PastOrTodayDate: type[date] = Annotated[date, AfterValidator(past_or_today_validator)] +PastOrTodayDatetime: type[datetime] = Annotated[ + datetime, AfterValidator(past_or_today_validator) +] # Custom type for UTC datetime serialization diff --git a/schemas/aquifer_system.py b/schemas/aquifer_system.py index 1e1961873..6dee0ff09 100644 --- a/schemas/aquifer_system.py +++ b/schemas/aquifer_system.py @@ -1,9 +1,10 @@ from typing import List from pydantic import BaseModel + +from core.enums import AquiferType, GeographicScale # Import specific Enums from schemas import BaseResponseModel from schemas.validators import GeometryMixin -from core.enums import AquiferType, GeographicScale # Import specific Enums # ------ CREATE ---------- diff --git a/schemas/contact.py b/schemas/contact.py index f98d8adc4..753982048 100644 --- a/schemas/contact.py +++ b/schemas/contact.py @@ -22,6 +22,7 @@ from core.enums import Role, ContactType, PhoneType, EmailType, AddressType from schemas import BaseResponseModel, BaseCreateModel, BaseUpdateModel +from schemas.notes import CreateNote, NoteResponse # -------- VALIDATORS ---------- @@ -156,6 +157,7 @@ class CreateContact(BaseCreateModel, ValidateContact): emails: list[CreateEmail] | None = None phones: list[CreatePhone] | None = None addresses: list[CreateAddress] | None = None + notes: list[CreateNote] | None = None # -------- RESPONSE ---------- @@ -220,6 +222,8 @@ class ContactResponse(BaseResponseModel): phones: List[PhoneResponse] = [] addresses: List[AddressResponse] = [] things: List[ThingResponseForContact] = [] + communication_notes: List[NoteResponse] = [] + general_notes: List[NoteResponse] = [] @field_validator("incomplete_nma_phones", mode="before") def make_incomplete_nma_phone_str(cls, v: list) -> list: diff --git a/schemas/geologic_formation.py b/schemas/geologic_formation.py index 67a3cb24a..2c78e7c92 100644 --- a/schemas/geologic_formation.py +++ b/schemas/geologic_formation.py @@ -2,9 +2,9 @@ from pydantic import BaseModel, field_validator, Field +from core.enums import FormationCode, Lithology from schemas import BaseResponseModel from schemas.validators import DepthIntervalMixin, GeometryMixin -from core.enums import FormationCode, Lithology # ------ CREATE ---------- diff --git a/schemas/location.py b/schemas/location.py index 2c8fe3126..596545287 100644 --- a/schemas/location.py +++ b/schemas/location.py @@ -14,8 +14,7 @@ # limitations under the License. # =============================================================================== from datetime import date -from typing import Any -from typing import List +from typing import List, Any from geoalchemy2 import WKBElement from geoalchemy2.shape import to_shape @@ -88,7 +87,7 @@ class GeoJSONGeometry(BaseModel): class GeoJSONUTMCoordinates(BaseModel): easting: float northing: float - utm_zone: int = 13 + utm_zone: str = "13N" horizontal_datum: str = "NAD83" model_config = ConfigDict( diff --git a/schemas/notes.py b/schemas/notes.py index 85c47ed9b..8b8d8c438 100644 --- a/schemas/notes.py +++ b/schemas/notes.py @@ -2,6 +2,9 @@ Pydantic models for the Notes table. """ +from core.enums import NoteType + +from pydantic import BaseModel from schemas import BaseCreateModel, BaseUpdateModel, BaseResponseModel # -------- BASE SCHEMA: ---------- @@ -10,8 +13,8 @@ """ -class BaseNote: - note_type: str +class BaseNote(BaseModel): + note_type: NoteType content: str diff --git a/schemas/permission_history.py b/schemas/permission_history.py index e0619d90e..d8f1f3ef2 100644 --- a/schemas/permission_history.py +++ b/schemas/permission_history.py @@ -1,7 +1,7 @@ from pydantic import BaseModel -from schemas import PastOrTodayDate from core.enums import PermissionType +from schemas import PastOrTodayDate # ------ RESPONSE ---------- diff --git a/schemas/thing.py b/schemas/thing.py index f4c3727a3..4c1588e97 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -27,6 +27,7 @@ WellConstructionMethod, WellPumpType, FormationCode, + OriginType, ) from schemas import BaseCreateModel, BaseUpdateModel, BaseResponseModel, PastOrTodayDate from schemas.group import GroupResponse @@ -42,6 +43,7 @@ class ValidateWell(BaseModel): hole_depth: float | None = None # in feet well_casing_depth: float | None = None # in feet measuring_point_height: float | None = None + well_pump_depth: float | None = None # in feet @model_validator(mode="after") def validate_values(self): @@ -58,6 +60,12 @@ def validate_values(self): "well casing depth must be less than or equal to hole depth" ) + if self.well_pump_depth is not None: + if self.well_depth is not None and self.well_pump_depth > self.well_depth: + raise ValueError("well pump depth must be less than well depth") + elif self.hole_depth is not None and self.well_pump_depth > self.hole_depth: + raise ValueError("well pump depth must be less than hole depth") + # if self.measuring_point_height is not None: # if ( # self.hole_depth is not None @@ -92,6 +100,12 @@ class CreateThingIdLink(BaseModel): alternate_organization: str +class CreateMonitoringFrequency(BaseModel): + monitoring_frequency: MonitoringFrequency + start_date: PastOrTodayDate + end_date: PastOrTodayDate | None = None + + class CreateBaseThing(BaseCreateModel): """ Developer's notes @@ -102,10 +116,25 @@ class CreateBaseThing(BaseCreateModel): e.g. POST /thing/water-well, POST /thing/spring determines the thing_type """ - location_id: int | None + location_id: int | None = None group_id: int | None = None # Optional group ID for the thing name: str # Name of the thing first_visit_date: PastOrTodayDate | None = None # Date of NMBGMR's first visit + notes: list[CreateNote] | None = None + alternate_ids: list[CreateThingIdLink] | None = None + monitoring_frequencies: list[CreateMonitoringFrequency] | None = None + + @field_validator("alternate_ids", mode="before") + def use_dummy_values(cls, v): + """ + When alternate IDs are provided they are assumed to be the same as + the thing being created. This gets handled in the function services/thing_helper.py::add_thing. + By using dummy values here we can avoid validation errors and then use the + thing's id when creating the actual links. + """ + for alternate_id in v: + alternate_id.thing_id = -1 # dummy value + return v class CreateWell(CreateBaseThing, ValidateWell): @@ -117,6 +146,7 @@ class CreateWell(CreateBaseThing, ValidateWell): well_depth: float | None = Field( default=None, gt=0, description="Well depth in feet" ) + well_depth_source: OriginType | None = None hole_depth: float | None = Field( default=None, gt=0, description="Hole depth in feet" ) @@ -127,17 +157,17 @@ class CreateWell(CreateBaseThing, ValidateWell): default=None, gt=0, description="Well casing depth in feet" ) well_casing_materials: list[CasingMaterial] | None = None - measuring_point_height: float = Field(description="Measuring point height in feet") measuring_point_description: str | None = None - notes: list[CreateNote] | None = None well_completion_date: PastOrTodayDate | None = None well_completion_date_source: str | None = None well_driller_name: str | None = None well_construction_method: WellConstructionMethod | None = None well_construction_method_source: str | None = None well_pump_type: WellPumpType | None = None - is_suitable_for_datalogger: bool | None + well_pump_depth: float | None = None + is_suitable_for_datalogger: bool | None = None + is_open: bool | None = None formation_completion_code: FormationCode | None = None nma_formation_zone: str | None = None @@ -238,8 +268,9 @@ class WellResponse(BaseThingResponse): well_pump_type: WellPumpType | None well_pump_depth: float | None well_pump_depth_unit: str = "ft" - is_suitable_for_datalogger: bool | None well_status: str | None + open_status: str | None + datalogger_suitability_status: str | None measuring_point_height: float measuring_point_height_unit: str = "ft" measuring_point_description: str | None diff --git a/schemas/validators.py b/schemas/validators.py index 963047bc2..4d9b56697 100644 --- a/schemas/validators.py +++ b/schemas/validators.py @@ -5,6 +5,7 @@ """ from pydantic import model_validator, field_validator, BaseModel, Field + from services.validation.geospatial import validate_wkt_geometry diff --git a/schemas/well_inventory.py b/schemas/well_inventory.py new file mode 100644 index 000000000..9ab31f5a9 --- /dev/null +++ b/schemas/well_inventory.py @@ -0,0 +1,388 @@ +# =============================================================================== +# Copyright 2025 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +import re +from datetime import datetime, date +from typing import Optional, Annotated, TypeAlias +from schemas import past_or_today_validator, PastOrTodayDatetime + +import phonenumbers +import utm +from pydantic import ( + BaseModel, + model_validator, + BeforeValidator, + validate_email, + AfterValidator, + field_validator, +) + +from core.constants import STATE_CODES +from core.enums import ( + ElevationMethod, + Role, + ContactType, + PhoneType, + EmailType, + AddressType, + WellPurpose as WellPurposeEnum, + MonitoringFrequency, +) +from services.util import convert_dt_tz_naive_to_tz_aware + + +def empty_str_to_none(v): + if isinstance(v, str) and v.strip() == "": + return None + return v + + +def blank_to_none(v): + if isinstance(v, str) and v.strip() == "": + return None + return v + + +def owner_default(v): + v = blank_to_none(v) + if v is None: + return "Owner" + return v + + +def primary_default(v): + v = blank_to_none(v) + if v is None: + return "Primary" + return v + + +US_POSTAL_REGEX = re.compile(r"^\d{5}(-\d{4})?$") + + +def postal_code_or_none(v): + if v is None or (isinstance(v, str) and v.strip() == ""): + return None + + if not US_POSTAL_REGEX.match(v): + raise ValueError("Invalid postal code") + + return v + + +def state_validator(v): + if v and len(v) != 2: + raise ValueError("State must be a 2 letter abbreviation") + + if v and v.upper() not in STATE_CODES: + raise ValueError("State must be a valid US state abbreviation") + return v + + +def phone_validator(phone_number_str): + phone_number_str = phone_number_str.strip() + if phone_number_str: + parsed_number = phonenumbers.parse(phone_number_str, "US") + if phonenumbers.is_valid_number(parsed_number): + formatted_number = phonenumbers.format_number( + parsed_number, phonenumbers.PhoneNumberFormat.E164 + ) + return formatted_number + else: + raise ValueError(f"Invalid phone number. {phone_number_str}") + + +def email_validator_function(email_str): + if email_str: + try: + validate_email(email_str) + return email_str + except ValueError as e: + raise ValueError(f"Invalid email format. {email_str}") from e + + +# Reusable type +PhoneTypeField: TypeAlias = Annotated[ + Optional[PhoneType], BeforeValidator(blank_to_none) +] +ContactTypeField: TypeAlias = Annotated[ + Optional[ContactType], BeforeValidator(blank_to_none) +] +EmailTypeField: TypeAlias = Annotated[ + Optional[EmailType], BeforeValidator(blank_to_none) +] +AddressTypeField: TypeAlias = Annotated[ + Optional[AddressType], BeforeValidator(blank_to_none) +] +ContactRoleField: TypeAlias = Annotated[Optional[Role], BeforeValidator(blank_to_none)] +OptionalFloat: TypeAlias = Annotated[ + Optional[float], BeforeValidator(empty_str_to_none) +] +MonitoryFrequencyField: TypeAlias = Annotated[ + Optional[MonitoringFrequency], BeforeValidator(blank_to_none) +] +WellPurposeField: TypeAlias = Annotated[ + Optional[WellPurposeEnum], BeforeValidator(blank_to_none) +] +PostalCodeField: TypeAlias = Annotated[ + Optional[str], BeforeValidator(postal_code_or_none) +] +StateField: TypeAlias = Annotated[Optional[str], BeforeValidator(state_validator)] +PhoneField: TypeAlias = Annotated[Optional[str], BeforeValidator(phone_validator)] +EmailField: TypeAlias = Annotated[ + Optional[str], BeforeValidator(email_validator_function) +] + +OptionalBool: TypeAlias = Annotated[Optional[bool], BeforeValidator(empty_str_to_none)] +OptionalPastOrTodayDateTime: TypeAlias = Annotated[ + Optional[datetime], + BeforeValidator(empty_str_to_none), + AfterValidator(past_or_today_validator), +] +OptionalPastOrTodayDate: TypeAlias = Annotated[ + Optional[date], + BeforeValidator(empty_str_to_none), + AfterValidator(past_or_today_validator), +] + + +# ============= EOF ============================================= +class WellInventoryRow(BaseModel): + # Required fields + project: str + well_name_point_id: str + site_name: str + date_time: PastOrTodayDatetime + field_staff: str + utm_easting: float + utm_northing: float + utm_zone: str + elevation_ft: float + elevation_method: ElevationMethod + measuring_point_height_ft: float + + # Optional fields + field_staff_2: Optional[str] = None + field_staff_3: Optional[str] = None + + contact_1_name: Optional[str] = None + contact_1_organization: Optional[str] = None + contact_1_role: ContactRoleField = None + contact_1_type: ContactTypeField = None + contact_1_phone_1: PhoneField = None + contact_1_phone_1_type: PhoneTypeField = None + contact_1_phone_2: PhoneField = None + contact_1_phone_2_type: PhoneTypeField = None + contact_1_email_1: EmailField = None + contact_1_email_1_type: EmailTypeField = None + contact_1_email_2: EmailField = None + contact_1_email_2_type: EmailTypeField = None + contact_1_address_1_line_1: Optional[str] = None + contact_1_address_1_line_2: Optional[str] = None + contact_1_address_1_type: AddressTypeField = None + contact_1_address_1_state: StateField = None + contact_1_address_1_city: Optional[str] = None + contact_1_address_1_postal_code: PostalCodeField = None + contact_1_address_2_line_1: Optional[str] = None + contact_1_address_2_line_2: Optional[str] = None + contact_1_address_2_type: AddressTypeField = None + contact_1_address_2_state: StateField = None + contact_1_address_2_city: Optional[str] = None + contact_1_address_2_postal_code: PostalCodeField = None + + contact_2_name: Optional[str] = None + contact_2_organization: Optional[str] = None + contact_2_role: ContactRoleField = None + contact_2_type: ContactTypeField = None + contact_2_phone_1: PhoneField = None + contact_2_phone_1_type: PhoneTypeField = None + contact_2_phone_2: PhoneField = None + contact_2_phone_2_type: PhoneTypeField = None + contact_2_email_1: EmailField = None + contact_2_email_1_type: EmailTypeField = None + contact_2_email_2: EmailField = None + contact_2_email_2_type: EmailTypeField = None + contact_2_address_1_line_1: Optional[str] = None + contact_2_address_1_line_2: Optional[str] = None + contact_2_address_1_type: AddressTypeField = None + contact_2_address_1_state: StateField = None + contact_2_address_1_city: Optional[str] = None + contact_2_address_1_postal_code: PostalCodeField = None + contact_2_address_2_line_1: Optional[str] = None + contact_2_address_2_line_2: Optional[str] = None + contact_2_address_2_type: AddressTypeField = None + contact_2_address_2_state: StateField = None + contact_2_address_2_city: Optional[str] = None + contact_2_address_2_postal_code: PostalCodeField = None + + directions_to_site: Optional[str] = None + specific_location_of_well: Optional[str] = None + repeat_measurement_permission: OptionalBool = None + sampling_permission: OptionalBool = None + datalogger_installation_permission: OptionalBool = None + public_availability_acknowledgement: OptionalBool = None # TODO: needs a home + special_requests: Optional[str] = None + ose_well_record_id: Optional[str] = None + date_drilled: OptionalPastOrTodayDate = None + completion_source: Optional[str] = None + total_well_depth_ft: OptionalFloat = None + historic_depth_to_water_ft: OptionalFloat = None + depth_source: Optional[str] = None + well_pump_type: Optional[str] = None + well_pump_depth_ft: OptionalFloat = None + is_open: OptionalBool = None + datalogger_possible: OptionalBool = None + casing_diameter_ft: OptionalFloat = None + measuring_point_description: Optional[str] = None + well_purpose: WellPurposeField = None + well_purpose_2: WellPurposeField = None + well_hole_status: Optional[str] = None + monitoring_frequency: MonitoryFrequencyField = None + + result_communication_preference: Optional[str] = None + contact_special_requests_notes: Optional[str] = None + sampling_scenario_notes: Optional[str] = None + well_measuring_notes: Optional[str] = None + sample_possible: OptionalBool = None # TODO: needs a home + + # water levels + sampler: Optional[str] = None + sample_method: Optional[str] = None + measurement_date_time: OptionalPastOrTodayDateTime = None + mp_height: Optional[float] = None + level_status: Optional[str] = None + depth_to_water_ft: Optional[float] = None + data_quality: Optional[str] = None + water_level_notes: Optional[str] = None # TODO: needs a home + + @field_validator("date_time", mode="before") + def make_date_time_tz_aware(cls, v): + if isinstance(v, str): + dt = datetime.fromisoformat(v) + elif isinstance(v, datetime): + dt = v + else: + raise ValueError("date_time must be a datetime or ISO format string") + + if dt.tzinfo is None: + aware_dt = convert_dt_tz_naive_to_tz_aware(dt, "America/Denver") + return aware_dt + else: + raise ValueError("date_time must be a timezone-naive datetime") + + @model_validator(mode="after") + def validate_model(self): + + optional_wl = ( + "sampler", + "sample_method", + "measurement_date_time", + "mp_height", + "level_status", + "depth_to_water_ft", + "data_quality", + "water_level_notes", + ) + + wl_fields = [getattr(self, a) for a in optional_wl] + if any(wl_fields): + if not all(wl_fields): + raise ValueError("All water level fields must be provided") + + # verify utm in NM + zone = int(self.utm_zone[:-1]) + northern = self.utm_zone[-1] + if northern.upper() not in ("S", "N"): + raise ValueError("Invalid utm zone. Must end in S or N. e.g 13N") + + northern = self.utm_zone[-1] == "N" + lat, lon = utm.to_latlon( + self.utm_easting, self.utm_northing, zone, northern=northern + ) + if not ((31.33 <= lat <= 37.00) and (-109.05 <= lon <= -103.00)): + raise ValueError( + f"UTM coordinates are outside of the NM. E={self.utm_easting} N={self.utm_northing}" + f" Zone={self.utm_zone}" + ) + + required_attrs = ("line_1", "type", "state", "city", "postal_code") + all_attrs = ("line_1", "line_2", "type", "state", "city", "postal_code") + for jdx in (1, 2): + key = f"contact_{jdx}" + # Check if any contact data is provided + name = getattr(self, f"{key}_name") + organization = getattr(self, f"{key}_organization") + has_contact_data = any( + [ + name, + organization, + getattr(self, f"{key}_role"), + getattr(self, f"{key}_type"), + *[getattr(self, f"{key}_email_{i}", None) for i in (1, 2)], + *[getattr(self, f"{key}_phone_{i}", None) for i in (1, 2)], + *[ + getattr(self, f"{key}_address_{i}_{a}", None) + for i in (1, 2) + for a in all_attrs + ], + ] + ) + + # If any contact data is provided, both name and organization are required + if has_contact_data: + if not name: + raise ValueError( + f"{key}_name is required when other contact fields are provided" + ) + if not organization: + raise ValueError( + f"{key}_organization is required when other contact fields are provided" + ) + for idx in (1, 2): + if any(getattr(self, f"{key}_address_{idx}_{a}") for a in all_attrs): + if not all( + getattr(self, f"{key}_address_{idx}_{a}") + for a in required_attrs + ): + raise ValueError("All contact address fields must be provided") + + name = getattr(self, f"{key}_name") + if name: + if not getattr(self, f"{key}_role"): + raise ValueError( + f"{key}_role must be provided if name is provided" + ) + if not getattr(self, f"{key}_type"): + raise ValueError( + f"{key}_type must be provided if name is provided" + ) + + phone = getattr(self, f"{key}_phone_{idx}") + tag = f"{key}_phone_{idx}_type" + phone_type = getattr(self, f"{key}_phone_{idx}_type") + if phone and not phone_type: + raise ValueError( + f"{tag} must be provided if phone number is provided" + ) + + email = getattr(self, f"{key}_email_{idx}") + tag = f"{key}_email_{idx}_type" + email_type = getattr(self, tag) + if email and not email_type: + raise ValueError( + f"{tag} type must be provided if email is provided" + ) + + return self diff --git a/services/contact_helper.py b/services/contact_helper.py index 942293e70..5e9766be9 100644 --- a/services/contact_helper.py +++ b/services/contact_helper.py @@ -13,15 +13,15 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== +from fastapi_pagination.ext.sqlalchemy import paginate +from sqlalchemy.orm import Session, joinedload + from db.contact import Contact, Email, Phone, Address, ThingContactAssociation from schemas.contact import ( CreateContact, ) -from services.query_helper import order_sort_filter from services.audit_helper import audit_add - -from fastapi_pagination.ext.sqlalchemy import paginate -from sqlalchemy.orm import Session, joinedload +from services.query_helper import order_sort_filter def get_db_contacts( @@ -62,6 +62,7 @@ def add_contact(session: Session, data: CreateContact | dict, user: dict) -> Con phone_data = data.pop("phones", []) address_data = data.pop("addresses", []) thing_id = data.pop("thing_id", None) + notes_data = data.pop("notes", None) contact_data = data """ @@ -96,20 +97,28 @@ def add_contact(session: Session, data: CreateContact | dict, user: dict) -> Con session.add(contact) session.flush() session.refresh(contact) + if thing_id is not None: + thing_contact_association = ThingContactAssociation() + thing_contact_association.thing_id = thing_id + thing_contact_association.contact_id = contact.id - location_contact_association = ThingContactAssociation() - location_contact_association.thing_id = thing_id - location_contact_association.contact_id = contact.id + audit_add(user, thing_contact_association) + session.add(thing_contact_association) - audit_add(user, location_contact_association) - - session.add(location_contact_association) - # owner_contact_association = OwnerContactAssociation() - # owner_contact_association.owner_id = owner.id - # owner_contact_association.contact_id = contact.id - # session.add(owner_contact_association) session.flush() session.commit() + + if notes_data is not None: + for n in notes_data: + note = contact.add_note(n["content"], n["note_type"]) + session.add(note) + + session.commit() + session.refresh(contact) + + for note in contact.notes: + session.refresh(note) + except Exception as e: session.rollback() raise e diff --git a/services/query_helper.py b/services/query_helper.py index 970ad1720..74835a33b 100644 --- a/services/query_helper.py +++ b/services/query_helper.py @@ -157,12 +157,15 @@ def order_sort_filter( return sql -def paginated_all_getter(session, table, sort=None, order=None, filter_=None) -> Any: +def paginated_all_getter( + session, table, sort=None, order=None, filter_=None, sql=None +) -> Any: """ Helper function to get all records from the database with pagination. """ + if sql is None: + sql = select(table) - sql = select(table) sql = order_sort_filter(sql, table, sort, order, filter_) return paginate(query=sql, conn=session) diff --git a/services/thing_helper.py b/services/thing_helper.py index 731db8429..e7177b041 100644 --- a/services/thing_helper.py +++ b/services/thing_helper.py @@ -28,7 +28,6 @@ from db import ( LocationThingAssociation, Thing, - Base, Location, WellScreen, WellPurpose, @@ -36,6 +35,10 @@ ThingAquiferAssociation, GroupThingAssociation, MeasuringPointHistory, + DataProvenance, + ThingIdLink, + MonitoringFrequencyHistory, + StatusHistory, ) from services.audit_helper import audit_add @@ -49,7 +52,7 @@ "well_casing_materials": (WellCasingMaterial, "material"), } -WELL_LOADER_OPTIONS = [ +WATER_WELL_LOADER_OPTIONS = [ selectinload(Thing.location_associations).selectinload( LocationThingAssociation.location ), @@ -63,7 +66,7 @@ ), ] -WELL_THING_TYPE = "water well" +WATER_WELL_THING_TYPE = "water well" def wkb_to_geojson(wkb_element): @@ -92,11 +95,11 @@ def get_db_things( if thing_type: sql = sql.where(Thing.thing_type == thing_type) - if thing_type == WELL_THING_TYPE: - sql = sql.options(*WELL_LOADER_OPTIONS) + if thing_type == WATER_WELL_THING_TYPE: + sql = sql.options(*WATER_WELL_LOADER_OPTIONS) else: # add all eager loads for generic thing query until/unless GET /thing is deprecated - sql = sql.options(*WELL_LOADER_OPTIONS) + sql = sql.options(*WATER_WELL_LOADER_OPTIONS) if name: sql = sql.where(Thing.name == name) @@ -161,8 +164,8 @@ def get_thing_of_a_thing_type_by_id(session: Session, request: Request, thing_id thing_type = get_thing_type_from_request(request) sql = select(Thing).where(Thing.id == thing_id) - if thing_type == WELL_THING_TYPE: - sql = sql.options(*WELL_LOADER_OPTIONS) + if thing_type == WATER_WELL_THING_TYPE: + sql = sql.options(*WATER_WELL_LOADER_OPTIONS) thing = session.execute(sql).scalar_one_or_none() @@ -183,25 +186,50 @@ def add_thing( user: dict = None, request: Request | None = None, thing_type: str | None = None, # to be used only for data transfers, not the API -) -> Base: +) -> Thing: if request is not None: thing_type = get_thing_type_from_request(request) - if isinstance(data, BaseModel): - well_descriptor_table_list = list(WELL_DESCRIPTOR_MODEL_MAP.keys()) - data = data.model_dump(exclude=well_descriptor_table_list) + # Extract data for related tables - notes = None - if "notes" in data: - notes = data.pop("notes") + # --------- + # BEGIN UNIVERSAL THING RELATED TABLES + # --------- + notes = data.pop("notes", None) + alternate_ids = data.pop("alternate_ids", None) location_id = data.pop("location_id", None) + effective_start = data.get("first_visit_date") group_id = data.pop("group_id", None) + monitoring_frequencies = data.pop("monitoring_frequencies", None) + datalogger_suitability_status = data.pop("is_suitable_for_datalogger", None) + open_status = data.pop("is_open", None) - # Extract measuring point data (stored in separate history table, not as Thing columns) + # ---------- + # END UNIVERSAL THING RELATED TABLES + # ---------- + + # ---------- + # BEGIN WATER WELL SPECIFIC RELATED TABLES + # ---------- + + # measuring point info measuring_point_height = data.pop("measuring_point_height", None) measuring_point_description = data.pop("measuring_point_description", None) + # data provenance info + well_completion_date_source = data.pop("well_completion_date_source", None) + well_construction_method_source = data.pop("well_construction_method_source", None) + well_depth_source = data.pop("well_depth_source", None) + + # descriptor tables + well_purposes = data.pop("well_purposes", None) + well_casing_materials = data.pop("well_casing_materials", None) + + # ---------- + # END WATER WELL SPECIFIC RELATED TABLES + # ---------- + try: thing = Thing(**data) thing.thing_type = thing_type @@ -212,17 +240,105 @@ def add_thing( session.flush() session.refresh(thing) - # Create MeasuringPointHistory record if measuring_point_height provided - if measuring_point_height is not None: - measuring_point_history = MeasuringPointHistory( - thing_id=thing.id, - measuring_point_height=measuring_point_height, - measuring_point_description=measuring_point_description, - start_date=datetime.now(tz=ZoneInfo("UTC")), - end_date=None, - ) - audit_add(user, measuring_point_history) - session.add(measuring_point_history) + # ---------- + # BEGIN WATER WELL SPECIFIC LOGIC + # ---------- + + if thing_type == WATER_WELL_THING_TYPE: + + # Create MeasuringPointHistory record if measuring_point_height provided + if measuring_point_height is not None: + measuring_point_history = MeasuringPointHistory( + thing_id=thing.id, + measuring_point_height=measuring_point_height, + measuring_point_description=measuring_point_description, + start_date=datetime.now(tz=ZoneInfo("UTC")), + end_date=None, + ) + audit_add(user, measuring_point_history) + session.add(measuring_point_history) + + if well_completion_date_source is not None: + dp = DataProvenance( + target_id=thing.id, + target_table="thing", + field_name="well_completion_date", + origin_type=well_completion_date_source, + ) + audit_add(user, dp) + session.add(dp) + + if well_depth_source is not None: + dp = DataProvenance( + target_id=thing.id, + target_table="thing", + field_name="well_depth", + origin_type=well_depth_source, + ) + audit_add(user, dp) + session.add(dp) + + if well_construction_method_source is not None: + dp = DataProvenance( + target_id=thing.id, + target_table="thing", + field_name="well_construction_method", + origin_source=well_construction_method_source, + ) + audit_add(user, dp) + session.add(dp) + + if well_purposes: + for purpose in well_purposes: + wp = WellPurpose(thing_id=thing.id, purpose=purpose) + audit_add(user, wp) + session.add(wp) + + if well_casing_materials: + for material in well_casing_materials: + wcm = WellCasingMaterial(thing_id=thing.id, material=material) + audit_add(user, wcm) + session.add(wcm) + + if datalogger_suitability_status is not None: + if datalogger_suitability_status is True: + status_value = "Datalogger can be installed" + else: + status_value = "Datalogger cannot be installed" + dlss = StatusHistory( + target_id=thing.id, + target_table="thing", + status_value=status_value, + status_type="Datalogger Suitability Status", + start_date=effective_start, + end_date=None, + ) + audit_add(user, dlss) + session.add(dlss) + + if open_status is not None: + if open_status is True: + status_value = "Open" + else: + status_value = "Closed" + os_status = StatusHistory( + target_id=thing.id, + target_table="thing", + status_value=status_value, + status_type="Open Status", + start_date=effective_start, + end_date=None, + ) + audit_add(user, os_status) + session.add(os_status) + + # ---------- + # END WATER WELL SPECIFIC LOGIC + # ---------- + + # ---------- + # BEGIN UNIVERSAL THING RELATED LOGIC + # ---------- # endpoint catches ProgrammingError if location_id or group_id do not exist if group_id: @@ -233,23 +349,50 @@ def add_thing( session.add(assoc) if location_id is not None: - # TODO: how do we want to handle effective_start? is it the date it gets entered? assoc = LocationThingAssociation() audit_add(user, assoc) assoc.location_id = location_id assoc.thing_id = thing.id + assoc.effective_start = effective_start session.add(assoc) - session.commit() - session.refresh(thing) - if notes: for n in notes: - nn = thing.add_note(n["content"], n["note_type"]) - session.add(nn) + thing_note = thing.add_note(n["content"], n["note_type"]) + session.add(thing_note) session.commit() session.refresh(thing) + if alternate_ids: + for aid in alternate_ids: + id_link = ThingIdLink( + thing_id=thing.id, + relation=aid["relation"], + alternate_id=aid["alternate_id"], + alternate_organization=aid["alternate_organization"], + ) + session.add(id_link) + + if monitoring_frequencies: + for mf in monitoring_frequencies: + mfh = MonitoringFrequencyHistory( + thing_id=thing.id, + monitoring_frequency=mf["monitoring_frequency"], + start_date=mf["start_date"], + end_date=mf.get("end_date", None), + ) + session.add(mfh) + + # ---------- + # END UNIVERSAL THING RELATED LOGIC + # ---------- + + session.commit() + session.refresh(thing) + + for note in thing.notes: + session.refresh(note) + except Exception as e: session.rollback() raise e diff --git a/services/util.py b/services/util.py index 46e2f4cc1..80c917603 100644 --- a/services/util.py +++ b/services/util.py @@ -2,7 +2,8 @@ import logging import os import time - +from zoneinfo import ZoneInfo +from datetime import datetime import httpx import pyproj from shapely.ops import transform @@ -79,6 +80,23 @@ def transform_srid(geometry, source_srid, target_srid): return transform(transformer.transform, geometry) +def convert_dt_tz_naive_to_tz_aware( + dt_naive: datetime, iana_timezone: str = "America/Denver" +): + """ + Adds a timezone to a timezone-naive datetime object using + the specified ZoneInfo string. Since the input datetime is naive, + it is assumed to already be in the specified timezone. This function + does not perform any conversion of the datetime value itself. + """ + if dt_naive.tzinfo is not None: + raise ValueError("Input datetime must be timezone-naive.") + + tz = ZoneInfo(iana_timezone) + dt_aware = dt_naive.replace(tzinfo=tz) + return dt_aware + + def convert_ft_to_m(feet: float | None, ndigits: int = 6) -> float | None: """Convert a length from feet to meters.""" if feet is None: diff --git a/tests/conftest.py b/tests/conftest.py index dc51737ca..50423ad8e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -444,6 +444,14 @@ def contact(water_well_thing): session.commit() session.refresh(association) + for content, note_type in [ + ("Communication note", "Communication"), + ("General note", "General"), + ]: + note = contact.add_note(content, note_type) + session.add(note) + session.commit() + yield contact session.delete(contact) session.delete(association) diff --git a/tests/features/data/well-inventory-duplicate-columns.csv b/tests/features/data/well-inventory-duplicate-columns.csv new file mode 100644 index 000000000..8188528b0 --- /dev/null +++ b/tests/features/data/well-inventory-duplicate-columns.csv @@ -0,0 +1,3 @@ +project,well_name_point_id,site_name,date_time,field_staff,utm_easting,utm_northing,utm_zone,elevation_ft,elevation_method,measuring_point_height_ft,field_staff_2,field_staff_3,contact_1_name,contact_1_organization,contact_1_role,contact_1_type,contact_1_phone_1,contact_1_phone_1_type,contact_1_phone_2,contact_1_phone_2_type,contact_1_email_1,contact_1_email_1_type,contact_1_email_2,contact_1_email_2_type,contact_1_address_1_line_1,contact_1_address_1_line_2,contact_1_address_1_type,contact_1_address_1_state,contact_1_address_1_city,contact_1_address_1_postal_code,contact_1_address_2_line_1,contact_1_address_2_line_2,contact_1_address_2_type,contact_1_address_2_state,contact_1_address_2_city,contact_1_address_2_postal_code,contact_2_name,contact_2_organization,contact_2_role,contact_2_type,contact_2_phone_1,contact_2_phone_1_type,contact_2_phone_2,contact_2_phone_2_type,contact_2_email_1,contact_2_email_1_type,contact_2_email_2,contact_2_email_2_type,contact_2_address_1_line_1,contact_2_address_1_line_2,contact_2_address_1_type,contact_2_address_1_state,contact_2_address_1_city,contact_2_address_1_postal_code,contact_2_address_2_line_1,contact_2_address_2_line_2,contact_2_address_2_type,contact_2_address_2_state,contact_2_address_2_city,contact_2_address_2_postal_code,directions_to_site,specific_location_of_well,repeat_measurement_permission,sampling_permission,datalogger_installation_permission,public_availability_acknowledgement,result_communication_preference,contact_special_requests_notes,ose_well_record_id,date_drilled,completion_source,total_well_depth_ft,historic_depth_to_water_ft,depth_source,well_pump_type,well_pump_depth_ft,is_open,datalogger_possible,casing_diameter_ft,measuring_point_description,well_purpose,well_purpose_2,well_hole_status,monitoring_frequency,sampling_scenario_notes,well_measuring_notes,sample_possible,contact_1_email_1 +Middle Rio Grande Groundwater Monitoring,MRG-001_MP1,Smith Farm Domestic Well,2025-02-15T10:30:00,A Lopez,250000,4000000,13N,5250,Survey-grade GPS,1.5,B Chen,,John Smith,NMBGMR,Owner,Primary,505-555-0101,Primary,,,john.smith@example.com,Primary,,,123 County Rd 7,,Mailing,NM,Los Lunas,87031,,,,,,,Maria Garcia,NMBGMR,Principal Investigator,Secondary,505-555-0123,Home,,,maria.garcia@mrgcd.nm.gov,Work,,,1931 2nd St SW,Suite 200,Mailing,NM,Albuquerque,87102,,,,,,,Gate off County Rd 7 0.4 miles south of canal crossing,Domestic well in pump house east of residence,True,True,True,True,email,Call before visits during irrigation season,OSE-123456,2010-06-15,OSE well record,280,45,owner estimate,submersible,200,True,True,0.5,Top of steel casing inside pump house marked with orange paint,Domestic,,active,Biannual,Sample only when pump has been off more than 12 hours,Measure before owner starts irrigation,True,john.smith@example.com +Middle Rio Grande Groundwater Monitoring,MRG-003_MP1,Old Orchard Well,2025-01-20T09:00:00,B Chen,250000,4000000,13N,5320,Global positioning system (GPS),1.8,,,Emily Davis,NMBGMR,Biologist,Primary,505-555-0303,Work,,,emily.davis@example.org,Work,,,78 Orchard Ln,,Mailing,NM,Los Lunas,87031,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,From Main St turn east on Orchard Ln well house at dead end,Abandoned irrigation well in small cinderblock building,False,False,False,True,phone,Owner prefers weekday visits,,1965-04-10,historic log scanned,350,60,historic log,vertical turbine inactive,280,False,False,0.75,Top of steel casing under removable hatch use fixed reference mark,Irrigation,,abandoned,Annual,Sampling not permitted water level only when owner present,Well house can be locked coordinate ahead,False,emily.davis@example.org diff --git a/tests/features/data/well-inventory-duplicate-header.csv b/tests/features/data/well-inventory-duplicate-header.csv new file mode 100644 index 000000000..166f0e4e3 --- /dev/null +++ b/tests/features/data/well-inventory-duplicate-header.csv @@ -0,0 +1,5 @@ +project,well_name_point_id,site_name,date_time,field_staff,utm_easting,utm_northing,utm_zone,elevation_ft,elevation_method,measuring_point_height_ft,field_staff_2,field_staff_3,contact_1_name,contact_1_organization,contact_1_role,contact_1_type,contact_1_phone_1,contact_1_phone_1_type,contact_1_phone_2,contact_1_phone_2_type,contact_1_email_1,contact_1_email_1_type,contact_1_email_2,contact_1_email_2_type,contact_1_address_1_line_1,contact_1_address_1_line_2,contact_1_address_1_type,contact_1_address_1_state,contact_1_address_1_city,contact_1_address_1_postal_code,contact_1_address_2_line_1,contact_1_address_2_line_2,contact_1_address_2_type,contact_1_address_2_state,contact_1_address_2_city,contact_1_address_2_postal_code,contact_2_name,contact_2_organization,contact_2_role,contact_2_type,contact_2_phone_1,contact_2_phone_1_type,contact_2_phone_2,contact_2_phone_2_type,contact_2_email_1,contact_2_email_1_type,contact_2_email_2,contact_2_email_2_type,contact_2_address_1_line_1,contact_2_address_1_line_2,contact_2_address_1_type,contact_2_address_1_state,contact_2_address_1_city,contact_2_address_1_postal_code,contact_2_address_2_line_1,contact_2_address_2_line_2,contact_2_address_2_type,contact_2_address_2_state,contact_2_address_2_city,contact_2_address_2_postal_code,directions_to_site,specific_location_of_well,repeat_measurement_permission,sampling_permission,datalogger_installation_permission,public_availability_acknowledgement,result_communication_preference,contact_special_requests_notes,ose_well_record_id,date_drilled,completion_source,total_well_depth_ft,historic_depth_to_water_ft,depth_source,well_pump_type,well_pump_depth_ft,is_open,datalogger_possible,casing_diameter_ft,measuring_point_description,well_purpose,well_purpose_2,well_hole_status,monitoring_frequency,sampling_scenario_notes,well_measuring_notes,sample_possible +Middle Rio Grande Groundwater Monitoring,MRG-001_MP1,Smith Farm Domestic Well,2025-02-15T10:30:00,A Lopez,250000,4000000,13N,5250,Survey-grade GPS,1.5,B Chen,,John Smith,NMBGMR,Owner,Primary,505-555-0101,Primary,,,john.smith@example.com,Primary,,,123 County Rd 7,,Mailing,NM,Los Lunas,87031,,,,,,,Maria Garcia,NMBGMR,Principal Investigator,Secondary,505-555-0123,Home,,,maria.garcia@mrgcd.nm.gov,Work,,,1931 2nd St SW,Suite 200,Mailing,NM,Albuquerque,87102,,,,,,,Gate off County Rd 7 0.4 miles south of canal crossing,Domestic well in pump house east of residence,True,True,True,True,email,Call before visits during irrigation season,OSE-123456,2010-06-15,OSE well record,280,45,owner estimate,submersible,200,True,True,0.5,Top of steel casing inside pump house marked with orange paint,Domestic,,active,Biannual,Sample only when pump has been off more than 12 hours,Measure before owner starts irrigation,True +Middle Rio Grande Groundwater Monitoring,MRG-003_MP1,Old Orchard Well,2025-01-20T09:00:00,B Chen,250000,4000000,13N,5320,Global positioning system (GPS),1.8,,,Emily Davis,NMBGMR,Biologist,Primary,505-555-0303,Work,,,emily.davis@example.org,Work,,,78 Orchard Ln,,Mailing,NM,Los Lunas,87031,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,From Main St turn east on Orchard Ln well house at dead end,Abandoned irrigation well in small cinderblock building,False,False,False,True,phone,Owner prefers weekday visits,,1965-04-10,historic log scanned,350,60,historic log,vertical turbine inactive,280,False,False,0.75,Top of steel casing under removable hatch use fixed reference mark,Irrigation,,abandoned,Annual,Sampling not permitted water level only when owner present,Well house can be locked coordinate ahead,False +project,well_name_point_id,site_name,date_time,field_staff,utm_easting,utm_northing,utm_zone,elevation_ft,elevation_method,measuring_point_height_ft,field_staff_2,field_staff_3,contact_1_name,contact_1_organization,contact_1_role,contact_1_type,contact_1_phone_1,contact_1_phone_1_type,contact_1_phone_2,contact_1_phone_2_type,contact_1_email_1,contact_1_email_1_type,contact_1_email_2,contact_1_email_2_type,contact_1_address_1_line_1,contact_1_address_1_line_2,contact_1_address_1_type,contact_1_address_1_state,contact_1_address_1_city,contact_1_address_1_postal_code,contact_1_address_2_line_1,contact_1_address_2_line_2,contact_1_address_2_type,contact_1_address_2_state,contact_1_address_2_city,contact_1_address_2_postal_code,contact_2_name,contact_2_organization,contact_2_role,contact_2_type,contact_2_phone_1,contact_2_phone_1_type,contact_2_phone_2,contact_2_phone_2_type,contact_2_email_1,contact_2_email_1_type,contact_2_email_2,contact_2_email_2_type,contact_2_address_1_line_1,contact_2_address_1_line_2,contact_2_address_1_type,contact_2_address_1_state,contact_2_address_1_city,contact_2_address_1_postal_code,contact_2_address_2_line_1,contact_2_address_2_line_2,contact_2_address_2_type,contact_2_address_2_state,contact_2_address_2_city,contact_2_address_2_postal_code,directions_to_site,specific_location_of_well,repeat_measurement_permission,sampling_permission,datalogger_installation_permission,public_availability_acknowledgement,result_communication_preference,contact_special_requests_notes,ose_well_record_id,date_drilled,completion_source,total_well_depth_ft,historic_depth_to_water_ft,depth_source,well_pump_type,well_pump_depth_ft,is_open,datalogger_possible,casing_diameter_ft,measuring_point_description,well_purpose,well_purpose_2,well_hole_status,monitoring_frequency,sampling_scenario_notes,well_measuring_notes,sample_possible +Middle Rio Grande Groundwater Monitoring,MRG-001_MP1f,Smith Farm Domestic Well,2025-02-15T10:30:00,A Lopez,250000,4000000,13N,5250,Survey-grade GPS,1.5,B Chen,,John Smith,NMBGMR,Owner,Primary,505-555-0101,Primary,,,john.smith@example.com,Primary,,,123 County Rd 7,,Mailing,NM,Los Lunas,87031,,,,,,,Maria Garcia,NMBGMR,Principal Investigator,Secondary,505-555-0123,Home,,,maria.garcia@mrgcd.nm.gov,Work,,,1931 2nd St SW,Suite 200,Mailing,NM,Albuquerque,87102,,,,,,,Gate off County Rd 7 0.4 miles south of canal crossing,Domestic well in pump house east of residence,True,True,True,True,email,Call before visits during irrigation season,OSE-123456,2010-06-15,OSE well record,280,45,owner estimate,submersible,200,True,True,0.5,Top of steel casing inside pump house marked with orange paint,Domestic,,active,Biannual,Sample only when pump has been off more than 12 hours,Measure before owner starts irrigation,True \ No newline at end of file diff --git a/tests/features/data/well-inventory-duplicate.csv b/tests/features/data/well-inventory-duplicate.csv new file mode 100644 index 000000000..4f8ac75ad --- /dev/null +++ b/tests/features/data/well-inventory-duplicate.csv @@ -0,0 +1,3 @@ +project,measuring_point_height_ft,well_name_point_id,site_name,date_time,field_staff,contact_role,utm_easting,utm_northing,utm_zone,elevation_ft,elevation_method +foo,10,WELL001,Site Alpha,2025-02-15T10:30:00,Jane Doe,Owner,250000,4000000,13N,5120.5,LiDAR DEM +foob,10,WELL001,Site Beta,2025-03-20T09:15:00,John Smith,Manager,250000,4000000,13N,5130.7,LiDAR DEM diff --git a/tests/features/data/well-inventory-empty.csv b/tests/features/data/well-inventory-empty.csv new file mode 100644 index 000000000..e69de29bb diff --git a/tests/features/data/well-inventory-invalid-boolean-value-maybe.csv b/tests/features/data/well-inventory-invalid-boolean-value-maybe.csv new file mode 100644 index 000000000..1f7c1184b --- /dev/null +++ b/tests/features/data/well-inventory-invalid-boolean-value-maybe.csv @@ -0,0 +1,3 @@ +project,well_name_point_id,site_name,date_time,field_staff,utm_easting,utm_northing,utm_zone,elevation_ft,elevation_method,measuring_point_height_ft,field_staff_2,field_staff_3,contact_1_name,contact_1_organization,contact_1_role,contact_1_type,contact_1_phone_1,contact_1_phone_1_type,contact_1_phone_2,contact_1_phone_2_type,contact_1_email_1,contact_1_email_1_type,contact_1_email_2,contact_1_email_2_type,contact_1_address_1_line_1,contact_1_address_1_line_2,contact_1_address_1_type,contact_1_address_1_state,contact_1_address_1_city,contact_1_address_1_postal_code,contact_1_address_2_line_1,contact_1_address_2_line_2,contact_1_address_2_type,contact_1_address_2_state,contact_1_address_2_city,contact_1_address_2_postal_code,contact_2_name,contact_2_organization,contact_2_role,contact_2_type,contact_2_phone_1,contact_2_phone_1_type,contact_2_phone_2,contact_2_phone_2_type,contact_2_email_1,contact_2_email_1_type,contact_2_email_2,contact_2_email_2_type,contact_2_address_1_line_1,contact_2_address_1_line_2,contact_2_address_1_type,contact_2_address_1_state,contact_2_address_1_city,contact_2_address_1_postal_code,contact_2_address_2_line_1,contact_2_address_2_line_2,contact_2_address_2_type,contact_2_address_2_state,contact_2_address_2_city,contact_2_address_2_postal_code,directions_to_site,specific_location_of_well,repeat_measurement_permission,sampling_permission,datalogger_installation_permission,public_availability_acknowledgement,result_communication_preference,contact_special_requests_notes,ose_well_record_id,date_drilled,completion_source,total_well_depth_ft,historic_depth_to_water_ft,depth_source,well_pump_type,well_pump_depth_ft,is_open,datalogger_possible,casing_diameter_ft,measuring_point_description,well_purpose,well_purpose_2,well_hole_status,monitoring_frequency,sampling_scenario_notes,well_measuring_notes,sample_possible +Middle Rio Grande Groundwater Monitoring,MRG-001_MP1,Smith Farm Domestic Well,2025-02-15T10:30:00,A Lopez,250000,4000000,13N,5250,Survey-grade GPS,1.5,B Chen,,John Smith,NMBGMR,Owner,Primary,505-555-0101,Primary,,,john.smith@example.com,Primary,,,123 County Rd 7,,Mailing,NM,Los Lunas,87031,,,,,,,Maria Garcia,NMBGMR,Principal Investigator,Secondary,505-555-0123,Home,,,maria.garcia@mrgcd.nm.gov,Work,,,1931 2nd St SW,Suite 200,Mailing,NM,Albuquerque,87102,,,,,,,Gate off County Rd 7 0.4 miles south of canal crossing,Domestic well in pump house east of residence,True,True,True,True,email,Call before visits during irrigation season,OSE-123456,2010-06-15,OSE well record,280,45,owner estimate,submersible,200,True,True,0.5,Top of steel casing inside pump house marked with orange paint,Domestic,,active,Biannual,Sample only when pump has been off more than 12 hours,Measure before owner starts irrigation,True +Middle Rio Grande Groundwater Monitoring,MRG-003_MP1,Old Orchard Well,2025-01-20T09:00:00,B Chen,250000,4000000,13N,5320,Global positioning system (GPS),1.8,,,Emily Davis,NMBGMR,Biologist,Primary,505-555-0303,Work,,,emily.davis@example.org,Work,,,78 Orchard Ln,,Mailing,NM,Los Lunas,87031,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,From Main St turn east on Orchard Ln well house at dead end,Abandoned irrigation well in small cinderblock building,False,False,False,True,phone,Owner prefers weekday visits,,1965-04-10,historic log scanned,350,60,historic log,vertical turbine inactive,280,maybe,False,0.75,Top of steel casing under removable hatch use fixed reference mark,Irrigation,,abandoned,Annual,Sampling not permitted water level only when owner present,Well house can be locked coordinate ahead,False diff --git a/tests/features/data/well-inventory-invalid-contact-type.csv b/tests/features/data/well-inventory-invalid-contact-type.csv new file mode 100644 index 000000000..90898e9b7 --- /dev/null +++ b/tests/features/data/well-inventory-invalid-contact-type.csv @@ -0,0 +1,3 @@ +project,well_name_point_id,site_name,date_time,field_staff,utm_easting,utm_northing,utm_zone,elevation_ft,elevation_method,measuring_point_height_ft,field_staff_2,field_staff_3,contact_1_name,contact_1_organization,contact_1_role,contact_1_type,contact_1_phone_1,contact_1_phone_1_type,contact_1_phone_2,contact_1_phone_2_type,contact_1_email_1,contact_1_email_1_type,contact_1_email_2,contact_1_email_2_type,contact_1_address_1_line_1,contact_1_address_1_line_2,contact_1_address_1_type,contact_1_address_1_state,contact_1_address_1_city,contact_1_address_1_postal_code,contact_1_address_2_line_1,contact_1_address_2_line_2,contact_1_address_2_type,contact_1_address_2_state,contact_1_address_2_city,contact_1_address_2_postal_code,contact_2_name,contact_2_organization,contact_2_role,contact_2_type,contact_2_phone_1,contact_2_phone_1_type,contact_2_phone_2,contact_2_phone_2_type,contact_2_email_1,contact_2_email_1_type,contact_2_email_2,contact_2_email_2_type,contact_2_address_1_line_1,contact_2_address_1_line_2,contact_2_address_1_type,contact_2_address_1_state,contact_2_address_1_city,contact_2_address_1_postal_code,contact_2_address_2_line_1,contact_2_address_2_line_2,contact_2_address_2_type,contact_2_address_2_state,contact_2_address_2_city,contact_2_address_2_postal_code,directions_to_site,specific_location_of_well,repeat_measurement_permission,sampling_permission,datalogger_installation_permission,public_availability_acknowledgement,result_communication_preference,contact_special_requests_notes,ose_well_record_id,date_drilled,completion_source,total_well_depth_ft,historic_depth_to_water_ft,depth_source,well_pump_type,well_pump_depth_ft,is_open,datalogger_possible,casing_diameter_ft,measuring_point_description,well_purpose,well_purpose_2,well_hole_status,monitoring_frequency,sampling_scenario_notes,well_measuring_notes,sample_possible +Middle Rio Grande Groundwater Monitoring,MRG-001_MP1,Smith Farm Domestic Well,2025-02-15T10:30:00,A Lopez,250000,4000000,13N,5250,Survey-grade GPS,1.5,B Chen,,John Smith,NMBGMR,Owner,foo,505-555-0101,Primary,,,john.smith@example.com,Primary,,,123 County Rd 7,,Mailing,NM,Los Lunas,87031,,,,,,,Maria Garcia,NMBGMR,Principal Investigator,Secondary,505-555-0123,Home,,,maria.garcia@mrgcd.nm.gov,Work,,,1931 2nd St SW,Suite 200,Mailing,NM,Albuquerque,87102,,,,,,,Gate off County Rd 7 0.4 miles south of canal crossing,Domestic well in pump house east of residence,True,True,True,True,email,Call before visits during irrigation season,OSE-123456,2010-06-15,OSE well record,280,45,owner estimate,submersible,200,True,True,0.5,Top of steel casing inside pump house marked with orange paint,Domestic,,active,Biannual,Sample only when pump has been off more than 12 hours,Measure before owner starts irrigation,True +Middle Rio Grande Groundwater Monitoring,MRG-003_MP1,Old Orchard Well,2025-01-20T09:00:00,B Chen,250000,4000000,13N,5320,Global positioning system (GPS),1.8,,,Emily Davis,NMBGMR,Biologist,Primary,505-555-0303,Work,,,emily.davis@example.org,Work,,,78 Orchard Ln,,Mailing,NM,Los Lunas,87031,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,From Main St turn east on Orchard Ln well house at dead end,Abandoned irrigation well in small cinderblock building,False,False,False,True,phone,Owner prefers weekday visits,,1965-04-10,historic log scanned,350,60,historic log,vertical turbine inactive,280,False,False,0.75,Top of steel casing under removable hatch use fixed reference mark,Irrigation,,abandoned,Annual,Sampling not permitted water level only when owner present,Well house can be locked coordinate ahead,False diff --git a/tests/features/data/well-inventory-invalid-date-format.csv b/tests/features/data/well-inventory-invalid-date-format.csv new file mode 100644 index 000000000..179f659e7 --- /dev/null +++ b/tests/features/data/well-inventory-invalid-date-format.csv @@ -0,0 +1,3 @@ +project,well_name_point_id,site_name,date_time,field_staff,utm_easting,utm_northing,utm_zone,elevation_ft,elevation_method,measuring_point_height_ft,field_staff_2,field_staff_3,contact_1_name,contact_1_organization,contact_1_role,contact_1_type,contact_1_phone_1,contact_1_phone_1_type,contact_1_phone_2,contact_1_phone_2_type,contact_1_email_1,contact_1_email_1_type,contact_1_email_2,contact_1_email_2_type,contact_1_address_1_line_1,contact_1_address_1_line_2,contact_1_address_1_type,contact_1_address_1_state,contact_1_address_1_city,contact_1_address_1_postal_code,contact_1_address_2_line_1,contact_1_address_2_line_2,contact_1_address_2_type,contact_1_address_2_state,contact_1_address_2_city,contact_1_address_2_postal_code,contact_2_name,contact_2_organization,contact_2_role,contact_2_type,contact_2_phone_1,contact_2_phone_1_type,contact_2_phone_2,contact_2_phone_2_type,contact_2_email_1,contact_2_email_1_type,contact_2_email_2,contact_2_email_2_type,contact_2_address_1_line_1,contact_2_address_1_line_2,contact_2_address_1_type,contact_2_address_1_state,contact_2_address_1_city,contact_2_address_1_postal_code,contact_2_address_2_line_1,contact_2_address_2_line_2,contact_2_address_2_type,contact_2_address_2_state,contact_2_address_2_city,contact_2_address_2_postal_code,directions_to_site,specific_location_of_well,repeat_measurement_permission,sampling_permission,datalogger_installation_permission,public_availability_acknowledgement,result_communication_preference,contact_special_requests_notes,ose_well_record_id,date_drilled,completion_source,total_well_depth_ft,historic_depth_to_water_ft,depth_source,well_pump_type,well_pump_depth_ft,is_open,datalogger_possible,casing_diameter_ft,measuring_point_description,well_purpose,well_purpose_2,well_hole_status,monitoring_frequency,sampling_scenario_notes,well_measuring_notes,sample_possible +Middle Rio Grande Groundwater Monitoring,MRG-001_MP1,Smith Farm Domestic Well,25-02-15T10:30:00,A Lopez,250000,4000000,13N,5250,Survey-grade GPS,1.5,B Chen,,John Smith,NMBGMR,Owner,Primary,505-555-0101,Primary,,,john.smith@example.com,Primary,,,123 County Rd 7,,Mailing,NM,Los Lunas,87031,,,,,,,Maria Garcia,NMBGMR,Principal Investigator,Secondary,505-555-0123,Home,,,maria.garcia@mrgcd.nm.gov,Work,,,1931 2nd St SW,Suite 200,Mailing,NM,Albuquerque,87102,,,,,,,Gate off County Rd 7 0.4 miles south of canal crossing,Domestic well in pump house east of residence,True,True,True,True,email,Call before visits during irrigation season,OSE-123456,2010-06-15,OSE well record,280,45,owner estimate,submersible,200,True,True,0.5,Top of steel casing inside pump house marked with orange paint,Domestic,,active,Biannual,Sample only when pump has been off more than 12 hours,Measure before owner starts irrigation,True +Middle Rio Grande Groundwater Monitoring,MRG-003_MP1,Old Orchard Well,2025-01-20T09:00:00,B Chen,250000,4000000,13N,5320,Global positioning system (GPS),1.8,,,Emily Davis,NMBGMR,Biologist,Primary,505-555-0303,Work,,,emily.davis@example.org,Work,,,78 Orchard Ln,,Mailing,NM,Los Lunas,87031,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,From Main St turn east on Orchard Ln well house at dead end,Abandoned irrigation well in small cinderblock building,False,False,False,True,phone,Owner prefers weekday visits,,1965-04-10,historic log scanned,350,60,historic log,vertical turbine inactive,280,False,False,0.75,Top of steel casing under removable hatch use fixed reference mark,Irrigation,,abandoned,Annual,Sampling not permitted water level only when owner present,Well house can be locked coordinate ahead,False diff --git a/tests/features/data/well-inventory-invalid-date.csv b/tests/features/data/well-inventory-invalid-date.csv new file mode 100644 index 000000000..697f9c296 --- /dev/null +++ b/tests/features/data/well-inventory-invalid-date.csv @@ -0,0 +1,5 @@ +well_name_point_id,site_name,date_time,field_staff,contact_role,utm_easting,utm_northing,utm_zone,elevation_ft,elevation_method +WELL005,Site Alpha,2025-02-30T10:30:0,Jane Doe,Owner,250000,4000000,13N,5120.5,GPS +WELL006,Site Beta,2025-13-20T09:15:00,John Smith,Manager,250000,4000000,13N,5130.7,Survey +WELL007,Site Gamma,not-a-date,Emily Clark,Supervisor,250000,4000000,13N,5150.3,Survey +WELL008,Site Delta,2025-04-10 11:00:00,Michael Lee,Technician,250000,4000000,13N,5160.4,GPS diff --git a/tests/features/data/well-inventory-invalid-email.csv b/tests/features/data/well-inventory-invalid-email.csv new file mode 100644 index 000000000..7e2ca2e3d --- /dev/null +++ b/tests/features/data/well-inventory-invalid-email.csv @@ -0,0 +1,3 @@ +project,well_name_point_id,site_name,date_time,field_staff,utm_easting,utm_northing,utm_zone,elevation_ft,elevation_method,measuring_point_height_ft,field_staff_2,field_staff_3,contact_1_name,contact_1_organization,contact_1_role,contact_1_type,contact_1_phone_1,contact_1_phone_1_type,contact_1_phone_2,contact_1_phone_2_type,contact_1_email_1,contact_1_email_1_type,contact_1_email_2,contact_1_email_2_type,contact_1_address_1_line_1,contact_1_address_1_line_2,contact_1_address_1_type,contact_1_address_1_state,contact_1_address_1_city,contact_1_address_1_postal_code,contact_1_address_2_line_1,contact_1_address_2_line_2,contact_1_address_2_type,contact_1_address_2_state,contact_1_address_2_city,contact_1_address_2_postal_code,contact_2_name,contact_2_organization,contact_2_role,contact_2_type,contact_2_phone_1,contact_2_phone_1_type,contact_2_phone_2,contact_2_phone_2_type,contact_2_email_1,contact_2_email_1_type,contact_2_email_2,contact_2_email_2_type,contact_2_address_1_line_1,contact_2_address_1_line_2,contact_2_address_1_type,contact_2_address_1_state,contact_2_address_1_city,contact_2_address_1_postal_code,contact_2_address_2_line_1,contact_2_address_2_line_2,contact_2_address_2_type,contact_2_address_2_state,contact_2_address_2_city,contact_2_address_2_postal_code,directions_to_site,specific_location_of_well,repeat_measurement_permission,sampling_permission,datalogger_installation_permission,public_availability_acknowledgement,result_communication_preference,contact_special_requests_notes,ose_well_record_id,date_drilled,completion_source,total_well_depth_ft,historic_depth_to_water_ft,depth_source,well_pump_type,well_pump_depth_ft,is_open,datalogger_possible,casing_diameter_ft,measuring_point_description,well_purpose,well_purpose_2,well_hole_status,monitoring_frequency,sampling_scenario_notes,well_measuring_notes,sample_possible +Middle Rio Grande Groundwater Monitoring,MRG-001_MP1,Smith Farm Domestic Well,2025-02-15T10:30:00,A Lopez,250000,4000000,13N,5250,Survey-grade GPS,1.5,B Chen,,John Smith,NMBGMR,Owner,Primary,505-555-0101,Primary,,,john.smithexample.com,Primary,,,123 County Rd 7,,Mailing,NM,Los Lunas,87031,,,,,,,Maria Garcia,NMBGMR,Principal Investigator,Secondary,505-555-0123,Home,,,maria.garcia@mrgcd.nm.gov,Work,,,1931 2nd St SW,Suite 200,Mailing,NM,Albuquerque,87102,,,,,,,Gate off County Rd 7 0.4 miles south of canal crossing,Domestic well in pump house east of residence,True,True,True,True,email,Call before visits during irrigation season,OSE-123456,2010-06-15,OSE well record,280,45,owner estimate,submersible,200,True,True,0.5,Top of steel casing inside pump house marked with orange paint,Domestic,,active,Biannual,Sample only when pump has been off more than 12 hours,Measure before owner starts irrigation,True +Middle Rio Grande Groundwater Monitoring,MRG-003_MP1,Old Orchard Well,2025-01-20T09:00:00,B Chen,250000,4000000,13N,5320,Global positioning system (GPS),1.8,,,Emily Davis,NMBGMR,Biologist,Primary,505-555-0303,Work,,,emily.davis@example.org,Work,,,78 Orchard Ln,,Mailing,NM,Los Lunas,87031,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,From Main St turn east on Orchard Ln well house at dead end,Abandoned irrigation well in small cinderblock building,False,False,False,True,phone,Owner prefers weekday visits,,1965-04-10,historic log scanned,350,60,historic log,vertical turbine inactive,280,False,False,0.75,Top of steel casing under removable hatch use fixed reference mark,Irrigation,,abandoned,Annual,Sampling not permitted water level only when owner present,Well house can be locked coordinate ahead,False diff --git a/tests/features/data/well-inventory-invalid-filetype.txt b/tests/features/data/well-inventory-invalid-filetype.txt new file mode 100644 index 000000000..e69de29bb diff --git a/tests/features/data/well-inventory-invalid-lexicon.csv b/tests/features/data/well-inventory-invalid-lexicon.csv new file mode 100644 index 000000000..f9f5dda43 --- /dev/null +++ b/tests/features/data/well-inventory-invalid-lexicon.csv @@ -0,0 +1,5 @@ +project,well_name_point_id,site_name,date_time,field_staff,utm_easting,utm_northing,utm_zone,elevation_ft,elevation_method,measuring_point_height_ft,contact_role,contact_type +ProjectA,WELL001,Site1,2025-02-15T10:30:00,John Doe,250000,4000000,13N,5000,Survey,2.5,INVALID_ROLE,owner +ProjectB,WELL002,Site2,2025-02-16T11:00:00,Jane Smith,250000,4000000,13N,5100,Survey,2.7,manager,INVALID_TYPE +ProjectC,WELL003,Site3,2025-02-17T09:45:00,Jim Beam,250000,4000000,13N,5200,INVALID_METHOD,2.6,manager,owner +ProjectD,WELL004,Site4,2025-02-18T08:20:00,Jack Daniels,250000,4000000,13N,5300,Survey,2.8,INVALID_ROLE,INVALID_TYPE diff --git a/tests/features/data/well-inventory-invalid-numeric.csv b/tests/features/data/well-inventory-invalid-numeric.csv new file mode 100644 index 000000000..40675dc6b --- /dev/null +++ b/tests/features/data/well-inventory-invalid-numeric.csv @@ -0,0 +1,6 @@ +project,well_name_point_id,site_name,date_time,field_staff,utm_easting,utm_northing,utm_zone,elevation_ft,elevation_method,measuring_point_height_ft +ProjectA,WELL001,Site1,2025-02-15T10:30:00,John Doe,250000,4000000,13N,5000,Survey,2.5 +ProjectB,WELL002,Site2,2025-02-16T11:00:00,Jane Smith,250000,4000000,13N,5100,Survey,2.7 +ProjectC,WELL003,Site3,2025-02-17T09:45:00,Jim Beam,250000,4000000,13N,5200,Survey,2.6 +ProjectD,WELL004,Site4,2025-02-18T08:20:00,Jack Daniels,250000,4000000,13N,elev_bad,Survey,2.8 +ProjectE,WELL005,Site5,2025-02-19T12:00:00,Jill Hill,250000,4000000,13N,5300,Survey,not_a_height diff --git a/tests/features/data/well-inventory-invalid-partial.csv b/tests/features/data/well-inventory-invalid-partial.csv new file mode 100644 index 000000000..301cafef1 --- /dev/null +++ b/tests/features/data/well-inventory-invalid-partial.csv @@ -0,0 +1,4 @@ +project,well_name_point_id,site_name,date_time,field_staff,utm_easting,utm_northing,utm_zone,elevation_ft,elevation_method,measuring_point_height_ft,field_staff_2,field_staff_3,contact_1_name,contact_1_organization,contact_1_role,contact_1_type,contact_1_phone_1,contact_1_phone_1_type,contact_1_phone_2,contact_1_phone_2_type,contact_1_email_1,contact_1_email_1_type,contact_1_email_2,contact_1_email_2_type,contact_1_address_1_line_1,contact_1_address_1_line_2,contact_1_address_1_type,contact_1_address_1_state,contact_1_address_1_city,contact_1_address_1_postal_code,contact_1_address_2_line_1,contact_1_address_2_line_2,contact_1_address_2_type,contact_1_address_2_state,contact_1_address_2_city,contact_1_address_2_postal_code,contact_2_name,contact_2_organization,contact_2_role,contact_2_type,contact_2_phone_1,contact_2_phone_1_type,contact_2_phone_2,contact_2_phone_2_type,contact_2_email_1,contact_2_email_1_type,contact_2_email_2,contact_2_email_2_type,contact_2_address_1_line_1,contact_2_address_1_line_2,contact_2_address_1_type,contact_2_address_1_state,contact_2_address_1_city,contact_2_address_1_postal_code,contact_2_address_2_line_1,contact_2_address_2_line_2,contact_2_address_2_type,contact_2_address_2_state,contact_2_address_2_city,contact_2_address_2_postal_code,directions_to_site,specific_location_of_well,repeat_measurement_permission,sampling_permission,datalogger_installation_permission,public_availability_acknowledgement,result_communication_preference,contact_special_requests_notes,ose_well_record_id,date_drilled,completion_source,total_well_depth_ft,historic_depth_to_water_ft,depth_source,well_pump_type,well_pump_depth_ft,is_open,datalogger_possible,casing_diameter_ft,measuring_point_description,well_purpose,well_purpose_2,well_hole_status,monitoring_frequency,sampling_scenario_notes,well_measuring_notes,sample_possible +Middle Rio Grande Groundwater Monitoring,MRG-001_MP3,Smith Farm Domestic Well,2025-02-15T10:30:00,A Lopez,250000,4000000,13N,5250,Survey-grade GPS,1.5,B Chen,,John Smith F,NMBGMR,Owner,Primary,505-555-0101,Primary,,,john.smith@example.com,Primary,,,123 County Rd 7,,Mailing,NM,Los Lunas,87031,,,,,,,Maria Garcia G,NMBGMR,Principal Investigator,Secondary,505-555-0123,Home,,,maria.garcia@mrgcd.nm.gov,Work,,,1931 2nd St SW,Suite 200,Mailing,NM,Albuquerque,87102,,,,,,,Gate off County Rd 7 0.4 miles south of canal crossing,Domestic well in pump house east of residence,True,True,True,True,email,Call before visits during irrigation season,OSE-123456,2010-06-15,OSE well record,280,45,owner estimate,submersible,200,True,True,0.5,Top of steel casing inside pump house marked with orange paint,Domestic,,active,Biannual,Sample only when pump has been off more than 12 hours,Measure before owner starts irrigation,True +Middle Rio Grande Groundwater Monitoring,MRG-003_MP3,Old Orchard Well,2025-01-20T09:00:00,B Chen,250000,4000000,13N,5320,Global positioning system (GPS),1.8,,,Emily Davis G,NMBGMR,Biologist,Primary,505-555-0303,Work,,,emily.davis@example.org,Work,,,78 Orchard Ln,,Mailing,NM,Los Lunas,87031,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,From Main St turn east on Orchard Ln well house at dead end,Abandoned irrigation well in small cinderblock building,False,False,False,True,phone,Owner prefers weekday visits,,1965-04-10,historic log scanned,350,60,historic log,vertical turbine inactive,280,False,False,0.75,Top of steel casing under removable hatch use fixed reference mark,Irrigation,,abandoned,Annual,Sampling not permitted water level only when owner present,Well house can be locked coordinate ahead,False +Middle Rio Grande Groundwater Monitoring,,Old Orchard Well1,2025-01-20T09:00:00,B Chen,250000,4000000,13N,5320,Global positioning system (GPS),1.8,,,Emily Davis F,NMBGMR,Biologist,Primary,505-555-0303,Work,,,emily.davis@example.org,Work,,,78 Orchard Ln,,Mailing,NM,Los Lunas,87031,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,From Main St turn east on Orchard Ln well house at dead end,Abandoned irrigation well in small cinderblock building,False,False,False,True,phone,Owner prefers weekday visits,,1965-04-10,historic log scanned,350,60,historic log,vertical turbine inactive,280,False,False,0.75,Top of steel casing under removable hatch use fixed reference mark,Irrigation,,abandoned,Annual,Sampling not permitted water level only when owner present,Well house can be locked coordinate ahead,False \ No newline at end of file diff --git a/tests/features/data/well-inventory-invalid-phone-number.csv b/tests/features/data/well-inventory-invalid-phone-number.csv new file mode 100644 index 000000000..9d4ab6b01 --- /dev/null +++ b/tests/features/data/well-inventory-invalid-phone-number.csv @@ -0,0 +1,3 @@ +project,well_name_point_id,site_name,date_time,field_staff,utm_easting,utm_northing,utm_zone,elevation_ft,elevation_method,measuring_point_height_ft,field_staff_2,field_staff_3,contact_1_name,contact_1_organization,contact_1_role,contact_1_type,contact_1_phone_1,contact_1_phone_1_type,contact_1_phone_2,contact_1_phone_2_type,contact_1_email_1,contact_1_email_1_type,contact_1_email_2,contact_1_email_2_type,contact_1_address_1_line_1,contact_1_address_1_line_2,contact_1_address_1_type,contact_1_address_1_state,contact_1_address_1_city,contact_1_address_1_postal_code,contact_1_address_2_line_1,contact_1_address_2_line_2,contact_1_address_2_type,contact_1_address_2_state,contact_1_address_2_city,contact_1_address_2_postal_code,contact_2_name,contact_2_organization,contact_2_role,contact_2_type,contact_2_phone_1,contact_2_phone_1_type,contact_2_phone_2,contact_2_phone_2_type,contact_2_email_1,contact_2_email_1_type,contact_2_email_2,contact_2_email_2_type,contact_2_address_1_line_1,contact_2_address_1_line_2,contact_2_address_1_type,contact_2_address_1_state,contact_2_address_1_city,contact_2_address_1_postal_code,contact_2_address_2_line_1,contact_2_address_2_line_2,contact_2_address_2_type,contact_2_address_2_state,contact_2_address_2_city,contact_2_address_2_postal_code,directions_to_site,specific_location_of_well,repeat_measurement_permission,sampling_permission,datalogger_installation_permission,public_availability_acknowledgement,result_communication_preference,contact_special_requests_notes,ose_well_record_id,date_drilled,completion_source,total_well_depth_ft,historic_depth_to_water_ft,depth_source,well_pump_type,well_pump_depth_ft,is_open,datalogger_possible,casing_diameter_ft,measuring_point_description,well_purpose,well_purpose_2,well_hole_status,monitoring_frequency,sampling_scenario_notes,well_measuring_notes,sample_possible +Middle Rio Grande Groundwater Monitoring,MRG-001_MP1,Smith Farm Domestic Well,2025-02-15T10:30:00,A Lopez,250000,4000000,13N,5250,Survey-grade GPS,1.5,B Chen,,John Smith,NMBGMR,Owner,Primary,55-555-0101,Primary,,,john.smith@example.com,Primary,,,123 County Rd 7,,Mailing,NM,Los Lunas,87031,,,,,,,Maria Garcia,NMBGMR,Principal Investigator,Secondary,505-555-0123,Home,,,maria.garcia@mrgcd.nm.gov,Work,,,1931 2nd St SW,Suite 200,Mailing,NM,Albuquerque,87102,,,,,,,Gate off County Rd 7 0.4 miles south of canal crossing,Domestic well in pump house east of residence,True,True,True,True,email,Call before visits during irrigation season,OSE-123456,2010-06-15,OSE well record,280,45,owner estimate,submersible,200,True,True,0.5,Top of steel casing inside pump house marked with orange paint,Domestic,,active,Biannual,Sample only when pump has been off more than 12 hours,Measure before owner starts irrigation,True +Middle Rio Grande Groundwater Monitoring,MRG-003_MP1,Old Orchard Well,2025-01-20T09:00:00,B Chen,250000,4000000,13N,5320,Global positioning system (GPS),1.8,,,Emily Davis,NMBGMR,Biologist,Primary,505-555-0303,Work,,,emily.davis@example.org,Work,,,78 Orchard Ln,,Mailing,NM,Los Lunas,87031,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,From Main St turn east on Orchard Ln well house at dead end,Abandoned irrigation well in small cinderblock building,False,False,False,True,phone,Owner prefers weekday visits,,1965-04-10,historic log scanned,350,60,historic log,vertical turbine inactive,280,False,False,0.75,Top of steel casing under removable hatch use fixed reference mark,Irrigation,,abandoned,Annual,Sampling not permitted water level only when owner present,Well house can be locked coordinate ahead,False diff --git a/tests/features/data/well-inventory-invalid-postal-code.csv b/tests/features/data/well-inventory-invalid-postal-code.csv new file mode 100644 index 000000000..f84a14253 --- /dev/null +++ b/tests/features/data/well-inventory-invalid-postal-code.csv @@ -0,0 +1,3 @@ +project,well_name_point_id,site_name,date_time,field_staff,utm_easting,utm_northing,utm_zone,elevation_ft,elevation_method,measuring_point_height_ft,field_staff_2,field_staff_3,contact_1_name,contact_1_organization,contact_1_role,contact_1_type,contact_1_phone_1,contact_1_phone_1_type,contact_1_phone_2,contact_1_phone_2_type,contact_1_email_1,contact_1_email_1_type,contact_1_email_2,contact_1_email_2_type,contact_1_address_1_line_1,contact_1_address_1_line_2,contact_1_address_1_type,contact_1_address_1_state,contact_1_address_1_city,contact_1_address_1_postal_code,contact_1_address_2_line_1,contact_1_address_2_line_2,contact_1_address_2_type,contact_1_address_2_state,contact_1_address_2_city,contact_1_address_2_postal_code,contact_2_name,contact_2_organization,contact_2_role,contact_2_type,contact_2_phone_1,contact_2_phone_1_type,contact_2_phone_2,contact_2_phone_2_type,contact_2_email_1,contact_2_email_1_type,contact_2_email_2,contact_2_email_2_type,contact_2_address_1_line_1,contact_2_address_1_line_2,contact_2_address_1_type,contact_2_address_1_state,contact_2_address_1_city,contact_2_address_1_postal_code,contact_2_address_2_line_1,contact_2_address_2_line_2,contact_2_address_2_type,contact_2_address_2_state,contact_2_address_2_city,contact_2_address_2_postal_code,directions_to_site,specific_location_of_well,repeat_measurement_permission,sampling_permission,datalogger_installation_permission,public_availability_acknowledgement,result_communication_preference,contact_special_requests_notes,ose_well_record_id,date_drilled,completion_source,total_well_depth_ft,historic_depth_to_water_ft,depth_source,well_pump_type,well_pump_depth_ft,is_open,datalogger_possible,casing_diameter_ft,measuring_point_description,well_purpose,well_purpose_2,well_hole_status,monitoring_frequency,sampling_scenario_notes,well_measuring_notes,sample_possible +Middle Rio Grande Groundwater Monitoring,MRG-001_MP1,Smith Farm Domestic Well,2025-02-15T10:30:00,A Lopez,250000,4000000,13N,5250,Survey-grade GPS,1.5,B Chen,,John Smith,NMBGMR,Owner,Primary,505-555-0101,Primary,,,john.smith@example.com,Primary,,,123 County Rd 7,,Mailing,NM,Los Lunas,8731,,,,,,,Maria Garcia,NMBGMR,Principal Investigator,Secondary,505-555-0123,Home,,,maria.garcia@mrgcd.nm.gov,Work,,,1931 2nd St SW,Suite 200,Mailing,NM,Albuquerque,87102,,,,,,,Gate off County Rd 7 0.4 miles south of canal crossing,Domestic well in pump house east of residence,True,True,True,True,email,Call before visits during irrigation season,OSE-123456,2010-06-15,OSE well record,280,45,owner estimate,submersible,200,True,True,0.5,Top of steel casing inside pump house marked with orange paint,Domestic,,active,Biannual,Sample only when pump has been off more than 12 hours,Measure before owner starts irrigation,True +Middle Rio Grande Groundwater Monitoring,MRG-003_MP1,Old Orchard Well,2025-01-20T09:00:00,B Chen,250000,4000000,13N,5320,Global positioning system (GPS),1.8,,,Jemily Javis,NMBGMR,Biologist,Primary,505-555-0303,Work,,,emily.davis@example.org,Work,,,78 Orchard Ln,,Mailing,NM,Los Lunas,87031,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,From Main St turn east on Orchard Ln well house at dead end,Abandoned irrigation well in small cinderblock building,False,False,False,True,phone,Owner prefers weekday visits,,1965-04-10,historic log scanned,350,60,historic log,vertical turbine inactive,280,False,False,0.75,Top of steel casing under removable hatch use fixed reference mark,Irrigation,,abandoned,Annual,Sampling not permitted water level only when owner present,Well house can be locked coordinate ahead,False diff --git a/tests/features/data/well-inventory-invalid-utm.csv b/tests/features/data/well-inventory-invalid-utm.csv new file mode 100644 index 000000000..b10a81a24 --- /dev/null +++ b/tests/features/data/well-inventory-invalid-utm.csv @@ -0,0 +1,3 @@ +project,well_name_point_id,site_name,date_time,field_staff,utm_easting,utm_northing,utm_zone,elevation_ft,elevation_method,measuring_point_height_ft,field_staff_2,field_staff_3,contact_1_name,contact_1_organization,contact_1_role,contact_1_type,contact_1_phone_1,contact_1_phone_1_type,contact_1_phone_2,contact_1_phone_2_type,contact_1_email_1,contact_1_email_1_type,contact_1_email_2,contact_1_email_2_type,contact_1_address_1_line_1,contact_1_address_1_line_2,contact_1_address_1_type,contact_1_address_1_state,contact_1_address_1_city,contact_1_address_1_postal_code,contact_1_address_2_line_1,contact_1_address_2_line_2,contact_1_address_2_type,contact_1_address_2_state,contact_1_address_2_city,contact_1_address_2_postal_code,contact_2_name,contact_2_organization,contact_2_role,contact_2_type,contact_2_phone_1,contact_2_phone_1_type,contact_2_phone_2,contact_2_phone_2_type,contact_2_email_1,contact_2_email_1_type,contact_2_email_2,contact_2_email_2_type,contact_2_address_1_line_1,contact_2_address_1_line_2,contact_2_address_1_type,contact_2_address_1_state,contact_2_address_1_city,contact_2_address_1_postal_code,contact_2_address_2_line_1,contact_2_address_2_line_2,contact_2_address_2_type,contact_2_address_2_state,contact_2_address_2_city,contact_2_address_2_postal_code,directions_to_site,specific_location_of_well,repeat_measurement_permission,sampling_permission,datalogger_installation_permission,public_availability_acknowledgement,result_communication_preference,contact_special_requests_notes,ose_well_record_id,date_drilled,completion_source,total_well_depth_ft,historic_depth_to_water_ft,depth_source,well_pump_type,well_pump_depth_ft,is_open,datalogger_possible,casing_diameter_ft,measuring_point_description,well_purpose,well_purpose_2,well_hole_status,monitoring_frequency,sampling_scenario_notes,well_measuring_notes,sample_possible +Middle Rio Grande Groundwater Monitoring,MRG-001_MP1,Smith Farm Domestic Well,2025-02-15T10:30:00,A Lopez,457100,4159020,13N,5250,Survey-grade GPS,1.5,B Chen,,John Smith,NMBGMR,Owner,Primary,505-555-0101,Primary,,,john.smith@example.com,Primary,,,123 County Rd 7,,Mailing,NM,Los Lunas,87031,,,,,,,Maria Garcia,NMBGMR,Principal Investigator,Secondary,505-555-0123,Home,,,maria.garcia@mrgcd.nm.gov,Work,,,1931 2nd St SW,Suite 200,Mailing,NM,Albuquerque,87102,,,,,,,Gate off County Rd 7 0.4 miles south of canal crossing,Domestic well in pump house east of residence,True,True,True,True,email,Call before visits during irrigation season,OSE-123456,2010-06-15,OSE well record,280,45,owner estimate,submersible,200,True,True,0.5,Top of steel casing inside pump house marked with orange paint,Domestic,,active,Biannual,Sample only when pump has been off more than 12 hours,Measure before owner starts irrigation,True +Middle Rio Grande Groundwater Monitoring,MRG-003_MP1,Old Orchard Well,2025-01-20T09:00:00,B Chen,250000,4000000,13S,5320,Global positioning system (GPS),1.8,,,Emily Davis,NMBGMR,Biologist,Primary,505-555-0303,Work,,,emily.davis@example.org,Work,,,78 Orchard Ln,,Mailing,NM,Los Lunas,87031,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,From Main St turn east on Orchard Ln well house at dead end,Abandoned irrigation well in small cinderblock building,False,False,False,True,phone,Owner prefers weekday visits,,1965-04-10,historic log scanned,350,60,historic log,vertical turbine inactive,280,False,False,0.75,Top of steel casing under removable hatch use fixed reference mark,Irrigation,,abandoned,Annual,Sampling not permitted water level only when owner present,Well house can be locked coordinate ahead,False diff --git a/tests/features/data/well-inventory-invalid.csv b/tests/features/data/well-inventory-invalid.csv new file mode 100644 index 000000000..41fe15a2a --- /dev/null +++ b/tests/features/data/well-inventory-invalid.csv @@ -0,0 +1,5 @@ +well_name_point_id,site_name,date_time,field_staff,contact_role,utm_easting,utm_northing,utm_zone,elevation_ft,elevation_method +,Site Alpha,2025-02-15T10:30:00,Jane Doe,Owner,250000,4000000,13N,5120.5,GPS +WELL003,Site Beta,invalid-date,John Smith,Manager,250000,4000000,13N,5130.7,Survey +WELL004,Site Gamma,2025-04-10T11:00:00,,Technician,250000,4000000,13N,5140.2,GPS +WELL004,Site Delta,2025-05-12T12:45:00,Emily Clark,Supervisor,250000,4000000,13N,5150.3,Survey diff --git a/tests/features/data/well-inventory-missing-address-type.csv b/tests/features/data/well-inventory-missing-address-type.csv new file mode 100644 index 000000000..f3e55965d --- /dev/null +++ b/tests/features/data/well-inventory-missing-address-type.csv @@ -0,0 +1,3 @@ +project,well_name_point_id,site_name,date_time,field_staff,utm_easting,utm_northing,utm_zone,elevation_ft,elevation_method,measuring_point_height_ft,field_staff_2,field_staff_3,contact_1_name,contact_1_organization,contact_1_role,contact_1_type,contact_1_phone_1,contact_1_phone_1_type,contact_1_phone_2,contact_1_phone_2_type,contact_1_email_1,contact_1_email_1_type,contact_1_email_2,contact_1_email_2_type,contact_1_address_1_line_1,contact_1_address_1_line_2,contact_1_address_1_type,contact_1_address_1_state,contact_1_address_1_city,contact_1_address_1_postal_code,contact_1_address_2_line_1,contact_1_address_2_line_2,contact_1_address_2_type,contact_1_address_2_state,contact_1_address_2_city,contact_1_address_2_postal_code,contact_2_name,contact_2_organization,contact_2_role,contact_2_type,contact_2_phone_1,contact_2_phone_1_type,contact_2_phone_2,contact_2_phone_2_type,contact_2_email_1,contact_2_email_1_type,contact_2_email_2,contact_2_email_2_type,contact_2_address_1_line_1,contact_2_address_1_line_2,contact_2_address_1_type,contact_2_address_1_state,contact_2_address_1_city,contact_2_address_1_postal_code,contact_2_address_2_line_1,contact_2_address_2_line_2,contact_2_address_2_type,contact_2_address_2_state,contact_2_address_2_city,contact_2_address_2_postal_code,directions_to_site,specific_location_of_well,repeat_measurement_permission,sampling_permission,datalogger_installation_permission,public_availability_acknowledgement,result_communication_preference,contact_special_requests_notes,ose_well_record_id,date_drilled,completion_source,total_well_depth_ft,historic_depth_to_water_ft,depth_source,well_pump_type,well_pump_depth_ft,is_open,datalogger_possible,casing_diameter_ft,measuring_point_description,well_purpose,well_purpose_2,well_hole_status,monitoring_frequency,sampling_scenario_notes,well_measuring_notes,sample_possible +Middle Rio Grande Groundwater Monitoring,MRG-001_MP1,Smith Farm Domestic Well,2025-02-15T10:30:00,A Lopez,250000,4000000,13N,5250,Survey-grade GPS,1.5,B Chen,,John Smith,NMBGMR,Owner,Primary,505-555-0101,Primary,,,john.smith@example.com,Primary,,,123 County Rd 7,,,NM,Los Lunas,87031,,,,,,,Maria Garcia,NMBGMR,Principal Investigator,Secondary,505-555-0123,Home,,,maria.garcia@mrgcd.nm.gov,Work,,,1931 2nd St SW,Suite 200,Mailing,NM,Albuquerque,87102,,,,,,,Gate off County Rd 7 0.4 miles south of canal crossing,Domestic well in pump house east of residence,True,True,True,True,email,Call before visits during irrigation season,OSE-123456,2010-06-15,OSE well record,280,45,owner estimate,submersible,200,True,True,0.5,Top of steel casing inside pump house marked with orange paint,Domestic,,active,Biannual,Sample only when pump has been off more than 12 hours,Measure before owner starts irrigation,True +Middle Rio Grande Groundwater Monitoring,MRG-003_MP1,Old Orchard Well,2025-01-20T09:00:00,B Chen,250000,4000000,13N,5320,Global positioning system (GPS),1.8,,,Emily Davis,NMBGMR,Biologist,Primary,505-555-0303,Work,,,emily.davis@example.org,Work,,,78 Orchard Ln,,Mailing,NM,Los Lunas,87031,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,From Main St turn east on Orchard Ln well house at dead end,Abandoned irrigation well in small cinderblock building,False,False,False,True,phone,Owner prefers weekday visits,,1965-04-10,historic log scanned,350,60,historic log,vertical turbine inactive,280,False,False,0.75,Top of steel casing under removable hatch use fixed reference mark,Irrigation,,abandoned,Annual,Sampling not permitted water level only when owner present,Well house can be locked coordinate ahead,False diff --git a/tests/features/data/well-inventory-missing-contact-role.csv b/tests/features/data/well-inventory-missing-contact-role.csv new file mode 100644 index 000000000..3775e8cbd --- /dev/null +++ b/tests/features/data/well-inventory-missing-contact-role.csv @@ -0,0 +1,3 @@ +project,well_name_point_id,site_name,date_time,field_staff,utm_easting,utm_northing,utm_zone,elevation_ft,elevation_method,measuring_point_height_ft,field_staff_2,field_staff_3,contact_1_name,contact_1_organization,contact_1_role,contact_1_type,contact_1_phone_1,contact_1_phone_1_type,contact_1_phone_2,contact_1_phone_2_type,contact_1_email_1,contact_1_email_1_type,contact_1_email_2,contact_1_email_2_type,contact_1_address_1_line_1,contact_1_address_1_line_2,contact_1_address_1_type,contact_1_address_1_state,contact_1_address_1_city,contact_1_address_1_postal_code,contact_1_address_2_line_1,contact_1_address_2_line_2,contact_1_address_2_type,contact_1_address_2_state,contact_1_address_2_city,contact_1_address_2_postal_code,contact_2_name,contact_2_organization,contact_2_role,contact_2_type,contact_2_phone_1,contact_2_phone_1_type,contact_2_phone_2,contact_2_phone_2_type,contact_2_email_1,contact_2_email_1_type,contact_2_email_2,contact_2_email_2_type,contact_2_address_1_line_1,contact_2_address_1_line_2,contact_2_address_1_type,contact_2_address_1_state,contact_2_address_1_city,contact_2_address_1_postal_code,contact_2_address_2_line_1,contact_2_address_2_line_2,contact_2_address_2_type,contact_2_address_2_state,contact_2_address_2_city,contact_2_address_2_postal_code,directions_to_site,specific_location_of_well,repeat_measurement_permission,sampling_permission,datalogger_installation_permission,public_availability_acknowledgement,result_communication_preference,contact_special_requests_notes,ose_well_record_id,date_drilled,completion_source,total_well_depth_ft,historic_depth_to_water_ft,depth_source,well_pump_type,well_pump_depth_ft,is_open,datalogger_possible,casing_diameter_ft,measuring_point_description,well_purpose,well_purpose_2,well_hole_status,monitoring_frequency,sampling_scenario_notes,well_measuring_notes,sample_possible +Middle Rio Grande Groundwater Monitoring,MRG-001_MP1,Smith Farm Domestic Well,2025-02-15T10:30:00,A Lopez,250000,4000000,13N,5250,Survey-grade GPS,1.5,B Chen,,John Smith,NMBGMR,,Primary,505-555-0101,Primary,,,john.smith@example.com,Primary,,,123 County Rd 7,,Mailing,NM,Los Lunas,87031,,,,,,,Maria Garcia,NMBGMR,Principal Investigator,Secondary,505-555-0123,Home,,,maria.garcia@mrgcd.nm.gov,Work,,,1931 2nd St SW,Suite 200,Mailing,NM,Albuquerque,87102,,,,,,,Gate off County Rd 7 0.4 miles south of canal crossing,Domestic well in pump house east of residence,True,True,True,True,email,Call before visits during irrigation season,OSE-123456,2010-06-15,OSE well record,280,45,owner estimate,submersible,200,True,True,0.5,Top of steel casing inside pump house marked with orange paint,Domestic,,active,Biannual,Sample only when pump has been off more than 12 hours,Measure before owner starts irrigation,True +Middle Rio Grande Groundwater Monitoring,MRG-003_MP1,Old Orchard Well,2025-01-20T09:00:00,B Chen,250000,4000000,13N,5320,Global positioning system (GPS),1.8,,,David Emily,NMBGMR,Biologist,Primary,505-555-0303,Work,,,emily.davis@example.org,Work,,,78 Orchard Ln,,Mailing,NM,Los Lunas,87031,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,From Main St turn east on Orchard Ln well house at dead end,Abandoned irrigation well in small cinderblock building,False,False,False,True,phone,Owner prefers weekday visits,,1965-04-10,historic log scanned,350,60,historic log,vertical turbine inactive,280,False,False,0.75,Top of steel casing under removable hatch use fixed reference mark,Irrigation,,abandoned,Annual,Sampling not permitted water level only when owner present,Well house can be locked coordinate ahead,False diff --git a/tests/features/data/well-inventory-missing-contact-type.csv b/tests/features/data/well-inventory-missing-contact-type.csv new file mode 100644 index 000000000..3cc7aeb59 --- /dev/null +++ b/tests/features/data/well-inventory-missing-contact-type.csv @@ -0,0 +1,3 @@ +project,well_name_point_id,site_name,date_time,field_staff,utm_easting,utm_northing,utm_zone,elevation_ft,elevation_method,measuring_point_height_ft,field_staff_2,field_staff_3,contact_1_name,contact_1_organization,contact_1_role,contact_1_type,contact_1_phone_1,contact_1_phone_1_type,contact_1_phone_2,contact_1_phone_2_type,contact_1_email_1,contact_1_email_1_type,contact_1_email_2,contact_1_email_2_type,contact_1_address_1_line_1,contact_1_address_1_line_2,contact_1_address_1_type,contact_1_address_1_state,contact_1_address_1_city,contact_1_address_1_postal_code,contact_1_address_2_line_1,contact_1_address_2_line_2,contact_1_address_2_type,contact_1_address_2_state,contact_1_address_2_city,contact_1_address_2_postal_code,contact_2_name,contact_2_organization,contact_2_role,contact_2_type,contact_2_phone_1,contact_2_phone_1_type,contact_2_phone_2,contact_2_phone_2_type,contact_2_email_1,contact_2_email_1_type,contact_2_email_2,contact_2_email_2_type,contact_2_address_1_line_1,contact_2_address_1_line_2,contact_2_address_1_type,contact_2_address_1_state,contact_2_address_1_city,contact_2_address_1_postal_code,contact_2_address_2_line_1,contact_2_address_2_line_2,contact_2_address_2_type,contact_2_address_2_state,contact_2_address_2_city,contact_2_address_2_postal_code,directions_to_site,specific_location_of_well,repeat_measurement_permission,sampling_permission,datalogger_installation_permission,public_availability_acknowledgement,result_communication_preference,contact_special_requests_notes,ose_well_record_id,date_drilled,completion_source,total_well_depth_ft,historic_depth_to_water_ft,depth_source,well_pump_type,well_pump_depth_ft,is_open,datalogger_possible,casing_diameter_ft,measuring_point_description,well_purpose,well_purpose_2,well_hole_status,monitoring_frequency,sampling_scenario_notes,well_measuring_notes,sample_possible +Middle Rio Grande Groundwater Monitoring,MRG-001_MP1,Smith Farm Domestic Well,2025-02-15T10:30:00,A Lopez,250000,4000000,13N,5250,Survey-grade GPS,1.5,B Chen,,John Smith,NMBGMR,Owner,,505-555-0101,Primary,,,john.smith@example.com,Primary,,,123 County Rd 7,,Mailing,NM,Los Lunas,87031,,,,,,,Maria Garcia,NMBGMR,Principal Investigator,Secondary,505-555-0123,Home,,,maria.garcia@mrgcd.nm.gov,Work,,,1931 2nd St SW,Suite 200,Mailing,NM,Albuquerque,87102,,,,,,,Gate off County Rd 7 0.4 miles south of canal crossing,Domestic well in pump house east of residence,True,True,True,True,email,Call before visits during irrigation season,OSE-123456,2010-06-15,OSE well record,280,45,owner estimate,submersible,200,True,True,0.5,Top of steel casing inside pump house marked with orange paint,Domestic,,active,Biannual,Sample only when pump has been off more than 12 hours,Measure before owner starts irrigation,True +Middle Rio Grande Groundwater Monitoring,MRG-003_MP1,Old Orchard Well,2025-01-20T09:00:00,B Chen,250000,4000000,13N,5320,Global positioning system (GPS),1.8,,,Emily Davis,NMBGMR,Biologist,Primary,505-555-0303,Work,,,emily.davis@example.org,Work,,,78 Orchard Ln,,Mailing,NM,Los Lunas,87031,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,From Main St turn east on Orchard Ln well house at dead end,Abandoned irrigation well in small cinderblock building,False,False,False,True,phone,Owner prefers weekday visits,,1965-04-10,historic log scanned,350,60,historic log,vertical turbine inactive,280,False,False,0.75,Top of steel casing under removable hatch use fixed reference mark,Irrigation,,abandoned,Annual,Sampling not permitted water level only when owner present,Well house can be locked coordinate ahead,False diff --git a/tests/features/data/well-inventory-missing-email-type.csv b/tests/features/data/well-inventory-missing-email-type.csv new file mode 100644 index 000000000..1ba864315 --- /dev/null +++ b/tests/features/data/well-inventory-missing-email-type.csv @@ -0,0 +1,3 @@ +project,well_name_point_id,site_name,date_time,field_staff,utm_easting,utm_northing,utm_zone,elevation_ft,elevation_method,measuring_point_height_ft,field_staff_2,field_staff_3,contact_1_name,contact_1_organization,contact_1_role,contact_1_type,contact_1_phone_1,contact_1_phone_1_type,contact_1_phone_2,contact_1_phone_2_type,contact_1_email_1,contact_1_email_1_type,contact_1_email_2,contact_1_email_2_type,contact_1_address_1_line_1,contact_1_address_1_line_2,contact_1_address_1_type,contact_1_address_1_state,contact_1_address_1_city,contact_1_address_1_postal_code,contact_1_address_2_line_1,contact_1_address_2_line_2,contact_1_address_2_type,contact_1_address_2_state,contact_1_address_2_city,contact_1_address_2_postal_code,contact_2_name,contact_2_organization,contact_2_role,contact_2_type,contact_2_phone_1,contact_2_phone_1_type,contact_2_phone_2,contact_2_phone_2_type,contact_2_email_1,contact_2_email_1_type,contact_2_email_2,contact_2_email_2_type,contact_2_address_1_line_1,contact_2_address_1_line_2,contact_2_address_1_type,contact_2_address_1_state,contact_2_address_1_city,contact_2_address_1_postal_code,contact_2_address_2_line_1,contact_2_address_2_line_2,contact_2_address_2_type,contact_2_address_2_state,contact_2_address_2_city,contact_2_address_2_postal_code,directions_to_site,specific_location_of_well,repeat_measurement_permission,sampling_permission,datalogger_installation_permission,public_availability_acknowledgement,result_communication_preference,contact_special_requests_notes,ose_well_record_id,date_drilled,completion_source,total_well_depth_ft,historic_depth_to_water_ft,depth_source,well_pump_type,well_pump_depth_ft,is_open,datalogger_possible,casing_diameter_ft,measuring_point_description,well_purpose,well_purpose_2,well_hole_status,monitoring_frequency,sampling_scenario_notes,well_measuring_notes,sample_possible +Middle Rio Grande Groundwater Monitoring,MRG-001_MP1,Smith Farm Domestic Well,2025-02-15T10:30:00,A Lopez,250000,4000000,13N,5250,Survey-grade GPS,1.5,B Chen,,John Smith,NMBGMR,Owner,Primary,505-555-0101,Primary,,,john.smith@example.com,,,,123 County Rd 7,,Mailing,NM,Los Lunas,87031,,,,,,,Maria Garcia,NMBGMR,Principal Investigator,Secondary,505-555-0123,Home,,,maria.garcia@mrgcd.nm.gov,Work,,,1931 2nd St SW,Suite 200,Mailing,NM,Albuquerque,87102,,,,,,,Gate off County Rd 7 0.4 miles south of canal crossing,Domestic well in pump house east of residence,True,True,True,True,email,Call before visits during irrigation season,OSE-123456,2010-06-15,OSE well record,280,45,owner estimate,submersible,200,True,True,0.5,Top of steel casing inside pump house marked with orange paint,Domestic,,active,Biannual,Sample only when pump has been off more than 12 hours,Measure before owner starts irrigation,True +Middle Rio Grande Groundwater Monitoring,MRG-003_MP1,Old Orchard Well,2025-01-20T09:00:00,B Chen,250000,4000000,13N,5320,Global positioning system (GPS),1.8,,,Emily Davis,NMBGMR,Biologist,Primary,505-555-0303,Work,,,emily.davis@example.org,Work,,,78 Orchard Ln,,Mailing,NM,Los Lunas,87031,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,From Main St turn east on Orchard Ln well house at dead end,Abandoned irrigation well in small cinderblock building,False,False,False,True,phone,Owner prefers weekday visits,,1965-04-10,historic log scanned,350,60,historic log,vertical turbine inactive,280,False,False,0.75,Top of steel casing under removable hatch use fixed reference mark,Irrigation,,abandoned,Annual,Sampling not permitted water level only when owner present,Well house can be locked coordinate ahead,False diff --git a/tests/features/data/well-inventory-missing-phone-type.csv b/tests/features/data/well-inventory-missing-phone-type.csv new file mode 100644 index 000000000..24a8ea40e --- /dev/null +++ b/tests/features/data/well-inventory-missing-phone-type.csv @@ -0,0 +1,3 @@ +project,well_name_point_id,site_name,date_time,field_staff,utm_easting,utm_northing,utm_zone,elevation_ft,elevation_method,measuring_point_height_ft,field_staff_2,field_staff_3,contact_1_name,contact_1_organization,contact_1_role,contact_1_type,contact_1_phone_1,contact_1_phone_1_type,contact_1_phone_2,contact_1_phone_2_type,contact_1_email_1,contact_1_email_1_type,contact_1_email_2,contact_1_email_2_type,contact_1_address_1_line_1,contact_1_address_1_line_2,contact_1_address_1_type,contact_1_address_1_state,contact_1_address_1_city,contact_1_address_1_postal_code,contact_1_address_2_line_1,contact_1_address_2_line_2,contact_1_address_2_type,contact_1_address_2_state,contact_1_address_2_city,contact_1_address_2_postal_code,contact_2_name,contact_2_organization,contact_2_role,contact_2_type,contact_2_phone_1,contact_2_phone_1_type,contact_2_phone_2,contact_2_phone_2_type,contact_2_email_1,contact_2_email_1_type,contact_2_email_2,contact_2_email_2_type,contact_2_address_1_line_1,contact_2_address_1_line_2,contact_2_address_1_type,contact_2_address_1_state,contact_2_address_1_city,contact_2_address_1_postal_code,contact_2_address_2_line_1,contact_2_address_2_line_2,contact_2_address_2_type,contact_2_address_2_state,contact_2_address_2_city,contact_2_address_2_postal_code,directions_to_site,specific_location_of_well,repeat_measurement_permission,sampling_permission,datalogger_installation_permission,public_availability_acknowledgement,result_communication_preference,contact_special_requests_notes,ose_well_record_id,date_drilled,completion_source,total_well_depth_ft,historic_depth_to_water_ft,depth_source,well_pump_type,well_pump_depth_ft,is_open,datalogger_possible,casing_diameter_ft,measuring_point_description,well_purpose,well_purpose_2,well_hole_status,monitoring_frequency,sampling_scenario_notes,well_measuring_notes,sample_possible +Middle Rio Grande Groundwater Monitoring,MRG-001_MP1,Smith Farm Domestic Well,2025-02-15T10:30:00,A Lopez,250000,4000000,13N,5250,Survey-grade GPS,1.5,B Chen,,John Smith,NMBGMR,Owner,Primary,505-555-0101,,,,john.smith@example.com,Primary,,,123 County Rd 7,,Mailing,NM,Los Lunas,87031,,,,,,,Maria Garcia,NMBGMR,Principal Investigator,Secondary,505-555-0123,Home,,,maria.garcia@mrgcd.nm.gov,Work,,,1931 2nd St SW,Suite 200,Mailing,NM,Albuquerque,87102,,,,,,,Gate off County Rd 7 0.4 miles south of canal crossing,Domestic well in pump house east of residence,True,True,True,True,email,Call before visits during irrigation season,OSE-123456,2010-06-15,OSE well record,280,45,owner estimate,submersible,200,True,True,0.5,Top of steel casing inside pump house marked with orange paint,Domestic,,active,Biannual,Sample only when pump has been off more than 12 hours,Measure before owner starts irrigation,True +Middle Rio Grande Groundwater Monitoring,MRG-003_MP1,Old Orchard Well,2025-01-20T09:00:00,B Chen,250000,4000000,13N,5320,Global positioning system (GPS),1.8,,,Emily Davis,NMBGMR,Biologist,Primary,505-555-0303,Work,,,emily.davis@example.org,Work,,,78 Orchard Ln,,Mailing,NM,Los Lunas,87031,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,From Main St turn east on Orchard Ln well house at dead end,Abandoned irrigation well in small cinderblock building,False,False,False,True,phone,Owner prefers weekday visits,,1965-04-10,historic log scanned,350,60,historic log,vertical turbine inactive,280,False,False,0.75,Top of steel casing under removable hatch use fixed reference mark,Irrigation,,abandoned,Annual,Sampling not permitted water level only when owner present,Well house can be locked coordinate ahead,False diff --git a/tests/features/data/well-inventory-missing-required.csv b/tests/features/data/well-inventory-missing-required.csv new file mode 100644 index 000000000..9105a830a --- /dev/null +++ b/tests/features/data/well-inventory-missing-required.csv @@ -0,0 +1,5 @@ +project,well_name_point_id,site_name,date_time,field_staff,utm_easting,utm_northing,utm_zone,elevation_ft,elevation_method,measuring_point_height_ft +ProjectA,,Site1,2025-02-15T10:30:00,John Doe,250000,4000000,13N,5000,Survey,2.5 +ProjectB,,Site2,2025-02-16T11:00:00,Jane Smith,250000,4000000,13N,5100,Survey,2.7 +ProjectC,WELL003,Site3,2025-02-17T09:45:00,Jim Beam,250000,4000000,13N,5200,Survey,2.6 +ProjectD,,Site4,2025-02-18T08:20:00,Jack Daniels,250000,4000000,13N,5300,Survey,2.8 diff --git a/tests/features/data/well-inventory-missing-wl-fields.csv b/tests/features/data/well-inventory-missing-wl-fields.csv new file mode 100644 index 000000000..c0b2562be --- /dev/null +++ b/tests/features/data/well-inventory-missing-wl-fields.csv @@ -0,0 +1,3 @@ +project,well_name_point_id,site_name,date_time,field_staff,utm_easting,utm_northing,utm_zone,elevation_ft,elevation_method,measuring_point_height_ft,field_staff_2,field_staff_3,contact_1_name,contact_1_organization,contact_1_role,contact_1_type,contact_1_phone_1,contact_1_phone_1_type,contact_1_phone_2,contact_1_phone_2_type,contact_1_email_1,contact_1_email_1_type,contact_1_email_2,contact_1_email_2_type,contact_1_address_1_line_1,contact_1_address_1_line_2,contact_1_address_1_type,contact_1_address_1_state,contact_1_address_1_city,contact_1_address_1_postal_code,contact_1_address_2_line_1,contact_1_address_2_line_2,contact_1_address_2_type,contact_1_address_2_state,contact_1_address_2_city,contact_1_address_2_postal_code,contact_2_name,contact_2_organization,contact_2_role,contact_2_type,contact_2_phone_1,contact_2_phone_1_type,contact_2_phone_2,contact_2_phone_2_type,contact_2_email_1,contact_2_email_1_type,contact_2_email_2,contact_2_email_2_type,contact_2_address_1_line_1,contact_2_address_1_line_2,contact_2_address_1_type,contact_2_address_1_state,contact_2_address_1_city,contact_2_address_1_postal_code,contact_2_address_2_line_1,contact_2_address_2_line_2,contact_2_address_2_type,contact_2_address_2_state,contact_2_address_2_city,contact_2_address_2_postal_code,directions_to_site,specific_location_of_well,repeat_measurement_permission,sampling_permission,datalogger_installation_permission,public_availability_acknowledgement,result_communication_preference,contact_special_requests_notes,ose_well_record_id,date_drilled,completion_source,total_well_depth_ft,historic_depth_to_water_ft,depth_source,well_pump_type,well_pump_depth_ft,is_open,datalogger_possible,casing_diameter_ft,measuring_point_description,well_purpose,well_purpose_2,well_hole_status,monitoring_frequency,sampling_scenario_notes,well_measuring_notes,sample_possible,depth_to_water_ft +Middle Rio Grande Groundwater Monitoring,MRG-001_MP1,Smith Farm Domestic Well,2025-02-15T10:30:00,A Lopez,250000,4000000,13N,5250,Survey-grade GPS,1.5,B Chen,,John Smith,NMBGMR,Owner,Primary,505-555-0101,Primary,,,john.smith@example.com,Primary,,,123 County Rd 7,,Mailing,NM,Los Lunas,87031,,,,,,,Maria Garcia,NMBGMR,Principal Investigator,Secondary,505-555-0123,Home,,,maria.garcia@mrgcd.nm.gov,Work,,,1931 2nd St SW,Suite 200,Mailing,NM,Albuquerque,87102,,,,,,,Gate off County Rd 7 0.4 miles south of canal crossing,Domestic well in pump house east of residence,True,True,True,True,email,Call before visits during irrigation season,OSE-123456,2010-06-15,Interpreted fr geophys logs by source agency,280,45,"Memory of owner, operator, driller",Submersible,200,True,True,0.5,Top of steel casing inside pump house marked with orange paint,Domestic,,active,Biannual,Sample only when pump has been off more than 12 hours,Measure before owner starts irrigation,True,100 +Middle Rio Grande Groundwater Monitoring,MRG-003_MP1,Old Orchard Well,2025-01-20T09:00:00,B Chen,250000,4000000,13N,5320,Global positioning system (GPS),1.8,,,Emily Davis,NMBGMR,Biologist,Primary,505-555-0303,Work,,,emily.davis@example.org,Work,,,78 Orchard Ln,,Mailing,NM,Los Lunas,87031,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,From Main St turn east on Orchard Ln well house at dead end,Abandoned irrigation well in small cinderblock building,False,False,False,True,phone,Owner prefers weekday visits,,1965-04-10,From driller's log or well report,350,60,From driller's log or well report,Jet,280,False,False,0.75,Top of steel casing under removable hatch use fixed reference mark,Irrigation,,abandoned,Annual,Sampling not permitted water level only when owner present,Well house can be locked coordinate ahead,False,200 diff --git a/tests/features/data/well-inventory-no-data-headers.csv b/tests/features/data/well-inventory-no-data-headers.csv new file mode 100644 index 000000000..9c4b9e81c --- /dev/null +++ b/tests/features/data/well-inventory-no-data-headers.csv @@ -0,0 +1 @@ +project,measuring_point_height_ft,well_name_point_id,site_name,date_time,field_staff,contact_role,utm_easting,utm_northing,utm_zone,elevation_ft,elevation_method diff --git a/tests/features/data/well-inventory-no-data.csv b/tests/features/data/well-inventory-no-data.csv new file mode 100644 index 000000000..6a644482a --- /dev/null +++ b/tests/features/data/well-inventory-no-data.csv @@ -0,0 +1 @@ +well_name_point_id,site_name,date_time,field_staff,contact_role,utm_easting,utm_northing,utm_zone,elevation_ft,elevation_method diff --git a/tests/features/data/well-inventory-valid-comma-in-quotes.csv b/tests/features/data/well-inventory-valid-comma-in-quotes.csv new file mode 100644 index 000000000..68bd1ef97 --- /dev/null +++ b/tests/features/data/well-inventory-valid-comma-in-quotes.csv @@ -0,0 +1,3 @@ +project,well_name_point_id,site_name,date_time,field_staff,utm_easting,utm_northing,utm_zone,elevation_ft,elevation_method,measuring_point_height_ft,field_staff_2,field_staff_3,contact_1_name,contact_1_organization,contact_1_role,contact_1_type,contact_1_phone_1,contact_1_phone_1_type,contact_1_phone_2,contact_1_phone_2_type,contact_1_email_1,contact_1_email_1_type,contact_1_email_2,contact_1_email_2_type,contact_1_address_1_line_1,contact_1_address_1_line_2,contact_1_address_1_type,contact_1_address_1_state,contact_1_address_1_city,contact_1_address_1_postal_code,contact_1_address_2_line_1,contact_1_address_2_line_2,contact_1_address_2_type,contact_1_address_2_state,contact_1_address_2_city,contact_1_address_2_postal_code,contact_2_name,contact_2_organization,contact_2_role,contact_2_type,contact_2_phone_1,contact_2_phone_1_type,contact_2_phone_2,contact_2_phone_2_type,contact_2_email_1,contact_2_email_1_type,contact_2_email_2,contact_2_email_2_type,contact_2_address_1_line_1,contact_2_address_1_line_2,contact_2_address_1_type,contact_2_address_1_state,contact_2_address_1_city,contact_2_address_1_postal_code,contact_2_address_2_line_1,contact_2_address_2_line_2,contact_2_address_2_type,contact_2_address_2_state,contact_2_address_2_city,contact_2_address_2_postal_code,directions_to_site,specific_location_of_well,repeat_measurement_permission,sampling_permission,datalogger_installation_permission,public_availability_acknowledgement,result_communication_preference,contact_special_requests_notes,ose_well_record_id,date_drilled,completion_source,total_well_depth_ft,historic_depth_to_water_ft,depth_source,well_pump_type,well_pump_depth_ft,is_open,datalogger_possible,casing_diameter_ft,measuring_point_description,well_purpose,well_purpose_2,well_hole_status,monitoring_frequency,sampling_scenario_notes,well_measuring_notes,sample_possible +Middle Rio Grande Groundwater Monitoring,MRG-001_MP1D,"""Smith Farm, Domestic Well""",2025-02-15T10:30:00,A Lopez,250000,4000000,13N,5250,Survey-grade GPS,1.5,B Chen,,John Smith T,NMBGMR,Owner,Primary,505-555-0101,Primary,,,john.smith@example.com,Primary,,,123 County Rd 7,,Mailing,NM,Los Lunas,87031,,,,,,,Maria Garcia G,NMBGMR,Principal Investigator,Secondary,505-555-0123,Home,,,maria.garcia@mrgcd.nm.gov,Work,,,1931 2nd St SW,Suite 200,Mailing,NM,Albuquerque,87102,,,,,,,Gate off County Rd 7 0.4 miles south of canal crossing,Domestic well in pump house east of residence,True,True,True,True,email,Call before visits during irrigation season,OSE-123456,2010-06-15,Interpreted fr geophys logs by source agency,280,45,"Memory of owner, operator, driller",Submersible,200,True,True,0.5,Top of steel casing inside pump house marked with orange paint,Domestic,,active,Biannual,Sample only when pump has been off more than 12 hours,Measure before owner starts irrigation,True +Middle Rio Grande Groundwater Monitoring,MRG-003_MP1G,Old Orchard Well,2025-01-20T09:00:00,B Chen,250000,4000000,13N,5320,Global positioning system (GPS),1.8,,,Emily Davis E,NMBGMR,Biologist,Primary,505-555-0303,Work,,,emily.davis@example.org,Work,,,78 Orchard Ln,,Mailing,NM,Los Lunas,87031,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,From Main St turn east on Orchard Ln well house at dead end,Abandoned irrigation well in small cinderblock building,False,False,False,True,phone,Owner prefers weekday visits,,1965-04-10,From driller's log or well report,350,60,From driller's log or well report,Jet,280,False,False,0.75,Top of steel casing under removable hatch use fixed reference mark,Irrigation,,abandoned,Annual,Sampling not permitted water level only when owner present,Well house can be locked coordinate ahead,False diff --git a/tests/features/data/well-inventory-valid-extra-columns.csv b/tests/features/data/well-inventory-valid-extra-columns.csv new file mode 100644 index 000000000..173a36678 --- /dev/null +++ b/tests/features/data/well-inventory-valid-extra-columns.csv @@ -0,0 +1,3 @@ +project,well_name_point_id,site_name,date_time,field_staff,utm_easting,utm_northing,utm_zone,elevation_ft,elevation_method,measuring_point_height_ft,field_staff_2,field_staff_3,contact_1_name,contact_1_organization,contact_1_role,contact_1_type,contact_1_phone_1,contact_1_phone_1_type,contact_1_phone_2,contact_1_phone_2_type,contact_1_email_1,contact_1_email_1_type,contact_1_email_2,contact_1_email_2_type,contact_1_address_1_line_1,contact_1_address_1_line_2,contact_1_address_1_type,contact_1_address_1_state,contact_1_address_1_city,contact_1_address_1_postal_code,contact_1_address_2_line_1,contact_1_address_2_line_2,contact_1_address_2_type,contact_1_address_2_state,contact_1_address_2_city,contact_1_address_2_postal_code,contact_2_name,contact_2_organization,contact_2_role,contact_2_type,contact_2_phone_1,contact_2_phone_1_type,contact_2_phone_2,contact_2_phone_2_type,contact_2_email_1,contact_2_email_1_type,contact_2_email_2,contact_2_email_2_type,contact_2_address_1_line_1,contact_2_address_1_line_2,contact_2_address_1_type,contact_2_address_1_state,contact_2_address_1_city,contact_2_address_1_postal_code,contact_2_address_2_line_1,contact_2_address_2_line_2,contact_2_address_2_type,contact_2_address_2_state,contact_2_address_2_city,contact_2_address_2_postal_code,directions_to_site,specific_location_of_well,repeat_measurement_permission,sampling_permission,datalogger_installation_permission,public_availability_acknowledgement,result_communication_preference,contact_special_requests_notes,ose_well_record_id,date_drilled,completion_source,total_well_depth_ft,historic_depth_to_water_ft,depth_source,well_pump_type,well_pump_depth_ft,is_open,datalogger_possible,casing_diameter_ft,measuring_point_description,well_purpose,well_purpose_2,well_hole_status,monitoring_frequency,sampling_scenario_notes,well_measuring_notes,sample_possible,extra_column1,extract_column2 +Middle Rio Grande Groundwater Monitoring,MRG-001_MP1v,Smith Farm Domestic Well,2025-02-15T10:30:00,A Lopez,250000,4000000,13N,5250,Survey-grade GPS,1.5,B Chen,,John Smith B,NMBGMR,Owner,Primary,505-555-0101,Primary,,,john.smith@example.com,Primary,,,123 County Rd 7,,Mailing,NM,Los Lunas,87031,,,,,,,Maria Garcia V,NMBGMR,Principal Investigator,Secondary,505-555-0123,Home,,,maria.garcia@mrgcd.nm.gov,Work,,,1931 2nd St SW,Suite 200,Mailing,NM,Albuquerque,87102,,,,,,,Gate off County Rd 7 0.4 miles south of canal crossing,Domestic well in pump house east of residence,True,True,True,True,email,Call before visits during irrigation season,OSE-123456,2010-06-15,Interpreted fr geophys logs by source agency,280,45,"Memory of owner, operator, driller",Submersible,200,True,True,0.5,Top of steel casing inside pump house marked with orange paint,Domestic,,active,Biannual,Sample only when pump has been off more than 12 hours,Measure before owner starts irrigation,True,, +Middle Rio Grande Groundwater Monitoring,MRG-003_MP1f,Old Orchard Well,2025-01-20T09:00:00,B Chen,250000,4000000,13N,5320,Global positioning system (GPS),1.8,,,Emily Davis B,NMBGMR,Biologist,Primary,505-555-0303,Work,,,emily.davis@example.org,Work,,,78 Orchard Ln,,Mailing,NM,Los Lunas,87031,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,From Main St turn east on Orchard Ln well house at dead end,Abandoned irrigation well in small cinderblock building,False,False,False,True,phone,Owner prefers weekday visits,,1965-04-10,From driller's log or well report,350,60,From driller's log or well report,Jet,280,False,False,0.75,Top of steel casing under removable hatch use fixed reference mark,Irrigation,,abandoned,Annual,Sampling not permitted water level only when owner present,Well house can be locked coordinate ahead,False,, diff --git a/tests/features/data/well-inventory-valid-reordered.csv b/tests/features/data/well-inventory-valid-reordered.csv new file mode 100644 index 000000000..86c22411b --- /dev/null +++ b/tests/features/data/well-inventory-valid-reordered.csv @@ -0,0 +1,3 @@ +well_name_point_id,project,site_name,date_time,field_staff,utm_northing,utm_easting,utm_zone,elevation_method,elevation_ft,field_staff_2,measuring_point_height_ft,field_staff_3,contact_1_name,contact_1_organization,contact_1_role,contact_1_type,contact_1_phone_1,contact_1_phone_1_type,contact_1_phone_2,contact_1_phone_2_type,contact_1_email_1,contact_1_email_1_type,contact_1_email_2,contact_1_email_2_type,contact_1_address_1_line_1,contact_1_address_1_line_2,contact_1_address_1_type,contact_1_address_1_state,contact_1_address_1_city,contact_1_address_1_postal_code,contact_1_address_2_line_1,contact_1_address_2_line_2,contact_1_address_2_type,contact_1_address_2_state,contact_1_address_2_city,contact_1_address_2_postal_code,contact_2_name,contact_2_organization,contact_2_role,contact_2_type,contact_2_phone_1,contact_2_phone_1_type,contact_2_phone_2,contact_2_phone_2_type,contact_2_email_1,contact_2_email_1_type,contact_2_email_2,contact_2_email_2_type,contact_2_address_1_line_1,contact_2_address_1_line_2,contact_2_address_1_type,contact_2_address_1_state,contact_2_address_1_city,contact_2_address_1_postal_code,contact_2_address_2_line_1,contact_2_address_2_line_2,contact_2_address_2_type,contact_2_address_2_state,contact_2_address_2_city,contact_2_address_2_postal_code,directions_to_site,specific_location_of_well,repeat_measurement_permission,sampling_permission,datalogger_installation_permission,public_availability_acknowledgement,result_communication_preference,contact_special_requests_notes,ose_well_record_id,date_drilled,completion_source,total_well_depth_ft,historic_depth_to_water_ft,depth_source,well_pump_type,well_pump_depth_ft,is_open,datalogger_possible,casing_diameter_ft,measuring_point_description,well_purpose,well_purpose_2,well_hole_status,monitoring_frequency,sampling_scenario_notes,well_measuring_notes,sample_possible +MRG-001_MP12,Middle Rio Grande Groundwater Monitoring,Smith Farm Domestic Well,2025-02-15T10:30:00,A Lopez,4000000,250000,13N,Survey-grade GPS,5250,B Chen,1.5,,John Smith A,NMBGMR,Owner,Primary,505-555-0101,Primary,,,john.smith@example.com,Primary,,,123 County Rd 7,,Mailing,NM,Los Lunas,87031,,,,,,,Maria Garcia A,NMBGMR,Principal Investigator,Secondary,505-555-0123,Home,,,maria.garcia@mrgcd.nm.gov,Work,,,1931 2nd St SW,Suite 200,Mailing,NM,Albuquerque,87102,,,,,,,Gate off County Rd 7 0.4 miles south of canal crossing,Domestic well in pump house east of residence,True,True,True,True,email,Call before visits during irrigation season,OSE-123456,2010-06-15,Interpreted fr geophys logs by source agency,280,45,"Memory of owner, operator, driller",Submersible,200,True,True,0.5,Top of steel casing inside pump house marked with orange paint,Domestic,,active,Biannual,Sample only when pump has been off more than 12 hours,Measure before owner starts irrigation,True +MRG-003_MP12,Middle Rio Grande Groundwater Monitoring,Old Orchard Well,2025-01-20T09:00:00,B Chen,4000000,250000,13N,Global positioning system (GPS),5320,,1.8,,Emily Davis A,NMBGMR,Biologist,Primary,505-555-0303,Work,,,emily.davis@example.org,Work,,,78 Orchard Ln,,Mailing,NM,Los Lunas,87031,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,From Main St turn east on Orchard Ln well house at dead end,Abandoned irrigation well in small cinderblock building,False,False,False,True,phone,Owner prefers weekday visits,,1965-04-10,From driller's log or well report,350,60,From driller's log or well report,Jet,280,False,False,0.75,Top of steel casing under removable hatch use fixed reference mark,Irrigation,,abandoned,Annual,Sampling not permitted water level only when owner present,Well house can be locked coordinate ahead,False diff --git a/tests/features/data/well-inventory-valid.csv b/tests/features/data/well-inventory-valid.csv new file mode 100644 index 000000000..0e6b7ecb2 --- /dev/null +++ b/tests/features/data/well-inventory-valid.csv @@ -0,0 +1,3 @@ +project,well_name_point_id,site_name,date_time,field_staff,utm_easting,utm_northing,utm_zone,elevation_ft,elevation_method,measuring_point_height_ft,field_staff_2,field_staff_3,contact_1_name,contact_1_organization,contact_1_role,contact_1_type,contact_1_phone_1,contact_1_phone_1_type,contact_1_phone_2,contact_1_phone_2_type,contact_1_email_1,contact_1_email_1_type,contact_1_email_2,contact_1_email_2_type,contact_1_address_1_line_1,contact_1_address_1_line_2,contact_1_address_1_type,contact_1_address_1_state,contact_1_address_1_city,contact_1_address_1_postal_code,contact_1_address_2_line_1,contact_1_address_2_line_2,contact_1_address_2_type,contact_1_address_2_state,contact_1_address_2_city,contact_1_address_2_postal_code,contact_2_name,contact_2_organization,contact_2_role,contact_2_type,contact_2_phone_1,contact_2_phone_1_type,contact_2_phone_2,contact_2_phone_2_type,contact_2_email_1,contact_2_email_1_type,contact_2_email_2,contact_2_email_2_type,contact_2_address_1_line_1,contact_2_address_1_line_2,contact_2_address_1_type,contact_2_address_1_state,contact_2_address_1_city,contact_2_address_1_postal_code,contact_2_address_2_line_1,contact_2_address_2_line_2,contact_2_address_2_type,contact_2_address_2_state,contact_2_address_2_city,contact_2_address_2_postal_code,directions_to_site,specific_location_of_well,repeat_measurement_permission,sampling_permission,datalogger_installation_permission,public_availability_acknowledgement,result_communication_preference,contact_special_requests_notes,ose_well_record_id,date_drilled,completion_source,total_well_depth_ft,historic_depth_to_water_ft,depth_source,well_pump_type,well_pump_depth_ft,is_open,datalogger_possible,casing_diameter_ft,measuring_point_description,well_purpose,well_purpose_2,well_hole_status,monitoring_frequency,sampling_scenario_notes,well_measuring_notes,sample_possible +Middle Rio Grande Groundwater Monitoring,MRG-001_MP1,Smith Farm Domestic Well,2025-02-15T10:30:00,A Lopez,250000,4000000,13N,5250,Survey-grade GPS,1.5,B Chen,,John Smith,NMBGMR,Owner,Primary,505-555-0101,Primary,,,john.smith@example.com,Primary,,,123 County Rd 7,,Mailing,NM,Los Lunas,87031,,,,,,,Maria Garcia,NMBGMR,Principal Investigator,Secondary,505-555-0123,Home,,,maria.garcia@mrgcd.nm.gov,Work,,,1931 2nd St SW,Suite 200,Mailing,NM,Albuquerque,87102,,,,,,,Gate off County Rd 7 0.4 miles south of canal crossing,Domestic well in pump house east of residence,True,True,True,True,email,Call before visits during irrigation season,OSE-123456,2010-06-15,Interpreted fr geophys logs by source agency,280,45,"Memory of owner, operator, driller",Submersible,200,True,True,0.5,Top of steel casing inside pump house marked with orange paint,Domestic,,active,Biannual,Sample only when pump has been off more than 12 hours,Measure before owner starts irrigation,True +Middle Rio Grande Groundwater Monitoring,MRG-003_MP1,Old Orchard Well,2025-10-20T09:00:00,B Chen,250000,4000000,13N,5320,Global positioning system (GPS),1.8,,,Emily Davis,NMBGMR,Biologist,Primary,505-555-0303,Work,,,emily.davis@example.org,Work,,,78 Orchard Ln,,Mailing,NM,Los Lunas,87031,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,From Main St turn east on Orchard Ln well house at dead end,Abandoned irrigation well in small cinderblock building,False,False,False,True,phone,Owner prefers weekday visits,,1965-04-10,From driller's log or well report,350,60,From driller's log or well report,Jet,280,False,False,0.75,Top of steel casing under removable hatch use fixed reference mark,Irrigation,,abandoned,Annual,Sampling not permitted water level only when owner present,Well house can be locked coordinate ahead,False diff --git a/tests/features/environment.py b/tests/features/environment.py index c8ddcb13b..f238e9d22 100644 --- a/tests/features/environment.py +++ b/tests/features/environment.py @@ -32,7 +32,6 @@ TransducerObservationBlock, WellCasingMaterial, PermissionHistory, - Contact, StatusHistory, ThingIdLink, WellPurpose, @@ -46,6 +45,7 @@ ThingGeologicFormationAssociation, Base, Asset, + Contact, Sample, ) from db.engine import session_ctx @@ -104,7 +104,6 @@ def add_well(context, session, location, name_num): well_construction_method="Driven", well_pump_type="Submersible", well_pump_depth=8, - is_suitable_for_datalogger=True, formation_completion_code="000EXRV", ) @@ -503,9 +502,8 @@ def add_geologic_formation(context, session, formation_code, well): def before_all(context): context.objects = {} - rebuild = False - # rebuild = True - erase_data = True + rebuild = True + erase_data = False if rebuild: erase_and_rebuild_db() elif erase_data: @@ -533,6 +531,8 @@ def before_all(context): sensor_1 = add_sensor(context, session) deployment = add_deployment(context, session, well_1.id, sensor_1.id) + for well in [well_1, well_2, well_3]: + add_measuring_point_history(context, session, well=well) add_well_casing_material(context, session, well_1) contact = add_contact(context, session) @@ -583,14 +583,31 @@ def before_all(context): target_table="thing", ) - for value, start, end in ( - ("Currently monitored", datetime(2020, 1, 1), datetime(2021, 1, 1)), - ("Not currently monitored", datetime(2021, 1, 1), None), + for value, status_type, start, end in ( + ( + "Currently monitored", + "Monitoring Status", + datetime(2020, 1, 1), + datetime(2021, 1, 1), + ), + ( + "Not currently monitored", + "Monitoring Status", + datetime(2021, 1, 1), + None, + ), + ("Open", "Open Status", datetime(2020, 1, 1), None), + ( + "Datalogger can be installed", + "Datalogger Suitability Status", + datetime(2020, 1, 1), + None, + ), ): add_status_history( context, session, - status_type="Monitoring Status", + status_type=status_type, status_value=value, start_date=start, end_date=end, diff --git a/tests/features/steps/common.py b/tests/features/steps/common.py index ccfe3b79f..79d8433cd 100644 --- a/tests/features/steps/common.py +++ b/tests/features/steps/common.py @@ -91,6 +91,15 @@ def step_impl(context): ), f"Unexpected response: {context.response.text}" +@then("the system returns a 201 Created status code") +def step_impl(context): + assert context.response.status_code == 201, ( + f"Unexpected response status code " + f"{context.response.status_code}. " + f"Response json: {context.response.json()}" + ) + + @then("the system should return a 200 status code") def step_impl(context): assert ( @@ -105,6 +114,20 @@ def step_impl(context): ), f"Unexpected response status code {context.response.status_code}" +@then("the system returns a 400 status code") +def step_impl(context): + assert ( + context.response.status_code == 400 + ), f"Unexpected response status code {context.response.status_code}" + + +@then("the system returns a 422 Unprocessable Entity status code") +def step_impl(context): + assert ( + context.response.status_code == 422 + ), f"Unexpected response status code {context.response.status_code}" + + @then("the response should be paginated") def step_impl(context): data = context.response.json() diff --git a/tests/features/steps/post_migration_legacy_data.py b/tests/features/steps/post_migration_legacy_data.py index 185b1a758..d10d8dc8e 100644 --- a/tests/features/steps/post_migration_legacy_data.py +++ b/tests/features/steps/post_migration_legacy_data.py @@ -14,11 +14,12 @@ # limitations under the License. # =============================================================================== from datetime import date, datetime, timezone + +import parse from behave import given, when, then, register_type from behave.runner import Context -import parse -from db import Location, Thing, LocationThingAssociation +from db import Location from db.engine import session_ctx @@ -275,7 +276,7 @@ def step_then_all_have_date_created_field(context: Context): """Assert all locations have the date created field.""" items = context.locations_response.get("items", []) for item in items: - assert "nma_date_created" in item, f"Location missing nma_date_created" + assert "nma_date_created" in item, "Location missing nma_date_created" @then("each location should have a site date field") @@ -283,7 +284,7 @@ def step_then_all_have_site_date_field(context: Context): """Assert all locations have the site date field.""" items = context.locations_response.get("items", []) for item in items: - assert "nma_site_date" in item, f"Location missing nma_site_date" + assert "nma_site_date" in item, "Location missing nma_site_date" @then("some locations should have null site date") diff --git a/tests/features/steps/water-levels-csv.py b/tests/features/steps/water-levels-csv.py index 06901f74d..2176e4ebc 100644 --- a/tests/features/steps/water-levels-csv.py +++ b/tests/features/steps/water-levels-csv.py @@ -121,18 +121,12 @@ def step_impl(context: Context): _set_rows(context, rows) -@given("my CSV file is encoded in UTF-8 and uses commas as separators") -def step_impl(context: Context): - assert context.csv_raw_text.encode("utf-8").decode("utf-8") == context.csv_raw_text - assert "," in context.csv_raw_text.splitlines()[0] - - @given("my CSV file contains multiple rows of water level entry data") def step_impl(context: Context): assert len(context.csv_rows) >= 2 -@given("the CSV includes required fields:") +@given("the water level CSV includes required fields:") def step_impl(context: Context): field_name = context.table.headings[0] expected_fields = [row[field_name].strip() for row in context.table] @@ -159,13 +153,13 @@ def step_impl(context: Context): assert "T" in row["measurement_date_time"] -@given("the CSV includes optional fields when available:") -def step_impl(context: Context): - field_name = context.table.headings[0] - optional_fields = [row[field_name].strip() for row in context.table] - headers = set(context.csv_headers) - missing = [field for field in optional_fields if field not in headers] - assert not missing, f"Missing optional headers: {missing}" +# @given("the water level CSV includes optional fields when available:") +# def step_impl(context: Context): +# field_name = context.table.headings[0] +# optional_fields = [row[field_name].strip() for row in context.table] +# headers = set(context.csv_headers) +# missing = [field for field in optional_fields if field not in headers] +# assert not missing, f"Missing optional headers: {missing}" @when("I run the CLI command:") @@ -225,7 +219,9 @@ def step_impl(context: Context): # ============================================================================ # Scenario: Upload succeeds when required columns are present but reordered # ============================================================================ -@given("my CSV file contains all required headers but in a different column order") +@given( + "my water level CSV file contains all required headers but in a different column order" +) def step_impl(context: Context): rows = _build_valid_rows(context) headers = list(reversed(list(rows[0].keys()))) @@ -244,7 +240,7 @@ def step_impl(context: Context): # ============================================================================ # Scenario: Upload succeeds when CSV contains extra columns # ============================================================================ -@given("my CSV file contains extra columns but is otherwise valid") +@given("my water level CSV file contains extra columns but is otherwise valid") def step_impl(context: Context): rows = _build_valid_rows(context) for idx, row in enumerate(rows): @@ -258,7 +254,7 @@ def step_impl(context: Context): # Scenario: No entries imported when any row fails validation # ============================================================================ @given( - 'my CSV file contains 3 rows of data with 2 valid rows and 1 row missing the required "well_name_point_id"' + 'my water level CSV contains 3 rows with 2 valid rows and 1 row missing the required "well_name_point_id"' ) def step_impl(context: Context): rows = _build_valid_rows(context, count=3) @@ -289,7 +285,9 @@ def step_impl(context: Context): # ============================================================================ # Scenario Outline: Upload fails when a required field is missing # ============================================================================ -@given('my CSV file contains a row missing the required "{required_field}" field') +@given( + 'my water level CSV file contains a row missing the required "{required_field}" field' +) def step_impl(context: Context, required_field: str): rows = _build_valid_rows(context, count=1) rows[0][required_field] = "" diff --git a/tests/features/steps/well-additional-information.py b/tests/features/steps/well-additional-information.py index 8b00f7eb7..8eecef159 100644 --- a/tests/features/steps/well-additional-information.py +++ b/tests/features/steps/well-additional-information.py @@ -221,10 +221,15 @@ def step_impl(context): "the response should include whether the well is open and suitable for a datalogger" ) def step_impl(context): - assert "is_suitable_for_datalogger" in context.water_well_data + assert "datalogger_installation_status" in context.water_well_data + assert "open_status" in context.water_well_data assert ( - context.water_well_data["is_suitable_for_datalogger"] - == context.objects["wells"][0].is_suitable_for_datalogger + context.water_well_data["datalogger_installation_status"] + == context.objects["wells"][0].datalogger_installation_status + ) + assert ( + context.water_well_data["open_status"] + == context.objects["wells"][0].open_status ) diff --git a/tests/features/steps/well-core-information.py b/tests/features/steps/well-core-information.py index 0ae559c2b..f389d6af9 100644 --- a/tests/features/steps/well-core-information.py +++ b/tests/features/steps/well-core-information.py @@ -294,7 +294,7 @@ def step_impl(context): ] == { "easting": point_utm_zone_13.x, "northing": point_utm_zone_13.y, - "utm_zone": 13, + "utm_zone": "13N", "horizontal_datum": "NAD83", } diff --git a/tests/features/steps/well-inventory-csv-given.py b/tests/features/steps/well-inventory-csv-given.py new file mode 100644 index 000000000..4889984bd --- /dev/null +++ b/tests/features/steps/well-inventory-csv-given.py @@ -0,0 +1,321 @@ +# =============================================================================== +# Copyright 2025 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +import csv +from io import StringIO +from pathlib import Path + +import pandas as pd +from behave import given +from behave.runner import Context + + +def _set_file_content(context: Context, name): + path = Path("tests") / "features" / "data" / name + context.file_path = path + with open(path, "r") as f: + context.file_name = name + context.file_content = f.read() + if name.endswith(".csv"): + context.rows = list(csv.DictReader(context.file_content.splitlines())) + context.row_count = len(context.rows) + context.file_type = "text/csv" + else: + context.rows = [] + context.row_count = 0 + context.file_type = "text/plain" + + +@given( + 'my CSV file contains a row with a contact but is missing the required "contact_role" field for that contact' +) +def step_impl(context: Context): + _set_file_content(context, "well-inventory-missing-contact-role.csv") + + +@given( + "my CSV file contains a row that has an invalid postal code format in contact_1_address_1_postal_code" +) +def step_impl(context: Context): + _set_file_content(context, "well-inventory-invalid-postal-code.csv") + + +@given("a valid CSV file for bulk well inventory upload") +def step_impl_valid_csv_file(context: Context): + _set_file_content(context, "well-inventory-valid.csv") + + +@given('my CSV file contains rows missing a required field "well_name_point_id"') +def step_impl(context: Context): + _set_file_content(context, "well-inventory-missing-required.csv") + + +@given('my CSV file contains one or more duplicate "well_name_point_id" values') +def step_impl(context: Context): + _set_file_content(context, "well-inventory-duplicate.csv") + + +@given( + 'my CSV file contains invalid lexicon values for "contact_role" or other lexicon fields' +) +def step_impl(context: Context): + _set_file_content(context, "well-inventory-invalid-lexicon.csv") + + +@given('my CSV file contains invalid ISO 8601 date values in the "date_time" field') +def step_impl(context: Context): + _set_file_content(context, "well-inventory-invalid-date.csv") + + +@given( + 'my CSV file contains values that cannot be parsed as numeric in numeric-required fields such as "utm_easting"' +) +def step_impl(context: Context): + _set_file_content(context, "well-inventory-invalid-numeric.csv") + + +@given("my CSV file contains column headers but no data rows") +def step_impl(context: Context): + _set_file_content(context, "well-inventory-no-data-headers.csv") + + +@given("my CSV file is empty") +def step_impl(context: Context): + # context.file_content = "" + # context.rows = [] + # context.file_type = "text/csv" + _set_file_content(context, "well-inventory-empty.csv") + + +@given("I have a non-CSV file") +def step_impl(context: Context): + _set_file_content(context, "well-inventory-invalid-filetype.txt") + + +@given("my CSV file contains multiple rows of well inventory data") +def step_impl_csv_file_contains_multiple_rows(context: Context): + """Sets up the CSV file with multiple rows of well inventory data.""" + assert len(context.rows) > 0, "CSV file contains no data rows" + + +@given("my CSV file is encoded in UTF-8 and uses commas as separators") +def step_impl_csv_file_is_encoded_utf8(context: Context): + assert context.file_content.encode("utf-8").decode("utf-8") == context.file_content + + # determine the separator from the file content + sample = context.file_content[:1024] + dialect = csv.Sniffer().sniff(sample) + assert dialect.delimiter == "," + + +@given( + "my CSV file contains a row with a contact with a phone number that is not in the valid format" +) +def step_impl(context: Context): + _set_file_content(context, "well-inventory-invalid-phone-number.csv") + + +@given( + "my CSV file contains a row with a contact with an email that is not in the valid format" +) +def step_impl(context: Context): + _set_file_content(context, "well-inventory-invalid-email.csv") + + +@given( + 'my CSV file contains a row with a contact but is missing the required "contact_type" field for that contact' +) +def step_impl(context: Context): + _set_file_content(context, "well-inventory-missing-contact-type.csv") + + +@given( + 'my CSV file contains a row with a contact_type value that is not in the valid lexicon for "contact_type"' +) +def step_impl(context: Context): + _set_file_content(context, "well-inventory-invalid-contact-type.csv") + + +@given( + 'my CSV file contains a row with a contact with an email but is missing the required "email_type" field for that email' +) +def step_impl(context: Context): + _set_file_content(context, "well-inventory-missing-email-type.csv") + + +@given( + 'my CSV file contains a row with a contact with a phone but is missing the required "phone_type" field for that phone' +) +def step_impl(context: Context): + _set_file_content(context, "well-inventory-missing-phone-type.csv") + + +@given( + 'my CSV file contains a row with a contact with an address but is missing the required "address_type" field for that address' +) +def step_impl(context: Context): + _set_file_content(context, "well-inventory-missing-address-type.csv") + + +@given( + "my CSV file contains a row with utm_easting utm_northing and utm_zone values that are not within New Mexico" +) +def step_impl(context: Context): + _set_file_content(context, "well-inventory-invalid-utm.csv") + + +@given( + 'my CSV file contains invalid ISO 8601 date values in the "date_time" or "date_drilled" field' +) +def step_impl(context: Context): + _set_file_content(context, "well-inventory-invalid-date-format.csv") + + +@given("my CSV file contains all required headers but in a different column order") +def step_impl(context: Context): + _set_file_content(context, "well-inventory-valid-reordered.csv") + + +@given("my CSV file contains extra columns but is otherwise valid") +def step_impl(context: Context): + _set_file_content(context, "well-inventory-valid-extra-columns.csv") + + +@given( + 'my CSV file contains 3 rows of data with 2 valid rows and 1 row missing the required "well_name_point_id"' +) +def step_impl(context: Context): + _set_file_content(context, "well-inventory-invalid-partial.csv") + + +@given('my CSV file contains a row missing the required "{required_field}" field') +def step_impl(context, required_field): + _set_file_content(context, "well-inventory-valid.csv") + + df = pd.read_csv(context.file_path, dtype={"contact_2_address_1_postal_code": str}) + df = df.drop(required_field, axis=1) + + buffer = StringIO() + df.to_csv(buffer, index=False) + + context.file_content = buffer.getvalue() + context.rows = list(csv.DictReader(context.file_content.splitlines())) + + +@given( + 'my CSV file contains a row with an invalid boolean value "maybe" in the "is_open" field' +) +def step_impl(context: Context): + _set_file_content(context, "well-inventory-invalid-boolean-value-maybe.csv") + + +@given("my CSV file contains a valid but duplicate header row") +def step_impl(context: Context): + _set_file_content(context, "well-inventory-duplicate-header.csv") + + +@given( + 'my CSV file header row contains the "contact_1_email_1" column name more than once' +) +def step_impl(context: Context): + _set_file_content(context, "well-inventory-duplicate-columns.csv") + + +def _get_valid_df(context: Context) -> pd.DataFrame: + _set_file_content(context, "well-inventory-valid.csv") + df = pd.read_csv(context.file_path, dtype={"contact_2_address_1_postal_code": str}) + return df + + +def _set_content_from_df(context: Context, df: pd.DataFrame, delimiter: str = ","): + buffer = StringIO() + df.to_csv(buffer, index=False, sep=delimiter) + context.file_content = buffer.getvalue() + context.rows = list(csv.DictReader(context.file_content.splitlines())) + + +@given("my CSV file contains more rows than the configured maximum for bulk upload") +def step_impl(context: Context): + df = _get_valid_df(context) + + df = pd.concat([df.iloc[:2]] * 1001, ignore_index=True) + + _set_content_from_df(context, df) + + +@given("my file is named with a .csv extension") +def step_impl(context: Context): + _set_file_content(context, "well-inventory-valid.csv") + + +@given( + 'my file uses "{delimiter_description}" as the field delimiter instead of commas' +) +def step_impl(context, delimiter_description: str): + df = _get_valid_df(context) + + if delimiter_description == "semicolons": + delimiter = ";" + else: + delimiter = "\t" + + context.delimiter = delimiter + _set_content_from_df(context, df, delimiter=delimiter) + + +@given("my CSV file header row contains all required columns") +def step_impl(context: Context): + _set_file_content(context, "well-inventory-valid.csv") + + +@given( + 'my CSV file contains a data row where the "site_name" field value includes a comma and is enclosed in quotes' +) +def step_impl(context: Context): + _set_file_content(context, "well-inventory-valid-comma-in-quotes.csv") + + +@given( + "my CSV file contains a data row where a field begins with a quote but does not have a matching closing quote" +) +def step_impl(context: Context): + df = _get_valid_df(context) + df.loc[0]["well_name_point_id"] = '"well-name-point-id' + _set_content_from_df(context, df) + + +@given( + 'my CSV file contains all valid columns but uses "XY-" prefix for well_name_point_id values' +) +def step_impl(context: Context): + df = _get_valid_df(context) + df["well_name_point_id"] = df["well_name_point_id"].apply(lambda x: "XY-") + + # change contact name + df.loc[0, "contact_1_name"] = "Contact 1" + df.loc[0, "contact_2_name"] = "Contact 2" + df.loc[1, "contact_1_name"] = "Contact 3" + + _set_content_from_df(context, df) + + +@given( + "my csv file contains a row where some but not all water level entry fields are filled" +) +def step_impl(context): + _set_file_content(context, "well-inventory-missing-wl-fields.csv") + + +# ============= EOF ============================================= diff --git a/tests/features/steps/well-inventory-csv-validation-error.py b/tests/features/steps/well-inventory-csv-validation-error.py new file mode 100644 index 000000000..10443ea5c --- /dev/null +++ b/tests/features/steps/well-inventory-csv-validation-error.py @@ -0,0 +1,209 @@ +# =============================================================================== +# Copyright 2025 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== + +from behave import then +from behave.runner import Context + + +def _handle_validation_error(context, expected_errors): + response_json = context.response.json() + validation_errors = response_json.get("validation_errors", []) + print(validation_errors) + n = len(validation_errors) + assert len(validation_errors) == n, f"Expected {n} validation error" + for v, e in zip(validation_errors, expected_errors): + assert v["field"] == e["field"], f"Expected {e['field']} for {v['field']}" + assert v["error"] == e["error"], f"Expected {e['error']} for {v['error']}" + if "value" in e: + assert v["value"] == e["value"], f"Expected {e['value']} for {v['value']}" + + +@then( + 'the response includes a validation error indicating the missing "address_type" value' +) +def step_impl(context: Context): + expected_errors = [ + { + "field": "composite field error", + "error": "Value error, All contact address fields must be provided", + } + ] + _handle_validation_error(context, expected_errors) + + +@then("the response includes a validation error indicating the invalid UTM coordinates") +def step_impl(context: Context): + expected_errors = [ + { + "field": "composite field error", + "error": "Value error, UTM coordinates are outside of the NM. E=457100.0 N=4159020.0 Zone=13N", + }, + { + "field": "composite field error", + "error": "Value error, UTM coordinates are outside of the NM. E=250000.0 N=4000000.0 Zone=13S", + }, + ] + _handle_validation_error(context, expected_errors) + + +@then( + 'the response includes a validation error indicating an invalid "contact_type" value' +) +def step_impl(context): + expected_errors = [ + { + "field": "contact_1_type", + "error": "Input should be 'Primary', 'Secondary' or 'Field Event Participant'", + } + ] + _handle_validation_error(context, expected_errors) + + +@then( + 'the response includes a validation error indicating the missing "email_type" value' +) +def step_impl(context): + expected_errors = [ + { + "field": "composite field error", + "error": "Value error, contact_1_email_1_type type must be provided if email is provided", + } + ] + _handle_validation_error(context, expected_errors) + + +@then( + 'the response includes a validation error indicating the missing "phone_type" value' +) +def step_impl(context): + expected_errors = [ + { + "field": "composite field error", + "error": "Value error, contact_1_phone_1_type must be provided if phone number is provided", + } + ] + _handle_validation_error(context, expected_errors) + + +@then( + 'the response includes a validation error indicating the missing "contact_role" field' +) +def step_impl(context): + expected_errors = [ + { + "field": "composite field error", + "error": "Value error, contact_1_role must be provided if name is provided", + } + ] + _handle_validation_error(context, expected_errors) + + +@then( + "the response includes a validation error indicating the invalid postal code format" +) +def step_impl(context): + expected_errors = [ + { + "field": "contact_1_address_1_postal_code", + "error": "Value error, Invalid postal code", + } + ] + _handle_validation_error(context, expected_errors) + + +@then( + "the response includes a validation error indicating the invalid phone number format" +) +def step_impl(context): + expected_errors = [ + { + "field": "contact_1_phone_1", + "error": "Value error, Invalid phone number. 55-555-0101", + } + ] + _handle_validation_error(context, expected_errors) + + +@then("the response includes a validation error indicating the invalid email format") +def step_impl(context): + expected_errors = [ + { + "field": "contact_1_email_1", + "error": "Value error, Invalid email format. john.smithexample.com", + } + ] + _handle_validation_error(context, expected_errors) + + +@then( + 'the response includes a validation error indicating the missing "contact_type" value' +) +def step_impl(context): + expected_errors = [ + { + "field": "composite field error", + "error": "Value error, contact_1_type must be provided if name is provided", + } + ] + _handle_validation_error(context, expected_errors) + + +@then("the response includes a validation error indicating a repeated header row") +def step_impl(context: Context): + expected_errors = [{"field": "header", "error": "Duplicate header row"}] + _handle_validation_error(context, expected_errors) + + +@then("the response includes a validation error indicating duplicate header names") +def step_impl(context: Context): + + expected_errors = [ + {"field": "['contact_1_email_1']", "error": "Duplicate columns found"} + ] + _handle_validation_error(context, expected_errors) + + +@then( + 'the response includes a validation error indicating an invalid boolean value for the "is_open" field' +) +def step_impl(context: Context): + expected_errors = [ + { + "field": "is_open", + "error": "Input should be a valid boolean, unable to interpret input", + } + ] + _handle_validation_error(context, expected_errors) + + +@then( + "the response includes validation errors for each missing water level entry field" +) +def step_impl(context): + expected_errors = [ + { + "field": "composite field error", + "error": "Value error, All water level fields must be provided", + }, + { + "field": "composite field error", + "error": "Value error, All water level fields must be provided", + }, + ] + _handle_validation_error(context, expected_errors) + + +# ============= EOF ============================================= diff --git a/tests/features/steps/well-inventory-csv.py b/tests/features/steps/well-inventory-csv.py new file mode 100644 index 000000000..8cd69b035 --- /dev/null +++ b/tests/features/steps/well-inventory-csv.py @@ -0,0 +1,334 @@ +from datetime import datetime, timedelta + +from behave import given, when, then +from behave.runner import Context + +from services.util import convert_dt_tz_naive_to_tz_aware + + +@given("valid lexicon values exist for:") +def step_impl_valid_lexicon_values(context: Context): + for row in context.table: + response = context.client.get( + "/lexicon/category", + params={"name": row[0]}, + ) + assert response.status_code == 200, f"Invalid lexicon category: {row[0]}" + + +@given("the CSV includes required fields:") +def step_impl_csv_includes_required_fields(context: Context): + """Sets up the CSV file with multiple rows of well inventory data.""" + context.required_fields = [row[0] for row in context.table] + keys = context.rows[0].keys() + for field in context.required_fields: + assert field in keys, f"Missing required field: {field}" + + +@given('each "well_name_point_id" value is unique per row') +def step_impl(context: Context): + """Verifies that each "well_name_point_id" value is unique per row.""" + seen_ids = set() + for row in context.rows: + if row["well_name_point_id"] in seen_ids: + raise ValueError( + f"Duplicate well_name_point_id: {row['well_name_point_id']}" + ) + seen_ids.add(row["well_name_point_id"]) + + +@given("the CSV includes optional fields when available:") +def step_impl(context: Context): + optional_fields = [row[0] for row in context.table] + keys = context.rows[0].keys() + + for key in keys: + if key not in context.required_fields: + assert key in optional_fields, f"Unexpected field found: {key}" + + +@given("the csv includes optional water level entry fields when available:") +def step_impl(context: Context): + optional_fields = [row[0] for row in context.table] + context.water_level_optional_fields = optional_fields + + +@given( + 'the required "date_time" values are valid ISO 8601 timezone-naive datetime strings (e.g. "2025-02-15T10:30:00")' +) +def step_impl(context: Context): + """Verifies that "date_time" values are valid ISO 8601 timezone-naive datetime strings.""" + for row in context.rows: + try: + date_time = datetime.fromisoformat(row["date_time"]) + assert ( + date_time.tzinfo is None + ), f"date_time should be timezone-naive: {row['date_time']}" + except ValueError as e: + raise ValueError(f"Invalid date_time: {row['date_time']}") from e + + +@given( + 'the optional "water_level_date_time" values are valid ISO 8601 timezone-naive datetime strings (e.g. "2025-02-15T10:30:00") when provided' +) +def step_impl(context: Context): + """Verifies that "water_level_date_time" values are valid ISO 8601 timezone-naive datetime strings.""" + for row in context.rows: + if row.get("water_level_date_time", None): + try: + date_time = datetime.fromisoformat(row["water_level_date_time"]) + assert ( + date_time.tzinfo is None + ), f"water_level_date_time should be timezone-naive: {row['water_level_date_time']}" + except ValueError as e: + raise ValueError( + f"Invalid water_level_date_time: {row['water_level_date_time']}" + ) from e + + +@when("I upload the file to the bulk upload endpoint") +def step_impl(context: Context): + context.response = context.client.post( + "/well-inventory-csv", + files={"file": (context.file_name, context.file_content, context.file_type)}, + ) + + +@then( + "all datetime objects are assigned the correct Mountain Time timezone offset based on the date value." +) +def step_impl(context: Context): + """Converts all datetime strings in the CSV rows to timezone-aware datetime objects with Mountain Time offset.""" + for i, row in enumerate(context.rows): + # Convert date_time field + date_time_naive = datetime.fromisoformat(row["date_time"]) + date_time_aware = convert_dt_tz_naive_to_tz_aware( + date_time_naive, "America/Denver" + ) + row["date_time"] = date_time_aware.isoformat() + + # confirm correct time zone and offset + if i == 0: + # MST, offset -07:00 + assert date_time_aware.utcoffset() == timedelta( + hours=-7 + ), "date_time offset is not -07:00" + else: + # MDT, offset -06:00 + assert date_time_aware.utcoffset() == timedelta( + hours=-6 + ), "date_time offset is not -06:00" + + # confirm the time was not changed from what was provided + assert ( + date_time_aware.replace(tzinfo=None) == date_time_naive + ), "date_time value was changed during timezone assignment" + + # Convert water_level_date_time field if it exists + if row.get("water_level_date_time", None): + wl_date_time_naive = datetime.fromisoformat(row["water_level_date_time"]) + wl_date_time_aware = convert_dt_tz_naive_to_tz_aware( + wl_date_time_naive, "America/Denver" + ) + row["water_level_date_time"] = wl_date_time_aware.isoformat() + + if wl_date_time_aware.dst(): + # MDT, offset -06:00 + assert wl_date_time_aware.utcoffset() == timedelta( + hours=-6 + ), "water_level_date_time offset is not -06:00" + else: + # MST, offset -07:00 + assert wl_date_time_aware.utcoffset() == timedelta( + hours=-7 + ), "water_level_date_time offset is not -07:00" + + assert ( + wl_date_time_aware.replace(tzinfo=None) == wl_date_time_naive + ), "water_level_date_time value was changed during timezone assignment" + + +@then("the response includes a summary containing:") +def step_impl(context: Context): + response_json = context.response.json() + summary = response_json.get("summary", {}) + for row in context.table: + field = row[0] + expected_value = int(row[1]) + actual_value = summary.get(field) + assert ( + actual_value == expected_value + ), f"Expected {expected_value} for {field}, but got {actual_value}" + + +@then("the response includes an array of created well objects") +def step_impl(context: Context): + response_json = context.response.json() + wells = response_json.get("wells", []) + assert ( + len(wells) == context.row_count + ), "Expected the same number of wells as rows in the CSV" + + +@then("the response includes validation errors for all rows missing required fields") +def step_impl(context: Context): + response_json = context.response.json() + validation_errors = response_json.get("validation_errors", []) + assert len(validation_errors) == len( + context.rows + ), "Expected the same number of validation errors as rows in the CSV" + error_fields = [ + e["row"] for e in validation_errors if e["field"] == "well_name_point_id" + ] + for i, row in enumerate(context.rows): + if row["well_name_point_id"] == "": + assert i + 1 in error_fields, f"Missing required field for row {row}" + + +@then("the response identifies the row and field for each error") +def step_impl(context: Context): + response_json = context.response.json() + validation_errors = response_json.get("validation_errors", []) + for error in validation_errors: + assert "row" in error, "Expected validation error to include row number" + assert "field" in error, "Expected validation error to include field name" + + +@then("no wells are imported") +def step_impl(context: Context): + response_json = context.response.json() + wells = response_json.get("wells", []) + assert len(wells) == 0, "Expected no wells to be imported" + + +@then("the response includes validation errors indicating duplicated values") +def step_impl(context: Context): + response_json = context.response.json() + validation_errors = response_json.get("validation_errors", []) + + assert len(validation_errors) == 1, "Expected 1 validation error" + + error_fields = [ + e["row"] for e in validation_errors if e["field"] == "well_name_point_id" + ] + assert error_fields == [2], f"Expected duplicated values for row {error_fields}" + assert ( + validation_errors[0]["error"] == "Duplicate value for well_name_point_id" + ), "Expected duplicated values for row 2" + + +@then("each error identifies the row and field") +def step_impl(context: Context): + response_json = context.response.json() + validation_errors = response_json.get("validation_errors", []) + for error in validation_errors: + assert "row" in error, "Expected validation error to include row number" + assert "field" in error, "Expected validation error to include field name" + + +@then("the response includes validation errors identifying the invalid field and row") +def step_impl(context: Context): + response_json = context.response.json() + validation_errors = response_json.get("validation_errors", []) + for error in validation_errors: + assert "field" in error, "Expected validation error to include field name" + assert "error" in error, "Expected validation error to include error message" + + +@then("the response includes an error message indicating unsupported file type") +def step_impl(context: Context): + response_json = context.response.json() + assert "detail" in response_json, "Expected response to include an detail object" + assert ( + response_json["detail"][0]["msg"] == "Unsupported file type" + ), "Expected error message to indicate unsupported file type" + + +@then("the response includes an error message indicating an empty file") +def step_impl(context: Context): + response_json = context.response.json() + assert "detail" in response_json, "Expected response to include an detail object" + assert ( + response_json["detail"][0]["msg"] == "Empty file" + ), "Expected error message to indicate an empty file" + + +@then("the response includes an error indicating that no data rows were found") +def step_impl(context: Context): + response_json = context.response.json() + assert "detail" in response_json, "Expected response to include an detail object" + assert ( + response_json["detail"][0]["msg"] == "No data rows found" + ), "Expected error message to indicate no data rows were found" + + +@then("all wells are imported") +def step_impl(context: Context): + response_json = context.response.json() + assert "wells" in response_json, "Expected response to include wells" + assert len(response_json["wells"]) == context.row_count + + +@then( + 'the response includes a validation error for the row missing "well_name_point_id"' +) +def step_impl(context: Context): + response_json = context.response.json() + assert "summary" in response_json, "Expected summary in response" + summary = response_json["summary"] + assert "total_rows_processed" in summary, "Expected total_rows_processed" + assert ( + summary["total_rows_processed"] == context.row_count + ), f"Expected total_rows_processed = {context.row_count}" + assert "total_rows_imported" in summary, "Expected total_rows_imported" + assert summary["total_rows_imported"] == 0, "Expected total_rows_imported=0" + assert ( + "validation_errors_or_warnings" in summary + ), "Expected validation_errors_or_warnings" + assert ( + summary["validation_errors_or_warnings"] == 1 + ), "Expected validation_errors_or_warnings = 1" + + assert "validation_errors" in response_json, "Expected validation_errors" + ve = response_json["validation_errors"] + assert ( + ve[0]["field"] == "well_name_point_id" + ), "Expected missing field well_name_point_id" + assert ve[0]["error"] == "Field required", "Expected Field required" + + +@then('the response includes a validation error for the "{required_field}" field') +def step_impl(context: Context, required_field: str): + response_json = context.response.json() + assert "validation_errors" in response_json, "Expected validation errors" + vs = response_json["validation_errors"] + assert len(vs) == 2, "Expected 2 validation error" + assert vs[0]["field"] == required_field + + +@then("the response includes an error message indicating the row limit was exceeded") +def step_impl(context: Context): + response_json = context.response.json() + assert "detail" in response_json, "Expected response to include an detail object" + assert ( + response_json["detail"][0]["msg"] == "Too many rows 2002>2000" + ), "Expected error message to indicate too many rows uploaded" + + +@then("the response includes an error message indicating an unsupported delimiter") +def step_impl(context: Context): + response_json = context.response.json() + assert "detail" in response_json, "Expected response to include an detail object" + assert ( + response_json["detail"][0]["msg"] + == f"Unsupported delimiter '{context.delimiter}'" + ), "Expected error message to indicate unsupported delimiter" + + +@then("all wells are imported with system-generated unique well_name_point_id values") +def step_impl(context: Context): + response_json = context.response.json() + assert "wells" in response_json, "Expected response to include wells" + wells = response_json["wells"] + assert len(wells) == context.row_count + assert len(wells) == len(set(wells)), "Expected unique well_name_point_id values" diff --git a/tests/features/water-level-csv.feature b/tests/features/water-level-csv.feature index 5974bdd21..d924da6f8 100644 --- a/tests/features/water-level-csv.feature +++ b/tests/features/water-level-csv.feature @@ -22,7 +22,7 @@ Feature: Bulk upload water level entries from CSV via CLI Given a valid CSV file for bulk water level entry upload And my CSV file is encoded in UTF-8 and uses commas as separators And my CSV file contains multiple rows of water level entry data - And the CSV includes required fields: + And the water level CSV includes required fields: | required field name | | field_staff | | well_name_point_id | @@ -57,7 +57,7 @@ Feature: Bulk upload water level entries from CSV via CLI @positive @validation @column_order @BDMS-TBD @cleanup_samples Scenario: Upload succeeds when required columns are present but in a different order - Given my CSV file contains all required headers but in a different column order + Given my water level CSV file contains all required headers but in a different column order And the CSV includes required fields: | required field name | | well_name_point_id | @@ -80,7 +80,7 @@ Feature: Bulk upload water level entries from CSV via CLI @positive @validation @extra_columns @BDMS-TBD @cleanup_samples Scenario: Upload succeeds when CSV contains extra, unknown columns - Given my CSV file contains extra columns but is otherwise valid + Given my water level CSV file contains extra columns but is otherwise valid When I run the CLI command: """ oco water-levels bulk-upload --file ./water_levels.csv @@ -95,7 +95,7 @@ Feature: Bulk upload water level entries from CSV via CLI @negative @validation @BDMS-TBD Scenario: No water level entries are imported when any row fails validation - Given my CSV file contains 3 rows of data with 2 valid rows and 1 row missing the required "well_name_point_id" + Given my water level CSV contains 3 rows with 2 valid rows and 1 row missing the required "well_name_point_id" When I run the CLI command: """ oco water-levels bulk-upload --file ./water_levels.csv @@ -106,7 +106,7 @@ Feature: Bulk upload water level entries from CSV via CLI @negative @validation @required_fields @BDMS-TBD Scenario Outline: Upload fails when a required field is missing - Given my CSV file contains a row missing the required "" field + Given my water level CSV file contains a row missing the required "" field When I run the CLI command: """ oco water-levels bulk-upload --file ./water_levels.csv diff --git a/tests/features/well-inventory-csv.feature b/tests/features/well-inventory-csv.feature new file mode 100644 index 000000000..38fb040b0 --- /dev/null +++ b/tests/features/well-inventory-csv.feature @@ -0,0 +1,483 @@ +@backend +@BDMS-TBD +@production +Feature: Bulk upload well inventory from CSV + As a hydrogeologist or data specialist + I want to upload a CSV file containing well inventory data for multiple wells + So that well records can be created efficiently and accurately in the system + + Background: + Given a functioning api + And valid lexicon values exist for: + | lexicon category | + | contact_role | + | contact_type | + | phone_type | + | email_type | + | address_type | + | elevation_method | + | well_pump_type | + | well_purpose | + | well_hole_status | + | monitoring_frequency | + | sample_method | + | level_status | + | data_quality | + + @positive @happy_path @BDMS-TBD + Scenario: Uploading a valid well inventory CSV containing required and optional fields + Given a valid CSV file for bulk well inventory upload + And my CSV file is encoded in UTF-8 and uses commas as separators + And my CSV file contains multiple rows of well inventory data + And the CSV includes required fields: + | required field name | + | project | + | well_name_point_id | + | site_name | + | date_time | + | field_staff | + | utm_easting | + | utm_northing | + | utm_zone | + | elevation_ft | + | elevation_method | + | measuring_point_height_ft | + And each "well_name_point_id" value is unique per row + And the CSV includes optional fields when available: + | optional field name | + | field_staff_2 | + | field_staff_3 | + | contact_1_name | + | contact_1_organization | + | contact_1_role | + | contact_1_type | + | contact_1_phone_1 | + | contact_1_phone_1_type | + | contact_1_phone_2 | + | contact_1_phone_2_type | + | contact_1_email_1 | + | contact_1_email_1_type | + | contact_1_email_2 | + | contact_1_email_2_type | + | contact_1_address_1_line_1 | + | contact_1_address_1_line_2 | + | contact_1_address_1_type | + | contact_1_address_1_state | + | contact_1_address_1_city | + | contact_1_address_1_postal_code | + | contact_1_address_2_line_1 | + | contact_1_address_2_line_2 | + | contact_1_address_2_type | + | contact_1_address_2_state | + | contact_1_address_2_city | + | contact_1_address_2_postal_code | + | contact_2_name | + | contact_2_organization | + | contact_2_role | + | contact_2_type | + | contact_2_phone_1 | + | contact_2_phone_1_type | + | contact_2_phone_2 | + | contact_2_phone_2_type | + | contact_2_email_1 | + | contact_2_email_1_type | + | contact_2_email_2 | + | contact_2_email_2_type | + | contact_2_address_1_line_1 | + | contact_2_address_1_line_2 | + | contact_2_address_1_type | + | contact_2_address_1_state | + | contact_2_address_1_city | + | contact_2_address_1_postal_code | + | contact_2_address_2_line_1 | + | contact_2_address_2_line_2 | + | contact_2_address_2_type | + | contact_2_address_2_state | + | contact_2_address_2_city | + | contact_2_address_2_postal_code | + | directions_to_site | + | specific_location_of_well | + | repeat_measurement_permission | + | sampling_permission | + | datalogger_installation_permission | + | public_availability_acknowledgement | + | result_communication_preference | + | contact_special_requests_notes | + | ose_well_record_id | + | date_drilled | + | completion_source | + | total_well_depth_ft | + | historic_depth_to_water_ft | + | depth_source | + | well_pump_type | + | well_pump_depth_ft | + | is_open | + | datalogger_possible | + | casing_diameter_ft | + | measuring_point_description | + | well_purpose | + | well_purpose_2 | + | well_hole_status | + | monitoring_frequency | + | sampling_scenario_notes | + | well_measuring_notes | + | sample_possible | + And the csv includes optional water level entry fields when available: + | water_level_entry fields | + | measuring_person | + | sample_method | + | water_level_date_time | + | mp_height | + | level_status | + | depth_to_water_ft | + | data_quality | + | water_level_notes | + And the required "date_time" values are valid ISO 8601 timezone-naive datetime strings (e.g. "2025-02-15T10:30:00") + And the optional "water_level_date_time" values are valid ISO 8601 timezone-naive datetime strings (e.g. "2025-02-15T10:30:00") when provided + +# And all optional lexicon fields contain valid lexicon values when provided +# And all optional numeric fields contain valid numeric values when provided +# And all optional date fields contain valid ISO 8601 timestamps when provided + + When I upload the file to the bulk upload endpoint + # assumes users are entering datetimes as Mountain Time becuase location is restricted to New Mexico + Then all datetime objects are assigned the correct Mountain Time timezone offset based on the date value. + And the system returns a 201 Created status code + And the system should return a response in JSON format +# And null values in the response are represented as JSON null + And the response includes a summary containing: + | summary_field | value | + | total_rows_processed | 2 | + | total_rows_imported | 2 | + | validation_errors_or_warnings | 0 | + And the response includes an array of created well objects + + @positive @validation @column_order @BDMS-TBD + Scenario: Upload succeeds when required columns are present but in a different order + Given my CSV file contains all required headers but in a different column order + And the CSV includes required fields: + | required field name | + | project | + | well_name_point_id | + | site_name | + | date_time | + | field_staff | + | utm_easting | + | utm_northing | + | utm_zone | + | elevation_ft | + | elevation_method | + | measuring_point_height_ft | + When I upload the file to the bulk upload endpoint + Then the system returns a 201 Created status code + And the system should return a response in JSON format + And all wells are imported + + @positive @validation @extra_columns @BDMS-TBD + Scenario: Upload succeeds when CSV contains extra, unknown columns + Given my CSV file contains extra columns but is otherwise valid + When I upload the file to the bulk upload endpoint + Then the system returns a 201 Created status code + And the system should return a response in JSON format + And all wells are imported + + @positive @validation @autogenerate_ids @BDMS-TBD + Scenario: Upload succeeds and system auto-generates well_name_point_id when prefixed with "XY- + Given my CSV file contains all valid columns but uses "XY-" prefix for well_name_point_id values + When I upload the file to the bulk upload endpoint + Then the system returns a 201 Created status code + And the system should return a response in JSON format + And all wells are imported with system-generated unique well_name_point_id values + + ########################################################################### + # NEGATIVE VALIDATION SCENARIOS + ########################################################################### + @negative @validation @transactional_import @BDMS-TBD + Scenario: No wells are imported when any row fails validation + Given my CSV file contains 3 rows of data with 2 valid rows and 1 row missing the required "well_name_point_id" + When I upload the file to the bulk upload endpoint + Then the system returns a 422 Unprocessable Entity status code + And the system should return a response in JSON format + And the response includes a validation error for the row missing "well_name_point_id" + And no wells are imported + + @negative @validation @BDMS-TBD + Scenario: Upload fails when a row has an invalid postal code format + Given my CSV file contains a row that has an invalid postal code format in contact_1_address_1_postal_code + When I upload the file to the bulk upload endpoint + Then the system returns a 422 Unprocessable Entity status code + And the system should return a response in JSON format + And the response includes a validation error indicating the invalid postal code format + And no wells are imported + + @negative @validation @BDMS-TBD + Scenario: Upload fails when a row has a contact with a invalid phone number format + Given my CSV file contains a row with a contact with a phone number that is not in the valid format + When I upload the file to the bulk upload endpoint + Then the system returns a 422 Unprocessable Entity status code + And the system should return a response in JSON format + And the response includes a validation error indicating the invalid phone number format + And no wells are imported + + @negative @validation @BDMS-TBD + Scenario: Upload fails when a row has a contact with a invalid email format + Given my CSV file contains a row with a contact with an email that is not in the valid format + When I upload the file to the bulk upload endpoint + Then the system returns a 422 Unprocessable Entity status code + And the system should return a response in JSON format + And the response includes a validation error indicating the invalid email format + And no wells are imported + + @negative @validation @BDMS-TBD + Scenario: Upload fails when a row has contact without a contact_role + Given my CSV file contains a row with a contact but is missing the required "contact_role" field for that contact + When I upload the file to the bulk upload endpoint + Then the system returns a 422 Unprocessable Entity status code + And the system should return a response in JSON format + And the response includes a validation error indicating the missing "contact_role" field + And no wells are imported + + @negative @validation @BDMS-TBD + Scenario: Upload fails when a row has contact without a "contact_type" + Given my CSV file contains a row with a contact but is missing the required "contact_type" field for that contact + When I upload the file to the bulk upload endpoint + Then the system returns a 422 Unprocessable Entity status code + And the system should return a response in JSON format + And the response includes a validation error indicating the missing "contact_type" value + And no wells are imported + + @negative @validation @BDMS-TBD + Scenario: Upload fails when a row has contact with an invalid "contact_type" + Given my CSV file contains a row with a contact_type value that is not in the valid lexicon for "contact_type" + When I upload the file to the bulk upload endpoint + Then the system returns a 422 Unprocessable Entity status code + And the system should return a response in JSON format + And the response includes a validation error indicating an invalid "contact_type" value + And no wells are imported + + @negative @validation @BDMS-TBD + Scenario: Upload fails when a row has contact with an email without an email_type + Given my CSV file contains a row with a contact with an email but is missing the required "email_type" field for that email + When I upload the file to the bulk upload endpoint + Then the system returns a 422 Unprocessable Entity status code + And the system should return a response in JSON format + And the response includes a validation error indicating the missing "email_type" value + And no wells are imported + + @negative @validation @BDMS-TBD + Scenario: Upload fails when a row has contact with a phone without a phone_type + Given my CSV file contains a row with a contact with a phone but is missing the required "phone_type" field for that phone + When I upload the file to the bulk upload endpoint + Then the system returns a 422 Unprocessable Entity status code + And the system should return a response in JSON format + And the response includes a validation error indicating the missing "phone_type" value + And no wells are imported + + @negative @validation @BDMS-TBD + Scenario: Upload fails when a row has contact with an address without an address_type + Given my CSV file contains a row with a contact with an address but is missing the required "address_type" field for that address + When I upload the file to the bulk upload endpoint + Then the system returns a 422 Unprocessable Entity status code + And the system should return a response in JSON format + And the response includes a validation error indicating the missing "address_type" value + And no wells are imported + + @negative @validation @BDMS-TBD + Scenario: Upload fails when a row has utm_easting utm_northing and utm_zone values that are not within New Mexico + Given my CSV file contains a row with utm_easting utm_northing and utm_zone values that are not within New Mexico + When I upload the file to the bulk upload endpoint + Then the system returns a 422 Unprocessable Entity status code + And the system should return a response in JSON format + And the response includes a validation error indicating the invalid UTM coordinates + And no wells are imported + + @negative @validation @BDMS-TBD + Scenario: Upload fails when required fields are missing + Given my CSV file contains rows missing a required field "well_name_point_id" + When I upload the file to the bulk upload endpoint + Then the system returns a 422 Unprocessable Entity status code + And the system should return a response in JSON format + And the response includes validation errors for all rows missing required fields + And the response identifies the row and field for each error + And no wells are imported + + @negative @validation @required_fields @BDMS-TBD + Scenario Outline: Upload fails when a required field is missing + Given my CSV file contains a row missing the required "" field + When I upload the file to the bulk upload endpoint + Then the system returns a 422 Unprocessable Entity status code + And the system should return a response in JSON format + And the response includes a validation error for the "" field + And no wells are imported + + Examples: + | required_field | + | project | + | well_name_point_id | + | site_name | + | date_time | + | field_staff | + | utm_easting | + | utm_northing | + | utm_zone | + | elevation_ft | + | elevation_method | + | measuring_point_height_ft | + + @negative @validation @boolean_fields @BDMS-TBD + Scenario: Upload fails due to invalid boolean field values + Given my CSV file contains a row with an invalid boolean value "maybe" in the "is_open" field +# And my CSV file contains other boolean fields such as "sample_possible" with valid boolean values + When I upload the file to the bulk upload endpoint + Then the system returns a 422 Unprocessable Entity status code + And the system should return a response in JSON format + And the response includes a validation error indicating an invalid boolean value for the "is_open" field + And no wells are imported + + @negative @validation @BDMS-TBD + Scenario: Upload fails when duplicate well_name_point_id values are present + Given my CSV file contains one or more duplicate "well_name_point_id" values + When I upload the file to the bulk upload endpoint + Then the system returns a 422 Unprocessable Entity status code + And the response includes validation errors indicating duplicated values + And each error identifies the row and field + And no wells are imported + + @negative @validation @BDMS-TBD + Scenario: Upload fails due to invalid lexicon values + Given my CSV file contains invalid lexicon values for "contact_role" or other lexicon fields + When I upload the file to the bulk upload endpoint + Then the system returns a 422 Unprocessable Entity status code + And the response includes validation errors identifying the invalid field and row + And no wells are imported + + @negative @validation @BDMS-TBD + Scenario: Upload fails due to invalid date formats + Given my CSV file contains invalid ISO 8601 date values in the "date_time" or "date_drilled" field + When I upload the file to the bulk upload endpoint + Then the system returns a 422 Unprocessable Entity status code + And the response includes validation errors identifying the invalid field and row + And no wells are imported + + @negative @validation @BDMS-TBD + Scenario: Upload fails due to invalid numeric fields + Given my CSV file contains values that cannot be parsed as numeric in numeric-required fields such as "utm_easting" + When I upload the file to the bulk upload endpoint + Then the system returns a 422 Unprocessable Entity status code + And the response includes validation errors identifying the invalid field and row + And no wells are imported + + +# ########################################################################### +# # FILE FORMAT SCENARIOS +# ########################################################################### + + @negative @file_format @limits @BDMS-TBD + Scenario: Upload fails when the CSV exceeds the maximum allowed number of rows + Given my CSV file contains more rows than the configured maximum for bulk upload + When I upload the file to the bulk upload endpoint + Then the system returns a 400 status code + And the system should return a response in JSON format + And the response includes an error message indicating the row limit was exceeded + And no wells are imported + + @negative @file_format @BDMS-TBD + Scenario: Upload fails when file type is unsupported + Given I have a non-CSV file + When I upload the file to the bulk upload endpoint + Then the system returns a 400 status code + And the response includes an error message indicating unsupported file type + And no wells are imported + + @negative @file_format @BDMS-TBD + Scenario: Upload fails when the CSV file is empty + Given my CSV file is empty + When I upload the file to the bulk upload endpoint + Then the system returns a 400 status code + And the response includes an error message indicating an empty file + And no wells are imported + + @negative @file_format @BDMS-TBD + Scenario: Upload fails when CSV contains only headers + Given my CSV file contains column headers but no data rows + When I upload the file to the bulk upload endpoint + Then the system returns a 400 status code + And the response includes an error indicating that no data rows were found + And no wells are imported + + ########################################################################### + # HEADER & SCHEMA INTEGRITY SCENARIOS + ########################################################################### + + @negative @validation @header_row @BDMS-TBD + Scenario: Upload fails when a header row is repeated in the middle of the file + Given my CSV file contains a valid but duplicate header row + When I upload the file to the bulk upload endpoint + Then the system returns a 422 Unprocessable Entity status code + And the system should return a response in JSON format + And the response includes a validation error indicating a repeated header row + And no wells are imported + + + @negative @validation @header_row @BDMS-TBD + Scenario: Upload fails when the header row contains duplicate column names + Given my CSV file header row contains the "contact_1_email_1" column name more than once + When I upload the file to the bulk upload endpoint + Then the system returns a 422 Unprocessable Entity status code + And the system should return a response in JSON format + And the response includes a validation error indicating duplicate header names + And no wells are imported + + + ########################################################################### + # DELIMITER & QUOTING / EXCEL-RELATED SCENARIOS + ########################################################################### + + @negative @file_format @delimiter @BDMS-TBD + Scenario Outline: Upload fails when CSV uses an unsupported delimiter + Given my file is named with a .csv extension + And my file uses "" as the field delimiter instead of commas + When I upload the file to the bulk upload endpoint + Then the system returns a 400 status code + And the system should return a response in JSON format + And the response includes an error message indicating an unsupported delimiter + And no wells are imported + + Examples: + | delimiter_description | + | semicolons | + | tab characters | + + @positive @file_format @quoting @BDMS-TBD + Scenario: Upload succeeds when fields contain commas inside properly quoted values + Given my CSV file header row contains all required columns + And my CSV file contains a data row where the "site_name" field value includes a comma and is enclosed in quotes +# And all other required fields are populated with valid values + When I upload the file to the bulk upload endpoint + Then the system returns a 201 Created status code + And the system should return a response in JSON format + And all wells are imported +# +# @negative @validation @numeric @excel @BDMS-TBD +# Scenario: Upload fails when numeric fields are provided in Excel scientific notation format +# Given my CSV file contains a numeric-required field such as "utm_easting" +# And Excel has exported the "utm_easting" value in scientific notation (for example "1.2345E+06") +# When I upload the file to the bulk upload endpoint +# Then the system returns a 422 Unprocessable Entity status code +# And the system should return a response in JSON format +# And the response includes a validation error indicating an invalid numeric format for "utm_easting" +# And no wells are imported + +########################################################################### + # WATER LEVEL ENTRY VALIDATION +########################################################################### + + # if one water level entry field is filled, then all are required + @negative @validation @BDMS-TBD + Scenario: Water level entry fields are all required if any are filled + Given my csv file contains a row where some but not all water level entry fields are filled + When I upload the file to the bulk upload endpoint + Then the system returns a 422 Unprocessable Entity status code + And the system should return a response in JSON format + And the response includes validation errors for each missing water level entry field + And no wells are imported \ No newline at end of file diff --git a/tests/test_contact.py b/tests/test_contact.py index 68422b0a6..2076168ad 100644 --- a/tests/test_contact.py +++ b/tests/test_contact.py @@ -108,6 +108,12 @@ def test_add_contact(spring_thing): "address_type": "Primary", } ], + "notes": [ + { + "note_type": "General", + "content": "This is a general note for the contact.", + } + ], } response = client.post("/contact", json=payload) data = response.json() @@ -158,6 +164,12 @@ def test_add_contact(spring_thing): ) assert data["release_status"] == payload["release_status"] + assert data["general_notes"][0]["note_type"] == "General" + assert ( + data["general_notes"][0]["content"] == "This is a general note for the contact." + ) + assert len(data["communication_notes"]) == 0 + cleanup_post_test(Contact, data["id"]) @@ -429,6 +441,11 @@ def test_get_contacts( assert data["items"][0]["addresses"][0]["address_type"] == address.address_type assert data["items"][0]["addresses"][0]["release_status"] == address.release_status + assert data["items"][0]["general_notes"][0]["note_type"] == "General" + assert data["items"][0]["general_notes"][0]["content"] == "General note" + assert data["items"][0]["communication_notes"][0]["note_type"] == "Communication" + assert data["items"][0]["communication_notes"][0]["content"] == "Communication note" + def test_get_contacts_by_thing_id(contact, second_contact, water_well_thing): response = client.get(f"/contact?thing_id={water_well_thing.id}") @@ -495,6 +512,11 @@ def test_get_contact_by_id( assert data["addresses"][0]["address_type"] == address.address_type assert data["addresses"][0]["release_status"] == address.release_status + assert data["general_notes"][0]["note_type"] == "General" + assert data["general_notes"][0]["content"] == "General note" + assert data["communication_notes"][0]["note_type"] == "Communication" + assert data["communication_notes"][0]["content"] == "Communication note" + def test_get_contact_by_id_404_not_found(contact): bad_contact_id = 99999 diff --git a/tests/test_well_inventory.py b/tests/test_well_inventory.py new file mode 100644 index 000000000..066877ce6 --- /dev/null +++ b/tests/test_well_inventory.py @@ -0,0 +1,1035 @@ +""" +The feature tests for the well inventory csv upload tests if the API can +successfully process a well inventory upload and create the appropriate +response, but it does not verify that the database contents are correct. + +This module contains tests that verify the correctness of the database +contents after a well inventory upload. +""" + +import csv +from datetime import datetime +from io import BytesIO +from pathlib import Path + +import pytest +from shapely import Point + +from core.constants import SRID_UTM_ZONE_13N, SRID_WGS84 +from core.dependencies import ( + admin_function, + editor_function, + amp_admin_function, + amp_editor_function, + viewer_function, + amp_viewer_function, +) +from db import ( + Group, + Location, + LocationThingAssociation, + Thing, + Contact, + ThingContactAssociation, + FieldEvent, + FieldActivity, + FieldEventParticipant, +) +from db.engine import session_ctx +from main import app +from services.util import transform_srid, convert_ft_to_m +from tests import client, override_authentication + + +@pytest.fixture(scope="module", autouse=True) +def override_authentication_dependency_fixture(): + app.dependency_overrides[admin_function] = override_authentication( + default={"name": "foobar", "sub": "1234567890"} + ) + app.dependency_overrides[editor_function] = override_authentication( + default={"name": "foobar", "sub": "1234567890"} + ) + app.dependency_overrides[viewer_function] = override_authentication() + app.dependency_overrides[amp_admin_function] = override_authentication( + default={"name": "foobar", "sub": "1234567890"} + ) + app.dependency_overrides[amp_editor_function] = override_authentication( + default={"name": "foobar", "sub": "1234567890"} + ) + app.dependency_overrides[amp_viewer_function] = override_authentication() + + yield + + app.dependency_overrides = {} + + +def test_well_inventory_db_contents(): + """ + Test that the well inventory upload creates the correct database contents. + + This test verifies that the well inventory upload creates the correct + database contents by checking for the presence of specific records in + the database. + """ + + file = Path("tests/features/data/well-inventory-valid.csv") + assert file.exists(), "Test data file does not exist." + + # read file into dictionary to compare values with DB objects + with open(file, "r", encoding="utf-8") as f: + reader = csv.DictReader(f) + file_dict = {} + + for row in reader: + file_dict[row["well_name_point_id"]] = row + + response = client.post( + "/well-inventory-csv", + files={"file": open(file, "rb")}, + ) + data = response.json() + print(data) + assert ( + response.status_code == 201 + ), f"Unexpected status code: {response.status_code}" + + # Validate that specific records exist in the database and then clean up + with session_ctx() as session: + # verify the correct number of records were created for each table + locations = session.query(Location).all() + assert len(locations) == 2, "Expected 2 locations in the database." + + things = session.query(Thing).all() + assert len(things) == 2, "Expected 2 things in the database." + + location_thing_associations = session.query(LocationThingAssociation).all() + assert ( + len(location_thing_associations) == 2 + ), "Expected 2 location-thing associations in the database." + + # new field staff & new contacts + contacts = session.query(Contact).all() + assert len(contacts) == 5, "Expected 5 contacts in the database." + + thing_contact_associations = session.query(ThingContactAssociation).all() + assert ( + len(thing_contact_associations) == 3 + ), "Expected 3 thing-contact associations in the database." + + field_events = session.query(FieldEvent).all() + assert len(field_events) == 2, "Expected 2 field events in the database." + + field_activities = session.query(FieldActivity).all() + assert ( + len(field_activities) == 2 + ), "Expected 2 field activities in the database." + + field_event_participants = session.query(FieldEventParticipant).all() + assert ( + len(field_event_participants) == 3 + ), "Expected 3 field event participants in the database." + + # verify the values of specific records + for point_id in file_dict.keys(): + file_content = file_dict[point_id] + + # THING AND RELATED RECORDS + + thing = session.query(Thing).filter(Thing.name == point_id).all() + assert len(thing) == 1, f"Expected 1 thing with name {point_id}." + thing = thing[0] + + assert thing.name == point_id + assert thing.thing_type == "water well" + assert ( + thing.first_visit_date + == datetime.fromisoformat(file_content["date_time"]).date() + ) + assert thing.well_depth == float(file_content["total_well_depth_ft"]) + assert thing.hole_depth is None + assert thing.well_casing_diameter == float( + file_content["casing_diameter_ft"] + ) + assert thing.well_casing_depth is None + assert ( + thing.well_completion_date + == datetime.fromisoformat(file_content["date_drilled"]).date() + ) + assert thing.well_construction_method is None + assert thing.well_driller_name is None + assert thing.well_pump_type == file_content["well_pump_type"] + assert thing.well_pump_depth == float(file_content["well_pump_depth_ft"]) + assert thing.formation_completion_code is None + + assert thing.notes is not None + assert sorted(c.content for c in thing._get_notes("Access")) == sorted( + [file_content["specific_location_of_well"]] + ) + assert sorted(c.content for c in thing._get_notes("General")) == sorted( + [file_content["contact_special_requests_notes"]] + ) + assert sorted( + c.content for c in thing._get_notes("Sampling Procedure") + ) == sorted( + [ + file_content["well_measuring_notes"], + file_content["sampling_scenario_notes"], + ] + ) + assert sorted(c.content for c in thing._get_notes("Historical")) == sorted( + [ + f"historic depth to water: {float(file_content['historic_depth_to_water_ft'])} ft - source: {file_content['depth_source'].lower()}" + ] + ) + + assert ( + thing.measuring_point_description + == file_content["measuring_point_description"] + ) + assert float(thing.measuring_point_height) == float( + file_content["measuring_point_height_ft"] + ) + + assert ( + thing.well_completion_date_source == file_content["completion_source"] + ) + + assert thing.well_depth_source == file_content["depth_source"] + + # well_purpose_2 is blank for both test records in the CSV + assert sorted(wp.purpose for wp in thing.well_purposes) == sorted( + [file_content["well_purpose"]] + ) + + assert sorted( + mf.monitoring_frequency for mf in thing.monitoring_frequencies + ) == sorted([file_content["monitoring_frequency"]]) + + assert len(thing.permissions) == 3 + for permission_type in [ + "Water Level Sample", + "Water Chemistry Sample", + "Datalogger Installation", + ]: + permission = next( + ( + p + for p in thing.permissions + if p.permission_type == permission_type + ), + None, + ) + assert ( + permission is not None + ), f"Expected permission type {permission_type} for thing {point_id}." + + if permission_type == "Water Level Sample": + assert permission.permission_allowed is bool( + file_content["repeat_measurement_permission"].lower() == "true" + ) + elif permission_type == "Water Chemistry Sample": + assert permission.permission_allowed is bool( + file_content["sampling_permission"].lower() == "true" + ) + else: + assert permission.permission_allowed is bool( + file_content["datalogger_installation_permission"].lower() + == "true" + ) + + # LOCATION AND RELATED RECORDS + location_thing_association = ( + session.query(LocationThingAssociation) + .filter(LocationThingAssociation.thing_id == thing.id) + .all() + ) + assert ( + len(location_thing_association) == 1 + ), f"Expected 1 location-thing association for thing {point_id}." + + location = ( + session.query(Location) + .filter(Location.id == location_thing_association[0].location_id) + .all() + ) + assert len(location) == 1, f"Expected 1 location for thing {point_id}." + location = location[0] + + point_utm_13n = Point( + float(file_content["utm_easting"]), float(file_content["utm_northing"]) + ) + point_wgs84 = transform_srid(point_utm_13n, SRID_UTM_ZONE_13N, SRID_WGS84) + assert location.latlon[0] == point_wgs84.y + assert location.latlon[1] == point_wgs84.x + + assert location.elevation == convert_ft_to_m( + float(file_content["elevation_ft"]) + ) + assert location.elevation_method == file_content["elevation_method"] + + assert ( + location._get_notes("Directions")[0].content + == file_content["directions_to_site"] + ) + + # CONTACTS AND RELATED RECORDS + thing_contact_associations = ( + session.query(ThingContactAssociation) + .filter(ThingContactAssociation.thing_id == thing.id) + .all() + ) + contacts = ( + session.query(Contact) + .filter( + Contact.id.in_( + [tca.contact_id for tca in thing_contact_associations] + ) + ) + .all() + ) + if point_id == "MRG-001_MP1": + assert ( + len(contacts) == 2 + ), f"Expected 2 thing-contact associations for thing {point_id}." + else: + # no second contact + assert ( + len(contacts) == 1 + ), f"Expected 1 thing-contact association for thing {point_id}." + + for contact in contacts: + assert ( + contact.general_notes[0].content + == file_content["contact_special_requests_notes"] + ) + assert ( + contact.communication_notes[0].content + == file_content["result_communication_preference"] + ) + if contact.contact_type == "Primary": + assert contact.name == file_content["contact_1_name"] + assert ( + contact.organization == file_content["contact_1_organization"] + ) + assert contact.role == file_content["contact_1_role"] + + # no second phone in test data + assert [(p.phone_number, p.phone_type) for p in contact.phones] == [ + ( + f"+1{file_content["contact_1_phone_1"]}".replace("-", ""), + file_content["contact_1_phone_1_type"], + ), + ] + + # no second email in test data + assert [(e.email, e.email_type) for e in contact.emails] == [ + ( + file_content["contact_1_email_1"], + file_content["contact_1_email_1_type"], + ), + ] + + # no second address in test data + assert [ + ( + a.address_line_1, + a.address_line_2, + a.city, + a.state, + a.postal_code, + a.country, + a.address_type, + ) + for a in contact.addresses + ] == [ + ( + file_content["contact_1_address_1_line_1"], + file_content["contact_1_address_1_line_2"], + file_content["contact_1_address_1_city"], + file_content["contact_1_address_1_state"], + file_content["contact_1_address_1_postal_code"], + "United States", + file_content["contact_1_address_1_type"], + ) + ] + else: + assert contact.name == file_content["contact_2_name"] + assert ( + contact.organization == file_content["contact_2_organization"] + ) + assert contact.role == file_content["contact_2_role"] + + # no second phone in test data + assert [(p.phone_number, p.phone_type) for p in contact.phones] == [ + ( + f"+1{file_content["contact_2_phone_1"]}".replace("-", ""), + file_content["contact_2_phone_1_type"], + ), + ] + + # no second email in test data + assert [(e.email, e.email_type) for e in contact.emails] == [ + ( + file_content["contact_2_email_1"], + file_content["contact_2_email_1_type"], + ), + ] + + # no second address in test data + assert [ + ( + a.address_line_1, + a.address_line_2, + a.city, + a.state, + a.postal_code, + a.country, + a.address_type, + ) + for a in contact.addresses + ] == [ + ( + file_content["contact_2_address_1_line_1"], + file_content["contact_2_address_1_line_2"], + file_content["contact_2_address_1_city"], + file_content["contact_2_address_1_state"], + file_content["contact_2_address_1_postal_code"], + "United States", + file_content["contact_2_address_1_type"], + ) + ] + + # FIELD EVENTS AND RELATED RECORDS + field_events = ( + session.query(FieldEvent).filter(FieldEvent.thing_id == thing.id).all() + ) + assert ( + len(field_events) == 1 + ), f"Expected 1 field event for thing {point_id}." + field_event = field_events[0] + assert field_event.notes == "Initial field event from well inventory import" + assert ( + field_event.event_date.date() + == datetime.fromisoformat(file_content["date_time"]).date() + ) + + field_activity = ( + session.query(FieldActivity) + .filter(FieldActivity.field_event_id == field_event.id) + .all() + ) + assert ( + len(field_activity) == 1 + ), f"Expected 1 field activity for thing {point_id}." + field_activity = field_activity[0] + assert field_activity.activity_type == "well inventory" + assert ( + field_activity.notes == "Well inventory conducted during field event." + ) + + field_event_participants = ( + session.query(FieldEventParticipant) + .filter(FieldEventParticipant.field_event_id == field_event.id) + .all() + ) + if point_id == "MRG-001_MP1": + assert ( + len(field_event_participants) == 2 + ), f"Expected 2 field event participants for thing {point_id}." + else: + assert ( + len(field_event_participants) == 1 + ), f"Expected 1 field event participant for thing {point_id}." + + for participant in field_event_participants: + if participant.participant_role == "Lead": + assert participant.participant.name == file_content["field_staff"] + else: + assert participant.participant.name == file_content["field_staff_2"] + + # CLEAN UP THE DATABASE AFTER TESTING + session.query(Thing).delete() + session.query(ThingContactAssociation).delete() + session.query(Contact).delete() + session.query(LocationThingAssociation).delete() + session.query(Location).delete() + session.query(FieldEventParticipant).delete() + session.query(FieldActivity).delete() + session.query(FieldEvent).delete() + session.commit() + + +# ============================================================================= +# Error Handling Tests - Cover API error paths +# ============================================================================= + + +@pytest.fixture(scope="class", autouse=True) +def error_handling_auth_override(): + """Override authentication for error handling test class.""" + app.dependency_overrides[admin_function] = override_authentication( + default={"name": "foobar", "sub": "1234567890"} + ) + app.dependency_overrides[editor_function] = override_authentication( + default={"name": "foobar", "sub": "1234567890"} + ) + app.dependency_overrides[amp_admin_function] = override_authentication( + default={"name": "foobar", "sub": "1234567890"} + ) + app.dependency_overrides[amp_editor_function] = override_authentication( + default={"name": "foobar", "sub": "1234567890"} + ) + yield + app.dependency_overrides = {} + + +class TestWellInventoryErrorHandling: + """Tests for well inventory CSV upload error handling.""" + + def test_upload_invalid_file_type(self): + """Upload fails with 400 when file is not a CSV.""" + content = b"This is not a CSV file" + response = client.post( + "/well-inventory-csv", + files={"file": ("test.txt", BytesIO(content), "text/plain")}, + ) + assert response.status_code == 400 + data = response.json() + assert "Unsupported file type" in str(data) + + def test_upload_empty_file(self): + """Upload fails with 400 when CSV file is empty.""" + response = client.post( + "/well-inventory-csv", + files={"file": ("test.csv", BytesIO(b""), "text/csv")}, + ) + assert response.status_code == 400 + data = response.json() + assert "Empty file" in str(data) + + def test_upload_headers_only(self): + """Upload fails with 400 when CSV has headers but no data rows.""" + file_path = Path("tests/features/data/well-inventory-no-data-headers.csv") + if file_path.exists(): + response = client.post( + "/well-inventory-csv", + files={"file": open(file_path, "rb")}, + ) + assert response.status_code == 400 + data = response.json() + assert "No data rows found" in str(data) + + def test_upload_duplicate_columns(self): + """Upload fails with 422 when CSV has duplicate column names.""" + file_path = Path("tests/features/data/well-inventory-duplicate-columns.csv") + if file_path.exists(): + response = client.post( + "/well-inventory-csv", + files={"file": open(file_path, "rb")}, + ) + assert response.status_code == 422 + data = response.json() + assert "Duplicate columns found" in str(data.get("validation_errors", [])) + + def test_upload_duplicate_well_ids(self): + """Upload fails with 422 when CSV has duplicate well_name_point_id values.""" + file_path = Path("tests/features/data/well-inventory-duplicate.csv") + if file_path.exists(): + response = client.post( + "/well-inventory-csv", + files={"file": open(file_path, "rb")}, + ) + assert response.status_code == 422 + data = response.json() + errors = data.get("validation_errors", []) + assert any("Duplicate" in str(e) for e in errors) + + def test_upload_missing_required_field(self): + """Upload fails with 422 when required field is missing.""" + file_path = Path("tests/features/data/well-inventory-missing-required.csv") + if file_path.exists(): + response = client.post( + "/well-inventory-csv", + files={"file": open(file_path, "rb")}, + ) + assert response.status_code == 422 + + def test_upload_invalid_date_format(self): + """Upload fails with 422 when date format is invalid.""" + file_path = Path("tests/features/data/well-inventory-invalid-date-format.csv") + if file_path.exists(): + response = client.post( + "/well-inventory-csv", + files={"file": open(file_path, "rb")}, + ) + assert response.status_code == 422 + + def test_upload_invalid_numeric_value(self): + """Upload fails with 422 when numeric field has invalid value.""" + file_path = Path("tests/features/data/well-inventory-invalid-numeric.csv") + if file_path.exists(): + response = client.post( + "/well-inventory-csv", + files={"file": open(file_path, "rb")}, + ) + assert response.status_code == 422 + + def test_upload_invalid_email(self): + """Upload fails with 422 when email format is invalid.""" + file_path = Path("tests/features/data/well-inventory-invalid-email.csv") + if file_path.exists(): + response = client.post( + "/well-inventory-csv", + files={"file": open(file_path, "rb")}, + ) + assert response.status_code == 422 + + def test_upload_invalid_phone_number(self): + """Upload fails with 422 when phone number format is invalid.""" + file_path = Path("tests/features/data/well-inventory-invalid-phone-number.csv") + if file_path.exists(): + response = client.post( + "/well-inventory-csv", + files={"file": open(file_path, "rb")}, + ) + assert response.status_code == 422 + + def test_upload_invalid_utm_coordinates(self): + """Upload fails with 422 when UTM coordinates are outside New Mexico.""" + file_path = Path("tests/features/data/well-inventory-invalid-utm.csv") + if file_path.exists(): + response = client.post( + "/well-inventory-csv", + files={"file": open(file_path, "rb")}, + ) + assert response.status_code == 422 + + def test_upload_invalid_lexicon_value(self): + """Upload fails with 422 when lexicon value is not in allowed set.""" + file_path = Path("tests/features/data/well-inventory-invalid-lexicon.csv") + if file_path.exists(): + response = client.post( + "/well-inventory-csv", + files={"file": open(file_path, "rb")}, + ) + assert response.status_code == 422 + + def test_upload_invalid_boolean_value(self): + """Upload fails with 422 when boolean field has invalid value.""" + file_path = Path( + "tests/features/data/well-inventory-invalid-boolean-value-maybe.csv" + ) + if file_path.exists(): + response = client.post( + "/well-inventory-csv", + files={"file": open(file_path, "rb")}, + ) + assert response.status_code == 422 + + def test_upload_missing_contact_type(self): + """Upload fails with 422 when contact is provided without contact_type.""" + file_path = Path("tests/features/data/well-inventory-missing-contact-type.csv") + if file_path.exists(): + response = client.post( + "/well-inventory-csv", + files={"file": open(file_path, "rb")}, + ) + assert response.status_code == 422 + + def test_upload_missing_contact_role(self): + """Upload fails with 422 when contact is provided without role.""" + file_path = Path("tests/features/data/well-inventory-missing-contact-role.csv") + if file_path.exists(): + response = client.post( + "/well-inventory-csv", + files={"file": open(file_path, "rb")}, + ) + assert response.status_code == 422 + + def test_upload_partial_water_level_fields(self): + """Upload fails with 422 when only some water level fields are provided.""" + file_path = Path("tests/features/data/well-inventory-missing-wl-fields.csv") + if file_path.exists(): + response = client.post( + "/well-inventory-csv", + files={"file": open(file_path, "rb")}, + ) + assert response.status_code == 422 + + def test_upload_non_utf8_encoding(self): + """Upload fails with 400 when file has invalid encoding.""" + # Create a file with invalid UTF-8 bytes + invalid_bytes = b"well_name_point_id,project\n\xff\xfe invalid" + response = client.post( + "/well-inventory-csv", + files={"file": ("test.csv", BytesIO(invalid_bytes), "text/csv")}, + ) + assert response.status_code == 400 + data = response.json() + assert "encoding" in str(data).lower() or "Empty" in str(data) + + def test_validation_error_structure_is_consistent(self): + """Validation errors have consistent structure with row, field, error keys.""" + content = ( + b"project,well_name_point_id,site_name,date_time,field_staff," + b"utm_easting,utm_northing,utm_zone,elevation_ft,elevation_method," + b"measuring_point_height_ft\n" + b"Test,,Site1,2025-01-01T10:00:00,Staff," + b"357000,3784000,13N,5000,GPS,3.5\n" + ) + response = client.post( + "/well-inventory-csv", + files={"file": ("test.csv", BytesIO(content), "text/csv")}, + ) + + assert response.status_code == 422 + data = response.json() + errors = data.get("validation_errors", []) + + assert len(errors) > 0, "Expected validation errors" + + for error in errors: + assert "row" in error, f"Missing 'row' key in error: {error}" + assert "field" in error, f"Missing 'field' key in error: {error}" + assert "error" in error, f"Missing 'error' key in error: {error}" + + +# ============================================================================= +# Unit Tests for Helper Functions +# ============================================================================= + + +class TestWellInventoryHelpers: + """Unit tests for well inventory helper functions.""" + + def test_make_location_utm_zone_13n(self): + """Test location creation with UTM zone 13N coordinates.""" + from api.well_inventory import _make_location + from unittest.mock import MagicMock + + model = MagicMock() + model.utm_easting = 357000.0 + model.utm_northing = 3784000.0 + model.utm_zone = "13N" + model.elevation_ft = 5000.0 + + location = _make_location(model) + + assert location is not None + assert location.point is not None + # Elevation should be converted from feet to meters + assert location.elevation is not None + assert location.elevation < 5000 # meters < feet + + def test_make_location_utm_zone_12n(self): + """Test location creation with UTM zone 12N coordinates.""" + from api.well_inventory import _make_location + from unittest.mock import MagicMock + + model = MagicMock() + model.utm_easting = 600000.0 + model.utm_northing = 3900000.0 + model.utm_zone = "12N" + model.elevation_ft = 4500.0 + + location = _make_location(model) + + assert location is not None + assert location.point is not None + assert location.elevation is not None + + def test_make_contact_with_full_info(self): + """Test contact dict creation with all fields populated.""" + from api.well_inventory import _make_contact + from unittest.mock import MagicMock + + model = MagicMock() + model.result_communication_preference = "Email preferred" + model.contact_special_requests_notes = "Call before visiting" + model.contact_1_name = "John Doe" + model.contact_1_organization = "Test Org" + model.contact_1_role = "Owner" + model.contact_1_type = "Primary" + model.contact_1_email_1 = "john@example.com" + model.contact_1_email_1_type = "Work" + model.contact_1_email_2 = None + model.contact_1_email_2_type = None + model.contact_1_phone_1 = "+15055551234" + model.contact_1_phone_1_type = "Mobile" + model.contact_1_phone_2 = None + model.contact_1_phone_2_type = None + model.contact_1_address_1_line_1 = "123 Main St" + model.contact_1_address_1_line_2 = "Suite 100" + model.contact_1_address_1_city = "Albuquerque" + model.contact_1_address_1_state = "NM" + model.contact_1_address_1_postal_code = "87101" + model.contact_1_address_1_type = "Mailing" + model.contact_1_address_2_line_1 = None + model.contact_1_address_2_line_2 = None + model.contact_1_address_2_city = None + model.contact_1_address_2_state = None + model.contact_1_address_2_postal_code = None + model.contact_1_address_2_type = None + + well = MagicMock() + well.id = 1 + + contact_dict = _make_contact(model, well, 1) + + assert contact_dict is not None + assert contact_dict["name"] == "John Doe" + assert contact_dict["organization"] == "Test Org" + assert contact_dict["thing_id"] == 1 + assert len(contact_dict["emails"]) == 1 + assert len(contact_dict["phones"]) == 1 + assert len(contact_dict["addresses"]) == 1 + assert len(contact_dict["notes"]) == 2 + + def test_make_contact_with_no_name(self): + """Test contact dict returns None when name is empty.""" + from api.well_inventory import _make_contact + from unittest.mock import MagicMock + + model = MagicMock() + model.result_communication_preference = None + model.contact_special_requests_notes = None + model.contact_1_name = None # No name provided + + well = MagicMock() + well.id = 1 + + contact_dict = _make_contact(model, well, 1) + + assert contact_dict is None + + def test_make_well_permission(self): + """Test well permission creation.""" + from api.well_inventory import _make_well_permission + from datetime import date + from unittest.mock import MagicMock + + well = MagicMock() + well.id = 1 + + contact = MagicMock() + contact.id = 2 + + permission = _make_well_permission( + well=well, + contact=contact, + permission_type="Water Level Sample", + permission_allowed=True, + start_date=date(2025, 1, 1), + ) + + assert permission is not None + assert permission.target_table == "thing" + assert permission.target_id == 1 + assert permission.permission_type == "Water Level Sample" + assert permission.permission_allowed is True + + def test_make_well_permission_no_contact_raises(self): + """Test that permission creation without contact raises error.""" + from api.well_inventory import _make_well_permission + from services.exceptions_helper import PydanticStyleException + from datetime import date + from unittest.mock import MagicMock + + well = MagicMock() + well.id = 1 + + with pytest.raises(PydanticStyleException) as exc_info: + _make_well_permission( + well=well, + contact=None, + permission_type="Water Level Sample", + permission_allowed=True, + start_date=date(2025, 1, 1), + ) + + assert exc_info.value.status_code == 400 + + def test_generate_autogen_well_id_first_well(self): + """Test auto-generation of well ID when no existing wells with prefix.""" + from api.well_inventory import generate_autogen_well_id + from unittest.mock import MagicMock + + session = MagicMock() + session.scalars.return_value.first.return_value = None + + well_id, offset = generate_autogen_well_id(session, "XY-") + + assert well_id == "XY-0001" + assert offset == 1 + + def test_generate_autogen_well_id_with_existing(self): + """Test auto-generation of well ID with existing wells.""" + from api.well_inventory import generate_autogen_well_id + from unittest.mock import MagicMock + + session = MagicMock() + existing_well = MagicMock() + existing_well.name = "XY-0005" + session.scalars.return_value.first.return_value = existing_well + + well_id, offset = generate_autogen_well_id(session, "XY-") + + assert well_id == "XY-0006" + assert offset == 6 + + def test_generate_autogen_well_id_with_offset(self): + """Test auto-generation with offset parameter.""" + from api.well_inventory import generate_autogen_well_id + from unittest.mock import MagicMock + + session = MagicMock() + + well_id, offset = generate_autogen_well_id(session, "XY-", offset=10) + + assert well_id == "XY-0011" + assert offset == 11 + + def test_autogen_regex_pattern(self): + """Test the AUTOGEN_REGEX pattern matches correctly.""" + from api.well_inventory import AUTOGEN_REGEX + + # Should match + assert AUTOGEN_REGEX.match("XY-") is not None + assert AUTOGEN_REGEX.match("AB-") is not None + assert AUTOGEN_REGEX.match("ab-") is not None + + # Should not match + assert AUTOGEN_REGEX.match("XY-001") is None + assert AUTOGEN_REGEX.match("XYZ-") is None + assert AUTOGEN_REGEX.match("X-") is None + assert AUTOGEN_REGEX.match("123-") is None + + def test_generate_autogen_well_id_non_numeric_suffix(self): + """Test auto-generation when existing well has non-numeric suffix.""" + from api.well_inventory import generate_autogen_well_id + from unittest.mock import MagicMock + + session = MagicMock() + existing_well = MagicMock() + existing_well.name = "XY-ABC" # Non-numeric suffix + session.scalars.return_value.first.return_value = existing_well + + well_id, offset = generate_autogen_well_id(session, "XY-") + + # Should default to 1 when suffix is not numeric + assert well_id == "XY-0001" + assert offset == 1 + + def test_group_query_with_multiple_conditions(self): + """Group query correctly uses SQLAlchemy and_() for multiple conditions.""" + from db import Group + from sqlalchemy import select, and_ + + with session_ctx() as session: + # Create test group + test_group = Group(name="TestProject", group_type="Monitoring Plan") + session.add(test_group) + session.commit() + + # Query using and_() - this is the pattern used in well_inventory.py + sql = select(Group).where( + and_( + Group.group_type == "Monitoring Plan", + Group.name == "TestProject", + ) + ) + found = session.scalars(sql).one_or_none() + + assert found is not None, "and_() query should find the group" + assert found.name == "TestProject" + assert found.group_type == "Monitoring Plan" + + # Clean up + session.delete(test_group) + session.commit() + + +class TestWellInventoryAPIEdgeCases: + """Additional edge case tests for API endpoints.""" + + def test_upload_too_many_rows(self): + """Upload fails with 400 when CSV has more than 2000 rows.""" + # Create a CSV with header + 2001 data rows + header = "project,well_name_point_id,site_name,date_time,field_staff,utm_easting,utm_northing,utm_zone,elevation_ft,elevation_method,measuring_point_height_ft\n" + row = "TestProject,WELL-{i},Site{i},2025-01-01T10:00:00,Staff,357000,3784000,13N,5000,GPS,3.5\n" + + rows = [header] + for i in range(2001): + rows.append(row.format(i=i)) + + content = "".join(rows).encode("utf-8") + + response = client.post( + "/well-inventory-csv", + files={"file": ("test.csv", BytesIO(content), "text/csv")}, + ) + assert response.status_code == 400 + data = response.json() + assert "Too many rows" in str(data) or "2000" in str(data) + + def test_upload_semicolon_delimiter(self): + """Upload fails with 400 when CSV uses semicolon delimiter.""" + content = b"project;well_name_point_id;site_name\nTest;WELL-001;Site1\n" + response = client.post( + "/well-inventory-csv", + files={"file": ("test.csv", BytesIO(content), "text/csv")}, + ) + assert response.status_code == 400 + data = response.json() + assert "delimiter" in str(data).lower() or "Unsupported" in str(data) + + def test_upload_tab_delimiter(self): + """Upload fails with 400 when CSV uses tab delimiter.""" + content = b"project\twell_name_point_id\tsite_name\nTest\tWELL-001\tSite1\n" + response = client.post( + "/well-inventory-csv", + files={"file": ("test.csv", BytesIO(content), "text/csv")}, + ) + assert response.status_code == 400 + data = response.json() + assert "delimiter" in str(data).lower() or "Unsupported" in str(data) + + def test_upload_duplicate_header_row_in_data(self): + """Upload fails with 422 when header row is duplicated in data.""" + file_path = Path("tests/features/data/well-inventory-duplicate-header.csv") + if file_path.exists(): + response = client.post( + "/well-inventory-csv", + files={"file": open(file_path, "rb")}, + ) + assert response.status_code == 422 + data = response.json() + errors = data.get("validation_errors", []) + assert any( + "Duplicate header" in str(e) or "header" in str(e).lower() + for e in errors + ) + + def test_upload_valid_with_comma_in_quotes(self): + """Upload succeeds when field value contains comma inside quotes.""" + file_path = Path("tests/features/data/well-inventory-valid-comma-in-quotes.csv") + if file_path.exists(): + response = client.post( + "/well-inventory-csv", + files={"file": open(file_path, "rb")}, + ) + # Should succeed - commas in quoted fields are valid CSV + assert response.status_code in (201, 422) # 422 if other validation fails + + # Clean up if records were created + if response.status_code == 201: + with session_ctx() as session: + session.query(Thing).delete() + session.query(Location).delete() + session.query(Contact).delete() + session.query(FieldEvent).delete() + session.query(FieldActivity).delete() + session.commit() + + +# ============= EOF ============================================= diff --git a/tests/transfers/test_contact_with_multiple_wells.py b/tests/transfers/test_contact_with_multiple_wells.py index 1ba7fa2db..835aafb3f 100644 --- a/tests/transfers/test_contact_with_multiple_wells.py +++ b/tests/transfers/test_contact_with_multiple_wells.py @@ -37,7 +37,7 @@ def test_multiple_wells(): def test_owner_comment_creates_notes_for_primary_only(): - point_id = "MG-033" + point_id = "MG-043" _run_contact_transfer([point_id]) with session_ctx() as sess: diff --git a/transfers/aquifer_system_transfer.py b/transfers/aquifer_system_transfer.py index 6d223b80e..f1a4a5d3c 100644 --- a/transfers/aquifer_system_transfer.py +++ b/transfers/aquifer_system_transfer.py @@ -1,6 +1,7 @@ import time -from sqlalchemy.orm import Session + from pydantic import ValidationError +from sqlalchemy.orm import Session from db import AquiferSystem from schemas.aquifer_system import CreateAquiferSystem diff --git a/transfers/contact_transfer.py b/transfers/contact_transfer.py index badef59ad..9a2040774 100644 --- a/transfers/contact_transfer.py +++ b/transfers/contact_transfer.py @@ -379,8 +379,7 @@ def _make_contact_and_assoc( from schemas.contact import CreateContact contact = CreateContact(**data) - contact_data = contact.model_dump() - contact_data.pop("thing_id") + contact_data = contact.model_dump(exclude=["thing_id", "notes"]) contact = Contact(**contact_data) session.add(contact) diff --git a/transfers/permissions_transfer.py b/transfers/permissions_transfer.py index 364eacc9a..346e9f147 100644 --- a/transfers/permissions_transfer.py +++ b/transfers/permissions_transfer.py @@ -1,5 +1,6 @@ from datetime import datetime + from pandas import isna from sqlalchemy.orm import Session diff --git a/transfers/tester.py b/transfers/tester.py new file mode 100644 index 000000000..1e54cd910 --- /dev/null +++ b/transfers/tester.py @@ -0,0 +1,25 @@ +# =============================================================================== +# Copyright 2025 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +from transfers.util import get_transferable_wells, read_csv + +df = read_csv("WellData") +wells = get_transferable_wells(df) +print(len(wells)) + +mp = wells[wells["MPHeight"].notna()] +print(len(mp)) + +# ============= EOF ============================================= diff --git a/transfers/well_transfer.py b/transfers/well_transfer.py index c57491de2..c8f84935f 100644 --- a/transfers/well_transfer.py +++ b/transfers/well_transfer.py @@ -73,6 +73,24 @@ ADDED = [] +# these fields are excluded when the CreateWell model is dumped to a dict for Thing creation +EXCLUDED_FIELDS = [ + "location_id", + "group_id", + "well_purposes", + "well_casing_materials", + "measuring_point_height", + "measuring_point_description", + "well_completion_date_source", + "well_construction_method_source", + "well_depth_source", + "alternate_ids", + "monitoring_frequencies", + "notes", + "is_suitable_for_datalogger", + "is_open", +] + class WellTransferer(Transferer): source_table = "WellData" @@ -278,10 +296,6 @@ def _step(self, session: Session, df: pd.DataFrame, i: int, row: pd.Series): row, f"LU_ConstructionMethod:{row.ConstructionMethod}", "Unknown" ) - is_suitable_for_datalogger = False - if notna(row.OpenWellLoggerOK): - is_suitable_for_datalogger = bool(row.OpenWellLoggerOK) - mpheight = row.MPHeight mpheight_description = row.MeasuringPoint if mpheight is None: @@ -320,7 +334,6 @@ def _step(self, session: Session, df: pd.DataFrame, i: int, row: pd.Series): well_driller_name=row.DrillerName, well_construction_method=wcm, well_pump_type=well_pump_type, - is_suitable_for_datalogger=is_suitable_for_datalogger, ) CreateWell.model_validate(data) @@ -330,22 +343,10 @@ def _step(self, session: Session, df: pd.DataFrame, i: int, row: pd.Series): well = None try: - well_data = data.model_dump( - exclude=[ - "location_id", - "group_id", - "well_purposes", - "well_casing_materials", - "measuring_point_height", - "measuring_point_description", - "well_completion_date_source", - "well_construction_method_source", - ] - ) + well_data = data.model_dump(exclude=EXCLUDED_FIELDS) well_data["thing_type"] = "water well" well_data["nma_pk_welldata"] = row.WellID - well_data.pop("notes") well = Thing(**well_data) session.add(well) @@ -424,6 +425,9 @@ def _extract_well_purposes(self, row) -> list[str]: else: purposes = [] for cui in cu: + if cui == "A": + # skip "Open, unequipped well" as that gets mapped to the status_history table + continue p = self._get_lexicon_value(row, f"LU_CurrentUse:{cui}") if p is not None: purposes.append(p) @@ -644,10 +648,6 @@ def _build_well_payload(self, row) -> CreateWell | None: [], ) - is_suitable_for_datalogger = ( - bool(row.OpenWellLoggerOK) if notna(row.OpenWellLoggerOK) else False - ) - mpheight = row.MPHeight mpheight_description = row.MeasuringPoint if mpheight is None: @@ -683,7 +683,6 @@ def _build_well_payload(self, row) -> CreateWell | None: well_driller_name=row.DrillerName, well_construction_method=wcm, well_pump_type=well_pump_type, - is_suitable_for_datalogger=is_suitable_for_datalogger, ) CreateWell.model_validate(data) @@ -706,18 +705,7 @@ def _persist_well( data: CreateWell = payload["data"] well = None try: - well_data = data.model_dump( - exclude=[ - "location_id", - "group_id", - "well_purposes", - "well_casing_materials", - "measuring_point_height", - "measuring_point_description", - "well_completion_date_source", - "well_construction_method_source", - ] - ) + well_data = data.model_dump(exclude=EXCLUDED_FIELDS) well_data["thing_type"] = "water well" well_data["nma_pk_welldata"] = row.WellID well_data.pop("notes", None) @@ -876,6 +864,32 @@ def _add_histories(self, session: Session, row, well: Thing) -> None: except KeyError: pass + if notna(row.OpenWellLoggerOK): + if bool(row.OpenWellLoggerOK): + status_value = "Datalogger can be installed" + else: + status_value = "Datalogger cannot be installed" + status_history = StatusHistory( + status_type="Datalogger Suitability Status", + status_value=status_value, + reason=None, + start_date=datetime.now(tz=UTC), + target_id=target_id, + target_table=target_table, + ) + session.add(status_history) + + if notna(row.CurrentUse) and "A" in row.CurrentUse: + status_history = StatusHistory( + status_type="Open Status", + status_value="Open", + reason=None, + start_date=datetime.now(tz=UTC), + target_id=target_id, + target_table=target_table, + ) + session.add(status_history) + def _step_parallel_complete( self, session: Session, diff --git a/uv.lock b/uv.lock index 67ea6ae0d..4f49f260d 100644 --- a/uv.lock +++ b/uv.lock @@ -1105,6 +1105,7 @@ dependencies = [ { name = "typing-inspection" }, { name = "tzdata" }, { name = "urllib3" }, + { name = "utm" }, { name = "uvicorn" }, { name = "yarl" }, ] @@ -1213,6 +1214,7 @@ requires-dist = [ { name = "typing-inspection", specifier = "==0.4.1" }, { name = "tzdata", specifier = "==2025.2" }, { name = "urllib3", specifier = "==2.6.0" }, + { name = "utm", specifier = ">=0.8.1" }, { name = "uvicorn", specifier = "==0.38.0" }, { name = "yarl", specifier = "==1.20.1" }, ] @@ -1991,6 +1993,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/56/1a/9ffe814d317c5224166b23e7c47f606d6e473712a2fad0f704ea9b99f246/urllib3-2.6.0-py3-none-any.whl", hash = "sha256:c90f7a39f716c572c4e3e58509581ebd83f9b59cced005b7db7ad2d22b0db99f", size = 131083, upload-time = "2025-12-05T15:08:45.983Z" }, ] +[[package]] +name = "utm" +version = "0.8.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/76/c4/f7662574e0d8c883cea257a59efdc2dbb21f19f4a78e7c54be570d740f24/utm-0.8.1.tar.gz", hash = "sha256:634d5b6221570ddc6a1e94afa5c51bae92bcead811ddc5c9bc0a20b847c2dafa", size = 13128, upload-time = "2025-03-06T11:40:56.022Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3b/a4/0698f3e5c397442ec9323a537e48cc63b846288b6878d38efd04e91005e3/utm-0.8.1-py3-none-any.whl", hash = "sha256:e3d5e224082af138e40851dcaad08d7f99da1cc4b5c413a7de34eabee35f434a", size = 8613, upload-time = "2025-03-06T11:40:54.273Z" }, +] + [[package]] name = "uvicorn" version = "0.38.0"