feat: enhance organization validation and improve logging verbosity in transfer process

Author: jirhiker

.pre-commit-config.yaml

@@ -27,8 +27,6 @@ repos:
         always_run: true
         args:
           - -x
-          - -vv
-          - --durations=20
 
   # - repo: https://github.com/pre-commit/mirrors-mypy
   #   rev: v1.10.0  # Use the latest stable version or pin to your preference

transfers/contact_transfer.py

@@ -219,7 +219,10 @@ def _get_organization(row, co_to_org_mapper):
     organization = co_to_org_mapper.get(row.Company, row.Company)
 
     # use Organization enum to catch validation errors
-    Organization(organization)
+    try:
+        Organization(organization)
+    except ValueError:
+        return None
 
     return organization
 
@@ -239,7 +242,7 @@ def _add_second_contact(session, row, thing, co_to_org_mapper, added):
 
     organization = _get_organization(row, co_to_org_mapper)
     if (name, organization) in added:
-        return
+        return None
 
     added.append((name, organization))
 

transfers/well_transfer.py

@@ -80,22 +80,22 @@
 
 def _get_first_visit_date(row) -> datetime | None:
     first_visit_date = None
+
+    def _extract_date(date_str: str) -> datetime:
+        return datetime.strptime(date_str, "%Y-%m-%d %H:%M:%S.%f").date()
+
     if row.DateCreated and row.SiteDate:
-        date_created = datetime.strptime(row.DateCreated, "%Y-%m-%d %H:%M:%S.%f").date()
-        site_date = datetime.strptime(row.SiteDate, "%Y-%m-%d %H:%M:%S.%f").date()
+        date_created = _extract_date(row.DateCreated)
+        site_date = _extract_date(row.SiteDate)
 
         if date_created < site_date:
             first_visit_date = date_created
         else:
             first_visit_date = site_date
     elif row.DateCreated and not row.SiteDate:
-        first_visit_date = datetime.strptime(
-            row.DateCreated, "%Y-%m-%d %H:%M:%S.%f"
-        ).date()
+        first_visit_date = _extract_date(row.DateCreated)
     elif not row.DateCreated and row.SiteDate:
-        first_visit_date = datetime.strptime(
-            row.SiteDate, "%Y-%m-%d %H:%M:%S.%f"
-        ).date()
+        first_visit_date = _extract_date(row.SiteDate)
 
     return first_visit_date
 
@@ -432,9 +432,6 @@ def _add_formation_zone(self, row, well, formations):
                     )
             else:
                 # Formation does NOT exist: Do not create new formation. Flag and log for review
-                logger.critical(
-                    f"MISSING FORMATION: Formation '{formation_code}' not found for well {well.name}. Flagged for review."
-                )
                 self._capture_error(
                     row.PointID, f"Unknown formation: {formation_code}", "FormationZone"
                 )
@@ -740,9 +737,10 @@ def _after_hook_chunk(self, well, formations, measuring_point_estimator):
                 target_table=target_table,
             )
             objs.append(status_history)
-            logger.info(
-                f"  Added monitoring status for well {well.name}: {status_value}"
-            )
+            if self.verbose:
+                logger.info(
+                    f"  Added monitoring status for well {well.name}: {status_value}"
+                )
 
             for code in NMA_MONITORING_FREQUENCY.keys():
                 if code in row.MonitoringStatus:
@@ -755,9 +753,10 @@ def _after_hook_chunk(self, well, formations, measuring_point_estimator):
                     )
 
                     objs.append(monitoring_frequency_history)
-                    logger.info(
-                        f"  Adding '{monitoring_frequency}' monitoring frequency for well {well.name}"
-                    )
+                    if self.verbose:
+                        logger.info(
+                            f"  Adding '{monitoring_frequency}' monitoring frequency for well {well.name}"
+                        )
 
         if notna(row.Status):
 
@@ -772,7 +771,10 @@ def _after_hook_chunk(self, well, formations, measuring_point_estimator):
                     target_table=target_table,
                 )
                 objs.append(status_history)
-                logger.info(f"  Added well status for well {well.name}: {status_value}")
+                if self.verbose:
+                    logger.info(
+                        f"  Added well status for well {well.name}: {status_value}"
+                    )
         return objs