Some maybe final fixes for notification and provenance tracking

.vscode/launch.json

@@ -495,7 +495,7 @@
             // Test time range of DeploymentPlots with ahi planktivore deployment April 2025
             //"args": ["-v", "1", "--auv_name", "ahi", "--start", "20250401", "--end", "20250501", "--update_ssds_provenance", "--force"]
             // Test web page building with a short deployment
-            "args": ["-v", "1", "--dlist", "ahi/missionlogs/2025/20251022_20251024.dlist", "--update_ssds_provenance", "--force", "--notify", "mccann@mbari.org"]
+            //"args": ["-v", "1", "--dlist", "ahi/missionlogs/2025/20251022_20251024.dlist", "--update_ssds_provenance", "--force", "--notify", "mccann@mbari.org"]
             // Test --force option for rebuilding web pages with a short deployment
             //"args": ["-v", "1", "--last_n_days", "10", "--update_ssds_provenance", "--force"]
             // Test --notify option
@@ -510,6 +510,8 @@
             //"args": ["-v", "1", "--dlist", "daphne/missionlogs/2026/20260316_20260318.dlist", "--force", "--notify"]
             // Test ESP data presentation
             //"args": ["-v", "1", "--dlist", "makai/missionlogs/2024/20240607_20240615.dlist", "--update_ssds_provenance", "--force", "--notify"]
+            // Test of using dotenv for command line execution and INFO log of provenance submission - short failed mission
+            "args": ["-v", "1", "--dlist", "daphne/missionlogs/2026/20260423_20260423.dlist", "--update_ssds_provenance", "--force"]
 
         },
 

pyproject.toml

@@ -16,6 +16,7 @@ dependencies = [
     "datashader>=0.18.1",
     "defusedxml>=0.7.1",
     "gitpython>=3.1.44",
+    "python-dotenv>=1.0.0",
     "gsw>=3.6.20",
     "hvplot>=0.11.3",
     "ipympl>=0.9.7",

src/data/create_products.py

@@ -3187,6 +3187,9 @@ def process_command_line(self):
 
 
 if __name__ == "__main__":
+    from dotenv import load_dotenv
+
+    load_dotenv()
     cp = CreateProducts()
     cp.process_command_line()
     p_start = time.time()

src/data/lrauv_deployment_plots.py

@@ -531,6 +531,26 @@ def _send_slack_file_upload(
                 msg = f"chat.postMessage: {d.get('error')}"
                 raise RuntimeError(msg)
 
+    def _resolve_notify_targets(self, targets: list[str] | None) -> list[str] | None:
+        """Return the resolved list of notification targets, or None to skip.
+
+        Returns ``None`` when ``--notify`` was not given (targets is ``None``).
+        When ``--notify`` was given without values, falls back to ``LRAUV_NOTIFY``.
+        Returns ``None`` (with a warning) if no targets can be resolved.
+        """
+        if targets is None:
+            return None
+        non_empty = [t for t in targets if t]
+        resolved = non_empty or [
+            t.strip() for t in os.environ.get(ENV_LRAUV_NOTIFY, "").split(",") if t.strip()
+        ]
+        if not resolved:
+            self.logger.warning(
+                "--notify given but no targets found; set LRAUV_NOTIFY or pass a value"
+            )
+            return None
+        return resolved
+
     def _notify(  # noqa: PLR0912
         self,
         targets: list[str] | None,
@@ -546,18 +566,14 @@ def _notify(  # noqa: PLR0912
         - starts with ``https://`` → treated as a Slack incoming-webhook URL
         - anything else → treated as an email address (sent via localhost SMTP)
 
-        When *targets* is ``None`` or empty (i.e. ``--notify`` was omitted), falls
-        back to the ``LRAUV_NOTIFY`` environment variable (comma-separated list).
-        Any explicitly provided targets override the environment variable entirely.
+        *targets* is ``None`` when ``--notify`` was not given at all — in that case
+        no notification is sent.  When ``--notify`` is given without an explicit value,
+        *targets* is an empty (or blank-only) list and the ``LRAUV_NOTIFY`` environment
+        variable is used as the target list.  Explicit ``--notify`` values always take
+        precedence over ``LRAUV_NOTIFY``.
         """
-        non_empty = [t for t in (targets or []) if t]
-        if non_empty:
-            notify_list = non_empty
-        else:
-            notify_list = [
-                t.strip() for t in os.environ.get(ENV_LRAUV_NOTIFY, "").split(",") if t.strip()
-            ]
-        if not notify_list:
+        notify_list = self._resolve_notify_targets(targets)
+        if notify_list is None:
             return
 
         for target in notify_list:
@@ -674,6 +690,7 @@ def _submit_provenance(  # noqa: PLR0913
                     script_name="src/data/lrauv_deployment_plots.py",
                     cmd_line_args=cmd_line,
                     additional_resources=png_resources,
+                    log=self.logger,
                 )
             except Exception:  # noqa: BLE001
                 self.logger.warning("Provenance submission failed for %s", png_path, exc_info=True)
@@ -993,9 +1010,10 @@ def process_command_line(self) -> None:
             metavar="EMAIL_OR_WEBHOOK",
             help=(
                 "Send a notification when new plots are written. Provide an email"
-                " address or a Slack incoming-webhook URL. Repeat to notify multiple"
-                f" targets. When omitted, falls back to the {ENV_LRAUV_NOTIFY}"
-                " environment variable (comma-separated list)."
+                " address or a Slack channel ID (C...) or incoming-webhook URL."
+                " Repeat for multiple targets. If given without a value, targets"
+                f" are read from the {ENV_LRAUV_NOTIFY} environment variable."
+                " Omitting --notify entirely suppresses all notifications."
             ),
         )
         self.args = parser.parse_args()
@@ -1004,6 +1022,9 @@ def process_command_line(self) -> None:
 
 
 if __name__ == "__main__":
+    from dotenv import load_dotenv
+
+    load_dotenv()
     dp = DeploymentPlotter()
     dp.process_command_line()
     args = dp.args

src/data/process.py

@@ -1613,6 +1613,9 @@ def process_command_line(self):
 
 
 if __name__ == "__main__":
+    from dotenv import load_dotenv
+
+    load_dotenv()
     AUV_NAME = "i2map"
     VEHICLE_DIR = "/Volumes/M3/master/i2MAP"
     CALIBRATION_DIR = "/Volumes/DMO/MDUC_CORE_CTD_200103/Calibration Files"

src/data/provenance.py

@@ -207,7 +207,7 @@ def submit_process_run(  # noqa: PLR0913
     additional_resources: list[dict] | None = None,
     api_key: str | None = None,
     api_key_header: str | None = None,
-    api_base: str = SSDS_API_BASE,
+    api_base: str | None = None,
     session: requests.Session | None = None,
     log: logging.Logger | None = None,
 ) -> dict | None:
@@ -219,6 +219,7 @@ def submit_process_run(  # noqa: PLR0913
     Returns the created ProcessRun dict on success, or raises on HTTP error.
     """
     log = log or logger
+    api_base = api_base or os.environ.get(ENV_SSDS_API_BASE, SSDS_API_BASE)
     session = session or build_authenticated_session(
         api_key=api_key,
         api_key_header=api_key_header,
@@ -323,6 +324,9 @@ def process_command_line() -> argparse.Namespace:
 
 
 if __name__ == "__main__":
+    from dotenv import load_dotenv
+
+    load_dotenv()
     args = process_command_line()
     logging.basicConfig(
         level=logging.DEBUG if args.verbose else logging.INFO,

src/data/test_lrauv_deployment_plots.py

@@ -695,7 +695,7 @@ def test_noop_when_no_target_and_no_env(self, dp, monkeypatch):
         mock_post.assert_not_called()  # noqa: S101
 
     def test_env_var_fallback_used(self, dp, tmp_path, monkeypatch):
-        """When targets is None but LRAUV_NOTIFY env var is set, that value is used."""
+        """When --notify is given with no value (empty list), LRAUV_NOTIFY is used."""
         monkeypatch.setenv("LRAUV_NOTIFY", "fallback@mbari.org")
         html_file = tmp_path / "test.html"
         html_file.touch()
@@ -704,12 +704,25 @@ def test_env_var_fallback_used(self, dp, tmp_path, monkeypatch):
             mock_smtp = MagicMock()
             mock_smtp_cls.return_value.__enter__ = MagicMock(return_value=mock_smtp)
             mock_smtp_cls.return_value.__exit__ = MagicMock(return_value=False)
-            dp._notify(None, "CANON_April_2025", [html_file])
+            dp._notify([], "CANON_April_2025", [html_file])
 
         mock_smtp.send_message.assert_called_once()  # noqa: S101
         msg = mock_smtp.send_message.call_args[0][0]
         assert "fallback@mbari.org" in msg["To"]  # noqa: S101
 
+    def test_none_targets_skips_notification(self, dp, monkeypatch):
+        """When targets is None (--notify not given), no notification is sent."""
+        monkeypatch.setenv("LRAUV_NOTIFY", "fallback@mbari.org")
+
+        with (
+            patch("smtplib.SMTP") as mock_smtp_cls,
+            patch("lrauv_deployment_plots.requests.post") as mock_post,
+        ):
+            dp._notify(None, "CANON_April_2025", [])
+
+        mock_smtp_cls.assert_not_called()  # noqa: S101
+        mock_post.assert_not_called()  # noqa: S101
+
 
 class TestSendSlackFileUpload:
     """Unit tests for DeploymentPlotter._send_slack_file_upload()."""