Use --notify to send Slack and/or email notifications via LRAUV_NOTIFY env var

.env.example

@@ -20,3 +20,22 @@ SSDS_API_BASE=https://mooring-ssds.shore.mbari.org/api
 # SSDS provenance auth stubs (API key)
 SSDS_API_KEY=
 SSDS_API_KEY_HEADER=X-API-Key
+
+# Notifications from lrauv_deployment_plots.py (comma-separated list of targets).
+# Each target is auto-detected by its format:
+#   Slack channel ID  (e.g. C0AUEA3LZD0 for #lrauv-data) — uploads the PNG
+#                     directly via the Files API so the image is always current.
+#                     Requires SLACK_BOT_TOKEN (below) and the bot invited to the
+#                     channel (/invite @auv-python).  Find the channel ID in Slack:
+#                     right-click the channel → View channel details → bottom.
+#   Webhook URL       (https://hooks.slack.com/...) — posts an image-block message
+#                     via an incoming webhook; image may be stale if re-plotted.
+#   Email address     — sends a plain-HTML email with the PNG inline via SMTP_HOST.
+LRAUV_NOTIFY=C0AUEA3LZD0
+
+# Bot User OAuth Token for the auv-python Slack app (xoxb-...).
+# Required when LRAUV_NOTIFY contains a Slack channel ID.
+# Scopes needed: files:write, chat:write.
+# Generate/rotate at https://api.slack.com/apps → OAuth & Permissions.
+# Keep this value out of version control — it grants write access to your workspace.
+SLACK_BOT_TOKEN=

.vscode/launch.json

@@ -471,9 +471,11 @@
             // Make per log file .html files to test with lrauv_deployment_plots and --update_ssds_provenance
             //"args": ["-v", "1", "--auv_name", "ahi", "--start", "20251022T000000", "--end", "20251024T000000", "--update_ssds_provenance", "--clobber"]
             // Make per log file .html files to test with lrauv_deployment_plots and --update_ssds_provenance to test dynamic map bounds
-            "args": ["-v", "1", "--auv_name", "ahi", "--start", "20260409T000000", "--end", "20260415T000000", "--update_ssds_provenance", "--clobber"]
+            //"args": ["-v", "1", "--auv_name", "ahi", "--start", "20260409T000000", "--end", "20260415T000000", "--update_ssds_provenance", "--clobber"]
             // Fixup labels on No Data plots
             //"args": ["-v", "1", "--log_file", "ahi/missionlogs/2026/20260406_20260412/20260411T145332/202604111453_202604111937.nc4", "--update_ssds_provenance", "--clobber"]
+            // Test ESP markers - Shallow log file from Denmark deployment in June 2024, has large depth values in self.ds.depth.values[6500:6800]
+            "args": ["-v", "1", "--log_file", "makai/missionlogs/2024/20240607_20240615/20240611T082709/202406110827_202406111026.nc4", "--update_ssds_provenance", "--clobber"]
         },
         {
             "name": "lrauv_deployment_plots",
@@ -490,15 +492,21 @@
             // Test time range of DeploymentPlots with ahi planktivore deployment April 2025
             //"args": ["-v", "1", "--auv_name", "ahi", "--start", "20250401", "--end", "20250501", "--update_ssds_provenance", "--force"]
             // Test web page building with a short deployment
-            //"args": ["-v", "1", "--dlist", "ahi/missionlogs/2025/20251022_20251024.dlist", "--update_ssds_provenance"]
+            "args": ["-v", "1", "--dlist", "ahi/missionlogs/2025/20251022_20251024.dlist", "--update_ssds_provenance", "--force", "--notify"]
             // Test --force option for rebuilding web pages with a short deployment
             //"args": ["-v", "1", "--last_n_days", "10", "--update_ssds_provenance", "--force"]
             // Test --notify option
             //"args": ["-v", "1", "--dlist", "ahi/missionlogs/2025/20251022_20251024.dlist", "--update_ssds_provenance", "--force", "--notify", "mccann@mbari.org"]
             // Test what's running in cron on kraken
             //"args": ["-v", "1", "--last_n_days", "30", "--update_ssds_provenance", "--force", "--notify", "mccann@mbari.org"]
             // Test far offshore ahi mission with --update_ssds_provenance and --notify --force
-            "args": ["-v", "1", "--dlist", "ahi/missionlogs/2026/20260406_20260412.dlist", "--update_ssds_provenance", "--force", "--notify", "mccann@mbari.org"]
+            //"args": ["-v", "1", "--dlist", "ahi/missionlogs/2026/20260406_20260412.dlist", "--update_ssds_provenance", "--force", "--notify", "mccann@mbari.org"]
+            // Test --notify with no argument to a message to the Slack web hook in the LRAUV_NOTIFY environment variable
+            //"args": ["-v", "1", "--dlist", "ahi/missionlogs/2025/20251022_20251024.dlist", "--force", "--notify"]
+            // Test Sipper data presentation
+            //"args": ["-v", "1", "--dlist", "daphne/missionlogs/2026/20260316_20260318.dlist", "--force", "--notify"]
+            // Test ESP data presentation
+            //"args": ["-v", "1", "--dlist", "makai/missionlogs/2024/20240607_20240615.dlist", "--update_ssds_provenance", "--force", "--notify"]
 
         },
 

src/data/create_products.py

@@ -8,6 +8,7 @@
 __copyright__ = "Copyright 2023, Monterey Bay Aquarium Research Institute"
 
 import argparse  # noqa: I001
+import contextlib
 import logging
 import os
 import re
@@ -821,7 +822,7 @@ def _plot_nighttime_indicator(  # noqa: PLR0915
                 )
             day += timedelta(days=1)
 
-    def _grid_dims(self) -> tuple:
+    def _grid_dims(self, plot_vars: list[str] | None = None) -> tuple:
         # From Matlab code in plot_sections.m:
         # auvnav positions are too fine for distance calculations, they resolve
         # spiral ascents and circling while on station
@@ -903,14 +904,26 @@ def _grid_dims(self) -> tuple:
             distnav.to_numpy()[-1],
             int(3 * self.ds["profile_number"].to_numpy()[-1]),
         )
-        # Vertical gridded to .5 m, rounded down to nearest 50m
-        max_depth = np.floor(self.ds.cf["depth"].max() / 50) * 50
-        iz = np.arange(2.0, max_depth, 0.5)
-        if not iz.any():
-            self.logger.warning(
-                "Gridding vertical for a surface only mission: {self.ds.cf['depth'].max() =}",
-            )
-            iz = np.arange(0, self.ds.cf["depth"].max(), 0.05)
+        # Vertical gridded to .5 m, rounded down to nearest 10m (minimum 10m)
+        # Use only depths where at least one sensor variable has valid data to
+        # exclude bogus depth values recorded when no valid sensor data was logged
+        # (e.g. from memory corruption events)
+        depth_values = self.ds.cf["depth"].to_numpy()
+        time_dim = self.ds.cf["depth"].dims[0]
+        nav_vars = {"depth", "latitude", "longitude", "profile_number"}
+        has_valid_sensor_data = np.zeros(len(depth_values), dtype=bool)
+        vars_to_check = [
+            v for v in (plot_vars or self.ds.data_vars) if v in self.ds and "pitch" not in v
+        ]
+        for var in vars_to_check:
+            if var not in nav_vars and time_dim in self.ds[var].dims and self.ds[var].ndim == 1:
+                has_valid_sensor_data |= ~np.isnan(self.ds[var].to_numpy())
+        depths_with_data = depth_values[has_valid_sensor_data]
+        if len(depths_with_data) > 0 and not np.all(np.isnan(depths_with_data)):
+            max_depth = max(np.floor(np.nanmax(depths_with_data) / 10) * 10, 10)
+        else:
+            max_depth = max(np.floor(np.nanmax(depth_values) / 10) * 10, 10)
+        iz = np.arange(0, max_depth, 0.5)
 
         return idist, iz, distnav
 
@@ -1028,6 +1041,9 @@ def _get_gulper_locations(self, distnav: xr.DataArray) -> dict:
     def _get_sipper_locations(self, distnav: xr.DataArray) -> dict:
         """Get sipper sample locations in distance/depth space.
 
+        For deployment plots (self.nc_files is set), scans the syslog of every
+        log directory so samples from all logs are captured.
+
         Returns:
             Dictionary mapping sample number to (distance_km, depth_m) tuple
         """
@@ -1036,12 +1052,30 @@ def _get_sipper_locations(self, distnav: xr.DataArray) -> dict:
 
         sipper = Sipper()
         sipper.args = argparse.Namespace()
-        sipper.args.log_file = self.log_file
         sipper.args.local = self.local
         sipper.args.verbose = 0  # Suppress sipper logging
         sipper.logger.setLevel(logging.WARNING)
 
-        sipper_times = sipper.parse_sippers()
+        if self.nc_files:
+            # Deployment mode: derive a log_file path for each nc_file so we
+            # can read the syslog from each individual log directory.
+            log_files = [
+                re.sub(
+                    rf"_{re.escape(self.freq)}\.nc$",
+                    ".nc4",
+                    nc.replace(LRAUV_OPENDAP_BASE.rstrip("/") + "/", ""),
+                )
+                for nc in self.nc_files
+            ]
+            sipper_times: dict = {}
+            for lf in log_files:
+                sipper.args.log_file = lf
+                with contextlib.suppress(FileNotFoundError):
+                    sipper_times.update(sipper.parse_sippers())
+        else:
+            sipper.args.log_file = self.log_file
+            sipper_times = sipper.parse_sippers()
+
         if not sipper_times:
             return {}
 
@@ -1668,9 +1702,10 @@ def _plot_var_scatter(  # noqa: C901, PLR0912, PLR0913, PLR0915
         else:
             curr_ax.set_ylabel("")
 
-        # Set y-axis ticks at 0, 50, 100, 150, etc.
+        # Set y-axis ticks adaptively based on depth range
         y_min, y_max = curr_ax.get_ylim()
-        y_ticks = np.arange(0, int(y_min) + 50, 50)
+        tick_step = 10 if y_min <= 50 else 50  # noqa: PLR2004
+        y_ticks = np.arange(0, int(y_min) + tick_step, tick_step)
         curr_ax.set_yticks(y_ticks)
 
         cb = fig.colorbar(scatter, ax=curr_ax, pad=0.01)
@@ -1912,10 +1947,11 @@ def _plot_var_contour(  # noqa: C901, PLR0912, PLR0913, PLR0915
         else:
             curr_ax.set_ylabel("")
 
-        # Set y-axis ticks at 0, 50, 100, 150, etc.
+        # Set y-axis ticks adaptively based on depth range
         y_min, y_max = curr_ax.get_ylim()
         # Since y-axis is inverted (max at bottom), y_min is the deeper value
-        y_ticks = np.arange(0, int(y_min) + 50, 50)
+        tick_step = 10 if y_min <= 50 else 50  # noqa: PLR2004
+        y_ticks = np.arange(0, int(y_min) + tick_step, tick_step)
         curr_ax.set_yticks(y_ticks)
 
         cb = fig.colorbar(cntrf, ax=curr_ax, pad=0.01)
@@ -2026,7 +2062,7 @@ def plot_2column(self) -> str:  # noqa: C901, PLR0912, PLR0915
             )
             return None
 
-        idist, iz, distnav = self._grid_dims()
+        idist, iz, distnav = self._grid_dims([var for var, _ in plot_variables])
         if idist.size == 0 or iz.size == 0 or distnav.size == 0:
             self.logger.warning("Skipping plot_2column due to missing gridding dimensions")
             return None
@@ -2172,7 +2208,7 @@ def plot_biolume_2column(self) -> str:  # noqa: C901, PLR0912, PLR0915
             )
             return None
 
-        idist, iz, distnav = self._grid_dims()
+        idist, iz, distnav = self._grid_dims([var for var, _ in plot_variables])
         if idist.size == 0 or iz.size == 0 or distnav.size == 0:
             self.logger.warning("Skipping plot_biolume_2column due to missing gridding dimensions")
             return None
@@ -2318,7 +2354,8 @@ def plot_planktivore_2column(self) -> str:  # noqa: C901, PLR0912, PLR0915
             )
             return None
 
-        idist, iz, distnav = self._grid_dims()
+        planktivore_plot_vars = [var for var, _ in self._get_planktivore_plot_variables()]
+        idist, iz, distnav = self._grid_dims(planktivore_plot_vars)
         if idist.size == 0 or iz.size == 0 or distnav.size == 0:
             self.logger.warning(
                 "Skipping plot_planktivore_2column due to missing gridding dimensions"