From afc5208ffa9b2586556ecec51b334f4f76cc8d6f Mon Sep 17 00:00:00 2001 From: Mike McCann Date: Tue, 21 Apr 2026 16:00:42 -0700 Subject: [PATCH 1/5] Fix --notify so that no args sends messages to LRAUV_NOTIFY env var setting. --- .env.example | 4 + .vscode/launch.json | 4 +- src/data/lrauv_deployment_plots.py | 116 +++++++++++++++++++---------- 3 files changed, 84 insertions(+), 40 deletions(-) diff --git a/.env.example b/.env.example index f06282e..0d97339 100644 --- a/.env.example +++ b/.env.example @@ -20,3 +20,7 @@ SSDS_API_BASE=https://mooring-ssds.shore.mbari.org/api # SSDS provenance auth stubs (API key) SSDS_API_KEY= SSDS_API_KEY_HEADER=X-API-Key + +# For lrauv_deployment_plots.py messages to be sent to Slack and/or email +LRAUV_NOTIFY= +SLACK_BOT_TOKEN= diff --git a/.vscode/launch.json b/.vscode/launch.json index 00c5b80..4d5350e 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -498,7 +498,9 @@ // Test what's running in cron on kraken //"args": ["-v", "1", "--last_n_days", "30", "--update_ssds_provenance", "--force", "--notify", "mccann@mbari.org"] // Test far offshore ahi mission with --update_ssds_provenance and --notify --force - "args": ["-v", "1", "--dlist", "ahi/missionlogs/2026/20260406_20260412.dlist", "--update_ssds_provenance", "--force", "--notify", "mccann@mbari.org"] + //"args": ["-v", "1", "--dlist", "ahi/missionlogs/2026/20260406_20260412.dlist", "--update_ssds_provenance", "--force", "--notify", "mccann@mbari.org"] + // Test --notify with no argument to a message to the Slack web hook in the LRAUV_NOTIFY environment variable + "args": ["-v", "1", "--dlist", "ahi/missionlogs/2025/20251022_20251024.dlist", "--force", "--notify"] }, diff --git a/src/data/lrauv_deployment_plots.py b/src/data/lrauv_deployment_plots.py index 5139adf..14ca85f 100755 --- a/src/data/lrauv_deployment_plots.py +++ b/src/data/lrauv_deployment_plots.py @@ -189,7 +189,7 @@ def plot_deployment( # noqa: C901, PLR0912, PLR0913, PLR0915 verbose: int = 0, update_ssds_provenance: bool = False, # noqa: FBT001, FBT002 force: bool = False, # noqa: FBT001, FBT002 - notify: str | None = None, + notify: list[str] | None = None, ) -> None: """Main entry point: generate deployment-level plots from a .dlist path. @@ -198,8 +198,9 @@ def plot_deployment( # noqa: C901, PLR0912, PLR0913, PLR0915 verbose: Verbosity level (0-2). update_ssds_provenance: Submit provenance records to SSDS_Metadata. force: Reprocess even when output PNGs already exist. - notify: Email address or Slack webhook URL to notify after completion. - Falls back to the ``LRAUV_NOTIFY`` environment variable. + notify: One or more email addresses or Slack webhook URLs to notify after + completion. Falls back to the ``LRAUV_NOTIFY`` environment variable + (comma-separated list). """ self.logger.setLevel(self._log_levels[min(verbose, 2)]) @@ -330,7 +331,7 @@ def _build_and_write_html( # noqa: PLR0913 verbose: int = 0, update_ssds_provenance: bool = False, # noqa: FBT001, FBT002 force: bool = False, # noqa: FBT001, FBT002 - notify: str | None = None, + notify: list[str] | None = None, ) -> None: """Fetch STOQS permalink and write per-PNG HTML pages.""" dlist_no_ext = str(Path(dlist).with_suffix("")) @@ -366,7 +367,7 @@ def _build_and_write_html( # noqa: PLR0913 html_paths = [ Path(p).with_suffix(".html") for p in png_paths if Path(p).with_suffix(".html").exists() ] - self._notify(notify or "", raw_name or plot_name_stem, html_paths, stoqs_url, force=force) + self._notify(notify, raw_name or plot_name_stem, html_paths, force=force) if update_ssds_provenance: self._submit_provenance( deployment_dir=deployment_dir, @@ -441,49 +442,82 @@ def _send_notify_email( except Exception as exc: # noqa: BLE001 self.logger.warning("Email notification failed: %s", exc) - def _notify( # noqa: PLR0913 + def _notify( self, - target: str, + targets: list[str] | None, deployment_name: str, html_paths: list[Path], - stoqs_url: str | None, force: bool = False, # noqa: FBT001, FBT002 ) -> None: - """Send an email or Slack notification with links to the new deployment HTML pages. + """Send email and/or Slack notifications for each target in *targets*. - *target* is auto-detected: + Each entry is auto-detected: - starts with ``https://`` → treated as a Slack incoming-webhook URL - anything else → treated as an email address (sent via localhost SMTP) - The ``LRAUV_NOTIFY`` environment variable can supply the target so it - stays out of shell history and cron job command lines. + When *targets* is ``None`` or empty (i.e. ``--notify`` was omitted), falls + back to the ``LRAUV_NOTIFY`` environment variable (comma-separated list). + Any explicitly provided targets override the environment variable entirely. """ - resolved = target or os.environ.get(ENV_LRAUV_NOTIFY, "") - if not resolved: + non_empty = [t for t in (targets or []) if t] + if non_empty: + notify_list = non_empty + else: + notify_list = [ + t.strip() for t in os.environ.get(ENV_LRAUV_NOTIFY, "").split(",") if t.strip() + ] + if not notify_list: return - if resolved.startswith("https://"): - # Slack incoming webhook — include all plot links and STOQS URL - import requests # noqa: PLC0415 - - plot_links = [(get_web_url(str(p)), self._plot_label(str(p))) for p in html_paths] - prefix = "" if force else "New " - lines = [f"{prefix}LRAUV deployment plots available: {deployment_name}", ""] - for url, label in plot_links: - lines.append(f" {label}: {url}") - if stoqs_url: - after_scheme = stoqs_url.split("//", 1)[-1] if "//" in stoqs_url else stoqs_url - stoqs_db_label = after_scheme.split("/")[1] if "/" in after_scheme else after_scheme - lines.append("") - lines.append(f" STOQS ({stoqs_db_label}): {stoqs_url}") - try: - resp = requests.post(resolved, json={"text": "\n".join(lines)}, timeout=10) # noqa: S113 - resp.raise_for_status() - self.logger.info("Slack notification sent to webhook") - except Exception as exc: # noqa: BLE001 - self.logger.warning("Slack notification failed: %s", exc) - else: - self._send_notify_email(resolved, deployment_name, html_paths, force=force) + for target in notify_list: + if target.startswith("https://"): + # Slack incoming webhook — mirror the email: image on top, web link, timestamp + import requests # noqa: PLC0415 + + std_html = next( + (p for p in html_paths if "2column_cmocean" in str(p)), + html_paths[0] if html_paths else None, + ) + std_png = std_html.with_suffix(".png") if std_html else None + if std_png and not std_png.exists(): + std_png = None + web_url = get_web_url(str(std_html)) if std_html else "" + + prefix = "" if force else "New " + _la = ZoneInfo("America/Los_Angeles") + sent_on = datetime.now(tz=UTC).astimezone(_la).strftime("%Y-%m-%d %H:%M:%S %Z") + + blocks: list[dict] = [] + if std_png: + std_png_url = get_web_url(str(std_png)) + blocks.append( + { + "type": "image", + "image_url": std_png_url, + "alt_text": f"{deployment_name} standard plot", + } + ) + blocks.append( + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": ( + f"*{prefix}LRAUV deployment plots: {deployment_name}*\n" + f"<{web_url}|View this and related information on the web>\n" + f"_Sent on: {sent_on}_" + ), + }, + } + ) + try: + resp = requests.post(target, json={"blocks": blocks}, timeout=10) # noqa: S113 + resp.raise_for_status() + self.logger.info("Slack notification sent to webhook") + except Exception as exc: # noqa: BLE001 + self.logger.warning("Slack notification failed: %s", exc) + else: + self._send_notify_email(target, deployment_name, html_paths, force=force) def _submit_provenance( # noqa: PLR0913 self, @@ -852,12 +886,16 @@ def process_command_line(self) -> None: ) parser.add_argument( "--notify", - default="", + action="append", + nargs="?", + const="", + default=None, metavar="EMAIL_OR_WEBHOOK", help=( "Send a notification when new plots are written. Provide an email" - " address or a Slack incoming-webhook URL. Falls back to the" - f" {ENV_LRAUV_NOTIFY} environment variable if not specified." + " address or a Slack incoming-webhook URL. Repeat to notify multiple" + f" targets. When omitted, falls back to the {ENV_LRAUV_NOTIFY}" + " environment variable (comma-separated list)." ), ) self.args = parser.parse_args() From e1f829b15c125fbe619bf576531e2103984a960a Mon Sep 17 00:00:00 2001 From: Mike McCann Date: Tue, 21 Apr 2026 16:13:44 -0700 Subject: [PATCH 2/5] Add Sipper markers to the Deployment plot(s). --- .vscode/launch.json | 4 +++- src/data/create_products.py | 26 ++++++++++++++++++++++++-- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 4d5350e..d57debd 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -500,7 +500,9 @@ // Test far offshore ahi mission with --update_ssds_provenance and --notify --force //"args": ["-v", "1", "--dlist", "ahi/missionlogs/2026/20260406_20260412.dlist", "--update_ssds_provenance", "--force", "--notify", "mccann@mbari.org"] // Test --notify with no argument to a message to the Slack web hook in the LRAUV_NOTIFY environment variable - "args": ["-v", "1", "--dlist", "ahi/missionlogs/2025/20251022_20251024.dlist", "--force", "--notify"] + //"args": ["-v", "1", "--dlist", "ahi/missionlogs/2025/20251022_20251024.dlist", "--force", "--notify"] + // Test Sipper data presentation + "args": ["-v", "1", "--dlist", "daphne/missionlogs/2026/20260316_20260318.dlist", "--force", "--notify"] }, diff --git a/src/data/create_products.py b/src/data/create_products.py index f040a6a..fcc5c58 100755 --- a/src/data/create_products.py +++ b/src/data/create_products.py @@ -8,6 +8,7 @@ __copyright__ = "Copyright 2023, Monterey Bay Aquarium Research Institute" import argparse # noqa: I001 +import contextlib import logging import os import re @@ -1028,6 +1029,9 @@ def _get_gulper_locations(self, distnav: xr.DataArray) -> dict: def _get_sipper_locations(self, distnav: xr.DataArray) -> dict: """Get sipper sample locations in distance/depth space. + For deployment plots (self.nc_files is set), scans the syslog of every + log directory so samples from all logs are captured. + Returns: Dictionary mapping sample number to (distance_km, depth_m) tuple """ @@ -1036,12 +1040,30 @@ def _get_sipper_locations(self, distnav: xr.DataArray) -> dict: sipper = Sipper() sipper.args = argparse.Namespace() - sipper.args.log_file = self.log_file sipper.args.local = self.local sipper.args.verbose = 0 # Suppress sipper logging sipper.logger.setLevel(logging.WARNING) - sipper_times = sipper.parse_sippers() + if self.nc_files: + # Deployment mode: derive a log_file path for each nc_file so we + # can read the syslog from each individual log directory. + log_files = [ + re.sub( + rf"_{re.escape(self.freq)}\.nc$", + ".nc4", + nc.replace(LRAUV_OPENDAP_BASE.rstrip("/") + "/", ""), + ) + for nc in self.nc_files + ] + sipper_times: dict = {} + for lf in log_files: + sipper.args.log_file = lf + with contextlib.suppress(FileNotFoundError): + sipper_times.update(sipper.parse_sippers()) + else: + sipper.args.log_file = self.log_file + sipper_times = sipper.parse_sippers() + if not sipper_times: return {} From ccf771e271b73621cc0cf675a4ae0dd4fa394d87 Mon Sep 17 00:00:00 2001 From: Mike McCann Date: Wed, 22 Apr 2026 11:21:43 -0700 Subject: [PATCH 3/5] Fix too deep depth axis for bogus values caused by memory corruption during logging. Seen in the June 2024 Deployment: Makai 62 Denmark. --- src/data/create_products.py | 47 ++++++++++++++++++++++++------------- 1 file changed, 31 insertions(+), 16 deletions(-) diff --git a/src/data/create_products.py b/src/data/create_products.py index fcc5c58..7351cd9 100755 --- a/src/data/create_products.py +++ b/src/data/create_products.py @@ -822,7 +822,7 @@ def _plot_nighttime_indicator( # noqa: PLR0915 ) day += timedelta(days=1) - def _grid_dims(self) -> tuple: + def _grid_dims(self, plot_vars: list[str] | None = None) -> tuple: # From Matlab code in plot_sections.m: # auvnav positions are too fine for distance calculations, they resolve # spiral ascents and circling while on station @@ -904,14 +904,26 @@ def _grid_dims(self) -> tuple: distnav.to_numpy()[-1], int(3 * self.ds["profile_number"].to_numpy()[-1]), ) - # Vertical gridded to .5 m, rounded down to nearest 50m - max_depth = np.floor(self.ds.cf["depth"].max() / 50) * 50 - iz = np.arange(2.0, max_depth, 0.5) - if not iz.any(): - self.logger.warning( - "Gridding vertical for a surface only mission: {self.ds.cf['depth'].max() =}", - ) - iz = np.arange(0, self.ds.cf["depth"].max(), 0.05) + # Vertical gridded to .5 m, rounded down to nearest 10m (minimum 10m) + # Use only depths where at least one sensor variable has valid data to + # exclude bogus depth values recorded when no valid sensor data was logged + # (e.g. from memory corruption events) + depth_values = self.ds.cf["depth"].to_numpy() + time_dim = self.ds.cf["depth"].dims[0] + nav_vars = {"depth", "latitude", "longitude", "profile_number"} + has_valid_sensor_data = np.zeros(len(depth_values), dtype=bool) + vars_to_check = [ + v for v in (plot_vars or self.ds.data_vars) if v in self.ds and "pitch" not in v + ] + for var in vars_to_check: + if var not in nav_vars and time_dim in self.ds[var].dims and self.ds[var].ndim == 1: + has_valid_sensor_data |= ~np.isnan(self.ds[var].to_numpy()) + depths_with_data = depth_values[has_valid_sensor_data] + if len(depths_with_data) > 0 and not np.all(np.isnan(depths_with_data)): + max_depth = max(np.floor(np.nanmax(depths_with_data) / 10) * 10, 10) + else: + max_depth = max(np.floor(np.nanmax(depth_values) / 10) * 10, 10) + iz = np.arange(0, max_depth, 0.5) return idist, iz, distnav @@ -1690,9 +1702,10 @@ def _plot_var_scatter( # noqa: C901, PLR0912, PLR0913, PLR0915 else: curr_ax.set_ylabel("") - # Set y-axis ticks at 0, 50, 100, 150, etc. + # Set y-axis ticks adaptively based on depth range y_min, y_max = curr_ax.get_ylim() - y_ticks = np.arange(0, int(y_min) + 50, 50) + tick_step = 10 if y_min <= 50 else 50 # noqa: PLR2004 + y_ticks = np.arange(0, int(y_min) + tick_step, tick_step) curr_ax.set_yticks(y_ticks) cb = fig.colorbar(scatter, ax=curr_ax, pad=0.01) @@ -1934,10 +1947,11 @@ def _plot_var_contour( # noqa: C901, PLR0912, PLR0913, PLR0915 else: curr_ax.set_ylabel("") - # Set y-axis ticks at 0, 50, 100, 150, etc. + # Set y-axis ticks adaptively based on depth range y_min, y_max = curr_ax.get_ylim() # Since y-axis is inverted (max at bottom), y_min is the deeper value - y_ticks = np.arange(0, int(y_min) + 50, 50) + tick_step = 10 if y_min <= 50 else 50 # noqa: PLR2004 + y_ticks = np.arange(0, int(y_min) + tick_step, tick_step) curr_ax.set_yticks(y_ticks) cb = fig.colorbar(cntrf, ax=curr_ax, pad=0.01) @@ -2048,7 +2062,7 @@ def plot_2column(self) -> str: # noqa: C901, PLR0912, PLR0915 ) return None - idist, iz, distnav = self._grid_dims() + idist, iz, distnav = self._grid_dims([var for var, _ in plot_variables]) if idist.size == 0 or iz.size == 0 or distnav.size == 0: self.logger.warning("Skipping plot_2column due to missing gridding dimensions") return None @@ -2194,7 +2208,7 @@ def plot_biolume_2column(self) -> str: # noqa: C901, PLR0912, PLR0915 ) return None - idist, iz, distnav = self._grid_dims() + idist, iz, distnav = self._grid_dims([var for var, _ in plot_variables]) if idist.size == 0 or iz.size == 0 or distnav.size == 0: self.logger.warning("Skipping plot_biolume_2column due to missing gridding dimensions") return None @@ -2340,7 +2354,8 @@ def plot_planktivore_2column(self) -> str: # noqa: C901, PLR0912, PLR0915 ) return None - idist, iz, distnav = self._grid_dims() + planktivore_plot_vars = [var for var, _ in self._get_planktivore_plot_variables()] + idist, iz, distnav = self._grid_dims(planktivore_plot_vars) if idist.size == 0 or iz.size == 0 or distnav.size == 0: self.logger.warning( "Skipping plot_planktivore_2column due to missing gridding dimensions" From 88b4b9be5bf16654086c2b4d9fe1cf47e7f73dc2 Mon Sep 17 00:00:00 2001 From: Mike McCann Date: Wed, 22 Apr 2026 11:22:33 -0700 Subject: [PATCH 4/5] Implement Slack lrauv-data channel notification. Co-authored-by: Copilot --- .env.example | 19 ++++- .vscode/launch.json | 10 ++- src/data/lrauv_deployment_plots.py | 122 +++++++++++++++++++++++++---- 3 files changed, 131 insertions(+), 20 deletions(-) diff --git a/.env.example b/.env.example index 0d97339..2c10713 100644 --- a/.env.example +++ b/.env.example @@ -21,6 +21,21 @@ SSDS_API_BASE=https://mooring-ssds.shore.mbari.org/api SSDS_API_KEY= SSDS_API_KEY_HEADER=X-API-Key -# For lrauv_deployment_plots.py messages to be sent to Slack and/or email -LRAUV_NOTIFY= +# Notifications from lrauv_deployment_plots.py (comma-separated list of targets). +# Each target is auto-detected by its format: +# Slack channel ID (e.g. C0AUEA3LZD0 for #lrauv-data) — uploads the PNG +# directly via the Files API so the image is always current. +# Requires SLACK_BOT_TOKEN (below) and the bot invited to the +# channel (/invite @auv-python). Find the channel ID in Slack: +# right-click the channel → View channel details → bottom. +# Webhook URL (https://hooks.slack.com/...) — posts an image-block message +# via an incoming webhook; image may be stale if re-plotted. +# Email address — sends a plain-HTML email with the PNG inline via SMTP_HOST. +LRAUV_NOTIFY=C0AUEA3LZD0 + +# Bot User OAuth Token for the auv-python Slack app (xoxb-...). +# Required when LRAUV_NOTIFY contains a Slack channel ID. +# Scopes needed: files:write, chat:write. +# Generate/rotate at https://api.slack.com/apps → OAuth & Permissions. +# Keep this value out of version control — it grants write access to your workspace. SLACK_BOT_TOKEN= diff --git a/.vscode/launch.json b/.vscode/launch.json index d57debd..003d02c 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -471,9 +471,11 @@ // Make per log file .html files to test with lrauv_deployment_plots and --update_ssds_provenance //"args": ["-v", "1", "--auv_name", "ahi", "--start", "20251022T000000", "--end", "20251024T000000", "--update_ssds_provenance", "--clobber"] // Make per log file .html files to test with lrauv_deployment_plots and --update_ssds_provenance to test dynamic map bounds - "args": ["-v", "1", "--auv_name", "ahi", "--start", "20260409T000000", "--end", "20260415T000000", "--update_ssds_provenance", "--clobber"] + //"args": ["-v", "1", "--auv_name", "ahi", "--start", "20260409T000000", "--end", "20260415T000000", "--update_ssds_provenance", "--clobber"] // Fixup labels on No Data plots //"args": ["-v", "1", "--log_file", "ahi/missionlogs/2026/20260406_20260412/20260411T145332/202604111453_202604111937.nc4", "--update_ssds_provenance", "--clobber"] + // Test ESP markers - Shallow log file from Denmark deployment in June 2024, has large depth values in self.ds.depth.values[6500:6800] + "args": ["-v", "1", "--log_file", "makai/missionlogs/2024/20240607_20240615/20240611T082709/202406110827_202406111026.nc4", "--update_ssds_provenance", "--clobber"] }, { "name": "lrauv_deployment_plots", @@ -490,7 +492,7 @@ // Test time range of DeploymentPlots with ahi planktivore deployment April 2025 //"args": ["-v", "1", "--auv_name", "ahi", "--start", "20250401", "--end", "20250501", "--update_ssds_provenance", "--force"] // Test web page building with a short deployment - //"args": ["-v", "1", "--dlist", "ahi/missionlogs/2025/20251022_20251024.dlist", "--update_ssds_provenance"] + "args": ["-v", "1", "--dlist", "ahi/missionlogs/2025/20251022_20251024.dlist", "--update_ssds_provenance", "--force", "--notify"] // Test --force option for rebuilding web pages with a short deployment //"args": ["-v", "1", "--last_n_days", "10", "--update_ssds_provenance", "--force"] // Test --notify option @@ -502,7 +504,9 @@ // Test --notify with no argument to a message to the Slack web hook in the LRAUV_NOTIFY environment variable //"args": ["-v", "1", "--dlist", "ahi/missionlogs/2025/20251022_20251024.dlist", "--force", "--notify"] // Test Sipper data presentation - "args": ["-v", "1", "--dlist", "daphne/missionlogs/2026/20260316_20260318.dlist", "--force", "--notify"] + //"args": ["-v", "1", "--dlist", "daphne/missionlogs/2026/20260316_20260318.dlist", "--force", "--notify"] + // Test ESP data presentation + //"args": ["-v", "1", "--dlist", "makai/missionlogs/2024/20240607_20240615.dlist", "--update_ssds_provenance", "--force", "--notify"] }, diff --git a/src/data/lrauv_deployment_plots.py b/src/data/lrauv_deployment_plots.py index 14ca85f..bdf671b 100755 --- a/src/data/lrauv_deployment_plots.py +++ b/src/data/lrauv_deployment_plots.py @@ -21,13 +21,19 @@ import logging import os import re +import smtplib import sys import time import urllib.error import urllib.request from datetime import UTC, datetime, timedelta -from zoneinfo import ZoneInfo +from email.mime.image import MIMEImage +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText from pathlib import Path +from zoneinfo import ZoneInfo + +import requests import xarray as xr @@ -40,6 +46,7 @@ from resample import FREQ, LRAUV_OPENDAP_BASE ENV_LRAUV_NOTIFY = "LRAUV_NOTIFY" +ENV_SLACK_BOT_TOKEN = "SLACK_BOT_TOKEN" # noqa: S105 ENV_SMTP_HOST = "SMTP_HOST" ENV_SMTP_PORT = "SMTP_PORT" @@ -386,11 +393,6 @@ def _send_notify_email( force: bool = False, # noqa: FBT001, FBT002 ) -> None: """Send a plain-HTML email with the standard inline PNG and a single web link.""" - import smtplib # noqa: PLC0415 - from email.mime.image import MIMEImage # noqa: PLC0415 - from email.mime.multipart import MIMEMultipart # noqa: PLC0415 - from email.mime.text import MIMEText # noqa: PLC0415 - std_html = next( (p for p in html_paths if "2column_cmocean" in str(p)), html_paths[0] if html_paths else None, @@ -442,7 +444,94 @@ def _send_notify_email( except Exception as exc: # noqa: BLE001 self.logger.warning("Email notification failed: %s", exc) - def _notify( + def _send_slack_file_upload( + self, + channel_id: str, + deployment_name: str, + html_paths: list[Path], + ) -> None: + """Upload the standard PNG to Slack via the Files API and post to a channel. + + *channel_id* is the Slack channel ID (e.g. ``C0123456789``). The bot token + is read from the ``SLACK_BOT_TOKEN`` environment variable. The three-step + upload flow (getUploadURLExternal → POST bytes → completeUploadExternal) + avoids the URL-caching issue of ``image`` blocks sent via incoming webhooks. + """ + token = os.environ.get(ENV_SLACK_BOT_TOKEN, "") + if not token: + self.logger.warning("SLACK_BOT_TOKEN is not set; skipping Slack file upload") + return + self.logger.debug("Slack file upload: channel_id=%s", channel_id) + + std_html = next( + (p for p in html_paths if "2column_cmocean" in str(p)), + html_paths[0] if html_paths else None, + ) + std_png = std_html.with_suffix(".png") if std_html else None + if std_png and not std_png.exists(): + std_png = None + web_url = get_web_url(str(std_html)) if std_html else "" + + _la = ZoneInfo("America/Los_Angeles") + sent_on = datetime.now(tz=UTC).astimezone(_la).strftime("%Y-%m-%d %H:%M:%S %Z") + comment = ( + f"*{deployment_name}*\n" + f"<{web_url}|View related information on the web>\n" + f"_Sent on: {sent_on}_" + ) + headers = {"Authorization": f"Bearer {token}"} + + if std_png: + png_bytes = std_png.read_bytes() + + # Step 1: obtain an upload URL + r1 = requests.post( + "https://slack.com/api/files.getUploadURLExternal", + headers=headers, + data={"filename": std_png.name, "length": len(png_bytes)}, + timeout=10, + ) + r1.raise_for_status() + d1 = r1.json() + if not d1.get("ok"): + msg = f"getUploadURLExternal: {d1.get('error')}" + raise RuntimeError(msg) + + # Step 2: upload the raw bytes + r2 = requests.post(d1["upload_url"], data=png_bytes, timeout=30) # noqa: S113 + r2.raise_for_status() + + # Step 3: complete and share to channel + r3 = requests.post( + "https://slack.com/api/files.completeUploadExternal", + headers=headers, + json={ + "files": [{"id": d1["file_id"]}], + "channel_id": channel_id, + "initial_comment": comment, + }, + timeout=10, + ) + r3.raise_for_status() + d3 = r3.json() + if not d3.get("ok"): + msg = f"completeUploadExternal: {d3.get('error')}" + raise RuntimeError(msg) + else: + # No image available — fall back to a plain text message + r = requests.post( + "https://slack.com/api/chat.postMessage", + headers=headers, + json={"channel": channel_id, "text": comment}, + timeout=10, + ) + r.raise_for_status() + d = r.json() + if not d.get("ok"): + msg = f"chat.postMessage: {d.get('error')}" + raise RuntimeError(msg) + + def _notify( # noqa: PLR0912 self, targets: list[str] | None, deployment_name: str, @@ -452,6 +541,8 @@ def _notify( """Send email and/or Slack notifications for each target in *targets*. Each entry is auto-detected: + - starts with ``C`` (Slack channel ID) → uploads PNG via the Files API using + the ``SLACK_BOT_TOKEN`` env var (avoids URL caching) - starts with ``https://`` → treated as a Slack incoming-webhook URL - anything else → treated as an email address (sent via localhost SMTP) @@ -470,10 +561,14 @@ def _notify( return for target in notify_list: - if target.startswith("https://"): + if target.startswith("C") and os.environ.get(ENV_SLACK_BOT_TOKEN): + try: + self._send_slack_file_upload(target, deployment_name, html_paths) + self.logger.info("Slack file-upload notification sent") + except Exception as exc: # noqa: BLE001 + self.logger.warning("Slack file upload failed: %s", exc) + elif target.startswith("https://"): # Slack incoming webhook — mirror the email: image on top, web link, timestamp - import requests # noqa: PLC0415 - std_html = next( (p for p in html_paths if "2column_cmocean" in str(p)), html_paths[0] if html_paths else None, @@ -483,7 +578,6 @@ def _notify( std_png = None web_url = get_web_url(str(std_html)) if std_html else "" - prefix = "" if force else "New " _la = ZoneInfo("America/Los_Angeles") sent_on = datetime.now(tz=UTC).astimezone(_la).strftime("%Y-%m-%d %H:%M:%S %Z") @@ -503,8 +597,8 @@ def _notify( "text": { "type": "mrkdwn", "text": ( - f"*{prefix}LRAUV deployment plots: {deployment_name}*\n" - f"<{web_url}|View this and related information on the web>\n" + f"*{deployment_name}*\n" + f"<{web_url}|View related information on the web>\n" f"_Sent on: {sent_on}_" ), }, @@ -533,8 +627,6 @@ def _submit_provenance( # noqa: PLR0913 Each PNG is recorded as the output; the input nc_files are its sources. The producer name and description are derived from the HTML title text. """ - from datetime import UTC, datetime # noqa: PLC0415 - dlist_no_ext = str(Path(dlist).with_suffix("")) producer_name = ( "auv-python - lrauv_deployment_plots.py producing " From e0b50450ca203bc5d52a368a42232f77b266d9b0 Mon Sep 17 00:00:00 2001 From: Mike McCann Date: Wed, 22 Apr 2026 11:31:08 -0700 Subject: [PATCH 5/5] Fix the notification tests. --- src/data/test_lrauv_deployment_plots.py | 119 +++++++++++++++++++----- 1 file changed, 95 insertions(+), 24 deletions(-) diff --git a/src/data/test_lrauv_deployment_plots.py b/src/data/test_lrauv_deployment_plots.py index 6e5cf18..fdaca0a 100644 --- a/src/data/test_lrauv_deployment_plots.py +++ b/src/data/test_lrauv_deployment_plots.py @@ -631,7 +631,7 @@ def test_depth_range_in_permalink(self): # =========================================================================== -# Tests for _notify() +# Tests for _notify() and _send_slack_file_upload() # =========================================================================== @@ -648,10 +648,9 @@ def test_email_sent(self, dp, tmp_path): mock_smtp_cls.return_value.__enter__ = MagicMock(return_value=mock_smtp) mock_smtp_cls.return_value.__exit__ = MagicMock(return_value=False) dp._notify( - "team@mbari.org", + ["team@mbari.org"], "CANON_April_2025", [html_file], - "https://stoqs.mbari.org/query/?permalink_id=abc", ) mock_smtp.send_message.assert_called_once() # noqa: S101 @@ -660,54 +659,126 @@ def test_email_sent(self, dp, tmp_path): assert "CANON_April_2025" in msg["Subject"] # noqa: S101 def test_slack_webhook_posts(self, dp, tmp_path): - """_notify() with a Slack webhook URL should POST to that URL.""" - html_file = tmp_path / "test.html" + """_notify() with a Slack webhook URL should POST blocks to that URL.""" + html_file = tmp_path / "test_2column_cmocean.html" html_file.touch() webhook_url = "https://hooks.slack.com/services/T0000/B0000/xxxx" - with patch("requests.post") as mock_post: + with patch("lrauv_deployment_plots.requests.post") as mock_post: mock_post.return_value.raise_for_status = MagicMock() - dp._notify( - "https://hooks.slack.com/services/T0000/B0000/xxxx", - "CANON_April_2025", - [html_file], - None, - ) + dp._notify([webhook_url], "CANON_April_2025", [html_file]) mock_post.assert_called_once() # noqa: S101 call_kwargs = mock_post.call_args assert call_kwargs[0][0] == webhook_url # noqa: S101 - assert "text" in call_kwargs[1]["json"] # noqa: S101 + assert "blocks" in call_kwargs[1]["json"] # noqa: S101 - def test_noop_when_no_target_and_no_env(self, dp, tmp_path, monkeypatch): - """_notify() must not call anything when target is empty and env var unset.""" + def test_noop_when_no_target_and_no_env(self, dp, monkeypatch): + """_notify() must not call anything when targets is empty and env var unset.""" monkeypatch.delenv("LRAUV_NOTIFY", raising=False) + monkeypatch.delenv("SLACK_BOT_TOKEN", raising=False) with ( patch("smtplib.SMTP") as mock_smtp_cls, - patch("requests.post") as mock_post, + patch("lrauv_deployment_plots.requests.post") as mock_post, ): - dp._notify("", "CANON_April_2025", [], None) + dp._notify([], "CANON_April_2025", []) mock_smtp_cls.assert_not_called() # noqa: S101 mock_post.assert_not_called() # noqa: S101 def test_env_var_fallback_used(self, dp, tmp_path, monkeypatch): - """When notify='' but LRAUV_NOTIFY env var is set, that value is used.""" + """When targets is None but LRAUV_NOTIFY env var is set, that value is used.""" monkeypatch.setenv("LRAUV_NOTIFY", "fallback@mbari.org") html_file = tmp_path / "test.html" html_file.touch() - with ( - patch("lrauv_deployment_plots.os") as mock_os, - patch("smtplib.SMTP") as mock_smtp_cls, - ): - mock_os.environ = {"LRAUV_NOTIFY": "fallback@mbari.org"} + with patch("smtplib.SMTP") as mock_smtp_cls: mock_smtp = MagicMock() mock_smtp_cls.return_value.__enter__ = MagicMock(return_value=mock_smtp) mock_smtp_cls.return_value.__exit__ = MagicMock(return_value=False) - dp._notify("", "CANON_April_2025", [html_file], None) + dp._notify(None, "CANON_April_2025", [html_file]) mock_smtp.send_message.assert_called_once() # noqa: S101 msg = mock_smtp.send_message.call_args[0][0] assert "fallback@mbari.org" in msg["To"] # noqa: S101 + + +class TestSendSlackFileUpload: + """Unit tests for DeploymentPlotter._send_slack_file_upload().""" + + def _make_responses(self, upload_url="https://files.slack.com/upload/v1/abc"): + """Return side_effect list for three requests.post calls in the upload flow.""" + get_url_resp = MagicMock() + get_url_resp.raise_for_status = MagicMock() + get_url_resp.json.return_value = { + "ok": True, + "upload_url": upload_url, + "file_id": "F0TEST123", + } + upload_resp = MagicMock() + upload_resp.raise_for_status = MagicMock() + complete_resp = MagicMock() + complete_resp.raise_for_status = MagicMock() + complete_resp.json.return_value = {"ok": True} + return [get_url_resp, upload_resp, complete_resp] + + def test_three_posts_made_when_png_present(self, dp, tmp_path, monkeypatch): + """Upload flow must make exactly three POST requests when a PNG exists.""" + monkeypatch.setenv("SLACK_BOT_TOKEN", "xoxb-test-token") + png = tmp_path / "depl_2column_cmocean.png" + png.write_bytes(b"fakepng") + html = tmp_path / "depl_2column_cmocean.html" + html.touch() + + _EXPECTED_POST_COUNT = 3 + with patch( + "lrauv_deployment_plots.requests.post", side_effect=self._make_responses() + ) as mock_post: + dp._send_slack_file_upload("C0TEST", "CANON April 2025", [html]) + + assert mock_post.call_count == _EXPECTED_POST_COUNT # noqa: S101 + + def test_channel_id_sent_in_complete_call(self, dp, tmp_path, monkeypatch): + """completeUploadExternal must include the channel_id in its payload.""" + monkeypatch.setenv("SLACK_BOT_TOKEN", "xoxb-test-token") + png = tmp_path / "depl_2column_cmocean.png" + png.write_bytes(b"fakepng") + html = tmp_path / "depl_2column_cmocean.html" + html.touch() + + with patch( + "lrauv_deployment_plots.requests.post", side_effect=self._make_responses() + ) as mock_post: + dp._send_slack_file_upload("C0MYCHANNEL", "CANON April 2025", [html]) + + complete_call = mock_post.call_args_list[2] + assert complete_call[1]["json"]["channel_id"] == "C0MYCHANNEL" # noqa: S101 + + def test_missing_token_skips_upload(self, dp, tmp_path, monkeypatch): + """When SLACK_BOT_TOKEN is unset, no requests should be made.""" + monkeypatch.delenv("SLACK_BOT_TOKEN", raising=False) + html = tmp_path / "depl_2column_cmocean.html" + html.touch() + + with patch("lrauv_deployment_plots.requests.post") as mock_post: + dp._send_slack_file_upload("C0TEST", "CANON April 2025", [html]) + + mock_post.assert_not_called() # noqa: S101 + + def test_no_png_falls_back_to_chat_post_message(self, dp, tmp_path, monkeypatch): + """When no PNG exists, a single chat.postMessage call must be made instead.""" + monkeypatch.setenv("SLACK_BOT_TOKEN", "xoxb-test-token") + html = tmp_path / "depl_2column_cmocean.html" + html.touch() + # no PNG created — std_png will be None + + chat_resp = MagicMock() + chat_resp.raise_for_status = MagicMock() + chat_resp.json.return_value = {"ok": True} + + with patch("lrauv_deployment_plots.requests.post", return_value=chat_resp) as mock_post: + dp._send_slack_file_upload("C0TEST", "CANON April 2025", [html]) + + assert mock_post.call_count == 1 # noqa: S101 + assert "chat.postMessage" in mock_post.call_args[0][0] # noqa: S101