From 3b598e4b1d240c6ad956d8744f7f056a878ba286 Mon Sep 17 00:00:00 2001 From: Mike McCann Date: Wed, 15 Apr 2026 10:11:45 -0700 Subject: [PATCH 1/4] Remove "opendap/" from URLs that don't need it. --- .vscode/launch.json | 4 +++- src/data/lrauv_deployment_plots.py | 10 +++++----- src/data/process.py | 4 ++-- src/data/provenance.py | 20 ++++++++++++++++++++ 4 files changed, 30 insertions(+), 8 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 574a766..0612445 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -484,13 +484,15 @@ // ahi planktivore deployment April 2025 add --update_ssds_provenance //"args": ["-v", "1", "--dlist", "ahi/missionlogs/2025/20250414_20250418.dlist", "--update_ssds_provenance"] // Test time range of DeploymentPlots with ahi planktivore deployment April 2025 - "args": ["-v", "1", "--auv_name", "ahi", "--start", "20250401", "--end", "20250501", "--update_ssds_provenance", "--force"] + //"args": ["-v", "1", "--auv_name", "ahi", "--start", "20250401", "--end", "20250501", "--update_ssds_provenance", "--force"] // Test web page building with a short deployment //"args": ["-v", "1", "--dlist", "ahi/missionlogs/2025/20251022_20251024.dlist", "--update_ssds_provenance"] // Test --force option for rebuilding web pages with a short deployment //"args": ["-v", "1", "--last_n_days", "10", "--update_ssds_provenance", "--force"] // Test --notify option //"args": ["-v", "1", "--dlist", "ahi/missionlogs/2025/20251022_20251024.dlist", "--update_ssds_provenance", "--force", "--notify", "mccann@mbari.org"] + // Test what's running in cron on kraken + "args": ["-v", "1", "--last_n_days", "30", "--update_ssds_provenance", "--force", "--notify", "mccann@mbari.org"] }, diff --git a/src/data/lrauv_deployment_plots.py b/src/data/lrauv_deployment_plots.py index 68b9a55..5f32cfb 100755 --- a/src/data/lrauv_deployment_plots.py +++ b/src/data/lrauv_deployment_plots.py @@ -35,7 +35,7 @@ from logs2netcdfs import AUV_NetCDF from make_permalink import stoqs_url_from_ds from nc42netcdfs import BASE_LRAUV_PATH, BASE_LRAUV_WEB -from provenance import get_dods_url, get_script_github_url, submit_process_run +from provenance import get_script_github_url, get_web_url, submit_process_run from resample import FREQ, LRAUV_OPENDAP_BASE ENV_LRAUV_NOTIFY = "LRAUV_NOTIFY" @@ -362,7 +362,7 @@ def _build_and_write_html( # noqa: PLR0913 per_png_html, html_title, Path(png_path).name, - get_dods_url(png_path), + get_web_url(png_path), stoqs_url, nc_files, auv_name=_auv_name, @@ -409,7 +409,7 @@ def _notify( lines = [f"New LRAUV deployment plots available: {deployment_name}"] for p in html_paths: - lines.append(f" {get_dods_url(str(p))}") # noqa: PERF401 + lines.append(f" {get_web_url(str(p))}") # noqa: PERF401 if stoqs_url: lines.append(f"STOQS: {stoqs_url}") body = "\n".join(lines) @@ -489,7 +489,7 @@ def _submit_provenance( # noqa: PLR0913 png_resources = additional_resources + [ { "name": Path(png_path).name, - "uristring": get_dods_url(png_path), + "uristring": get_web_url(png_path), "description": f"Deployment quick look plot: {Path(png_path).name}", "resourcetype_name": "Quick Look Plot", } @@ -499,7 +499,7 @@ def _submit_provenance( # noqa: PLR0913 png_resources.append( { "name": per_png_html.name, - "uristring": get_dods_url(str(per_png_html)), + "uristring": get_web_url(str(per_png_html)), "description": f"Per-PNG HTML page for {per_png_html.name}", "resourcetype_name": "html", } diff --git a/src/data/process.py b/src/data/process.py index 5b79fd9..9213d25 100755 --- a/src/data/process.py +++ b/src/data/process.py @@ -75,7 +75,7 @@ class data are: download_process and calibrate, while for LRAUV class data from logs2netcdfs import BASE_PATH, MISSIONLOGS, MISSIONNETCDFS, AUV_NetCDF from lopcToNetCDF import LOPC_Processor, UnexpectedAreaOfCode from nc42netcdfs import BASE_LRAUV_PATH, BASE_LRAUV_WEB, GROUP, Extract -from provenance import get_dods_url, submit_process_run +from provenance import get_dods_url, get_web_url, submit_process_run from resample import ( AUVCTD_OPENDAP_BASE, FLASH_THRESHOLD, @@ -779,7 +779,7 @@ def _collect_lrauv_netcdf_resources( resources.append( { "name": plot_file.name, - "uristring": get_dods_url(str(plot_file)), + "uristring": get_web_url(str(plot_file)), "description": f"Created by create_products.py: {description}", "resourcetype_name": "Quick Look Plot", } diff --git a/src/data/provenance.py b/src/data/provenance.py index 8a2a17b..c6d06bf 100644 --- a/src/data/provenance.py +++ b/src/data/provenance.py @@ -41,6 +41,11 @@ str(_PROJECT_DATA / "auv_data"): "http://dods.mbari.org/opendap/data/auvctd", str(_PROJECT_DATA / "lrauv_data"): "http://dods.mbari.org/opendap/data/lrauv", } +# Web-serving URLs (no opendap/ path prefix) — for PNG, HTML, and other static files. +_PATH_TO_WEB_MAP: dict[str, str] = { + str(_PROJECT_DATA / "auv_data"): "https://dods.mbari.org/data/auvctd", + str(_PROJECT_DATA / "lrauv_data"): "https://dods.mbari.org/data/lrauv", +} # --------------------------------------------------------------------------- @@ -97,6 +102,21 @@ def get_dods_url(nc_file_path: str) -> str: return resolved +def get_web_url(file_path: str) -> str: + """Translate a local file path to its web-accessible URL (no OPeNDAP prefix). + + Use this for PNG, HTML, and other static files served over HTTP. + Walks ``_PATH_TO_WEB_MAP`` looking for a matching prefix in the + *resolved* path string. Returns the original path unchanged if no + match is found. + """ + resolved = str(Path(file_path).resolve()) + for local_prefix, url_prefix in _PATH_TO_WEB_MAP.items(): + if resolved.startswith(local_prefix): + return resolved.replace(local_prefix, url_prefix, 1) + return resolved + + def get_git_url(script_name: str, version: str) -> str: """Return a GitHub web URL for *script_name* at *version*.""" return f"{GIT_WEB_BASE}/{version}/{script_name}" From 6dec726df8f17f53b1216d2009e8db03429a359b Mon Sep 17 00:00:00 2001 From: Mike McCann Date: Wed, 15 Apr 2026 10:22:02 -0700 Subject: [PATCH 2/4] Remove duplicate "Provenance recorded:" log message. --- src/data/lrauv_deployment_plots.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/data/lrauv_deployment_plots.py b/src/data/lrauv_deployment_plots.py index 5f32cfb..c92119d 100755 --- a/src/data/lrauv_deployment_plots.py +++ b/src/data/lrauv_deployment_plots.py @@ -514,7 +514,6 @@ def _submit_provenance( # noqa: PLR0913 script_name="src/data/lrauv_deployment_plots.py", cmd_line_args=cmd_line, additional_resources=png_resources, - log=self.logger, ) except Exception: # noqa: BLE001 self.logger.warning("Provenance submission failed for %s", png_path, exc_info=True) From 3923b0f8691c3e88046dca8a143251d17b1530f0 Mon Sep 17 00:00:00 2001 From: Mike McCann Date: Wed, 15 Apr 2026 10:50:21 -0700 Subject: [PATCH 3/4] Simplify email message sent with --notify: 1. Remove attachments 2. Have one link to "View this on the web" where more links are available --- .vscode/launch.json | 4 +- src/data/lrauv_deployment_plots.py | 108 +++++++++++++++++++++++------ 2 files changed, 88 insertions(+), 24 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 0612445..71250ea 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -492,7 +492,9 @@ // Test --notify option //"args": ["-v", "1", "--dlist", "ahi/missionlogs/2025/20251022_20251024.dlist", "--update_ssds_provenance", "--force", "--notify", "mccann@mbari.org"] // Test what's running in cron on kraken - "args": ["-v", "1", "--last_n_days", "30", "--update_ssds_provenance", "--force", "--notify", "mccann@mbari.org"] + //"args": ["-v", "1", "--last_n_days", "30", "--update_ssds_provenance", "--force", "--notify", "mccann@mbari.org"] + // Test far offshore ahi mission with --update_ssds_provenance and --notify --force + "args": ["-v", "1", "--dlist", "ahi/missionlogs/2026/20260406_20260412.dlist", "--update_ssds_provenance", "--force", "--notify", "mccann@mbari.org"] }, diff --git a/src/data/lrauv_deployment_plots.py b/src/data/lrauv_deployment_plots.py index c92119d..1d9f450 100755 --- a/src/data/lrauv_deployment_plots.py +++ b/src/data/lrauv_deployment_plots.py @@ -376,7 +376,7 @@ def _build_and_write_html( # noqa: PLR0913 html_paths = [ Path(p).with_suffix(".html") for p in png_paths if Path(p).with_suffix(".html").exists() ] - self._notify(notify or "", raw_name or plot_name_stem, html_paths, stoqs_url) + self._notify(notify or "", raw_name or plot_name_stem, html_paths, png_paths, stoqs_url) if update_ssds_provenance: self._submit_provenance( deployment_dir=deployment_dir, @@ -387,11 +387,49 @@ def _build_and_write_html( # noqa: PLR0913 nc_files=nc_files, ) - def _notify( + def _build_notify_email( # noqa: PLR0913 + self, + deployment_name: str, + plain_body: str, + plot_links: list[tuple[str, str]], + stoqs_url: str | None, + stoqs_db_label: str, + std_png: Path | None, + recipient: str, + ): + """Build a multipart/related email message with an inline standard PNG.""" + from email.mime.image import MIMEImage # noqa: PLC0415 + from email.mime.multipart import MIMEMultipart # noqa: PLC0415 + from email.mime.text import MIMEText # noqa: PLC0415 + + img_tag = 'Standard plot
' if std_png else "" + web_url = plot_links[0][0] if plot_links else "" + view_link = f'View this on the web.' if web_url else "" + stoqs_html = ( + f'

STOQS: {stoqs_db_label}

' if stoqs_url else "" + ) + html_body = f"

{deployment_name}

\n{img_tag}\n

{view_link}

\n{stoqs_html}\n" + outer = MIMEMultipart("related") + outer["Subject"] = f"New LRAUV deployment plots: {deployment_name}" + outer["From"] = "auv-python@mbari.org" + outer["To"] = recipient + alt = MIMEMultipart("alternative") + outer.attach(alt) + alt.attach(MIMEText(plain_body, "plain")) + alt.attach(MIMEText(html_body, "html")) + if std_png: + img = MIMEImage(std_png.read_bytes()) + img.add_header("Content-ID", "") + img.add_header("Content-Disposition", "inline", filename=std_png.name) + outer.attach(img) + return outer + + def _notify( # noqa: PLR0912, PLR0913 self, target: str, deployment_name: str, html_paths: list[Path], + png_paths: list[str], stoqs_url: str | None, ) -> None: """Send an email or Slack notification with links to the new deployment HTML pages. @@ -407,40 +445,52 @@ def _notify( if not resolved: return - lines = [f"New LRAUV deployment plots available: {deployment_name}"] - for p in html_paths: - lines.append(f" {get_web_url(str(p))}") # noqa: PERF401 + # Standard (cmocean) PNG for inline embedding in email + std_png = next( + (Path(p) for p in png_paths if "2column_cmocean" in p and Path(p).exists()), + None, + ) + + # Labelled (url, label) pairs for each HTML page + plot_links = [(get_web_url(str(p)), self._plot_label(str(p))) for p in html_paths] + + # STOQS label (database name from URL path) + stoqs_db_label = "" + if stoqs_url: + after_scheme = stoqs_url.split("//", 1)[-1] if "//" in stoqs_url else stoqs_url + stoqs_db_label = after_scheme.split("/")[1] if "/" in after_scheme else after_scheme + + # Plain-text body (used by both Slack and the email text/plain part) + lines = [f"New LRAUV deployment plots available: {deployment_name}", ""] + for url, label in plot_links: + lines.append(f" {label}: {url}") if stoqs_url: - lines.append(f"STOQS: {stoqs_url}") - body = "\n".join(lines) + lines.append("") + lines.append(f" STOQS ({stoqs_db_label}): {stoqs_url}") + plain_body = "\n".join(lines) if resolved.startswith("https://"): # Slack incoming webhook import requests # noqa: PLC0415 try: - resp = requests.post(resolved, json={"text": body}, timeout=10) # noqa: S113 + resp = requests.post(resolved, json={"text": plain_body}, timeout=10) # noqa: S113 resp.raise_for_status() self.logger.info("Slack notification sent to webhook") except Exception as exc: # noqa: BLE001 self.logger.warning("Slack notification failed: %s", exc) else: - # Email via localhost SMTP relay import smtplib # noqa: PLC0415 - from email.message import EmailMessage # noqa: PLC0415 - - msg = EmailMessage() - msg["Subject"] = f"New LRAUV deployment plots: {deployment_name}" - msg["From"] = "auv-python@mbari.org" - msg["To"] = resolved - msg.set_content(body) - for p in html_paths: - if p.exists(): - msg.add_attachment( - p.read_text(encoding="utf-8"), - subtype="html", - filename=p.name, - ) + + msg = self._build_notify_email( + deployment_name, + plain_body, + plot_links, + stoqs_url, + stoqs_db_label, + std_png, + resolved, + ) try: smtp_host = os.environ.get(ENV_SMTP_HOST, "localhost") smtp_port = int(os.environ.get(ENV_SMTP_PORT, "587")) @@ -662,6 +712,11 @@ def _write_per_png_html( # noqa: C901, PLR0913 html_path.write_text(html, encoding="utf-8") _PLOT_KINDS = ("2column_cmocean", "2column_biolume", "2column_planktivore") + _PLOT_KIND_LABELS = { + "2column_cmocean": "Standard", + "2column_biolume": "Bioluminescence", + "2column_planktivore": "Planktivore", + } def _png_urls_for_nc(self, nc_url: str) -> list[str]: """Return web-accessible PNG URLs for all plot kinds for one OPeNDAP nc URL.""" @@ -669,6 +724,13 @@ def _png_urls_for_nc(self, nc_url: str) -> list[str]: base = BASE_LRAUV_WEB.rstrip("/") + "/" + rel[: -len(".nc")] return [f"{base}_{kind}.png" for kind in self._PLOT_KINDS] + def _plot_label(self, path_str: str) -> str: + """Return a human-readable label for a plot PNG or HTML path.""" + for kind, label in self._PLOT_KIND_LABELS.items(): + if kind in path_str: + return label + return Path(path_str).stem + def _url_exists(self, url: str) -> bool: """Return True if the URL responds with HTTP 200 to a HEAD request.""" try: From 51d43d500cba659f34a8b16a6e415469c8bad7bf Mon Sep 17 00:00:00 2001 From: Mike McCann Date: Wed, 15 Apr 2026 12:51:03 -0700 Subject: [PATCH 4/4] Simplify the email message, just the image and a single link. --- src/data/lrauv_deployment_plots.py | 117 +++++++++++++---------------- 1 file changed, 51 insertions(+), 66 deletions(-) diff --git a/src/data/lrauv_deployment_plots.py b/src/data/lrauv_deployment_plots.py index 1d9f450..9370238 100755 --- a/src/data/lrauv_deployment_plots.py +++ b/src/data/lrauv_deployment_plots.py @@ -376,7 +376,7 @@ def _build_and_write_html( # noqa: PLR0913 html_paths = [ Path(p).with_suffix(".html") for p in png_paths if Path(p).with_suffix(".html").exists() ] - self._notify(notify or "", raw_name or plot_name_stem, html_paths, png_paths, stoqs_url) + self._notify(notify or "", raw_name or plot_name_stem, html_paths, stoqs_url) if update_ssds_provenance: self._submit_provenance( deployment_dir=deployment_dir, @@ -387,49 +387,68 @@ def _build_and_write_html( # noqa: PLR0913 nc_files=nc_files, ) - def _build_notify_email( # noqa: PLR0913 + def _send_notify_email( self, - deployment_name: str, - plain_body: str, - plot_links: list[tuple[str, str]], - stoqs_url: str | None, - stoqs_db_label: str, - std_png: Path | None, recipient: str, - ): - """Build a multipart/related email message with an inline standard PNG.""" + deployment_name: str, + html_paths: list[Path], + ) -> None: + """Send a plain-HTML email with the standard inline PNG and a single web link.""" + import smtplib # noqa: PLC0415 from email.mime.image import MIMEImage # noqa: PLC0415 from email.mime.multipart import MIMEMultipart # noqa: PLC0415 from email.mime.text import MIMEText # noqa: PLC0415 - img_tag = 'Standard plot
' if std_png else "" - web_url = plot_links[0][0] if plot_links else "" - view_link = f'View this on the web.' if web_url else "" - stoqs_html = ( - f'

STOQS: {stoqs_db_label}

' if stoqs_url else "" + std_html = next( + (p for p in html_paths if "2column_cmocean" in str(p)), + html_paths[0] if html_paths else None, + ) + std_png = std_html.with_suffix(".png") if std_html else None + if std_png and not std_png.exists(): + std_png = None + web_url = get_web_url(str(std_html)) if std_html else "" + + plain = ( + f"New LRAUV deployment plots: {deployment_name}\n\n" + f" View this and related information on the web:\n {web_url}" ) - html_body = f"

{deployment_name}

\n{img_tag}\n

{view_link}

\n{stoqs_html}\n" + img_tag = ( + 'Standard plot
' + if std_png + else "" + ) + html_body = ( + f'{img_tag}

View this and related information on the web

' + ) + outer = MIMEMultipart("related") outer["Subject"] = f"New LRAUV deployment plots: {deployment_name}" outer["From"] = "auv-python@mbari.org" outer["To"] = recipient alt = MIMEMultipart("alternative") outer.attach(alt) - alt.attach(MIMEText(plain_body, "plain")) + alt.attach(MIMEText(plain, "plain")) alt.attach(MIMEText(html_body, "html")) if std_png: img = MIMEImage(std_png.read_bytes()) img.add_header("Content-ID", "") img.add_header("Content-Disposition", "inline", filename=std_png.name) outer.attach(img) - return outer + try: + smtp_host = os.environ.get(ENV_SMTP_HOST, "localhost") + smtp_port = int(os.environ.get(ENV_SMTP_PORT, "587")) + with smtplib.SMTP(smtp_host, smtp_port) as s: + s.starttls() + s.send_message(outer) + self.logger.info("Email notification sent to %s", recipient) + except Exception as exc: # noqa: BLE001 + self.logger.warning("Email notification failed: %s", exc) - def _notify( # noqa: PLR0912, PLR0913 + def _notify( self, target: str, deployment_name: str, html_paths: list[Path], - png_paths: list[str], stoqs_url: str | None, ) -> None: """Send an email or Slack notification with links to the new deployment HTML pages. @@ -445,61 +464,27 @@ def _notify( # noqa: PLR0912, PLR0913 if not resolved: return - # Standard (cmocean) PNG for inline embedding in email - std_png = next( - (Path(p) for p in png_paths if "2column_cmocean" in p and Path(p).exists()), - None, - ) - - # Labelled (url, label) pairs for each HTML page - plot_links = [(get_web_url(str(p)), self._plot_label(str(p))) for p in html_paths] - - # STOQS label (database name from URL path) - stoqs_db_label = "" - if stoqs_url: - after_scheme = stoqs_url.split("//", 1)[-1] if "//" in stoqs_url else stoqs_url - stoqs_db_label = after_scheme.split("/")[1] if "/" in after_scheme else after_scheme - - # Plain-text body (used by both Slack and the email text/plain part) - lines = [f"New LRAUV deployment plots available: {deployment_name}", ""] - for url, label in plot_links: - lines.append(f" {label}: {url}") - if stoqs_url: - lines.append("") - lines.append(f" STOQS ({stoqs_db_label}): {stoqs_url}") - plain_body = "\n".join(lines) - if resolved.startswith("https://"): - # Slack incoming webhook + # Slack incoming webhook — include all plot links and STOQS URL import requests # noqa: PLC0415 + plot_links = [(get_web_url(str(p)), self._plot_label(str(p))) for p in html_paths] + lines = [f"New LRAUV deployment plots available: {deployment_name}", ""] + for url, label in plot_links: + lines.append(f" {label}: {url}") + if stoqs_url: + after_scheme = stoqs_url.split("//", 1)[-1] if "//" in stoqs_url else stoqs_url + stoqs_db_label = after_scheme.split("/")[1] if "/" in after_scheme else after_scheme + lines.append("") + lines.append(f" STOQS ({stoqs_db_label}): {stoqs_url}") try: - resp = requests.post(resolved, json={"text": plain_body}, timeout=10) # noqa: S113 + resp = requests.post(resolved, json={"text": "\n".join(lines)}, timeout=10) # noqa: S113 resp.raise_for_status() self.logger.info("Slack notification sent to webhook") except Exception as exc: # noqa: BLE001 self.logger.warning("Slack notification failed: %s", exc) else: - import smtplib # noqa: PLC0415 - - msg = self._build_notify_email( - deployment_name, - plain_body, - plot_links, - stoqs_url, - stoqs_db_label, - std_png, - resolved, - ) - try: - smtp_host = os.environ.get(ENV_SMTP_HOST, "localhost") - smtp_port = int(os.environ.get(ENV_SMTP_PORT, "587")) - with smtplib.SMTP(smtp_host, smtp_port) as s: - s.starttls() - s.send_message(msg) - self.logger.info("Email notification sent to %s", resolved) - except Exception as exc: # noqa: BLE001 - self.logger.warning("Email notification failed: %s", exc) + self._send_notify_email(resolved, deployment_name, html_paths) def _submit_provenance( # noqa: PLR0913 self,