diff --git a/.vscode/launch.json b/.vscode/launch.json index 574a766..71250ea 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -484,13 +484,17 @@ // ahi planktivore deployment April 2025 add --update_ssds_provenance //"args": ["-v", "1", "--dlist", "ahi/missionlogs/2025/20250414_20250418.dlist", "--update_ssds_provenance"] // Test time range of DeploymentPlots with ahi planktivore deployment April 2025 - "args": ["-v", "1", "--auv_name", "ahi", "--start", "20250401", "--end", "20250501", "--update_ssds_provenance", "--force"] + //"args": ["-v", "1", "--auv_name", "ahi", "--start", "20250401", "--end", "20250501", "--update_ssds_provenance", "--force"] // Test web page building with a short deployment //"args": ["-v", "1", "--dlist", "ahi/missionlogs/2025/20251022_20251024.dlist", "--update_ssds_provenance"] // Test --force option for rebuilding web pages with a short deployment //"args": ["-v", "1", "--last_n_days", "10", "--update_ssds_provenance", "--force"] // Test --notify option //"args": ["-v", "1", "--dlist", "ahi/missionlogs/2025/20251022_20251024.dlist", "--update_ssds_provenance", "--force", "--notify", "mccann@mbari.org"] + // Test what's running in cron on kraken + //"args": ["-v", "1", "--last_n_days", "30", "--update_ssds_provenance", "--force", "--notify", "mccann@mbari.org"] + // Test far offshore ahi mission with --update_ssds_provenance and --notify --force + "args": ["-v", "1", "--dlist", "ahi/missionlogs/2026/20260406_20260412.dlist", "--update_ssds_provenance", "--force", "--notify", "mccann@mbari.org"] }, diff --git a/src/data/lrauv_deployment_plots.py b/src/data/lrauv_deployment_plots.py index 68b9a55..9370238 100755 --- a/src/data/lrauv_deployment_plots.py +++ b/src/data/lrauv_deployment_plots.py @@ -35,7 +35,7 @@ from logs2netcdfs import AUV_NetCDF from make_permalink import stoqs_url_from_ds from nc42netcdfs import BASE_LRAUV_PATH, BASE_LRAUV_WEB -from provenance import get_dods_url, get_script_github_url, submit_process_run +from provenance import get_script_github_url, get_web_url, submit_process_run from resample import FREQ, LRAUV_OPENDAP_BASE ENV_LRAUV_NOTIFY = "LRAUV_NOTIFY" @@ -362,7 +362,7 @@ def _build_and_write_html( # noqa: PLR0913 per_png_html, html_title, Path(png_path).name, - get_dods_url(png_path), + get_web_url(png_path), stoqs_url, nc_files, auv_name=_auv_name, @@ -387,6 +387,63 @@ def _build_and_write_html( # noqa: PLR0913 nc_files=nc_files, ) + def _send_notify_email( + self, + recipient: str, + deployment_name: str, + html_paths: list[Path], + ) -> None: + """Send a plain-HTML email with the standard inline PNG and a single web link.""" + import smtplib # noqa: PLC0415 + from email.mime.image import MIMEImage # noqa: PLC0415 + from email.mime.multipart import MIMEMultipart # noqa: PLC0415 + from email.mime.text import MIMEText # noqa: PLC0415 + + std_html = next( + (p for p in html_paths if "2column_cmocean" in str(p)), + html_paths[0] if html_paths else None, + ) + std_png = std_html.with_suffix(".png") if std_html else None + if std_png and not std_png.exists(): + std_png = None + web_url = get_web_url(str(std_html)) if std_html else "" + + plain = ( + f"New LRAUV deployment plots: {deployment_name}\n\n" + f" View this and related information on the web:\n {web_url}" + ) + img_tag = ( + 'Standard plot
' + if std_png + else "" + ) + html_body = ( + f'{img_tag}

View this and related information on the web

' + ) + + outer = MIMEMultipart("related") + outer["Subject"] = f"New LRAUV deployment plots: {deployment_name}" + outer["From"] = "auv-python@mbari.org" + outer["To"] = recipient + alt = MIMEMultipart("alternative") + outer.attach(alt) + alt.attach(MIMEText(plain, "plain")) + alt.attach(MIMEText(html_body, "html")) + if std_png: + img = MIMEImage(std_png.read_bytes()) + img.add_header("Content-ID", "") + img.add_header("Content-Disposition", "inline", filename=std_png.name) + outer.attach(img) + try: + smtp_host = os.environ.get(ENV_SMTP_HOST, "localhost") + smtp_port = int(os.environ.get(ENV_SMTP_PORT, "587")) + with smtplib.SMTP(smtp_host, smtp_port) as s: + s.starttls() + s.send_message(outer) + self.logger.info("Email notification sent to %s", recipient) + except Exception as exc: # noqa: BLE001 + self.logger.warning("Email notification failed: %s", exc) + def _notify( self, target: str, @@ -407,49 +464,27 @@ def _notify( if not resolved: return - lines = [f"New LRAUV deployment plots available: {deployment_name}"] - for p in html_paths: - lines.append(f" {get_dods_url(str(p))}") # noqa: PERF401 - if stoqs_url: - lines.append(f"STOQS: {stoqs_url}") - body = "\n".join(lines) - if resolved.startswith("https://"): - # Slack incoming webhook + # Slack incoming webhook — include all plot links and STOQS URL import requests # noqa: PLC0415 + plot_links = [(get_web_url(str(p)), self._plot_label(str(p))) for p in html_paths] + lines = [f"New LRAUV deployment plots available: {deployment_name}", ""] + for url, label in plot_links: + lines.append(f" {label}: {url}") + if stoqs_url: + after_scheme = stoqs_url.split("//", 1)[-1] if "//" in stoqs_url else stoqs_url + stoqs_db_label = after_scheme.split("/")[1] if "/" in after_scheme else after_scheme + lines.append("") + lines.append(f" STOQS ({stoqs_db_label}): {stoqs_url}") try: - resp = requests.post(resolved, json={"text": body}, timeout=10) # noqa: S113 + resp = requests.post(resolved, json={"text": "\n".join(lines)}, timeout=10) # noqa: S113 resp.raise_for_status() self.logger.info("Slack notification sent to webhook") except Exception as exc: # noqa: BLE001 self.logger.warning("Slack notification failed: %s", exc) else: - # Email via localhost SMTP relay - import smtplib # noqa: PLC0415 - from email.message import EmailMessage # noqa: PLC0415 - - msg = EmailMessage() - msg["Subject"] = f"New LRAUV deployment plots: {deployment_name}" - msg["From"] = "auv-python@mbari.org" - msg["To"] = resolved - msg.set_content(body) - for p in html_paths: - if p.exists(): - msg.add_attachment( - p.read_text(encoding="utf-8"), - subtype="html", - filename=p.name, - ) - try: - smtp_host = os.environ.get(ENV_SMTP_HOST, "localhost") - smtp_port = int(os.environ.get(ENV_SMTP_PORT, "587")) - with smtplib.SMTP(smtp_host, smtp_port) as s: - s.starttls() - s.send_message(msg) - self.logger.info("Email notification sent to %s", resolved) - except Exception as exc: # noqa: BLE001 - self.logger.warning("Email notification failed: %s", exc) + self._send_notify_email(resolved, deployment_name, html_paths) def _submit_provenance( # noqa: PLR0913 self, @@ -489,7 +524,7 @@ def _submit_provenance( # noqa: PLR0913 png_resources = additional_resources + [ { "name": Path(png_path).name, - "uristring": get_dods_url(png_path), + "uristring": get_web_url(png_path), "description": f"Deployment quick look plot: {Path(png_path).name}", "resourcetype_name": "Quick Look Plot", } @@ -499,7 +534,7 @@ def _submit_provenance( # noqa: PLR0913 png_resources.append( { "name": per_png_html.name, - "uristring": get_dods_url(str(per_png_html)), + "uristring": get_web_url(str(per_png_html)), "description": f"Per-PNG HTML page for {per_png_html.name}", "resourcetype_name": "html", } @@ -514,7 +549,6 @@ def _submit_provenance( # noqa: PLR0913 script_name="src/data/lrauv_deployment_plots.py", cmd_line_args=cmd_line, additional_resources=png_resources, - log=self.logger, ) except Exception: # noqa: BLE001 self.logger.warning("Provenance submission failed for %s", png_path, exc_info=True) @@ -663,6 +697,11 @@ def _write_per_png_html( # noqa: C901, PLR0913 html_path.write_text(html, encoding="utf-8") _PLOT_KINDS = ("2column_cmocean", "2column_biolume", "2column_planktivore") + _PLOT_KIND_LABELS = { + "2column_cmocean": "Standard", + "2column_biolume": "Bioluminescence", + "2column_planktivore": "Planktivore", + } def _png_urls_for_nc(self, nc_url: str) -> list[str]: """Return web-accessible PNG URLs for all plot kinds for one OPeNDAP nc URL.""" @@ -670,6 +709,13 @@ def _png_urls_for_nc(self, nc_url: str) -> list[str]: base = BASE_LRAUV_WEB.rstrip("/") + "/" + rel[: -len(".nc")] return [f"{base}_{kind}.png" for kind in self._PLOT_KINDS] + def _plot_label(self, path_str: str) -> str: + """Return a human-readable label for a plot PNG or HTML path.""" + for kind, label in self._PLOT_KIND_LABELS.items(): + if kind in path_str: + return label + return Path(path_str).stem + def _url_exists(self, url: str) -> bool: """Return True if the URL responds with HTTP 200 to a HEAD request.""" try: diff --git a/src/data/process.py b/src/data/process.py index 5b79fd9..9213d25 100755 --- a/src/data/process.py +++ b/src/data/process.py @@ -75,7 +75,7 @@ class data are: download_process and calibrate, while for LRAUV class data from logs2netcdfs import BASE_PATH, MISSIONLOGS, MISSIONNETCDFS, AUV_NetCDF from lopcToNetCDF import LOPC_Processor, UnexpectedAreaOfCode from nc42netcdfs import BASE_LRAUV_PATH, BASE_LRAUV_WEB, GROUP, Extract -from provenance import get_dods_url, submit_process_run +from provenance import get_dods_url, get_web_url, submit_process_run from resample import ( AUVCTD_OPENDAP_BASE, FLASH_THRESHOLD, @@ -779,7 +779,7 @@ def _collect_lrauv_netcdf_resources( resources.append( { "name": plot_file.name, - "uristring": get_dods_url(str(plot_file)), + "uristring": get_web_url(str(plot_file)), "description": f"Created by create_products.py: {description}", "resourcetype_name": "Quick Look Plot", } diff --git a/src/data/provenance.py b/src/data/provenance.py index 8a2a17b..c6d06bf 100644 --- a/src/data/provenance.py +++ b/src/data/provenance.py @@ -41,6 +41,11 @@ str(_PROJECT_DATA / "auv_data"): "http://dods.mbari.org/opendap/data/auvctd", str(_PROJECT_DATA / "lrauv_data"): "http://dods.mbari.org/opendap/data/lrauv", } +# Web-serving URLs (no opendap/ path prefix) — for PNG, HTML, and other static files. +_PATH_TO_WEB_MAP: dict[str, str] = { + str(_PROJECT_DATA / "auv_data"): "https://dods.mbari.org/data/auvctd", + str(_PROJECT_DATA / "lrauv_data"): "https://dods.mbari.org/data/lrauv", +} # --------------------------------------------------------------------------- @@ -97,6 +102,21 @@ def get_dods_url(nc_file_path: str) -> str: return resolved +def get_web_url(file_path: str) -> str: + """Translate a local file path to its web-accessible URL (no OPeNDAP prefix). + + Use this for PNG, HTML, and other static files served over HTTP. + Walks ``_PATH_TO_WEB_MAP`` looking for a matching prefix in the + *resolved* path string. Returns the original path unchanged if no + match is found. + """ + resolved = str(Path(file_path).resolve()) + for local_prefix, url_prefix in _PATH_TO_WEB_MAP.items(): + if resolved.startswith(local_prefix): + return resolved.replace(local_prefix, url_prefix, 1) + return resolved + + def get_git_url(script_name: str, version: str) -> str: """Return a GitHub web URL for *script_name* at *version*.""" return f"{GIT_WEB_BASE}/{version}/{script_name}"