diff --git a/src/data/lrauv_deployment_plots.py b/src/data/lrauv_deployment_plots.py index 19321dd..d258ae5 100755 --- a/src/data/lrauv_deployment_plots.py +++ b/src/data/lrauv_deployment_plots.py @@ -313,6 +313,7 @@ def plot_deployment( # noqa: C901, PLR0912, PLR0913, PLR0915 nc_files, verbose=verbose, update_ssds_provenance=update_ssds_provenance, + force=force, notify=notify, ) @@ -327,6 +328,7 @@ def _build_and_write_html( # noqa: PLR0913 nc_files: list[str], verbose: int = 0, update_ssds_provenance: bool = False, # noqa: FBT001, FBT002 + force: bool = False, # noqa: FBT001, FBT002 notify: str | None = None, ) -> None: """Fetch STOQS permalink and write per-PNG HTML pages.""" @@ -363,7 +365,7 @@ def _build_and_write_html( # noqa: PLR0913 html_paths = [ Path(p).with_suffix(".html") for p in png_paths if Path(p).with_suffix(".html").exists() ] - self._notify(notify or "", raw_name or plot_name_stem, html_paths, stoqs_url) + self._notify(notify or "", raw_name or plot_name_stem, html_paths, stoqs_url, force=force) if update_ssds_provenance: self._submit_provenance( deployment_dir=deployment_dir, @@ -379,6 +381,7 @@ def _send_notify_email( recipient: str, deployment_name: str, html_paths: list[Path], + force: bool = False, # noqa: FBT001, FBT002 ) -> None: """Send a plain-HTML email with the standard inline PNG and a single web link.""" import smtplib # noqa: PLC0415 @@ -395,9 +398,10 @@ def _send_notify_email( std_png = None web_url = get_web_url(str(std_html)) if std_html else "" + prefix = "" if force else "New " sent_on = datetime.now(tz=UTC).astimezone().strftime("%Y-%m-%d %H:%M:%S %Z") plain = ( - f"New LRAUV deployment plots: {deployment_name}\n\n" + f"{prefix}LRAUV deployment plots: {deployment_name}\n\n" f" View this and related information on the web:\n {web_url}\n\n" f"Sent on: {sent_on}" ) @@ -413,7 +417,7 @@ def _send_notify_email( ) outer = MIMEMultipart("related") - outer["Subject"] = f"New LRAUV deployment plots: {deployment_name}" + outer["Subject"] = f"{prefix}LRAUV deployment plots: {deployment_name}" outer["From"] = "auv-python@mbari.org" outer["To"] = recipient alt = MIMEMultipart("alternative") @@ -435,12 +439,13 @@ def _send_notify_email( except Exception as exc: # noqa: BLE001 self.logger.warning("Email notification failed: %s", exc) - def _notify( + def _notify( # noqa: PLR0913 self, target: str, deployment_name: str, html_paths: list[Path], stoqs_url: str | None, + force: bool = False, # noqa: FBT001, FBT002 ) -> None: """Send an email or Slack notification with links to the new deployment HTML pages. @@ -460,7 +465,8 @@ def _notify( import requests # noqa: PLC0415 plot_links = [(get_web_url(str(p)), self._plot_label(str(p))) for p in html_paths] - lines = [f"New LRAUV deployment plots available: {deployment_name}", ""] + prefix = "" if force else "New " + lines = [f"{prefix}LRAUV deployment plots available: {deployment_name}", ""] for url, label in plot_links: lines.append(f" {label}: {url}") if stoqs_url: @@ -475,7 +481,7 @@ def _notify( except Exception as exc: # noqa: BLE001 self.logger.warning("Slack notification failed: %s", exc) else: - self._send_notify_email(resolved, deployment_name, html_paths) + self._send_notify_email(resolved, deployment_name, html_paths, force=force) def _submit_provenance( # noqa: PLR0913 self, diff --git a/src/data/process.py b/src/data/process.py index 9213d25..3c12fb2 100755 --- a/src/data/process.py +++ b/src/data/process.py @@ -75,7 +75,7 @@ class data are: download_process and calibrate, while for LRAUV class data from logs2netcdfs import BASE_PATH, MISSIONLOGS, MISSIONNETCDFS, AUV_NetCDF from lopcToNetCDF import LOPC_Processor, UnexpectedAreaOfCode from nc42netcdfs import BASE_LRAUV_PATH, BASE_LRAUV_WEB, GROUP, Extract -from provenance import get_dods_url, get_web_url, submit_process_run +from provenance import ds_geobounds, get_dods_url, get_web_url, submit_process_run from resample import ( AUVCTD_OPENDAP_BASE, FLASH_THRESHOLD, @@ -811,6 +811,15 @@ def _submit_provenance( # noqa: PLR0913 self.logger.debug("Output %s not found, skipping provenance", full_nc) return log_url = get_dods_url(log_file) if log_file else None + import xarray as xr # noqa: PLC0415 + + try: + _ds = xr.open_dataset(full_nc) + _geobounds = ds_geobounds(_ds) + _ds.close() + except Exception: # noqa: BLE001 + self.logger.debug("Could not extract geobounds from %s", full_nc, exc_info=True) + _geobounds = {} submit_process_run( producer_name=( f"auv-python - Execution of {Path(script_name).name}" @@ -826,6 +835,7 @@ def _submit_provenance( # noqa: PLR0913 log_file_url=log_url, additional_resources=additional_resources, log=self.logger, + **_geobounds, ) except Exception: # noqa: BLE001 self.logger.warning("Provenance submission failed", exc_info=True) diff --git a/src/data/provenance.py b/src/data/provenance.py index c6d06bf..4f94629 100644 --- a/src/data/provenance.py +++ b/src/data/provenance.py @@ -127,6 +127,37 @@ def get_script_github_url(script_name: str) -> str: return get_git_url(script_name, _get_git_version()) +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- +def ds_geobounds(ds: object) -> dict: + """Return output_* kwargs for submit_process_run extracted from an xarray Dataset. + + Silently skips any variable that is missing or raises on min/max. + """ + import pandas as pd # noqa: PLC0415 + + bounds: dict = {} + for var, out_min, out_max in ( + ("latitude", "output_minlatitude", "output_maxlatitude"), + ("longitude", "output_minlongitude", "output_maxlongitude"), + ("depth", "output_mindepth", "output_maxdepth"), + ): + if var in ds: # type: ignore[operator] + try: + bounds[out_min] = float(ds[var].min()) # type: ignore[index] + bounds[out_max] = float(ds[var].max()) # type: ignore[index] + except Exception: # noqa: BLE001 + logger.debug("Could not extract %s/%s bounds", out_min, out_max, exc_info=True) + try: + t = ds["time"] # type: ignore[index] + bounds["output_startdate"] = pd.Timestamp(t.min().values).isoformat() + bounds["output_enddate"] = pd.Timestamp(t.max().values).isoformat() + except Exception: # noqa: BLE001 + logger.debug("Could not extract temporal bounds", exc_info=True) + return bounds + + # --------------------------------------------------------------------------- # Core function # --------------------------------------------------------------------------- @@ -139,6 +170,14 @@ def submit_process_run( # noqa: PLR0913 poc_email: str = "mccann@mbari.org", pr_start: str | None = None, pr_end: str | None = None, + output_startdate: str | None = None, + output_enddate: str | None = None, + output_minlatitude: float | None = None, + output_maxlatitude: float | None = None, + output_minlongitude: float | None = None, + output_maxlongitude: float | None = None, + output_mindepth: float | None = None, + output_maxdepth: float | None = None, software_name: str = "auv-python", software_version: str | None = None, script_name: str = "src/data/process.py", @@ -212,6 +251,14 @@ def submit_process_run( # noqa: PLR0913 "person_email": poc_email, "startdate": pr_start, "enddate": pr_end, + "output_startdate": output_startdate, + "output_enddate": output_enddate, + "output_minlatitude": output_minlatitude, + "output_maxlatitude": output_maxlatitude, + "output_minlongitude": output_minlongitude, + "output_maxlongitude": output_maxlongitude, + "output_mindepth": output_mindepth, + "output_maxdepth": output_maxdepth, "resources": resources, } if nc_file_path is not None: