From 140b4129ad776026abe4064f3917248eeafac7e5 Mon Sep 17 00:00:00 2001 From: Mike McCann Date: Tue, 14 Apr 2026 17:10:00 -0700 Subject: [PATCH 1/3] Make sure all local directories exist for the plots. --- .vscode/launch.json | 8 ++++---- src/data/create_products.py | 3 +++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 2d93b6c..574a766 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -467,9 +467,9 @@ // Test --update_ssds_provenance with a short log_file //"args": ["-v", "1", "--log_file", "ahi/missionlogs/2025/20250128_20250131/20250131T051404/202501310514_202501310535.nc4", "--update_ssds_provenance"] // Test ahi mission that has Backseat Planktivore data with --update_ssds_provenance - //"args": ["-v", "1", "--log_file", "ahi/missionlogs/2025/20250414_20250418/20250415T040019/202504150400_202504152346.nc4", "--update_ssds_provenance"] + "args": ["-v", "1", "--log_file", "ahi/missionlogs/2025/20250414_20250418/20250415T040019/202504150400_202504152346.nc4", "--update_ssds_provenance"] // Make per log file .html files to test with lrauv_deployment_plots and --update_ssds_provenance - "args": ["-v", "1", "--auv_name", "ahi", "--start", "20251022T000000", "--end", "20251024T000000", "--update_ssds_provenance", "--clobber"] + //"args": ["-v", "1", "--auv_name", "ahi", "--start", "20251022T000000", "--end", "20251024T000000", "--update_ssds_provenance", "--clobber"] }, { "name": "lrauv_deployment_plots", @@ -484,13 +484,13 @@ // ahi planktivore deployment April 2025 add --update_ssds_provenance //"args": ["-v", "1", "--dlist", "ahi/missionlogs/2025/20250414_20250418.dlist", "--update_ssds_provenance"] // Test time range of DeploymentPlots with ahi planktivore deployment April 2025 - //"args": ["-v", "1", "--auv_name", "ahi", "--start", "20251001", "--end", "20251231", "--update_ssds_provenance", "--force"] + "args": ["-v", "1", "--auv_name", "ahi", "--start", "20250401", "--end", "20250501", "--update_ssds_provenance", "--force"] // Test web page building with a short deployment //"args": ["-v", "1", "--dlist", "ahi/missionlogs/2025/20251022_20251024.dlist", "--update_ssds_provenance"] // Test --force option for rebuilding web pages with a short deployment //"args": ["-v", "1", "--last_n_days", "10", "--update_ssds_provenance", "--force"] // Test --notify option - "args": ["-v", "1", "--dlist", "ahi/missionlogs/2025/20251022_20251024.dlist", "--update_ssds_provenance", "--force", "--notify", "mccann@mbari.org"] + //"args": ["-v", "1", "--dlist", "ahi/missionlogs/2025/20251022_20251024.dlist", "--update_ssds_provenance", "--force", "--notify", "mccann@mbari.org"] }, diff --git a/src/data/create_products.py b/src/data/create_products.py index 253046a..55b5345 100755 --- a/src/data/create_products.py +++ b/src/data/create_products.py @@ -2030,6 +2030,7 @@ def plot_2column(self) -> str: # noqa: C901, PLR0912, PLR0915 if self.output_dir is not None else Path(BASE_LRAUV_PATH, f"{Path(self.log_file).parent}") ) + out_dir.mkdir(parents=True, exist_ok=True) stem = ( self.plot_name_stem if self.plot_name_stem is not None else Path(self.log_file).stem ) @@ -2170,6 +2171,7 @@ def plot_biolume_2column(self) -> str: # noqa: C901, PLR0912, PLR0915 if self.output_dir is not None else Path(BASE_LRAUV_PATH, f"{Path(self.log_file).parent}") ) + out_dir.mkdir(parents=True, exist_ok=True) stem = ( self.plot_name_stem if self.plot_name_stem is not None else Path(self.log_file).stem ) @@ -2311,6 +2313,7 @@ def plot_planktivore_2column(self) -> str: # noqa: C901, PLR0912, PLR0915 if self.output_dir is not None else Path(BASE_LRAUV_PATH, f"{Path(self.log_file).parent}") ) + out_dir.mkdir(parents=True, exist_ok=True) stem = ( self.plot_name_stem if self.plot_name_stem is not None else Path(self.log_file).stem ) From 07914ba3aa5bc347d84e1dff3a97daab6d1bf733 Mon Sep 17 00:00:00 2001 From: Mike McCann Date: Tue, 14 Apr 2026 17:10:59 -0700 Subject: [PATCH 2/3] Replace any '/'s in the Deployment name with '_'s; web page fixups. --- src/data/lrauv_deployment_plots.py | 100 +++++++++++++++++------------ 1 file changed, 58 insertions(+), 42 deletions(-) diff --git a/src/data/lrauv_deployment_plots.py b/src/data/lrauv_deployment_plots.py index b54e1bf..68b9a55 100755 --- a/src/data/lrauv_deployment_plots.py +++ b/src/data/lrauv_deployment_plots.py @@ -247,7 +247,7 @@ def plot_deployment( # noqa: C901, PLR0912, PLR0913, PLR0915 # Deployment name (spaces → underscores for filenames) raw_name = self._parse_deployment_name(dlist_content) if dlist_content else None if raw_name: - plot_name_stem = raw_name.replace(" ", "_") + plot_name_stem = raw_name.replace(" ", "_").replace("/", "_") self.logger.info("Deployment name: %s", raw_name) else: plot_name_stem = dlist_rel.stem @@ -367,6 +367,7 @@ def _build_and_write_html( # noqa: PLR0913 nc_files, auv_name=_auv_name, png_file_path=Path(png_path), + other_png_paths=[p for p in png_paths if p != png_path], ) self.logger.info("Per-PNG HTML written to %s", per_png_html) archiver = Archiver(add_handlers=True, clobber=True) @@ -527,16 +528,6 @@ def _stoqs_url_for_nc_url(self, nc_url: str, auv_name: str) -> str | None: self.logger.debug("Could not generate per-log STOQS URL for %s: %s", nc_url, exc) return None - def _per_log_html_url(self, nc_urls: list[str]) -> str: - """Return the first reachable per-log HTML URL for a list of nc URLs, or empty string.""" - for nc_url in nc_urls: - candidate = nc_url.replace( - LRAUV_OPENDAP_BASE.rstrip("/"), BASE_LRAUV_WEB.rstrip("/") - ).replace(f"_{FREQ}.nc", f"_{FREQ}.html") - if self._url_exists(candidate): - return candidate - return "" - def _per_log_stoqs_url( self, nc_urls: list[str], auv_name: str, fallback: str | None ) -> str | None: @@ -548,15 +539,15 @@ def _per_log_stoqs_url( return url return fallback - def _per_log_png_links(self, nc_urls: list[str]) -> str: - """Return HTML anchor tags for each existing per-log PNG, pipe-separated.""" - parts: list[str] = [] + def _per_log_png_links(self, nc_urls: list[str]) -> list[tuple[str, str]]: + """Return (url, label) pairs for each existing per-log PNG.""" + parts: list[tuple[str, str]] = [] for nc_url in nc_urls: for png_url in self._png_urls_for_nc(nc_url): if self._url_exists(png_url): pname = png_url.rsplit("/", 1)[1] - parts.append(f'{pname}') - return " | ".join(parts) + parts.append((png_url, pname)) + return parts def _write_per_png_html( # noqa: C901, PLR0913 self, @@ -568,6 +559,7 @@ def _write_per_png_html( # noqa: C901, PLR0913 nc_files: list[str], auv_name: str = "", png_file_path: Path | None = None, + other_png_paths: list[str] | None = None, ) -> None: """Write a plain HTML page for one deployment PNG. @@ -581,37 +573,59 @@ def _write_per_png_html( # noqa: C901, PLR0913 log_dir = url.rsplit("/", 2)[1] grouped.setdefault(log_dir, []).append(url) - log_items = "" + # Collect per-row data first so we can suppress empty columns + rows: list[dict] = [] for log_dir in sorted(grouped): nc_urls = grouped[log_dir] - per_log_html_url = self._per_log_html_url(nc_urls) log_stoqs_url = self._per_log_stoqs_url(nc_urls, auv_name, stoqs_url) - - links = "" - if per_log_html_url: - links += f' image' png_links = self._per_log_png_links(nc_urls) - if png_links: - if links: - links += " | " - links += png_links - for nc_url in nc_urls: - dap_form_url = nc_url + ".html" - if links: - links += " | " - links += f'OPeNDAP Data Access Form' - if log_stoqs_url: - if links: - links += " | " - links += f'STOQS' + rows.append( + { + "dir": log_dir, + "plots": "
".join(f'{lbl}' for u, lbl in png_links), + "dap": "".join(f'OPeNDAP' for nc_url in nc_urls), + "stoqs": f'STOQS' if log_stoqs_url else "", + } + ) - log_items += f"
  • {log_dir} — {links}
    • \n" + # Only show columns that have at least one non-empty cell + show = {col: any(r[col] for r in rows) for col in ("plots", "dap", "stoqs")} + col_labels = {"plots": "Quick Look Plots", "dap": "OPeNDAP", "stoqs": "STOQS"} + + header_cells = "Log directory" + "".join( + f"{col_labels[c]}" for c in col_labels if show[c] + ) + log_rows = "" + for r in rows: + data_cells = "".join(f"{r[c]}" for c in col_labels if show[c]) + log_rows += f" {r['dir']}{data_cells}\n" + + log_table = ( + " \n" + f" {header_cells}\n" + " \n" + f"{log_rows}" + " \n" + "
    \n" + ) + + other_plots_line = "" + if other_png_paths: + sibling_links = [ + f'{Path(p).name}' + for p in other_png_paths + if Path(p).exists() + ] + if sibling_links: + other_plots_line = ( + "

    Other plots for this deployment: " + " | ".join(sibling_links) + "

    \n" + ) stoqs_line = "" if stoqs_url: after_scheme = stoqs_url.split("//", 1)[-1] if "//" in stoqs_url else stoqs_url db_label = after_scheme.split("/")[1] if "/" in after_scheme else after_scheme - stoqs_line = f'

    View these data in {db_label}

    \n' + stoqs_line = f'

    View these data in {db_label}

    \n' if png_file_path is not None and png_file_path.exists(): b64 = base64.b64encode(png_file_path.read_bytes()).decode("ascii") @@ -638,11 +652,10 @@ def _write_per_png_html( # noqa: C901, PLR0913 "\n" f"

    {html_title_single}

    \n" f' {png_name}\n' + f" {other_plots_line}" f" {stoqs_line}" "

    Log files

    \n" - " \n" + f"{log_table}" f"{footer}" "\n" "\n" @@ -680,8 +693,11 @@ def _dlist_list( # noqa: C901, PLR0912 window. The sibling ``.dlist`` file (``YYYYMMDD_YYYYMMDD.dlist``) is returned for each matching directory. """ - _vol = Path(LRAUV_VOL) - base = _vol if _vol.is_dir() else Path(BASE_LRAUV_PATH) + base = Path(LRAUV_VOL) + if not base.is_dir(): + self.logger.error("%s does not exist.", LRAUV_VOL) + self.logger.info("Is %s mounted?", LRAUV_VOL) + sys.exit(1) dlists: list[str] = [] auv_dirs = ( sorted(base.glob("*/missionlogs/")) From 1758391b3ec3a03ed9e4d804268bb72bdbbc0d26 Mon Sep 17 00:00:00 2001 From: Mike McCann Date: Tue, 14 Apr 2026 17:23:11 -0700 Subject: [PATCH 3/3] Fix the tests following web page construction cleanup. --- src/data/test_lrauv_deployment_plots.py | 32 +++++-------------------- 1 file changed, 6 insertions(+), 26 deletions(-) diff --git a/src/data/test_lrauv_deployment_plots.py b/src/data/test_lrauv_deployment_plots.py index 0b389cc..6e5cf18 100644 --- a/src/data/test_lrauv_deployment_plots.py +++ b/src/data/test_lrauv_deployment_plots.py @@ -210,7 +210,7 @@ def test_per_log_html_link_when_url_exists(self, dp, tmp_path): None, [_NC_URL], ) - assert "image" in html_path.read_text() # noqa: S101 + assert "2column_cmocean" in html_path.read_text() # noqa: S101 def test_per_log_png_links_included(self, dp, tmp_path): html_path = tmp_path / "depl.html" @@ -220,7 +220,7 @@ def test_per_log_png_links_included(self, dp, tmp_path): patch.object( dp, "_per_log_png_links", - return_value=f'ahi_1S_2column_cmocean.png', + return_value=[(fake_png_url, "ahi_1S_2column_cmocean.png")], ), patch.object(dp, "_stoqs_url_for_nc_url", return_value=None), ): @@ -279,25 +279,6 @@ def test_footer_contains_script_link_and_timestamp(self, dp, tmp_path): # --------------------------------------------------------------------------- -class TestPerLogHtmlUrl: - def test_returns_first_reachable(self, dp): - # Compute the expected candidate the same way _per_log_html_url does - from nc42netcdfs import BASE_LRAUV_WEB # noqa: PLC0415 - from resample import FREQ, LRAUV_OPENDAP_BASE # noqa: PLC0415 - - nc_url_a = _NC_URL - expected = nc_url_a.replace( - LRAUV_OPENDAP_BASE.rstrip("/"), BASE_LRAUV_WEB.rstrip("/") - ).replace(f"_{FREQ}.nc", f"_{FREQ}.html") - with patch.object(dp, "_url_exists", side_effect=lambda u: u == expected): - result = dp._per_log_html_url([nc_url_a]) - assert result == expected # noqa: S101 - - def test_returns_empty_when_none_reachable(self, dp): - with patch.object(dp, "_url_exists", return_value=False): - assert dp._per_log_html_url([_NC_URL]) == "" # noqa: S101 - - class TestPerLogStoqsUrl: def test_returns_nc_scoped_url_when_available(self, dp): scoped = _STOQS_URL.replace("abc123", "scoped") @@ -322,17 +303,16 @@ class TestPerLogPngLinks: def test_returns_links_for_existing_pngs(self, dp): with patch.object(dp, "_url_exists", return_value=True): result = dp._per_log_png_links([_NC_URL]) - assert "2column_cmocean" in result # noqa: S101 - assert " 1 # noqa: S101 # ---------------------------------------------------------------------------