Skip to content

Improve provenance data and quick look plots #167

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
MBARIMike merged 3 commits into from
Apr 27, 2026
Merged

Improve provenance data and quick look plots #167

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
0a09d1b
Use short output file name (without path) in the Processrun name.
MBARIMike Apr 27, 2026
bd4e0fb
Fix uristring Resource paths.
MBARIMike Apr 27, 2026
2c8abe6
Decrease sample marker size and turn off clipping for them.
MBARIMike Apr 27, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 12 additions & 4 deletions src/data/create_products.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1155,11 +1155,12 @@ def _overlay_gulper_on_map(self, map_ax: matplotlib.axes.Axes, transformer) -> N
gx,
gy,
"o",
markersize=7,
markersize=5,
zorder=7,
markerfacecolor="white",
markeredgecolor="black",
label="Gulper" if first else "",
clip_on=False,
)
map_ax.annotate(
str(bottle),
Expand All @@ -1170,6 +1171,7 @@ def _overlay_gulper_on_map(self, map_ax: matplotlib.axes.Axes, transformer) -> N
ha="left",
va="center",
color="black",
annotation_clip=False,
)
first = False

Expand Down Expand Up @@ -1208,11 +1210,12 @@ def _overlay_sipper_on_map(self, map_ax: matplotlib.axes.Axes, transformer) -> N
sx,
sy,
"o",
markersize=7,
markersize=5,
zorder=7,
markerfacecolor="white",
markeredgecolor="black",
label="Sipper" if first else "",
clip_on=False,
)
map_ax.annotate(
str(sample),
Expand All @@ -1223,6 +1226,7 @@ def _overlay_sipper_on_map(self, map_ax: matplotlib.axes.Axes, transformer) -> N
ha="left",
va="center",
color="black",
annotation_clip=False,
)
first = False

Expand Down Expand Up @@ -1797,10 +1801,11 @@ def _plot_var_scatter( # noqa: C901, PLR0912, PLR0913, PLR0915
dist,
depth,
"o",
markersize=7,
markersize=5,
zorder=5,
markerfacecolor="white",
markeredgecolor="black",
clip_on=False,
)
curr_ax.annotate(
str(bottle),
Expand All @@ -1813,6 +1818,7 @@ def _plot_var_scatter( # noqa: C901, PLR0912, PLR0913, PLR0915
color="black",
fontweight="bold",
zorder=6,
annotation_clip=False,
)

# Only show y-label on left column or top of right column
Expand Down Expand Up @@ -2052,10 +2058,11 @@ def _plot_var_contour( # noqa: C901, PLR0912, PLR0913, PLR0915
dist,
depth,
"o",
markersize=7,
markersize=5,
zorder=5,
markerfacecolor="white",
markeredgecolor="black",
clip_on=False,
)
curr_ax.annotate(
str(bottle),
Expand All @@ -2068,6 +2075,7 @@ def _plot_var_contour( # noqa: C901, PLR0912, PLR0913, PLR0915
color="black",
fontweight="bold",
zorder=6,
annotation_clip=False,
)

# Only show y-label on left column or top of right column
Expand Down
4 changes: 2 additions & 2 deletions src/data/process.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -839,7 +839,7 @@ def _submit_provenance( # noqa: PLR0913
if not Path(full_nc).exists():
self.logger.debug("Output %s not found, skipping provenance", full_nc)
return
log_url = get_dods_url(log_file) if log_file else None
log_url = get_web_url(log_file) if log_file else None
import xarray as xr # noqa: PLC0415

try:
Expand All @@ -852,7 +852,7 @@ def _submit_provenance( # noqa: PLR0913
submit_process_run(
producer_name=(
f"auv-python - Execution of {Path(script_name).name}"
f" to produce {Path(output_nc)}"
f" to produce {Path(output_nc).name}"
),
producer_description=self.commandline,
nc_file_path=full_nc,
Expand Down
46 changes: 30 additions & 16 deletions src/data/provenance.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,10 @@
import argparse
import logging
import os
import re
import sys
from datetime import UTC, datetime
from pathlib import Path
from socket import gethostname

import git
import requests
Expand Down Expand Up @@ -88,31 +88,52 @@ def build_authenticated_session(
return session


_MISSION_NETCDFS_RE = re.compile(r"[^/]+/missionnetcdfs/(\d{4})\.[^/]+/(.+)$")
_MISSION_IMAGES_RE = re.compile(r"[^/]+/missionimages/(\d{4})\.[^/]+/(.+)$")


def get_dods_url(nc_file_path: str) -> str:
"""Translate a local NetCDF path to its OPeNDAP URL.

Walks ``PATH_TO_URL_MAP`` looking for a matching prefix in the
*resolved* path string. Returns the original path unchanged if no
match is found.
Dorado/i2map NetCDF files are stored locally under
``{auv_name}/missionnetcdfs/{YYYY}.{DDD}.{SS}/`` but are archived to
``surveys/{YYYY}/netcdf/`` on the OPeNDAP server, so that mapping is
applied when the pattern is detected.
"""
resolved = str(Path(nc_file_path).resolve())
for local_prefix, url_prefix in PATH_TO_URL_MAP.items():
if resolved.startswith(local_prefix):
rel = resolved[len(local_prefix) :].lstrip("/")
m = _MISSION_NETCDFS_RE.match(rel)
if m:
year, filename = m.group(1), m.group(2)
return f"{url_prefix}/surveys/{year}/netcdf/{filename}"
return resolved.replace(local_prefix, url_prefix, 1)
return resolved


def get_web_url(file_path: str) -> str:
"""Translate a local file path to its web-accessible URL (no OPeNDAP prefix).

Use this for PNG, HTML, and other static files served over HTTP.
Walks ``_PATH_TO_WEB_MAP`` looking for a matching prefix in the
*resolved* path string. Returns the original path unchanged if no
match is found.
Use this for PNG, HTML, log, and other static files served over HTTP.

Dorado/i2map files are stored locally under mission-specific subdirectories
but are archived to a flat ``surveys/{YYYY}/`` layout on the web server:
- ``{auv_name}/missionnetcdfs/{YYYY}.*/`` → ``surveys/{YYYY}/netcdf/``
- ``{auv_name}/missionimages/{YYYY}.*/`` → ``surveys/{YYYY}/images/``
"""
resolved = str(Path(file_path).resolve())
for local_prefix, url_prefix in _PATH_TO_WEB_MAP.items():
if resolved.startswith(local_prefix):
rel = resolved[len(local_prefix) :].lstrip("/")
m = _MISSION_NETCDFS_RE.match(rel)
if m:
year, filename = m.group(1), m.group(2)
return f"{url_prefix}/surveys/{year}/netcdf/{filename}"
m = _MISSION_IMAGES_RE.match(rel)
if m:
year, filename = m.group(1), m.group(2)
return f"{url_prefix}/surveys/{year}/images/{filename}"
return resolved.replace(local_prefix, url_prefix, 1)
return resolved

Expand Down Expand Up @@ -221,14 +242,7 @@ def submit_process_run( # noqa: PLR0913
"description": f"Processing script: {script_name}",
},
]
if cmd_line_args:
resources.append(
{
"name": "command_line",
"uristring": f"urn:cmdline:{gethostname()}",
"description": cmd_line_args,
}
)

if log_file_url:
resources.append(
{
Expand Down
Loading