Skip to content

Add Resources to the process-run for --update_ssds_provenance #146

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
MBARIMike merged 2 commits into from
Apr 6, 2026
Merged

Add Resources to the process-run for --update_ssds_provenance #146

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
41350a1
Add associated NetCDF Reources to the Processrun for process_lrauv.py.
MBARIMike Apr 6, 2026
a3a18c0
Add Quick Look plots to Resource list and test it.
MBARIMike Apr 6, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion .vscode/launch.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -465,7 +465,9 @@
// Test not writing _2column_cmocean.png file if there is no data to plot
//"args": ["-v", "1", "--log_file", "ahi/missionlogs/2025/20250128_20250131/20250131T051404/202501310514_202501310535.nc4", "--clobber"]
// Test --update_ssds_provenance with a short log_file
"args": ["-v", "1", "--log_file", "ahi/missionlogs/2025/20250128_20250131/20250131T051404/202501310514_202501310535.nc4", "--update_ssds_provenance"]
//"args": ["-v", "1", "--log_file", "ahi/missionlogs/2025/20250128_20250131/20250131T051404/202501310514_202501310535.nc4", "--update_ssds_provenance"]
// Test ahi mission that has Backseat Planktivore data with --update_ssds_provenance
"args": ["-v", "1", "--log_file", "ahi/missionlogs/2025/20250414_20250418/20250415T040019/202504150400_202504152346.nc4", "--update_ssds_provenance"]
},

]
Expand Down
85 changes: 84 additions & 1 deletion src/data/process.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ class data are: download_process and calibrate, while for LRAUV class data
from emailer import NOTIFICATION_EMAIL, Emailer
from logs2netcdfs import BASE_PATH, MISSIONLOGS, MISSIONNETCDFS, AUV_NetCDF
from lopcToNetCDF import LOPC_Processor, UnexpectedAreaOfCode
from nc42netcdfs import BASE_LRAUV_PATH, BASE_LRAUV_WEB, Extract
from nc42netcdfs import BASE_LRAUV_PATH, BASE_LRAUV_WEB, GROUP, Extract
from provenance import get_dods_url, submit_process_run
from resample import (
AUVCTD_OPENDAP_BASE,
Expand Down Expand Up @@ -710,6 +710,82 @@ def create_products(self, mission: str = None, log_file: str = None) -> None:
cp.sipper_odv()
cp.logger.removeHandler(self.log_handler)

def _collect_lrauv_netcdf_resources(
self,
log_file: str,
netcdfs_dir: Path,
) -> list[dict]:
"""Build Resource dicts for all intermediate NetCDF files produced by the LRAUV pipeline."""
stem = Path(log_file).stem
resources: list[dict] = []

# nc42netcdfs.py: {stem}_{GROUP}_*.nc files
resources.extend(
{
"name": nc_file.name,
"uristring": get_dods_url(str(nc_file)),
"description": f"Extracted by nc42netcdfs.py from {Path(log_file).name}",
"resourcetype_name": "NetCDF",
}
for nc_file in sorted(netcdfs_dir.glob(f"{stem}_{GROUP}_*.nc"))
)

# combine.py: {stem}_combined.nc4
combined = netcdfs_dir / f"{stem}_combined.nc4"
if combined.exists():
resources.append(
{
"name": combined.name,
"uristring": get_dods_url(str(combined)),
"description": "Combined by combine.py",
"resourcetype_name": "NetCDF",
}
)

# align.py: {stem}_align.nc4
align = netcdfs_dir / f"{stem}_align.nc4"
if align.exists():
resources.append(
{
"name": align.name,
"uristring": get_dods_url(str(align)),
"description": "Aligned by align.py",
"resourcetype_name": "NetCDF",
}
)

# resample.py: {stem}_{FREQ}.nc
resampled = netcdfs_dir / f"{stem}_{FREQ}.nc"
if resampled.exists():
resources.append(
{
"name": resampled.name,
"uristring": get_dods_url(str(resampled)),
"description": f"Resampled by resample.py at {FREQ} frequency",
"resourcetype_name": "NetCDF",
}
)

# create_products.py: quick look plots
plot_suffixes = [
("2column_cmocean", "Standard 2-column plot"),
("2column_biolume", "Bioluminescence 2-column plot"),
("2column_planktivore", "Planktivore 2-column plot"),
]
for suffix, description in plot_suffixes:
plot_file = netcdfs_dir / f"{stem}_{FREQ}_{suffix}.png"
if plot_file.exists():
resources.append(
{
"name": plot_file.name,
"uristring": get_dods_url(str(plot_file)),
"description": f"Created by create_products.py: {description}",
"resourcetype_name": "Quick Look Plot",
}
)

return resources

def _submit_provenance( # noqa: PLR0913
self,
output_nc: str,
Expand All @@ -719,6 +795,7 @@ def _submit_provenance( # noqa: PLR0913
pr_end: str,
script_name: str = "src/data/process.py",
log_file: str | None = None,
additional_resources: list[dict] | None = None,
) -> None:
"""Submit a provenance record — failures are logged, never raised.

Expand Down Expand Up @@ -746,6 +823,7 @@ def _submit_provenance( # noqa: PLR0913
script_name=script_name,
cmd_line_args=self.commandline,
log_file_url=log_url,
additional_resources=additional_resources,
log=self.logger,
)
except Exception: # noqa: BLE001
Expand Down Expand Up @@ -1154,6 +1232,10 @@ def process_log_file(self, log_file: str) -> None:
if resampled_file.exists():
self.create_products(log_file=log_file)
if self.config["update_ssds_provenance"]:
netcdf_resources = self._collect_lrauv_netcdf_resources(
log_file,
netcdfs_dir,
)
self._submit_provenance(
output_nc=str(Path(log_file).parent / f"{Path(log_file).stem}_{FREQ}.nc"),
base_path=str(BASE_LRAUV_PATH),
Expand All @@ -1168,6 +1250,7 @@ def process_log_file(self, log_file: str) -> None:
f"{Path(log_file).stem}_processing.log",
)
),
additional_resources=netcdf_resources,
)
else:
self.logger.warning(
Expand Down
Loading