diff --git a/.vscode/launch.json b/.vscode/launch.json index 98a94ff..ca4e38e 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -465,7 +465,9 @@ // Test not writing _2column_cmocean.png file if there is no data to plot //"args": ["-v", "1", "--log_file", "ahi/missionlogs/2025/20250128_20250131/20250131T051404/202501310514_202501310535.nc4", "--clobber"] // Test --update_ssds_provenance with a short log_file - "args": ["-v", "1", "--log_file", "ahi/missionlogs/2025/20250128_20250131/20250131T051404/202501310514_202501310535.nc4", "--update_ssds_provenance"] + //"args": ["-v", "1", "--log_file", "ahi/missionlogs/2025/20250128_20250131/20250131T051404/202501310514_202501310535.nc4", "--update_ssds_provenance"] + // Test ahi mission that has Backseat Planktivore data with --update_ssds_provenance + "args": ["-v", "1", "--log_file", "ahi/missionlogs/2025/20250414_20250418/20250415T040019/202504150400_202504152346.nc4", "--update_ssds_provenance"] }, ] diff --git a/src/data/process.py b/src/data/process.py index 33e881e..179a3a4 100755 --- a/src/data/process.py +++ b/src/data/process.py @@ -74,7 +74,7 @@ class data are: download_process and calibrate, while for LRAUV class data from emailer import NOTIFICATION_EMAIL, Emailer from logs2netcdfs import BASE_PATH, MISSIONLOGS, MISSIONNETCDFS, AUV_NetCDF from lopcToNetCDF import LOPC_Processor, UnexpectedAreaOfCode -from nc42netcdfs import BASE_LRAUV_PATH, BASE_LRAUV_WEB, Extract +from nc42netcdfs import BASE_LRAUV_PATH, BASE_LRAUV_WEB, GROUP, Extract from provenance import get_dods_url, submit_process_run from resample import ( AUVCTD_OPENDAP_BASE, @@ -710,6 +710,82 @@ def create_products(self, mission: str = None, log_file: str = None) -> None: cp.sipper_odv() cp.logger.removeHandler(self.log_handler) + def _collect_lrauv_netcdf_resources( + self, + log_file: str, + netcdfs_dir: Path, + ) -> list[dict]: + """Build Resource dicts for all intermediate NetCDF files produced by the LRAUV pipeline.""" + stem = Path(log_file).stem + resources: list[dict] = [] + + # nc42netcdfs.py: {stem}_{GROUP}_*.nc files + resources.extend( + { + "name": nc_file.name, + "uristring": get_dods_url(str(nc_file)), + "description": f"Extracted by nc42netcdfs.py from {Path(log_file).name}", + "resourcetype_name": "NetCDF", + } + for nc_file in sorted(netcdfs_dir.glob(f"{stem}_{GROUP}_*.nc")) + ) + + # combine.py: {stem}_combined.nc4 + combined = netcdfs_dir / f"{stem}_combined.nc4" + if combined.exists(): + resources.append( + { + "name": combined.name, + "uristring": get_dods_url(str(combined)), + "description": "Combined by combine.py", + "resourcetype_name": "NetCDF", + } + ) + + # align.py: {stem}_align.nc4 + align = netcdfs_dir / f"{stem}_align.nc4" + if align.exists(): + resources.append( + { + "name": align.name, + "uristring": get_dods_url(str(align)), + "description": "Aligned by align.py", + "resourcetype_name": "NetCDF", + } + ) + + # resample.py: {stem}_{FREQ}.nc + resampled = netcdfs_dir / f"{stem}_{FREQ}.nc" + if resampled.exists(): + resources.append( + { + "name": resampled.name, + "uristring": get_dods_url(str(resampled)), + "description": f"Resampled by resample.py at {FREQ} frequency", + "resourcetype_name": "NetCDF", + } + ) + + # create_products.py: quick look plots + plot_suffixes = [ + ("2column_cmocean", "Standard 2-column plot"), + ("2column_biolume", "Bioluminescence 2-column plot"), + ("2column_planktivore", "Planktivore 2-column plot"), + ] + for suffix, description in plot_suffixes: + plot_file = netcdfs_dir / f"{stem}_{FREQ}_{suffix}.png" + if plot_file.exists(): + resources.append( + { + "name": plot_file.name, + "uristring": get_dods_url(str(plot_file)), + "description": f"Created by create_products.py: {description}", + "resourcetype_name": "Quick Look Plot", + } + ) + + return resources + def _submit_provenance( # noqa: PLR0913 self, output_nc: str, @@ -719,6 +795,7 @@ def _submit_provenance( # noqa: PLR0913 pr_end: str, script_name: str = "src/data/process.py", log_file: str | None = None, + additional_resources: list[dict] | None = None, ) -> None: """Submit a provenance record — failures are logged, never raised. @@ -746,6 +823,7 @@ def _submit_provenance( # noqa: PLR0913 script_name=script_name, cmd_line_args=self.commandline, log_file_url=log_url, + additional_resources=additional_resources, log=self.logger, ) except Exception: # noqa: BLE001 @@ -1154,6 +1232,10 @@ def process_log_file(self, log_file: str) -> None: if resampled_file.exists(): self.create_products(log_file=log_file) if self.config["update_ssds_provenance"]: + netcdf_resources = self._collect_lrauv_netcdf_resources( + log_file, + netcdfs_dir, + ) self._submit_provenance( output_nc=str(Path(log_file).parent / f"{Path(log_file).stem}_{FREQ}.nc"), base_path=str(BASE_LRAUV_PATH), @@ -1168,6 +1250,7 @@ def process_log_file(self, log_file: str) -> None: f"{Path(log_file).stem}_processing.log", ) ), + additional_resources=netcdf_resources, ) else: self.logger.warning(