Add --update_ssds_provenance option and its basic implementation

.env.example

@@ -0,0 +1,18 @@
+# Copy this file to .env and fill in values for your host/container.
+# .env is ignored by git.
+
+# Existing docker-compose volume vars
+M3_VOL=/Volumes/M3
+AUVCTD_VOL=/Volumes/AUVCTD
+LRAUV_VOL=/Volumes/LRAUV
+CALIBRATION_VOL=/Volumes/DMO
+WORK_VOL=/opt/docker_auv-python_vols/data
+HOST_NAME=example.shore.mbari.org
+GMT_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/
+
+# SSDS API endpoint (set to your dev server as needed)
+SSDS_API_BASE=https://mooring-ssds.shore.mbari.org/api
+
+# SSDS provenance auth stubs (API key)
+SSDS_API_KEY=
+SSDS_API_KEY_HEADER=X-API-Key

.vscode/launch.json

@@ -394,6 +394,7 @@
             "request": "launch",
             "program": "${workspaceFolder}/src/data/process_lrauv.py",
             "console": "integratedTerminal",
+            "envFile": "${workspaceFolder}/.env",
             // Lots bad time values in brizo 20250914T080941 due to memory corruption on the vehicle
             //"args": ["-v", "1", "--log_file", "brizo/missionlogs/2025/20250909_20250915/20250914T080941/202509140809_202509150109.nc4"]
             //"args": ["-v", "2", "--log_file", "brizo/missionlogs/2025/20250909_20250915/20250914T080941/202509140809_202509150109.nc4", "--clobber"]
@@ -459,8 +460,12 @@
             //"args": ["-v", "1", "--log_file", "pontus/missionlogs/2025/20250107_20250123/20250111T233141/202501112331_202501131744.nc4", "--no_cleanup"]
             // Test sipper_odv() with a log_file that has an ESP Sample
             //"args": ["-v", "1", "--log_file", "daphne/missionlogs/2026/20260316_20260318/20260317T191958/202603171919_202603181628.nc4", "--no_cleanup"]
-            // Test sipper_odv() with a log_file that has an ESP Sample
-            "args": ["-v", "1", "--log_file", "ahi/missionlogs/2025/20250414_20250418/20250415T040019/202504150400_202504152346.nc4", "--no_cleanup"]
+            // Test ahi mission that has Backseat Planktivore data
+            //"args": ["-v", "1", "--log_file", "ahi/missionlogs/2025/20250414_20250418/20250415T040019/202504150400_202504152346.nc4", "--no_cleanup"]
+            // Test not writing _2column_cmocean.png file if there is no data to plot
+            //"args": ["-v", "1", "--log_file", "ahi/missionlogs/2025/20250128_20250131/20250131T051404/202501310514_202501310535.nc4", "--clobber"]
+            // Test --update_ssds_provenance with a short log_file 
+            "args": ["-v", "1", "--log_file", "ahi/missionlogs/2025/20250128_20250131/20250131T051404/202501310514_202501310535.nc4", "--update_ssds_provenance"]
         },
 
     ]

README.md

@@ -103,28 +103,38 @@ First time use with Docker on a server using a service account:
 * cd /opt   # There should be an `auv-python` directory here that is writable by docker_user 
 * git clone git@github.com:mbari-org/auv-python.git
 * cd auv-python
-* Create a .env file in `/opt/auv-python` with the following contents:   
-    `M3_VOL=<mount_location>`
-    `AUVCTD_VOL=<mount_location>`
-    `LRAUV_VOL=<mount_location>`
-    `CALIBRATION_VOL=<mount_location>`
-    `WORK_VOL=<auv-python_home>/data`
-    `HOST_NAME=<name_of_host>`
-    `GMT_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu/`
+* Create a .env file in `/opt/auv-python` using .env.example as a guide
+
 After installation and when logging into the server again mission data can be processed thusly:
-* Setting up environment and printing help message:   
-    `sudo -u docker_user -i`  
-    `cd /opt/auv-python`     
-    `git pull`      # To get new changes, e.g. mission added to src/data/dorado_info.py   
-    `export DOCKER_USER_ID=$(id -u)`  
-    `docker compose build`   
-    `docker compose run --rm auvpython src/data/process_i2map.py --help`   
-* To actually process a mission and have the processed data copied to the archive use the `-v` and `--clobber` options, e.g.:   
-    `docker compose run --rm auvpython src/data/process_dorado.py --mission 2025.139.04 -v --clobber --noinput`   
-* To process LRAUV data for a specific vehicle and time range:   
-    `docker compose run --rm auvpython src/data/process_lrauv.py --auv_name tethys --start 20250401T000000 --end 20250502T000000 -v --noinput`   
-* To process a specific LRAUV log file:   
-    `docker compose run --rm auvpython src/data/process_lrauv.py --log_file tethys/missionlogs/2012/20120908_20120920/20120917T025522/201209170255_201209171110.nc4 -v --noinput`   
+* Setting up environment and printing help message:
+```
+    sudo -u docker_user -i
+    cd /opt/auv-python
+    git pull
+    export DOCKER_USER_ID=$(id -u)
+    docker compose build
+    docker compose run --rm auvpython src/data/process_i2map.py --help
+```
+
+* To actually process a mission and have the processed data copied to the archive use the `-v` and `--clobber` options, e.g.:
+```
+    docker compose run --rm auvpython src/data/process_dorado.py --mission 2025.139.04 -v --clobber --noinput
+```
+
+* To process LRAUV data for a specific vehicle and time range:
+```  
+    docker compose run --rm auvpython src/data/process_lrauv.py --auv_name tethys --start 20250401T000000 --end 20250502T000000 -v --clobber --noinput
+```
+
+* For missions/log_files to be processed sequentially add the `--num_cores 1` option:
+```
+    docker compose run --rm auvpython src/data/process_lrauv.py --start 20250101T000000 --end 20260101T000000 -v --clobber --noinput --num_cores 1
+```
+
+* To process a specific LRAUV log file:
+```
+    docker compose run --rm auvpython src/data/process_lrauv.py --log_file tethys/missionlogs/2012/20120908_20120920/20120917T025522/201209170255_201209171110.nc4 -v --clobber --noinput
+``` 
 
 
 --

docker-compose.yml

@@ -1,3 +1,4 @@
+# Create .env from .env.example, then set values for your environment.
 # Env variables required in .env, e.g. for Mac:
 # M3_VOL=/Volumes/M3
 # AUVCTD_VOL=/Volumes/AUVCTD

pyproject.toml

@@ -28,6 +28,7 @@ dependencies = [
     "pygmt==0.16",
     "pyproj>=3.7.1",
     "pysolar>=0.13",
+    "requests>=2.31.0",
     "rolling>=0.5.0",
     "seawater>=3.3.5",
     "statsmodels>=0.14.4",

src/data/common_args.py

@@ -87,6 +87,11 @@ def get_processing_parser():
             action="store_true",
             help="Don't re-process existing output files",
         )
+        parser.add_argument(
+            "--update_ssds_provenance",
+            action="store_true",
+            help="Submit/update provenance records in the SSDS_Metadata database",
+        )
 
         return parser
 

src/data/create_products.py

@@ -1638,6 +1638,15 @@ def plot_2column(self) -> str:  # noqa: C901, PLR0912, PLR0915
 
         self._open_ds()
 
+        # Early return if no plot variables present in dataset
+        # Use a quick pre-check with LRAUV or Dorado variables (excluding computed 'density')
+        plot_variables = self._get_plot_variables(None if self._is_lrauv() else "ctd1")
+        if not any(var in self.ds for var, _ in plot_variables if var != "density"):
+            self.logger.warning(
+                "No plot variables found in dataset, skipping plot_2column",
+            )
+            return None
+
         idist, iz, distnav = self._grid_dims()
         if idist.size == 0 or iz.size == 0 or distnav.size == 0:
             self.logger.warning("Skipping plot_2column due to missing gridding dimensions")

src/data/process.py

@@ -75,6 +75,7 @@ class data are: download_process and calibrate, while for LRAUV class data
 from logs2netcdfs import BASE_PATH, MISSIONLOGS, MISSIONNETCDFS, AUV_NetCDF
 from lopcToNetCDF import LOPC_Processor, UnexpectedAreaOfCode
 from nc42netcdfs import BASE_LRAUV_PATH, BASE_LRAUV_WEB, Extract
+from provenance import get_dods_url, submit_process_run
 from resample import (
     AUVCTD_OPENDAP_BASE,
     FLASH_THRESHOLD,
@@ -177,6 +178,7 @@ def __init__(self, auv_name, vehicle_dir, mount_dir, calibration_dir, config=Non
         "no_cleanup": False,
         "skip_download_process": False,
         "archive_only_products": False,
+        "update_ssds_provenance": False,
         "num_cores": None,
         # Filtering/processing params (only used in from_args, not common_config)
         "start_year": None,
@@ -708,6 +710,47 @@ def create_products(self, mission: str = None, log_file: str = None) -> None:
             cp.sipper_odv()
         cp.logger.removeHandler(self.log_handler)
 
+    def _submit_provenance(  # noqa: PLR0913
+        self,
+        output_nc: str,
+        base_path: str,
+        input_files: list[str],
+        pr_start: str,
+        pr_end: str,
+        script_name: str = "src/data/process.py",
+        log_file: str | None = None,
+    ) -> None:
+        """Submit a provenance record — failures are logged, never raised.
+
+        *output_nc* is a path relative to *base_path* (e.g.
+        ``ahi/missionlogs/.../file_1S.nc``), matching the ``--log_file``
+        convention.  *base_path* is used internally to locate the file and
+        build the OPeNDAP URL.
+        """
+        try:
+            full_nc = str(Path(base_path, output_nc))
+            if not Path(full_nc).exists():
+                self.logger.debug("Output %s not found, skipping provenance", full_nc)
+                return
+            log_url = get_dods_url(log_file) if log_file else None
+            submit_process_run(
+                producer_name=(
+                    f"auv-python - Execution of {Path(script_name).name}"
+                    f" to produce {Path(output_nc)}"
+                ),
+                producer_description=self.commandline,
+                nc_file_path=full_nc,
+                input_uris=input_files,
+                pr_start=pr_start,
+                pr_end=pr_end,
+                script_name=script_name,
+                cmd_line_args=self.commandline,
+                log_file_url=log_url,
+                log=self.logger,
+            )
+        except Exception:  # noqa: BLE001
+            self.logger.warning("Provenance submission failed", exc_info=True)
+
     def email(self, mission: str) -> None:
         self.logger.info("Sending notification email for %s", mission)
         email = Emailer()
@@ -771,6 +814,7 @@ def cleanup(self, mission: str = None, log_file: str = None) -> None:
             self.logger.error("Either mission or log_file must be provided for cleanup.")
 
     def process_mission(self, mission: str, src_dir: str = "") -> None:  # noqa: C901, PLR0912, PLR0915
+        _pr_start = datetime.now(tz=UTC).isoformat()
         netcdfs_dir = Path(
             self.config["base_path"],
             self.auv_name,
@@ -854,6 +898,40 @@ def process_mission(self, mission: str, src_dir: str = "") -> None:  # noqa: C90
             self.align(mission)
             self.resample(mission)
             self.create_products(mission)
+            if self.config["update_ssds_provenance"]:
+                self._submit_provenance(
+                    output_nc=str(
+                        Path(
+                            self.auv_name,
+                            MISSIONNETCDFS,
+                            mission,
+                            f"{self.auv_name}_{mission}_{FREQ}.nc",
+                        )
+                    ),
+                    base_path=self.config["base_path"],
+                    input_files=[
+                        str(
+                            Path(
+                                self.config["base_path"],
+                                self.auv_name,
+                                MISSIONLOGS,
+                                mission,
+                            )
+                        )
+                    ],
+                    pr_start=_pr_start,
+                    pr_end=datetime.now(tz=UTC).isoformat(),
+                    script_name="src/data/process_dorado.py",
+                    log_file=str(
+                        Path(
+                            self.config["base_path"],
+                            self.auv_name,
+                            MISSIONNETCDFS,
+                            mission,
+                            f"{self.auv_name}_{mission}_{LOG_NAME}",
+                        )
+                    ),
+                )
             # self.archive() is called in finally: blocks in process_missions()
 
     def process_mission_job(self, mission: str, src_dir: str = "") -> None:
@@ -1041,6 +1119,7 @@ def combine(self, log_file: str) -> None:
 
     @log_file_processor
     def process_log_file(self, log_file: str) -> None:
+        _pr_start = datetime.now(tz=UTC).isoformat()
         netcdfs_dir = Path(BASE_LRAUV_PATH, Path(log_file).parent)
         Path(netcdfs_dir).mkdir(parents=True, exist_ok=True)
         self.log_handler = logging.FileHandler(
@@ -1074,6 +1153,22 @@ def process_log_file(self, log_file: str) -> None:
         )
         if resampled_file.exists():
             self.create_products(log_file=log_file)
+            if self.config["update_ssds_provenance"]:
+                self._submit_provenance(
+                    output_nc=str(Path(log_file).parent / f"{Path(log_file).stem}_{FREQ}.nc"),
+                    base_path=str(BASE_LRAUV_PATH),
+                    input_files=[get_dods_url(str(Path(BASE_LRAUV_PATH, log_file)))],
+                    pr_start=_pr_start,
+                    pr_end=datetime.now(tz=UTC).isoformat(),
+                    script_name="src/data/process_lrauv.py",
+                    log_file=str(
+                        Path(
+                            BASE_LRAUV_PATH,
+                            Path(log_file).parent,
+                            f"{Path(log_file).stem}_processing.log",
+                        )
+                    ),
+                )
         else:
             self.logger.warning(
                 "Resampled file %s not found, skipping create_products step",
@@ -1340,6 +1435,11 @@ def process_command_line(self):
                 "and append to the netCDF file name"
             ),
         )
+        parser.add_argument(
+            "--update_ssds_provenance",
+            action="store_true",
+            help="Submit/update provenance records in the SSDS_Metadata database",
+        )
         parser.add_argument(
             "--num_cores",
             action="store",