From fcdaec0681cba0b3a0cbf8e55e5389b5a485ff91 Mon Sep 17 00:00:00 2001 From: Mike McCann Date: Wed, 25 Mar 2026 15:30:31 -0700 Subject: [PATCH 1/5] Add planktivore varaibles from the Backseat (_) Group. --- .vscode/launch.json | 4 +++- src/data/nc42netcdfs.py | 42 +++++++++++++++++++++++++++++++++++++++-- 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 48f2c05..c3c725b 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -458,7 +458,9 @@ // Processing truncated at 2025-01-12T15:15:43 instead of 2025-01-13T17:43:58 - seems to happen with several other log_files too //"args": ["-v", "1", "--log_file", "pontus/missionlogs/2025/20250107_20250123/20250111T233141/202501112331_202501131744.nc4", "--no_cleanup"] // Test sipper_odv() with a log_file that has an ESP Sample - "args": ["-v", "1", "--log_file", "daphne/missionlogs/2026/20260316_20260318/20260317T191958/202603171919_202603181628.nc4", "--no_cleanup"] + //"args": ["-v", "1", "--log_file", "daphne/missionlogs/2026/20260316_20260318/20260317T191958/202603171919_202603181628.nc4", "--no_cleanup"] + // Test sipper_odv() with a log_file that has an ESP Sample + "args": ["-v", "1", "--log_file", "ahi/missionlogs/2025/20250414_20250418/20250415T040019/202504150400_202504152346.nc4", "--no_cleanup"] }, ] diff --git a/src/data/nc42netcdfs.py b/src/data/nc42netcdfs.py index be31aa4..75f35d7 100755 --- a/src/data/nc42netcdfs.py +++ b/src/data/nc42netcdfs.py @@ -77,6 +77,15 @@ {"name": "VolumeScatCoeff117deg700nm"}, {"name": "mass_concentration_of_petroleum_hydrocarbons_in_sea_water"}, ], + "_": [ + {"name": "planktivore_diatoms"}, + {"name": "planktivore_dinoflagellates"}, + {"name": "planktivore_caseTemp"}, + {"name": "planktivore_caseHumidity"}, + {"name": "planktivore_casePress"}, + {"name": "planktivore_HM_AvgRois"}, + {"name": "planktivore_LM_AvgRois"}, + ], "WetLabsUBAT": [ {"name": "average_bioluminescence"}, {"name": "flow_rate"}, @@ -205,9 +214,12 @@ def extract_groups_to_files_netcdf4(self, log_file: str) -> Path: # Extract all other groups all_groups = list(src_dataset.groups.keys()) + if "_" in all_groups: + self._extract_backseat_group(log_file, src_dataset, netcdfs_dir) + for group_name in sorted(SCIENG_PARMS): - if group_name == "/" or group_name not in all_groups: - if group_name != "/" and group_name not in all_groups: + if group_name in {"/", "_"} or group_name not in all_groups: + if group_name not in {"/", "_"} and group_name not in all_groups: self.logger.warning("Group %s not found in %s", group_name, input_file) continue self._extract_single_group(log_file, group_name, src_dataset, netcdfs_dir) @@ -243,6 +255,32 @@ def _extract_root_group( else: self.logger.warning("No requested variables found in root group '/'") + def _extract_backseat_group( + self, log_file: str, src_dataset: netCDF4.Dataset, output_dir: Path + ): + """Extract variables from the '_' group to _{GROUP}_Backseat.nc.""" + backseat_parms = SCIENG_PARMS.get("_", []) + if not backseat_parms: + return + + try: + src_group = src_dataset.groups["_"] + except KeyError: + self.logger.warning("Backseat group '_' not found in %s", Path(log_file).name) + return + + self.logger.info("Extracting backseat group '_' as Backseat") + vars_to_extract, requested_vars = self._get_available_variables(src_group, backseat_parms) + + if vars_to_extract: + output_file = output_dir / f"{Path(log_file).stem}_{GROUP}_Backseat.nc" + self._create_netcdf_file(log_file, "Backseat", src_group, vars_to_extract, output_file) + self.logger.info("Extracted backseat group '_' to %s", output_file) + else: + self.logger.warning( + "No requested variables (%s) found in backseat group '_'", requested_vars + ) + def _extract_single_group( self, log_file: str, From ff88c2068711b553197a483317ad3f7a15d6544a Mon Sep 17 00:00:00 2001 From: Mike McCann Date: Wed, 25 Mar 2026 15:31:02 -0700 Subject: [PATCH 2/5] Update EXPECTED_SIZE_LOCAL --- src/data/test_process_i2map.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/data/test_process_i2map.py b/src/data/test_process_i2map.py index 533ee52..88d0026 100644 --- a/src/data/test_process_i2map.py +++ b/src/data/test_process_i2map.py @@ -32,7 +32,7 @@ def test_process_i2map(complete_i2map_processing): # update the expected size here. EXPECTED_SIZE_GITHUB = 63137 EXPECTED_SIZE_ACT = 63106 - EXPECTED_SIZE_LOCAL = 64642 + EXPECTED_SIZE_LOCAL = 64650 if str(proc.args.base_path).startswith("/home/runner"): # The size is different in GitHub Actions, maybe due to different metadata assert nc_file.stat().st_size == EXPECTED_SIZE_GITHUB # noqa: S101 From e63b1a04f51f72acd54e750c317ab66b6fb01f7a Mon Sep 17 00:00:00 2001 From: Mike McCann Date: Wed, 25 Mar 2026 15:34:16 -0700 Subject: [PATCH 3/5] fix(resample): handle mixed time axes within an instrument MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reset df_o per variable in resample_variable() so variables with different time coordinates are not reindexed against a prior variable’s index. This fixes all-NaN outputs for Backseat planktivore channels where diatoms (10s) and case* (1s) use different time axes. --- src/data/resample.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/src/data/resample.py b/src/data/resample.py index b5dcaf0..0485c18 100755 --- a/src/data/resample.py +++ b/src/data/resample.py @@ -1987,10 +1987,14 @@ def resample_variable( # noqa: PLR0913 surface_exclusion_depth_m, ) else: - self.df_o[variable] = self.ds[variable].to_pandas() - self.df_o[f"{variable}_mf"] = ( + orig_series = self.ds[variable].to_pandas() + mf_series = ( self.ds[variable].rolling(**{timevar: mf_width}, center=True).median().to_pandas() ) + # Reset df_o per variable to avoid index alignment issues when + # variables within the same instrument have different time axes + # (e.g., Backseat group has diatoms at ~10s and case* at ~1s) + self.df_o = pd.DataFrame({variable: orig_series, f"{variable}_mf": mf_series}) # Resample to center of freq https://stackoverflow.com/a/69945592/1281657 self.logger.info( "Resampling %s with frequency %s following %d point median filter", @@ -2006,19 +2010,11 @@ def resample_variable( # noqa: PLR0913 ) dt_index = pd.date_range(mission_start, mission_end, freq=freq.lower()) self.df_r[variable] = pd.Series(np.nan, index=dt_index) - instr_data = ( - self.df_o[f"{variable}_mf"] - .shift(0.5, freq=freq.lower()) - .resample(freq.lower()) - .mean() - ) + instr_data = mf_series.shift(0.5, freq=freq.lower()).resample(freq.lower()).mean() self.df_r.loc[instr_data.index, variable] = instr_data else: self.df_r[variable] = ( - self.df_o[f"{variable}_mf"] - .shift(0.5, freq=freq.lower()) - .resample(freq.lower()) - .mean() + mf_series.shift(0.5, freq=freq.lower()).resample(freq.lower()).mean() ) return ".mean() aggregator" From 46c6fa466deef422f0a2e4ba31635d3042e3256b Mon Sep 17 00:00:00 2001 From: Mike McCann Date: Wed, 25 Mar 2026 16:09:41 -0700 Subject: [PATCH 4/5] Add plot page for planktivore variables. --- src/data/create_products.py | 151 ++++++++++++++++++++++++++++++++++++ src/data/process.py | 1 + 2 files changed, 152 insertions(+) diff --git a/src/data/create_products.py b/src/data/create_products.py index 1eae538..e46d985 100755 --- a/src/data/create_products.py +++ b/src/data/create_products.py @@ -190,6 +190,10 @@ def __init__( # noqa: PLR0913 "proxy_adinos": "algae", "proxy_diatoms": "Purples", "proxy_hdinos": "Blues", + "avgrois": "YlOrRd", + "casetemp": "thermal", + "casehumidity": "PuBu", + "casepress": "deep", } def _open_ds(self): @@ -407,6 +411,20 @@ def _get_biolume_plot_variables(self) -> list: return self._get_lrauv_biolume_variables() return self._get_dorado_biolume_variables() + def _get_planktivore_plot_variables(self) -> list: + """Get planktivore + context variables for plot_planktivore_2column().""" + return [ + ("backseat_planktivore_hm_avgrois", "linear"), + ("backseat_planktivore_lm_avgrois", "linear"), + ("backseat_planktivore_casetemp", "linear"), + ("backseat_planktivore_casehumidity", "linear"), + ("density", "linear"), + ("wetlabsbb2fl_particulatebackscatteringcoeff470nm", "linear"), + ("wetlabsbb2fl_particulatebackscatteringcoeff650nm", "linear"), + ("wetlabsbb2fl_mass_concentration_of_chlorophyll_in_sea_water", "linear"), + ("backseat_planktivore_casepress", "linear"), + ] + def _plot_nighttime_indicator( self, fig: matplotlib.figure.Figure, @@ -1868,6 +1886,138 @@ def plot_biolume_2column(self) -> str: # noqa: C901, PLR0912, PLR0915 self.logger.info("Saved biolume 2column plot to %s", output_file) return str(output_file) + def plot_planktivore_2column(self) -> str: # noqa: C901, PLR0912, PLR0915 + """Create 2-column planktivore plot with map, ROIs, engineering, and context. + + Layout (5 rows x 2 columns, column-major order): + (0,0) track map (0,1) density + (1,0) hm_avgrois (1,1) backscatter470 + (2,0) lm_avgrois (2,1) backscatter650 + (3,0) casetemp (3,1) chlorophyll + (4,0) casehumidity (4,1) casepress + """ + # Skip plotting in pytest environment - too many prerequisites for CI + if "pytest" in sys.modules: + self.logger.info("Skipping plot_planktivore_2column in pytest environment") + return None + + self._open_ds() + + # Early return if no planktivore variables present + if not any(v.startswith("backseat_planktivore_") for v in self.ds.variables): + self.logger.warning( + "No backseat_planktivore_* variables found in dataset, " + "skipping plot_planktivore_2column", + ) + return None + + idist, iz, distnav = self._grid_dims() + if idist.size == 0 or iz.size == 0 or distnav.size == 0: + self.logger.warning( + "Skipping plot_planktivore_2column due to missing gridding dimensions" + ) + return None + + fig, ax = plt.subplots(nrows=5, ncols=2, figsize=(18, 10)) + plt.subplots_adjust(hspace=0.15, wspace=0.04, left=0.05, right=0.97, top=0.96, bottom=0.06) + + best_ctd = None + if self._is_lrauv(): + self.logger.info("LRAUV mission detected for planktivore 2column plot") + self._compute_density_lrauv() + else: + self.logger.info("Dorado mission detected for planktivore 2column plot") + best_ctd = self._get_best_ctd() + self._compute_density(best_ctd) + + self._plot_track_map(ax[0, 0], ax[1, 0]) + + if self.auv_name and self.mission: + try: + gulper_locations = self._get_gulper_locations(distnav) + except FileNotFoundError as e: + self.logger.warning("Error retrieving gulper locations: %s", e) # noqa: TRY400 + gulper_locations = {} + else: + try: + gulper_locations = self._get_sipper_locations(distnav) + except FileNotFoundError as e: + self.logger.warning("Error retrieving sipper locations: %s", e) # noqa: TRY400 + gulper_locations = {} + + try: + profile_bottoms = self._profile_bottoms(distnav) + except (TypeError, ValueError) as e: + self.logger.warning("Error computing profile bottoms: %s", e) # noqa: TRY400 + profile_bottoms = None + + try: + bottom_depths = self._get_bathymetry( + self.ds.cf["longitude"].to_numpy(), + self.ds.cf["latitude"].to_numpy(), + ) + except ValueError as e: # noqa: BLE001 + self.logger.warning("Error retrieving bathymetry: %s", e) # noqa: TRY400 + bottom_depths = None + + row = 1 + col = 0 + + plot_variables = self._get_planktivore_plot_variables() + + for var, scale in plot_variables: + self.logger.info("Plotting %s...", var) + if var not in self.ds: + self.logger.warning("%s not in dataset, plotting with no data", var) + + self._plot_var( + var, + idist, + iz, + distnav, + fig, + ax, + row, + col, + profile_bottoms, + scale=scale, + gulper_locations=gulper_locations, + bottom_depths=bottom_depths, + best_ctd=best_ctd, + ) + if row != 4: # noqa: PLR2004 + ax[row, col].get_xaxis().set_visible(False) + else: + ax[row, col].set_xlabel("Distance along track (km)") + + if row == 4 and col == 0: # noqa: PLR2004 + row = 0 + col = 1 + else: + row += 1 + + self._plot_nighttime_indicator(fig, ax[0, 1], distnav) + + if self._is_lrauv(): + netcdfs_dir = Path(BASE_LRAUV_PATH, f"{Path(self.log_file).parent}") + output_file = Path( + netcdfs_dir, + f"{Path(self.log_file).stem}_{self.freq}_2column_planktivore.png", + ) + else: + images_dir = Path(BASE_PATH, self.auv_name, MISSIONIMAGES, self.mission) + Path(images_dir).mkdir(parents=True, exist_ok=True) + output_file = Path( + images_dir, + f"{self.auv_name}_{self.mission}_{self.freq}_2column_planktivore.png", + ) + plt.savefig(output_file, dpi=100, bbox_inches="tight") + plt.show() + plt.close(fig) + + self.logger.info("Saved planktivore 2column plot to %s", output_file) + return str(output_file) + def _get_best_ctd(self) -> str: """Determine best CTD to use for ODV lookup table based on metadata""" # LRAUV doesn't use multiple CTDs, return None @@ -2333,6 +2483,7 @@ def process_command_line(self): p_start = time.time() cp.plot_2column() cp.plot_biolume_2column() + cp.plot_planktivore_2column() if cp.mission and cp.auv_name: cp.gulper_odv() if cp.log_file: diff --git a/src/data/process.py b/src/data/process.py index d7661df..572d61c 100755 --- a/src/data/process.py +++ b/src/data/process.py @@ -700,6 +700,7 @@ def create_products(self, mission: str = None, log_file: str = None) -> None: cp.logger.addHandler(self.log_handler) cp.plot_biolume_2column() + cp.plot_planktivore_2column() cp.plot_2column() if mission and "dorado" in cp.auv_name.lower(): cp.gulper_odv() From 287308299fad0e19b9f091f9932b212e77d441c6 Mon Sep 17 00:00:00 2001 From: Mike McCann Date: Wed, 25 Mar 2026 16:15:13 -0700 Subject: [PATCH 5/5] fix(create_products): skip biolume/planktivore plots when no relevant data Check for actual plot variables in the dataset instead of prefix-matching, which was triggering false positives (e.g. wetlabsubat_flowratecalibcoeff passing the wetlabsubat_ prefix check when no biolume data existed). --- src/data/create_products.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/data/create_products.py b/src/data/create_products.py index e46d985..6887817 100755 --- a/src/data/create_products.py +++ b/src/data/create_products.py @@ -1768,12 +1768,11 @@ def plot_biolume_2column(self) -> str: # noqa: C901, PLR0912, PLR0915 self._open_ds() - # Early return if no biolume variables present - biolume_prefix = "wetlabsubat_" if self._is_lrauv() else "biolume_" - if not any(v.startswith(biolume_prefix) for v in self.ds.variables): + # Early return if no biolume plot variables present in dataset + plot_variables = self._get_biolume_plot_variables() + if not any(var in self.ds for var, _ in plot_variables): self.logger.warning( - "No %s* variables found in dataset, skipping plot_biolume_2column", - biolume_prefix, + "No biolume plot variables found in dataset, skipping plot_biolume_2column", ) return None @@ -1903,10 +1902,15 @@ def plot_planktivore_2column(self) -> str: # noqa: C901, PLR0912, PLR0915 self._open_ds() - # Early return if no planktivore variables present - if not any(v.startswith("backseat_planktivore_") for v in self.ds.variables): + # Early return if no planktivore plot variables present in dataset + planktivore_vars = [ + v + for v, _ in self._get_planktivore_plot_variables() + if v.startswith("backseat_planktivore_") + ] + if not any(var in self.ds for var in planktivore_vars): self.logger.warning( - "No backseat_planktivore_* variables found in dataset, " + "No backseat_planktivore plot variables found in dataset, " "skipping plot_planktivore_2column", ) return None