From e558478f16d872cef29e37d502b63a915e2be2cb Mon Sep 17 00:00:00 2001 From: Mike McCann Date: Wed, 18 Mar 2026 10:35:44 -0700 Subject: [PATCH 1/2] fix(resample): apply surface-depth filter to proxy correction correlation logic in correct_biolume_proxies(), compute adinos threshold counts using only points deeper than surface_exclusion_depth_m compute fluo/bg_biolume correlation on the deep-only subset (and depth-limited portion) instead of full profile data update debug messaging to report deep-only adinos counts used for correction decisions --- src/data/resample.py | 43 ++++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/src/data/resample.py b/src/data/resample.py index 38d5ca9..4fa7866 100755 --- a/src/data/resample.py +++ b/src/data/resample.py @@ -1764,6 +1764,9 @@ def _interval_contains_sunevent( # print(f'no corrections possible for {iprofil_=}') continue auv_profil = df_p.loc[iprofil] + deep_mask = auv_profil.depth > surface_exclusion_depth_m + deep_adinos = auv_profil.loc[deep_mask, f"{prefix}_proxy_adinos"] + valid_adinos_count = np.sum(deep_adinos > adinos_threshold) self.logger.debug( "Processing profile=%d for proxy correction: total_points=%d > thresh=%d ?", iprofil_, @@ -1771,38 +1774,36 @@ def _interval_contains_sunevent( correction_threshold, ) if auv_profil.shape[0] > correction_threshold: - if ( - np.sum(auv_profil[f"{prefix}_proxy_adinos"] > adinos_threshold) - < correction_threshold - ): + if valid_adinos_count < correction_threshold: # all proxies are NaN so skip - if auv_profil[f"{prefix}_proxy_adinos"].count() == 0: + if deep_adinos.count() == 0: self.logger.debug( - "Correcting proxies: valid adinos=%d < thresh=%d -- all NaN so skip", - np.sum(auv_profil[f"{prefix}_proxy_adinos"] > adinos_threshold), + "Correcting proxies: valid deep adinos=%d < thresh=%d -- all NaN so skip", # noqa: E501 + valid_adinos_count, correction_threshold, ) continue # no correction for low fluo & biolum values fluoBL_corr = 1.0 self.logger.debug( - "Correcting proxies: valid adinos=%d < thresh=%d" + "Correcting proxies: valid deep adinos=%d < thresh=%d" " -- using fluoBL_corr=%.4f, total_size_adinos=%d, nans=%d", - np.sum(auv_profil[f"{prefix}_proxy_adinos"] > adinos_threshold), + valid_adinos_count, correction_threshold, fluoBL_corr, - auv_profil[f"{prefix}_proxy_adinos"].shape[0], - auv_profil[f"{prefix}_proxy_adinos"].isna().sum(), + deep_adinos.shape[0], + deep_adinos.isna().sum(), ) else: - # correlation between fluo and bg_biolum computed on high - # adino values for each profile - idepth = ( - auv_profil.depth - <= auv_profil.depth[ - auv_profil[f"{prefix}_proxy_adinos"] > adinos_threshold - ].max() + # Correlation between fluo and bg_biolum is computed on the + # profile part below surface_exclusion_depth_m, truncated at + # the max depth where adinos exceeds threshold. + deep_high_adinos = deep_mask & ( + auv_profil[f"{prefix}_proxy_adinos"] > adinos_threshold ) + idepth = ( + auv_profil.depth <= auv_profil.depth[deep_high_adinos].max() + ) & deep_mask auv_profil_idepth = auv_profil[ [f"{prefix}_fluo", f"{prefix}_bg_biolume", "depth"] ].loc[idepth] @@ -1811,14 +1812,14 @@ def _interval_contains_sunevent( auv_profil_idepth[f"{prefix}_bg_biolume"], method=corr_type ) self.logger.debug( - "Correcting proxies: valid adinos=%d > thresh=%d" + "Correcting proxies: valid deep adinos=%d > thresh=%d" " -- using fluoBL_corr=%.4f, total_size_idepth=%d, nans=%d," " min_depth=%.4f, max_depth=%.4f", - np.sum(auv_profil[f"{prefix}_proxy_adinos"] > adinos_threshold), + valid_adinos_count, correction_threshold, fluoBL_corr, auv_profil_idepth.shape[0], - auv_profil[f"{prefix}_proxy_adinos"].isna().sum(), + deep_adinos.isna().sum(), auv_profil_idepth.depth.min(), auv_profil_idepth.depth.max(), ) From 5c66dc611970cb64de49121f5905d8bfefc01ead Mon Sep 17 00:00:00 2001 From: Mike McCann Date: Wed, 18 Mar 2026 10:45:04 -0700 Subject: [PATCH 2/2] Instrument correct_biolume_proxies() and log summary info. --- src/data/resample.py | 42 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/src/data/resample.py b/src/data/resample.py index 4fa7866..b5dcaf0 100755 --- a/src/data/resample.py +++ b/src/data/resample.py @@ -1741,10 +1741,17 @@ def _interval_contains_sunevent( # compute correlation per profil and then correct proxies profil = df_p.profile_number dt_5mins = np.timedelta64(timedelta(minutes=minutes_from_surface_threshold)) + sun_event_skipped = 0 + profile_window_skipped = 0 + threshold_skipped = 0 + all_nan_skipped = 0 + fluo_corr_messages = 0 + corrected_profiles = 0 for iprofil_ in range(1, int(np.max(profil)) + 1): iprofil = profil == iprofil_ has_sunevent = df_p.loc[iprofil, "has_sunevent"].any() if has_sunevent: # set proxies for this profile to NaN + sun_event_skipped += 1 self.logger.info( "Processing profile=%d for proxy correction: found sun event -- set NaN", iprofil_, @@ -1761,7 +1768,12 @@ def _interval_contains_sunevent( ) iprofil = iprofil & itime if not np.any(iprofil): - # print(f'no corrections possible for {iprofil_=}') + profile_window_skipped += 1 + self.logger.debug( + "profile=%d skipped for proxy correction: no points in deep ±%d minute window", # noqa: E501 + iprofil_, + minutes_from_surface_threshold, + ) continue auv_profil = df_p.loc[iprofil] deep_mask = auv_profil.depth > surface_exclusion_depth_m @@ -1777,6 +1789,7 @@ def _interval_contains_sunevent( if valid_adinos_count < correction_threshold: # all proxies are NaN so skip if deep_adinos.count() == 0: + all_nan_skipped += 1 self.logger.debug( "Correcting proxies: valid deep adinos=%d < thresh=%d -- all NaN so skip", # noqa: E501 valid_adinos_count, @@ -1794,6 +1807,7 @@ def _interval_contains_sunevent( deep_adinos.shape[0], deep_adinos.isna().sum(), ) + fluo_corr_messages += 1 else: # Correlation between fluo and bg_biolum is computed on the # profile part below surface_exclusion_depth_m, truncated at @@ -1823,6 +1837,7 @@ def _interval_contains_sunevent( auv_profil_idepth.depth.min(), auv_profil_idepth.depth.max(), ) + fluo_corr_messages += 1 # save correlation df_p.loc[iprofil, "fluoBL_corr"] = fluoBL_corr @@ -1867,7 +1882,9 @@ def _interval_contains_sunevent( iprofil ] self.df_r.loc[target_indices, f"{prefix}_proxy_hdinos"] = df_p.hdinosN.loc[iprofil] + corrected_profiles += 1 else: + threshold_skipped += 1 self.logger.debug( "profile=%d skipped for proxy correction", iprofil_, @@ -1880,6 +1897,29 @@ def _interval_contains_sunevent( profiles_processed, fluo_bl_threshold, ) + corrected_inferred = ( + profiles_processed + - sun_event_skipped + - profile_window_skipped + - all_nan_skipped + - threshold_skipped + ) + self.logger.info( + "Proxy correction summary: processed_profiles=%d, sun_event_skipped=%d, " + "profile_window_skipped=%d, all_nan_skipped=%d, corrected_inferred=%d, " + "corrected_profiles=%d, fluo_corr_messages=%d, threshold_skipped=%d", + profiles_processed, + sun_event_skipped, + profile_window_skipped, + all_nan_skipped, + corrected_inferred, + corrected_profiles, + fluo_corr_messages, + threshold_skipped, + ) + self.logger.info( + "Proxy correction sanity check: deep-only logic active (valid deep adinos)." + ) # Restore attrs for every df_r column – assignments above cleared the # entire _item_cache, losing attrs on columns we never intended to modify.