From 6c805818194c620c4fc6c031c3163f7cd6f3fdb7 Mon Sep 17 00:00:00 2001 From: Pradeep L Date: Sun, 10 May 2026 15:41:22 +0530 Subject: [PATCH 01/10] searchBack Pressure Signed-off-by: Pradeep L # Conflicts: # sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/AnalyticsSearchBackendPlugin.java # sandbox/plugins/analytics-backend-datafusion/rust/src/query_tracker.rs # sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java # sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java # sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java # Conflicts: # server/src/main/java/org/opensearch/monitor/os/OsProbe.java --- .../spi/AnalyticsSearchBackendPlugin.java | 14 + .../analytics/spi/QueryExecutionMetrics.java | 26 ++ .../libs/dataformat-native/rust/Cargo.toml | 2 +- .../rust/src/ffm.rs | 47 ++++ .../rust/src/query_tracker.rs | 247 ++++++++++++++++- .../DataFusionAnalyticsBackendPlugin.java | 9 + .../be/datafusion/DataFusionPlugin.java | 82 ++++++ .../be/datafusion/nativelib/NativeBridge.java | 55 ++++ .../nativelib/QueryMemoryRegistry.java | 82 ++++++ .../nativelib/QueryMemoryUsage.java | 25 ++ .../nativelib/QueryRegistryLayout.java | 70 +++++ .../common/settings/ClusterSettings.java | 5 + .../SearchBackpressureService.java | 180 +++++++++++-- .../settings/NodeDuressSettings.java | 36 +++ .../settings/SearchShardTaskSettings.java | 51 ++++ .../settings/SearchTaskSettings.java | 51 ++++ .../stats/SearchShardTaskStats.java | 10 + .../backpressure/stats/SearchTaskStats.java | 10 + .../trackers/NativeMemoryUsageTracker.java | 248 ++++++++++++++++++ .../trackers/NodeDuressTrackers.java | 19 +- .../TaskResourceUsageTrackerType.java | 5 +- .../trackers/TaskResourceUsageTrackers.java | 11 + .../java/org/opensearch/wlm/ResourceType.java | 11 +- .../tracker/NativeMemoryUsageCalculator.java | 40 +++ ...kloadGroupResourceUsageTrackerService.java | 7 +- .../opensearch/monitor/os/OsProbeTests.java | 112 ++++++++ .../SearchBackpressureServiceTests.java | 90 +++++++ .../stats/SearchShardTaskStatsTests.java | 5 +- .../stats/SearchTaskStatsTests.java | 5 +- 29 files changed, 1527 insertions(+), 28 deletions(-) create mode 100644 sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/QueryExecutionMetrics.java create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/QueryMemoryRegistry.java create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/QueryMemoryUsage.java create mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/QueryRegistryLayout.java create mode 100644 server/src/main/java/org/opensearch/search/backpressure/trackers/NativeMemoryUsageTracker.java create mode 100644 server/src/main/java/org/opensearch/wlm/tracker/NativeMemoryUsageCalculator.java diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/AnalyticsSearchBackendPlugin.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/AnalyticsSearchBackendPlugin.java index 59f9b1f899f92..707284f93a5a1 100644 --- a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/AnalyticsSearchBackendPlugin.java +++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/AnalyticsSearchBackendPlugin.java @@ -114,6 +114,20 @@ default void configureFilterDelegation(FilterDelegationHandle handle, BackendExe throw new UnsupportedOperationException("configureFilterDelegation not implemented for [" + name() + "]"); } + /** + * Returns a snapshot of this backend's currently-tracked queries, keyed by {@code contextId}. + * + *

The map is a point-in-time view — entries can register or drain concurrently on the + * backend side. Implementations MUST return a non-null map (empty when nothing is tracked) + * and SHOULD make it unmodifiable so callers cannot mutate backend state. + * + *

Default implementation returns an empty map so backends that do not track per-query + * metrics don't have to opt in. + */ + default Map getActiveQueryMetrics() { + return Collections.emptyMap(); + } + /** * Install a thread tracker for attribution of delegation callbacks executing on foreign threads. * Called after {@link #configureFilterDelegation}. Pass {@code null} to clear. diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/QueryExecutionMetrics.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/QueryExecutionMetrics.java new file mode 100644 index 0000000000000..8a246d74c86dc --- /dev/null +++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/QueryExecutionMetrics.java @@ -0,0 +1,26 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.analytics.spi; + +/** + * Snapshot of per-query execution metrics reported by an analytics backend. + * + *

The context id (the map key in {@link AnalyticsSearchBackendPlugin#getActiveQueryMetrics()}) + * is not repeated here — this record holds only the remaining parameters: memory accounting, + * wall time, and whether the query has completed but not yet been drained. + * + * @param currentBytes bytes currently reserved by the query's memory pool + * @param peakBytes high-water mark of bytes reserved during the query's lifetime + * @param wallNanos live or frozen wall-clock duration in nanoseconds + * @param completed {@code true} when the query has finished executing but its entry has not + * yet been drained from the backend's internal registry + * + * @opensearch.internal + */ +public record QueryExecutionMetrics(long currentBytes, long peakBytes, long wallNanos, boolean completed) {} diff --git a/sandbox/libs/dataformat-native/rust/Cargo.toml b/sandbox/libs/dataformat-native/rust/Cargo.toml index 4f391153203eb..0c890cc18177e 100644 --- a/sandbox/libs/dataformat-native/rust/Cargo.toml +++ b/sandbox/libs/dataformat-native/rust/Cargo.toml @@ -81,7 +81,7 @@ opensearch-repository-azure = { path = "../../../plugins/native-repository-azure opensearch-repository-fs = { path = "../../../plugins/native-repository-fs/src/main/rust" } [profile.release] -lto = true +lto = "thin" codegen-units = 1 incremental = true debug = "line-tables-only" diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs index 140f3e8bbc15a..e6f54e2db5799 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs @@ -12,6 +12,7 @@ use std::slice; use std::str; use std::sync::Arc; +use log::info; use native_bridge_common::ffm_safe; use parking_lot::RwLock; @@ -206,6 +207,52 @@ pub extern "C" fn df_cancel_query(context_id: i64) { api::cancel_query(context_id); } +// --------------------------------------------------------------------------- +// Per-query registry snapshot (two-phase) +// +// Java calls `df_query_registry_len` to size a buffer, then `df_query_registry_snapshot` +// to populate it. See `query_tracker::WireQueryMetric` for the wire layout. +// --------------------------------------------------------------------------- + +/// Returns the current number of entries in the query registry. +/// Value is racy — treat it as a sizing hint only. +#[no_mangle] +pub extern "C" fn df_query_registry_len() -> i64 { + let len = crate::query_tracker::query_registry_len(); + info!("[nativemem-bp] ffm.df_query_registry_len -> {}", len); + len as i64 +} + +/// Copies up to `cap_entries` `WireQueryMetric`s into the caller-provided buffer. +/// Returns the number of entries actually written. +/// +/// Safety: `out_ptr` must be non-null, 8-byte aligned, and point to storage for +/// at least `cap_entries * size_of::()` bytes. +#[ffm_safe] +#[no_mangle] +pub unsafe extern "C" fn df_query_registry_snapshot(out_ptr: *mut u8, cap_entries: i64) -> i64 { + use crate::query_tracker::{snapshot_query_registry, WireQueryMetric}; + + if cap_entries < 0 { + return Err(format!("negative capacity: {cap_entries}")); + } + if cap_entries == 0 { + info!("[nativemem-bp] ffm.df_query_registry_snapshot: capacity=0, nothing to write"); + return Ok(0); + } + if out_ptr.is_null() { + return Err("null snapshot buffer".to_string()); + } + let out: &mut [WireQueryMetric] = + slice::from_raw_parts_mut(out_ptr as *mut WireQueryMetric, cap_entries as usize); + let written = snapshot_query_registry(out); + info!( + "[nativemem-bp] ffm.df_query_registry_snapshot: wrote {} entries (capacity {})", + written, cap_entries + ); + Ok(written as i64) +} + #[ffm_safe] #[no_mangle] pub unsafe extern "C" fn df_sql_to_substrait( diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/query_tracker.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/query_tracker.rs index 90a59a64f0aa8..9f5ece0bf578e 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/query_tracker.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/query_tracker.rs @@ -22,7 +22,7 @@ use std::sync::Arc; use std::time::Instant; use dashmap::DashMap; -use log::debug; +use log::{debug, info}; use once_cell::sync::Lazy; use tokio_util::sync::CancellationToken; @@ -132,6 +132,21 @@ impl QueryTracker { } } + /// Wall-clock duration in nanoseconds, as an `i64` for FFM transport. + /// Returns the frozen snapshot if completed, otherwise live elapsed time. + /// Elapsed nanos is `u128` internally; saturates at `i64::MAX` (~292 years) + /// so it can always be represented as an `i64`. + pub fn elapsed_nanos(&self) -> i64 { + let frozen = self.wall_nanos.load(Ordering::Acquire); + if frozen > 0 { + // `AtomicU64` → `i64`: `frozen` was produced from `elapsed().as_nanos() as u64` + // so its high bit is effectively clear. Still clamp defensively. + frozen.min(i64::MAX as u64) as i64 + } else { + self.start_time.elapsed().as_nanos().min(i64::MAX as u128) as i64 + } + } + pub fn is_completed(&self) -> bool { self.completed.load(Ordering::Acquire) } @@ -150,6 +165,105 @@ impl QueryTracker { static QUERY_REGISTRY: Lazy>> = Lazy::new(DashMap::new); +/// Remove a completed tracker from the registry and return it. +/// Called from JNI after Java has consumed the metrics. +pub fn drain_completed_query(context_id: i64) -> Option> { + QUERY_REGISTRY + .remove_if(&context_id, |_, t| t.is_completed()) + .map(|(_, t)| t) +} + +// --------------------------------------------------------------------------- +// Registry snapshot — two-phase FFM export +// --------------------------------------------------------------------------- + +/// Wire representation of a single query's tracker, used by the two-phase +/// snapshot API. Fields are `i64` so the struct crosses the FFM boundary with +/// a stable, alignment-safe layout (8-byte aligned on every target we support). +/// +/// | Field | Meaning | +/// |---------------|-----------------------------------------------------------| +/// | context_id | `QueryTracker::context_id` | +/// | current_bytes | `QueryMemoryPool::current_bytes`, clamped to `i64::MAX` | +/// | peak_bytes | `QueryMemoryPool::peak_bytes`, clamped to `i64::MAX` | +/// | wall_nanos | live elapsed or frozen wall time (see `elapsed_nanos()`) | +/// | completed | 1 if the tracker has been marked completed, else 0 | +#[repr(C)] +#[derive(Debug, Clone, Copy)] +pub struct WireQueryMetric { + pub context_id: i64, + pub current_bytes: i64, + pub peak_bytes: i64, + pub wall_nanos: i64, + pub completed: i64, +} + +const _: () = assert!(std::mem::size_of::() == 5 * 8); + +fn usize_to_i64_saturating(value: usize) -> i64 { + if value > i64::MAX as usize { + i64::MAX + } else { + value as i64 + } +} + +impl WireQueryMetric { + fn from_tracker(tracker: &QueryTracker) -> Self { + Self { + context_id: tracker.context_id, + current_bytes: usize_to_i64_saturating(tracker.memory_pool.current_bytes()), + peak_bytes: usize_to_i64_saturating(tracker.memory_pool.peak_bytes()), + wall_nanos: tracker.elapsed_nanos(), + completed: if tracker.is_completed() { 1 } else { 0 }, + } + } +} + +/// Phase 1 of the snapshot API: return the current number of entries in the +/// registry. Java uses this to allocate a buffer for phase 2. +/// +/// The value is racy by design — queries may register or be drained between +/// phase 1 and phase 2. Phase 2 handles both cases: if there are more entries +/// than the buffer holds, the extra entries are truncated; if there are +/// fewer, the unused tail of the buffer is left untouched and the actual +/// count is returned. +pub fn query_registry_len() -> usize { + let n = QUERY_REGISTRY.len(); + info!("[nativemem-bp] rust.query_registry_len = {}", n); + n +} + +/// Phase 2 of the snapshot API: copy up to `cap_entries` entries from the +/// registry into `out`. Returns the number of entries actually written. +/// +/// `out` may be a raw slice pointing at a Java-owned buffer; it must be +/// valid for writes of `cap_entries * size_of::()` bytes +/// and properly aligned for `WireQueryMetric` (8-byte aligned). Entries are +/// collected in the order DashMap's iterator returns them, which is not +/// otherwise specified. +pub fn snapshot_query_registry(out: &mut [WireQueryMetric]) -> usize { + let mut written = 0usize; + for entry in QUERY_REGISTRY.iter() { + if written >= out.len() { + break; + } + let metric = WireQueryMetric::from_tracker(entry.value()); + info!( + "[nativemem-bp] rust.snapshot entry[{}]: ctx={}, current={}B, peak={}B, wall_ns={}, completed={}", + written, metric.context_id, metric.current_bytes, metric.peak_bytes, metric.wall_nanos, metric.completed + ); + out[written] = metric; + written += 1; + } + info!( + "[nativemem-bp] rust.snapshot_query_registry: wrote {} entries (buffer cap {})", + written, + out.len() + ); + written +} + /// Fire the cancellation token for the given context_id. /// No-op for unknown or already-completed queries. pub fn cancel_query(context_id: i64) { @@ -199,6 +313,11 @@ impl QueryTrackingContext { wall_nanos: std::sync::atomic::AtomicU64::new(0), }); QUERY_REGISTRY.insert(context_id, Arc::clone(&tracker)); + info!( + "[nativemem-bp] rust.QueryTrackingContext::new: registered ctx={} (registry_size={})", + context_id, + QUERY_REGISTRY.len() + ); Self { tracker: Some(tracker), } @@ -220,6 +339,14 @@ impl Drop for QueryTrackingContext { fn drop(&mut self) { if let Some(tracker) = &self.tracker { tracker.mark_completed(); + info!( + "[nativemem-bp] rust.QueryTrackingContext::drop: ctx={} completed (wall={:.3}s, mem_current={}B, mem_peak={}B)", + tracker.context_id, + tracker.wall_secs(), + tracker.memory_pool.current_bytes(), + tracker.memory_pool.peak_bytes(), + ); + // Keep the debug line for operators who already tail the Rust debug log. debug!( "Query completed ctx={}: wall={:.3}s, mem_current={}B, mem_peak={}B", tracker.context_id, @@ -467,4 +594,122 @@ mod tests { assert!(!QUERY_REGISTRY.contains_key(&ctx_id)); } + + // ----------------------------------------------------------------------- + // Two-phase snapshot tests + // ----------------------------------------------------------------------- + + #[test] + fn test_snapshot_captures_live_and_completed() { + let global = make_global_pool(1_000_000); + let live_id = 60_000; + let done_id = 60_001; + + let live_ctx = QueryTrackingContext::new(live_id, Arc::clone(&global)); + let live_pool = live_ctx.memory_pool().unwrap(); + let pool: Arc = live_pool.clone(); + let mut reservation = make_reservation(&pool, "live"); + reservation.try_grow(4096).unwrap(); + + let done_ctx = QueryTrackingContext::new(done_id, Arc::clone(&global)); + let done_pool = done_ctx.memory_pool().unwrap(); + let done_pool_dyn: Arc = done_pool.clone(); + let mut done_reservation = make_reservation(&done_pool_dyn, "done"); + done_reservation.try_grow(2048).unwrap(); + drop(done_reservation); + drop(done_ctx); + + // Phase 1 + let len = query_registry_len(); + assert!(len >= 2, "expected at least 2 entries, got {len}"); + + // Phase 2 with exact capacity + let mut buf = vec![ + WireQueryMetric { + context_id: 0, + current_bytes: 0, + peak_bytes: 0, + wall_nanos: 0, + completed: 0, + }; + len + ]; + let written = snapshot_query_registry(&mut buf); + assert_eq!(written, len); + + let live = buf.iter().find(|m| m.context_id == live_id).expect("live not captured"); + assert_eq!(live.current_bytes, 4096); + assert_eq!(live.peak_bytes, 4096); + assert_eq!(live.completed, 0); + assert!(live.wall_nanos >= 0); + + let done = buf.iter().find(|m| m.context_id == done_id).expect("done not captured"); + assert_eq!(done.current_bytes, 0); + assert_eq!(done.peak_bytes, 2048); + assert_eq!(done.completed, 1); + assert!(done.wall_nanos > 0); + + drop(reservation); + drop(live_ctx); + QUERY_REGISTRY.remove(&live_id); + QUERY_REGISTRY.remove(&done_id); + } + + #[test] + fn test_snapshot_truncates_when_buffer_is_smaller_than_registry() { + let global = make_global_pool(1_000_000); + let ids: Vec = (60_100..60_105).collect(); + let contexts: Vec = ids + .iter() + .map(|id| QueryTrackingContext::new(*id, Arc::clone(&global))) + .collect(); + + let mut buf = vec![ + WireQueryMetric { + context_id: 0, + current_bytes: 0, + peak_bytes: 0, + wall_nanos: 0, + completed: 0, + }; + 2 + ]; + let written = snapshot_query_registry(&mut buf); + assert_eq!(written, 2, "buffer should cap the write count"); + // QUERY_REGISTRY is process-wide. Other parallel tests may contribute entries to + // this 2-slot buffer, so the only invariant we can assert here is "no more than + // two entries were written". The more specific "every entry belongs to this + // test's id range" check is inherently racy; don't re-introduce it. + + drop(contexts); + for id in &ids { + QUERY_REGISTRY.remove(id); + } + } + + #[test] + fn test_snapshot_into_oversized_buffer_leaves_tail_untouched() { + let global = make_global_pool(1_000_000); + let id = 60_200; + let ctx = QueryTrackingContext::new(id, global); + + let sentinel = WireQueryMetric { + context_id: -1, + current_bytes: -1, + peak_bytes: -1, + wall_nanos: -1, + completed: -1, + }; + let mut buf = vec![sentinel; query_registry_len() + 4]; + let written = snapshot_query_registry(&mut buf); + assert!(written >= 1); + assert!(written < buf.len(), "should not fill the oversized buffer"); + // Trailing slots keep the sentinel — verifies snapshot did not touch them. + for entry in &buf[written..] { + assert_eq!(entry.context_id, -1); + } + + drop(ctx); + QUERY_REGISTRY.remove(&id); + } } diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java index d677195e8c2aa..bb1e24ec57114 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java @@ -30,6 +30,7 @@ import org.opensearch.analytics.spi.JoinCapability; import org.opensearch.analytics.spi.NumericToDoubleAdapter; import org.opensearch.analytics.spi.ProjectCapability; +import org.opensearch.analytics.spi.QueryExecutionMetrics; import org.opensearch.analytics.spi.ScalarFunction; import org.opensearch.analytics.spi.ScalarFunctionAdapter; import org.opensearch.analytics.spi.ScanCapability; @@ -704,6 +705,14 @@ public void configureFilterDelegation(FilterDelegationHandle handle, BackendExec FilterTreeCallbacks.setHandle(handle); } + @Override + public Map getActiveQueryMetrics() { + // Delegate to the plugin that owns the DataFusionService and native runtime. + // Keeping the real implementation on DataFusionPlugin lets operators call it + // directly (e.g., from a REST action) without going through the SPI. + return plugin.getActiveQueryMetrics(); + } + @Override public void setDelegationThreadTracker(org.opensearch.analytics.spi.DelegationThreadTracker tracker) { FilterTreeCallbacks.setThreadTracker(tracker); diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java index bad7879a96322..f3be8042fc32c 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java @@ -11,7 +11,11 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.analytics.spi.AnalyticsSearchBackendPlugin; +import org.opensearch.analytics.spi.QueryExecutionMetrics; import org.opensearch.be.datafusion.action.DataFusionStatsAction; +import org.opensearch.be.datafusion.cache.CacheSettings; +import org.opensearch.be.datafusion.nativelib.QueryMemoryRegistry; +import org.opensearch.be.datafusion.nativelib.QueryMemoryUsage; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.service.ClusterService; @@ -35,6 +39,7 @@ import org.opensearch.rest.RestController; import org.opensearch.rest.RestHandler; import org.opensearch.script.ScriptService; +import org.opensearch.search.backpressure.trackers.NativeMemoryUsageTracker; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.client.Client; import org.opensearch.watcher.ResourceWatcherService; @@ -42,7 +47,10 @@ import java.io.IOException; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.function.Supplier; import io.substrait.extension.DefaultExtensionCatalog; @@ -153,11 +161,53 @@ public Collection createComponents( this.datafusionSettings = new DatafusionSettings(clusterService); + // Expose per-task native-memory usage to search backpressure. The tracker calls + // this supplier once per refresh (invoked by the backpressure service at the top of + // doRun() and nodeStats()), snapshotting all live queries in one FFM call. Per-task + // evaluation then reads from the tracker's cached map — no FFM call per task. + // + // The OpenSearch task id is used as the DataFusion context_id at query launch + // (see ShardScanInstructionHandler / DatafusionSearchExecEngine), so the map is + // already keyed by Task#getId on the consumer side. + logger.info("[nativemem-bp] plugin: installing native-memory snapshot supplier for backpressure tracker"); + NativeMemoryUsageTracker.setSnapshotSupplier(this::snapshotNativeMemoryByTaskId); + this.substraitExtensions = loadSubstraitExtensions(); return Collections.singletonList(dataFusionService); } + /** + * Project the active-query metrics map down to {@code taskId -> currentBytes} for the + * backpressure native-memory tracker. One FFM snapshot per call. Returns an empty map + * when the service isn't running, so startup/shutdown races don't surface bad data. + */ + private Map snapshotNativeMemoryByTaskId() { + if (dataFusionService == null) { + logger.info("[nativemem-bp] plugin.snapshot: service not running, returning empty map"); + return Collections.emptyMap(); + } + long t0 = System.nanoTime(); + Map metrics = getActiveQueryMetrics(); + if (metrics.isEmpty()) { + logger.info( + "[nativemem-bp] plugin.snapshot: empty registry (elapsedMicros={})", + (System.nanoTime() - t0) / 1000L + ); + return Collections.emptyMap(); + } + Map out = new HashMap<>(metrics.size()); + for (Map.Entry e : metrics.entrySet()) { + out.put(e.getKey(), e.getValue().currentBytes()); + } + logger.info( + "[nativemem-bp] plugin.snapshot: built taskId->currentBytes map, size={}, elapsedMicros={}", + out.size(), + (System.nanoTime() - t0) / 1000L + ); + return out; + } + /** * Loads the Substrait default extension catalog with the plugin's classloader as the * thread context classloader. Jackson polymorphic deserialization (used by Substrait @@ -284,4 +334,36 @@ public void close() throws IOException { dataFusionService.close(); } } + + /** + * Snapshot the native DataFusion per-query registry and return a {@code contextId -> metrics} + * map. Returns an empty map when the service is not yet running (startup) or has been stopped + * (shutdown), so callers never see a half-initialized view. + * + *

Each entry mirrors one {@code QueryTracker} on the Rust side — current and peak memory + * reservation, wall time, and whether the query has completed but not yet been drained. + * Ordering follows {@link QueryMemoryRegistry#snapshot()} (insertion-agnostic); a + * {@link LinkedHashMap} preserves that ordering for callers that want deterministic iteration + * in tests or logs. + */ + @Override + public Map getActiveQueryMetrics() { + if (dataFusionService == null) { + return Collections.emptyMap(); + } + List usages = QueryMemoryRegistry.snapshot(); + if (usages.isEmpty()) { + logger.info("[nativemem-bp] plugin.getActiveQueryMetrics: native registry empty"); + return Collections.emptyMap(); + } + Map result = new LinkedHashMap<>(usages.size()); + for (QueryMemoryUsage u : usages) { + // DashMap keys are unique, but defensively keep the first snapshot — a duplicate + // would only occur if context ids were reused, which QueryTrackingContext::new + // already treats as a caller bug. + result.putIfAbsent(u.contextId(), new QueryExecutionMetrics(u.currentBytes(), u.peakBytes(), u.wallNanos(), u.completed())); + } + logger.info("[nativemem-bp] plugin.getActiveQueryMetrics: decoded {} entries from native registry", result.size()); + return Collections.unmodifiableMap(result); + } } diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java index 45440e3c6f3d5..50f8161ae6e79 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java @@ -19,6 +19,7 @@ import java.lang.foreign.FunctionDescriptor; import java.lang.foreign.Linker; +import java.lang.foreign.MemorySegment; import java.lang.foreign.SymbolLookup; import java.lang.foreign.ValueLayout; import java.lang.invoke.MethodHandle; @@ -87,6 +88,8 @@ public final class NativeBridge { private static final MethodHandle PREPARE_PARTIAL_PLAN; private static final MethodHandle PREPARE_FINAL_PLAN; private static final MethodHandle EXECUTE_LOCAL_PREPARED_PLAN; + private static final MethodHandle QUERY_REGISTRY_LEN; + private static final MethodHandle QUERY_REGISTRY_SNAPSHOT; static { SymbolLookup lib = NativeLibraryLoader.symbolLookup(); @@ -425,6 +428,18 @@ public final class NativeBridge { lib.find("df_execute_local_prepared_plan").orElseThrow(), FunctionDescriptor.of(ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG) ); + + // i64 df_query_registry_len() + QUERY_REGISTRY_LEN = linker.downcallHandle( + lib.find("df_query_registry_len").orElseThrow(), + FunctionDescriptor.of(ValueLayout.JAVA_LONG) + ); + + // i64 df_query_registry_snapshot(out_ptr, cap_entries) + QUERY_REGISTRY_SNAPSHOT = linker.downcallHandle( + lib.find("df_query_registry_snapshot").orElseThrow(), + FunctionDescriptor.of(ValueLayout.JAVA_LONG, ValueLayout.ADDRESS, ValueLayout.JAVA_LONG) + ); } private NativeBridge() {} @@ -709,6 +724,46 @@ public static DataFusionStats stats() { } } + // ---- Per-query registry snapshot (two-phase) ---- + + /** + * Phase 1: return the current size of the per-query registry on the native side. + * + *

The value is a sizing hint only — it can go stale the moment it's returned + * because queries may register or drain concurrently. {@link #queryRegistrySnapshot} + * handles both over- and under-sized buffers safely. + */ + public static int queryRegistryLen() { + try { + long result = NativeLibraryLoader.checkResult((long) QUERY_REGISTRY_LEN.invokeExact()); + if (result > Integer.MAX_VALUE) { + throw new IllegalStateException("Native registry length exceeds Integer.MAX_VALUE: " + result); + } + return (int) result; + } catch (Throwable t) { + throw t instanceof RuntimeException re ? re : new RuntimeException(t); + } + } + + /** + * Phase 2: copy up to {@code capacity} entries from the native registry into the + * given segment and return the number of entries actually written. + * + *

{@code out} must be 8-byte aligned and large enough to hold + * {@code capacity * QueryRegistryLayout.ENTRY_BYTES} bytes. If the native + * registry is larger than {@code capacity}, the extra entries are silently + * truncated; if smaller, the unused tail of the buffer is left untouched. + */ + public static int queryRegistrySnapshot(MemorySegment out, int capacity) { + if (capacity < 0) { + throw new IllegalArgumentException("capacity must be non-negative: " + capacity); + } + try (var call = new NativeCall()) { + long written = call.invoke(QUERY_REGISTRY_SNAPSHOT, out, (long) capacity); + return (int) written; + } + } + // ---- Stubs ---- public static byte[] sqlToSubstrait(long readerPtr, String tableName, String sql, long runtimePtr) { diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/QueryMemoryRegistry.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/QueryMemoryRegistry.java new file mode 100644 index 0000000000000..82db18052ddff --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/QueryMemoryRegistry.java @@ -0,0 +1,82 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion.nativelib; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.nativebridge.spi.NativeCall; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * Java-side view of the Rust {@code QUERY_REGISTRY}, exposing per-query memory + * tracking metrics (current + peak bytes, wall time, completion status). + * + *

Two-phase snapshot

+ *
    + *
  1. Ask the native side how many entries the registry holds via + * {@link NativeBridge#queryRegistryLen()}.
    2. + *
  2. Allocate a buffer sized for that count and hand it to + * {@link NativeBridge#queryRegistrySnapshot(java.lang.foreign.MemorySegment, int)}, + * which writes entries back-to-back in the {@link QueryRegistryLayout} + * format and returns the actual number written.
    3. + *
+ * + *

Between phase 1 and phase 2 the registry can grow — new queries register + * or complete. The native snapshot truncates silently when there are more + * entries than the buffer holds, and Java exposes the actual written count, so + * callers never see a partially-populated entry. If the registry is smaller + * than the buffer at snapshot time, only the written prefix is decoded and + * the remainder is ignored. + */ +public final class QueryMemoryRegistry { + + private static final Logger logger = LogManager.getLogger(QueryMemoryRegistry.class); + + private QueryMemoryRegistry() {} + + /** + * Snapshot the current per-query memory registry. + * + *

Returns an unmodifiable {@link List} of {@link QueryMemoryUsage} + * records — one per live or completed-but-undrained query. Ordering is + * unspecified (mirrors DashMap iteration order on the Rust side). + * + * @return zero-or-more entries; never {@code null}. + */ + public static List snapshot() { + int len = NativeBridge.queryRegistryLen(); + logger.info("[nativemem-bp] ffm.snapshot: phase-1 queryRegistryLen={}", len); + if (len <= 0) { + return Collections.emptyList(); + } + try (var call = new NativeCall()) { + // buf() returns an 8-byte aligned segment (Arena.allocate honours the + // alignment of the native layout we stride through in QueryRegistryLayout). + long bytes = (long) len * QueryRegistryLayout.ENTRY_BYTES; + if (bytes > Integer.MAX_VALUE) { + throw new IllegalStateException("Native registry too large for a single snapshot: " + len + " entries"); + } + var seg = call.buf((int) bytes); + int written = NativeBridge.queryRegistrySnapshot(seg, len); + logger.info( + "[nativemem-bp] ffm.snapshot: phase-2 queryRegistrySnapshot wrote={} entries into buffer of capacity={}", + written, + len + ); + List out = new ArrayList<>(written); + for (int i = 0; i < written; i++) { + out.add(QueryRegistryLayout.readEntry(seg, i)); + } + return Collections.unmodifiableList(out); + } + } +} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/QueryMemoryUsage.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/QueryMemoryUsage.java new file mode 100644 index 0000000000000..e3dc0f5d3a3ce --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/QueryMemoryUsage.java @@ -0,0 +1,25 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion.nativelib; + +/** + * Per-query memory tracking snapshot, decoded from the native {@code QUERY_REGISTRY}. + * + *

A single record represents one query's tracker at the moment the native snapshot + * was taken. Completed queries stay in the registry until + * {@code query_tracker::drain_completed_query} removes them, so {@link #completed()} + * distinguishes live from finished queries. + * + * @param contextId the {@code context_id} passed to {@code QueryTrackingContext::new} + * @param currentBytes bytes currently reserved by this query (saturates at {@code Long.MAX_VALUE}) + * @param peakBytes peak bytes reserved over the query's lifetime + * @param wallNanos live or frozen wall-clock duration in nanoseconds + * @param completed {@code true} once the {@code QueryTrackingContext} has been dropped + */ +public record QueryMemoryUsage(long contextId, long currentBytes, long peakBytes, long wallNanos, boolean completed) {} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/QueryRegistryLayout.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/QueryRegistryLayout.java new file mode 100644 index 0000000000000..8584d72834807 --- /dev/null +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/QueryRegistryLayout.java @@ -0,0 +1,70 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.be.datafusion.nativelib; + +import java.lang.foreign.MemoryLayout; +import java.lang.foreign.MemoryLayout.PathElement; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.StructLayout; +import java.lang.foreign.ValueLayout; + +/** + * Mirrors the Rust {@code query_tracker::WireQueryMetric} {@code #[repr(C)]} + * struct (5 × i64 = 40 bytes). A {@code df_query_registry_snapshot} call fills + * an array of these structs back-to-back into a caller-provided buffer. + * + *

Provides a decoder that reads one entry at a time by striding over the + * buffer at multiples of {@link #ENTRY_BYTES}. Field offsets are derived from + * the {@link StructLayout} so they stay in sync with any future reshuffle of + * {@code WireQueryMetric}. + */ +public final class QueryRegistryLayout { + + /** Layout of a single wire entry. */ + public static final StructLayout ENTRY_LAYOUT = MemoryLayout.structLayout( + ValueLayout.JAVA_LONG.withName("context_id"), + ValueLayout.JAVA_LONG.withName("current_bytes"), + ValueLayout.JAVA_LONG.withName("peak_bytes"), + ValueLayout.JAVA_LONG.withName("wall_nanos"), + ValueLayout.JAVA_LONG.withName("completed") + ); + + /** Byte size of one wire entry. Matches {@code size_of::()} on the Rust side. */ + public static final long ENTRY_BYTES = ENTRY_LAYOUT.byteSize(); + + private static final long OFF_CONTEXT_ID = ENTRY_LAYOUT.byteOffset(PathElement.groupElement("context_id")); + private static final long OFF_CURRENT_BYTES = ENTRY_LAYOUT.byteOffset(PathElement.groupElement("current_bytes")); + private static final long OFF_PEAK_BYTES = ENTRY_LAYOUT.byteOffset(PathElement.groupElement("peak_bytes")); + private static final long OFF_WALL_NANOS = ENTRY_LAYOUT.byteOffset(PathElement.groupElement("wall_nanos")); + private static final long OFF_COMPLETED = ENTRY_LAYOUT.byteOffset(PathElement.groupElement("completed")); + + static { + long expected = 5L * Long.BYTES; + if (ENTRY_BYTES != expected) { + throw new AssertionError("QueryRegistryLayout entry size mismatch: expected " + expected + " but got " + ENTRY_BYTES); + } + } + + private QueryRegistryLayout() {} + + /** + * Decode a single entry at zero-based index {@code i} from a buffer populated + * by {@code df_query_registry_snapshot}. + */ + public static QueryMemoryUsage readEntry(MemorySegment seg, int i) { + long base = i * ENTRY_BYTES; + return new QueryMemoryUsage( + seg.get(ValueLayout.JAVA_LONG, base + OFF_CONTEXT_ID), + seg.get(ValueLayout.JAVA_LONG, base + OFF_CURRENT_BYTES), + seg.get(ValueLayout.JAVA_LONG, base + OFF_PEAK_BYTES), + seg.get(ValueLayout.JAVA_LONG, base + OFF_WALL_NANOS), + seg.get(ValueLayout.JAVA_LONG, base + OFF_COMPLETED) != 0L + ); + } +} diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index 01fbf335a3ca0..0d0c3646287f7 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -725,6 +725,7 @@ public void apply(Settings value, Settings current, Settings previous) { NodeDuressSettings.SETTING_NUM_SUCCESSIVE_BREACHES, NodeDuressSettings.SETTING_CPU_THRESHOLD, NodeDuressSettings.SETTING_HEAP_THRESHOLD, + NodeDuressSettings.SETTING_NATIVE_MEMORY_THRESHOLD, SearchTaskSettings.SETTING_CANCELLATION_RATIO, SearchTaskSettings.SETTING_CANCELLATION_RATE, SearchTaskSettings.SETTING_CANCELLATION_BURST, @@ -734,6 +735,8 @@ public void apply(Settings value, Settings current, Settings previous) { SearchTaskSettings.SETTING_CPU_TIME_MILLIS_THRESHOLD, SearchTaskSettings.SETTING_ELAPSED_TIME_MILLIS_THRESHOLD, SearchTaskSettings.SETTING_TOTAL_HEAP_PERCENT_THRESHOLD, + SearchTaskSettings.SETTING_TOTAL_NATIVE_MEMORY_BYTES_THRESHOLD, + SearchTaskSettings.SETTING_NATIVE_MEMORY_BYTES_THRESHOLD, SearchShardTaskSettings.SETTING_CANCELLATION_RATIO, SearchShardTaskSettings.SETTING_CANCELLATION_RATE, SearchShardTaskSettings.SETTING_CANCELLATION_BURST, @@ -743,6 +746,8 @@ public void apply(Settings value, Settings current, Settings previous) { SearchShardTaskSettings.SETTING_CPU_TIME_MILLIS_THRESHOLD, SearchShardTaskSettings.SETTING_ELAPSED_TIME_MILLIS_THRESHOLD, SearchShardTaskSettings.SETTING_TOTAL_HEAP_PERCENT_THRESHOLD, + SearchShardTaskSettings.SETTING_TOTAL_NATIVE_MEMORY_BYTES_THRESHOLD, + SearchShardTaskSettings.SETTING_NATIVE_MEMORY_BYTES_THRESHOLD, SearchBackpressureSettings.SETTING_CANCELLATION_RATIO, // deprecated SearchBackpressureSettings.SETTING_CANCELLATION_RATE, // deprecated SearchBackpressureSettings.SETTING_CANCELLATION_BURST, // deprecated diff --git a/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java b/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java index 6c12857477768..cb89a299c19fb 100644 --- a/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java +++ b/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java @@ -16,7 +16,9 @@ import org.opensearch.common.lifecycle.AbstractLifecycleComponent; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; +import org.opensearch.common.util.TimeBasedExpiryTracker; import org.opensearch.monitor.jvm.JvmStats; +import org.opensearch.monitor.os.OsProbe; import org.opensearch.monitor.process.ProcessProbe; import org.opensearch.search.backpressure.settings.SearchBackpressureMode; import org.opensearch.search.backpressure.settings.SearchBackpressureSettings; @@ -28,6 +30,7 @@ import org.opensearch.search.backpressure.trackers.CpuUsageTracker; import org.opensearch.search.backpressure.trackers.ElapsedTimeTracker; import org.opensearch.search.backpressure.trackers.HeapUsageTracker; +import org.opensearch.search.backpressure.trackers.NativeMemoryUsageTracker; import org.opensearch.search.backpressure.trackers.NodeDuressTrackers; import org.opensearch.search.backpressure.trackers.NodeDuressTrackers.NodeDuressTracker; import org.opensearch.search.backpressure.trackers.TaskResourceUsageTrackerType; @@ -49,16 +52,19 @@ import java.util.ArrayList; import java.util.Collections; import java.util.EnumMap; +import java.util.EnumSet; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.function.DoubleSupplier; import java.util.function.Function; import java.util.function.LongSupplier; import java.util.stream.Collectors; import static org.opensearch.search.backpressure.trackers.HeapUsageTracker.isHeapTrackingSupported; +import static org.opensearch.search.backpressure.trackers.NativeMemoryUsageTracker.isNativeTrackingSupported; /** * SearchBackpressureService is responsible for monitoring and cancelling in-flight search tasks if they are @@ -68,13 +74,27 @@ */ public class SearchBackpressureService extends AbstractLifecycleComponent implements TaskCompletionListener { private static final Logger logger = LogManager.getLogger(SearchBackpressureService.class); + // Tracker-apply rules: + // - When native-memory duress is active, we run ONLY the native-memory and elapsed-time + // trackers. CPU and heap trackers are intentionally skipped so that a runaway native + // allocator doesn't trigger unrelated cancellation paths (heap is still low, CPU may + // not be pegged). See the dedicated short-circuit in doRun(). + // - Otherwise, CPU and heap trackers run when their corresponding resource is in duress, + // elapsed-time always runs, and the native-memory tracker runs only under its own + // native-memory duress condition. private static final Map> trackerApplyConditions = Map.of( TaskResourceUsageTrackerType.CPU_USAGE_TRACKER, (nodeDuressTrackers) -> nodeDuressTrackers.isResourceInDuress(ResourceType.CPU), TaskResourceUsageTrackerType.HEAP_USAGE_TRACKER, (nodeDuressTrackers) -> isHeapTrackingSupported() && nodeDuressTrackers.isResourceInDuress(ResourceType.MEMORY), TaskResourceUsageTrackerType.ELAPSED_TIME_TRACKER, - (nodeDuressTrackers) -> true + (nodeDuressTrackers) -> true, + TaskResourceUsageTrackerType.NATIVE_MEMORY_USAGE_TRACKER, + (nodeDuressTrackers) -> isNativeTrackingSupported() && nodeDuressTrackers.isNativeMemoryInDuress() + ); + private static final Set NATIVE_MEMORY_DURESS_TRACKERS = EnumSet.of( + TaskResourceUsageTrackerType.NATIVE_MEMORY_USAGE_TRACKER, + TaskResourceUsageTrackerType.ELAPSED_TIME_TRACKER ); private volatile Scheduler.Cancellable scheduledFuture; @@ -114,14 +134,64 @@ public SearchBackpressureService( () -> settings.getNodeDuressSettings().getNumSuccessiveBreaches() ) ); + put(ResourceType.NATIVE_MEMORY, new NodeDuressTracker(() -> { + // POC native-memory duress probe. We want a "native-only" view — the + // off-heap footprint owned by DataFusion's pool + direct buffers + + // metaspace, without the JVM heap contribution. RssAnon on Linux is the + // private-anonymous portion of the process RSS, which includes BOTH the + // JVM heap pages AND the native allocator pages. Subtracting the + // configured heap-max gives an approximation of the non-heap anonymous + // footprint; clamp to zero because early-lifecycle RssAnon can be below + // heap-max (pages not yet touched / committed). + // + // nativeBytes = max(0, rssAnon - jvmHeapMax) + // usedFraction = nativeBytes / 10 GiB + // + // The denominator is a fixed 10 GiB budget for off-heap growth — a POC + // knob that makes the short-circuit path easy to exercise end-to-end. + // Replace with a real denominator (total physical memory, cgroup limit, + // etc.) before shipping. + // + // getProcessRssAnonBytes() returns -1 on non-Linux or older kernels where + // RssAnon isn't exposed; in that case we stay out of duress so we don't + // flip the cluster into cancellation on platforms where the signal isn't + // reliable. + OsProbe osProbe = OsProbe.getInstance(); + long rssAnon = osProbe.getProcessRssAnonBytes(); + if (rssAnon < 0L) { + logger.info( + "[nativemem-bp] duress-probe: RssAnon signal unavailable (rssAnon={}B, platform or older kernel)", + rssAnon + ); + return false; + } + long jvmHeapMax = JvmStats.jvmStats().getMem().getHeapMax().getBytes(); + long nativeBytes = Math.max(0L, rssAnon - jvmHeapMax); + final long nativeDenomBytes = 10L * 1024L * 1024L * 1024L; // 10 GiB + double usedFraction = (double) nativeBytes / (double) nativeDenomBytes; + double threshold = settings.getNodeDuressSettings().getNativeMemoryThreshold(); + boolean breached = usedFraction >= threshold; + logger.info( + "[nativemem-bp] duress-probe: rssAnon={}B, jvmHeapMax={}B, nativeBytes={}B / denom=10GiB " + + "(usedFraction={}, threshold={}, breached={})", + rssAnon, + jvmHeapMax, + nativeBytes, + String.format(java.util.Locale.ROOT, "%.4f", usedFraction), + String.format(java.util.Locale.ROOT, "%.4f", threshold), + breached + ); + return breached; + }, () -> settings.getNodeDuressSettings().getNumSuccessiveBreaches())); } - }), + }, new TimeBasedExpiryTracker(System::nanoTime)), getTrackers( settings.getSearchTaskSettings()::getCpuTimeNanosThreshold, settings.getSearchTaskSettings()::getHeapVarianceThreshold, settings.getSearchTaskSettings()::getHeapPercentThreshold, settings.getSearchTaskSettings().getHeapMovingAverageWindowSize(), settings.getSearchTaskSettings()::getElapsedTimeNanosThreshold, + settings.getSearchTaskSettings()::getNativeMemoryBytesThreshold, settings.getClusterSettings(), SearchTaskSettings.SETTING_HEAP_MOVING_AVERAGE_WINDOW_SIZE ), @@ -131,6 +201,7 @@ public SearchBackpressureService( settings.getSearchShardTaskSettings()::getHeapPercentThreshold, settings.getSearchShardTaskSettings().getHeapMovingAverageWindowSize(), settings.getSearchShardTaskSettings()::getElapsedTimeNanosThreshold, + settings.getSearchShardTaskSettings()::getNativeMemoryBytesThreshold, settings.getClusterSettings(), SearchShardTaskSettings.SETTING_HEAP_MOVING_AVERAGE_WINDOW_SIZE ), @@ -195,20 +266,45 @@ void doRun() { List searchTasks = getTaskByType(SearchTask.class); List searchShardTasks = getTaskByType(SearchShardTask.class); - boolean isHeapUsageDominatedBySearchTasks = isHeapUsageDominatedBySearch( - searchTasks, - getSettings().getSearchTaskSettings().getTotalHeapPercentThreshold() - ); - boolean isHeapUsageDominatedBySearchShardTasks = isHeapUsageDominatedBySearch( - searchShardTasks, - getSettings().getSearchShardTaskSettings().getTotalHeapPercentThreshold() - ); - final Map, List> cancellableTasks = Map.of( - SearchTask.class, - isHeapUsageDominatedBySearchTasks ? searchTasks : Collections.emptyList(), - SearchShardTask.class, - isHeapUsageDominatedBySearchShardTasks ? searchShardTasks : Collections.emptyList() - ); + // Native-memory duress takes precedence: it's a symptom of off-heap pressure the heap + // share check isn't designed to diagnose, so we skip the heap-dominance gate and only + // run the native-memory + elapsed-time trackers. Heap/CPU trackers would target the + // wrong workload here. + final boolean inNativeMemoryDuress = nodeDuressTrackers.isNativeMemoryInDuress(); + if (inNativeMemoryDuress) { + logger.info( + "[nativemem-bp] doRun: native-memory duress active, short-circuiting to " + + "[NATIVE_MEMORY_USAGE_TRACKER, ELAPSED_TIME_TRACKER] — searchTasks={}, searchShardTasks={}", + searchTasks.size(), + searchShardTasks.size() + ); + } else { + logger.info( + "[nativemem-bp] doRun: node in duress (non-native-memory) — searchTasks={}, searchShardTasks={}", + searchTasks.size(), + searchShardTasks.size() + ); + } + + final Map, List> cancellableTasks; + if (inNativeMemoryDuress) { + cancellableTasks = Map.of(SearchTask.class, searchTasks, SearchShardTask.class, searchShardTasks); + } else { + boolean isHeapUsageDominatedBySearchTasks = isHeapUsageDominatedBySearch( + searchTasks, + getSettings().getSearchTaskSettings().getTotalHeapPercentThreshold() + ); + boolean isHeapUsageDominatedBySearchShardTasks = isHeapUsageDominatedBySearch( + searchShardTasks, + getSettings().getSearchShardTaskSettings().getTotalHeapPercentThreshold() + ); + cancellableTasks = Map.of( + SearchTask.class, + isHeapUsageDominatedBySearchTasks ? searchTasks : Collections.emptyList(), + SearchShardTask.class, + isHeapUsageDominatedBySearchShardTasks ? searchShardTasks : Collections.emptyList() + ); + } // Force-refresh usage stats of these tasks before making a cancellation decision. taskResourceTrackingService.refreshResourceStats(searchTasks.toArray(new Task[0])); @@ -216,7 +312,24 @@ void doRun() { List taskCancellations = new ArrayList<>(); - for (TaskResourceUsageTrackerType trackerType : TaskResourceUsageTrackerType.values()) { + // When native-memory duress is active, run ONLY the native-memory + elapsed-time + // trackers. Otherwise defer to the per-tracker duress conditions above. + Set activeTrackers = inNativeMemoryDuress + ? NATIVE_MEMORY_DURESS_TRACKERS + : EnumSet.allOf(TaskResourceUsageTrackerType.class); + // Refresh trackers that batch expensive cross-boundary reads (e.g. the + // native-memory tracker pulling a DataFusion snapshot via FFM). One refresh + // per active tracker per tick amortises the cost across every candidate task. + for (TaskResourceUsageTrackerType trackerType : activeTrackers) { + if (shouldApply(trackerType) == false) { + continue; + } + logger.info("[nativemem-bp] doRun: refreshing tracker [{}]", trackerType.getName()); + for (TaskResourceUsageTrackers trackers : taskTrackers.values()) { + trackers.getTracker(trackerType).ifPresent(TaskResourceUsageTracker::refresh); + } + } + for (TaskResourceUsageTrackerType trackerType : activeTrackers) { if (shouldApply(trackerType)) { addResourceTrackerBasedCancellations(trackerType, taskCancellations, cancellableTasks); } @@ -373,8 +486,9 @@ public static TaskResourceUsageTrackers getTrackers( DoubleSupplier heapPercentThresholdSupplier, int heapMovingAverageWindowSize, LongSupplier ElapsedTimeNanosSupplier, + LongSupplier nativeMemoryBytesThresholdSupplier, ClusterSettings clusterSettings, - Setting windowSizeSetting + Setting heapWindowSizeSetting ) { TaskResourceUsageTrackers trackers = new TaskResourceUsageTrackers(); trackers.addTracker(new CpuUsageTracker(cpuThresholdSupplier), TaskResourceUsageTrackerType.CPU_USAGE_TRACKER); @@ -385,7 +499,7 @@ public static TaskResourceUsageTrackers getTrackers( heapPercentThresholdSupplier, heapMovingAverageWindowSize, clusterSettings, - windowSizeSetting + heapWindowSizeSetting ), TaskResourceUsageTrackerType.HEAP_USAGE_TRACKER ); @@ -396,6 +510,21 @@ public static TaskResourceUsageTrackers getTrackers( new ElapsedTimeTracker(ElapsedTimeNanosSupplier, System::nanoTime), TaskResourceUsageTrackerType.ELAPSED_TIME_TRACKER ); + // Native-memory tracker pulls per-task bytes from a snapshot installed via + // NativeMemoryUsageTracker#setSnapshotSupplier (typically by a backend plugin). + // Per-task evaluation reads from the snapshot map — no FFI call per task. The + // service calls tracker.refresh() once per tick to rebuild the snapshot. + // + // Gate on isNativeTrackingSupported() so we only register the tracker on platforms + // where the duress probe and total-physical-memory readings are meaningful. + if (isNativeTrackingSupported()) { + trackers.addTracker( + new NativeMemoryUsageTracker(nativeMemoryBytesThresholdSupplier), + TaskResourceUsageTrackerType.NATIVE_MEMORY_USAGE_TRACKER + ); + } else { + logger.warn("native memory tracking not supported on this platform"); + } return trackers; } @@ -452,6 +581,19 @@ protected void doClose() throws IOException {} public SearchBackpressureStats nodeStats() { List searchTasks = getTaskByType(SearchTask.class); List searchShardTasks = getTaskByType(SearchShardTask.class); + // One refresh per tracker before stats() iterates activeTasks twice (max + avg). + // Mirrors the refresh pass in doRun() so both paths stay consistent — a tracker + // that caches an expensive cross-boundary read never has to re-read per task. + logger.info( + "[nativemem-bp] nodeStats: refreshing all trackers — searchTasks={}, searchShardTasks={}", + searchTasks.size(), + searchShardTasks.size() + ); + for (TaskResourceUsageTrackers trackers : taskTrackers.values()) { + for (TaskResourceUsageTracker tracker : trackers.all()) { + tracker.refresh(); + } + } SearchTaskStats searchTaskStats = new SearchTaskStats( searchBackpressureStates.get(SearchTask.class).getCancellationCount(), searchBackpressureStates.get(SearchTask.class).getLimitReachedCount(), diff --git a/server/src/main/java/org/opensearch/search/backpressure/settings/NodeDuressSettings.java b/server/src/main/java/org/opensearch/search/backpressure/settings/NodeDuressSettings.java index 09c1e4fcef46c..72acdb0eaeb87 100644 --- a/server/src/main/java/org/opensearch/search/backpressure/settings/NodeDuressSettings.java +++ b/server/src/main/java/org/opensearch/search/backpressure/settings/NodeDuressSettings.java @@ -22,6 +22,10 @@ private static class Defaults { private static final int NUM_SUCCESSIVE_BREACHES = 3; private static final double CPU_THRESHOLD = 0.9; private static final double HEAP_THRESHOLD = 0.7; + // Trip native-memory duress when (totalPhysical - memAvailable) / totalPhysical + // exceeds this fraction. Default 0.85 — tighter than HEAP_THRESHOLD because + // native allocations bypass GC and hit the OS scheduler directly. + private static final double NATIVE_MEMORY_THRESHOLD = 0.85; } /** @@ -62,6 +66,27 @@ private static class Defaults { Setting.Property.NodeScope ); + /** + * Defines the physical-memory usage threshold (as a fraction of total physical memory) above + * which a node is considered "in duress" for native memory. The probe computes used bytes as + * {@code totalPhysicalMemory - memAvailableFromProcMeminfo} and compares + * {@code used / totalPhysicalMemory} against this threshold; when the comparison holds for + * {@link #numSuccessiveBreaches} consecutive observations the node is marked in duress. + * + *

This gate is independent of heap duress: backends that manage memory outside the JVM + * heap (e.g. DataFusion's memory pool) can cancel search tasks when physical RAM is tight + * even if the heap itself is nowhere near its threshold. + */ + private volatile double nativeMemoryThreshold; + public static final Setting SETTING_NATIVE_MEMORY_THRESHOLD = Setting.doubleSetting( + "search_backpressure.node_duress.native_memory_threshold", + Defaults.NATIVE_MEMORY_THRESHOLD, + 0.0, + 1.0, + Setting.Property.Dynamic, + Setting.Property.NodeScope + ); + public NodeDuressSettings(Settings settings, ClusterSettings clusterSettings) { numSuccessiveBreaches = SETTING_NUM_SUCCESSIVE_BREACHES.get(settings); clusterSettings.addSettingsUpdateConsumer(SETTING_NUM_SUCCESSIVE_BREACHES, this::setNumSuccessiveBreaches); @@ -71,6 +96,9 @@ public NodeDuressSettings(Settings settings, ClusterSettings clusterSettings) { heapThreshold = SETTING_HEAP_THRESHOLD.get(settings); clusterSettings.addSettingsUpdateConsumer(SETTING_HEAP_THRESHOLD, this::setHeapThreshold); + + nativeMemoryThreshold = SETTING_NATIVE_MEMORY_THRESHOLD.get(settings); + clusterSettings.addSettingsUpdateConsumer(SETTING_NATIVE_MEMORY_THRESHOLD, this::setNativeMemoryThreshold); } public int getNumSuccessiveBreaches() { @@ -96,4 +124,12 @@ public double getHeapThreshold() { private void setHeapThreshold(double heapThreshold) { this.heapThreshold = heapThreshold; } + + public double getNativeMemoryThreshold() { + return nativeMemoryThreshold; + } + + private void setNativeMemoryThreshold(double nativeMemoryThreshold) { + this.nativeMemoryThreshold = nativeMemoryThreshold; + } } diff --git a/server/src/main/java/org/opensearch/search/backpressure/settings/SearchShardTaskSettings.java b/server/src/main/java/org/opensearch/search/backpressure/settings/SearchShardTaskSettings.java index 38213506c55b7..7125369ff185d 100644 --- a/server/src/main/java/org/opensearch/search/backpressure/settings/SearchShardTaskSettings.java +++ b/server/src/main/java/org/opensearch/search/backpressure/settings/SearchShardTaskSettings.java @@ -34,6 +34,10 @@ private static class Defaults { private static final double HEAP_PERCENT_THRESHOLD = 0.005; private static final double HEAP_VARIANCE_THRESHOLD = 2.0; private static final int HEAP_MOVING_AVERAGE_WINDOW_SIZE = 100; + // See SearchTaskSettings.Defaults for rationale: 0 keeps the native-memory tracker + // inert until an operator or backend plugin opts in by setting a non-zero threshold. + private static final long TOTAL_NATIVE_MEMORY_BYTES_THRESHOLD = 0L; + private static final long NATIVE_MEMORY_BYTES_THRESHOLD = 0L; } /** @@ -161,6 +165,33 @@ private static class Defaults { Setting.Property.NodeScope ); + /** + * Defines the native-memory threshold (in bytes) for the sum of native-memory usages across + * all search shard tasks before in-flight cancellation is applied. {@code 0} disables the + * check. + */ + private volatile long totalNativeMemoryBytesThreshold; + public static final Setting SETTING_TOTAL_NATIVE_MEMORY_BYTES_THRESHOLD = Setting.longSetting( + "search_backpressure.search_shard_task.total_native_memory_bytes_threshold", + Defaults.TOTAL_NATIVE_MEMORY_BYTES_THRESHOLD, + 0L, + Setting.Property.Dynamic, + Setting.Property.NodeScope + ); + + /** + * Defines the native-memory threshold (in bytes) for an individual search shard task before + * it is considered for cancellation. {@code 0} disables the check. + */ + private volatile long nativeMemoryBytesThreshold; + public static final Setting SETTING_NATIVE_MEMORY_BYTES_THRESHOLD = Setting.longSetting( + "search_backpressure.search_shard_task.native_memory_bytes_threshold", + Defaults.NATIVE_MEMORY_BYTES_THRESHOLD, + 0L, + Setting.Property.Dynamic, + Setting.Property.NodeScope + ); + public SearchShardTaskSettings(Settings settings, ClusterSettings clusterSettings) { totalHeapPercentThreshold = SETTING_TOTAL_HEAP_PERCENT_THRESHOLD.get(settings); this.cpuTimeMillisThreshold = SETTING_CPU_TIME_MILLIS_THRESHOLD.get(settings); @@ -171,6 +202,8 @@ public SearchShardTaskSettings(Settings settings, ClusterSettings clusterSetting this.cancellationRatio = SETTING_CANCELLATION_RATIO.get(settings); this.cancellationRate = SETTING_CANCELLATION_RATE.get(settings); this.cancellationBurst = SETTING_CANCELLATION_BURST.get(settings); + this.totalNativeMemoryBytesThreshold = SETTING_TOTAL_NATIVE_MEMORY_BYTES_THRESHOLD.get(settings); + this.nativeMemoryBytesThreshold = SETTING_NATIVE_MEMORY_BYTES_THRESHOLD.get(settings); clusterSettings.addSettingsUpdateConsumer(SETTING_TOTAL_HEAP_PERCENT_THRESHOLD, this::setTotalHeapPercentThreshold); clusterSettings.addSettingsUpdateConsumer(SETTING_CPU_TIME_MILLIS_THRESHOLD, this::setCpuTimeMillisThreshold); @@ -181,6 +214,8 @@ public SearchShardTaskSettings(Settings settings, ClusterSettings clusterSetting clusterSettings.addSettingsUpdateConsumer(SETTING_CANCELLATION_RATIO, this::setCancellationRatio); clusterSettings.addSettingsUpdateConsumer(SETTING_CANCELLATION_RATE, this::setCancellationRate); clusterSettings.addSettingsUpdateConsumer(SETTING_CANCELLATION_BURST, this::setCancellationBurst); + clusterSettings.addSettingsUpdateConsumer(SETTING_TOTAL_NATIVE_MEMORY_BYTES_THRESHOLD, this::setTotalNativeMemoryBytesThreshold); + clusterSettings.addSettingsUpdateConsumer(SETTING_NATIVE_MEMORY_BYTES_THRESHOLD, this::setNativeMemoryBytesThreshold); } public double getTotalHeapPercentThreshold() { @@ -231,6 +266,22 @@ public void setHeapMovingAverageWindowSize(int heapMovingAverageWindowSize) { this.heapMovingAverageWindowSize = heapMovingAverageWindowSize; } + public long getTotalNativeMemoryBytesThreshold() { + return totalNativeMemoryBytesThreshold; + } + + private void setTotalNativeMemoryBytesThreshold(long totalNativeMemoryBytesThreshold) { + this.totalNativeMemoryBytesThreshold = totalNativeMemoryBytesThreshold; + } + + public long getNativeMemoryBytesThreshold() { + return nativeMemoryBytesThreshold; + } + + private void setNativeMemoryBytesThreshold(long nativeMemoryBytesThreshold) { + this.nativeMemoryBytesThreshold = nativeMemoryBytesThreshold; + } + public double getCancellationRatio() { return cancellationRatio; } diff --git a/server/src/main/java/org/opensearch/search/backpressure/settings/SearchTaskSettings.java b/server/src/main/java/org/opensearch/search/backpressure/settings/SearchTaskSettings.java index f9af7f9b59fdb..62bc8c0e1e017 100644 --- a/server/src/main/java/org/opensearch/search/backpressure/settings/SearchTaskSettings.java +++ b/server/src/main/java/org/opensearch/search/backpressure/settings/SearchTaskSettings.java @@ -38,6 +38,11 @@ private static class Defaults { private static final double HEAP_PERCENT_THRESHOLD = 0.02; private static final double HEAP_VARIANCE_THRESHOLD = 2.0; private static final int HEAP_MOVING_AVERAGE_WINDOW_SIZE = 100; + // Native-memory tracking is opt-in. Zero keeps the tracker inert (see + // NativeMemoryUsageTracker); raise either threshold to engage cancellation for + // native-memory-heavy tasks while the node is in native-memory duress. + private static final long TOTAL_NATIVE_MEMORY_BYTES_THRESHOLD = 0L; + private static final long NATIVE_MEMORY_BYTES_THRESHOLD = 0L; } /** @@ -165,6 +170,32 @@ private static class Defaults { Setting.Property.NodeScope ); + /** + * Defines the native-memory threshold (in bytes) for the sum of native-memory usages across all + * search tasks before in-flight cancellation is applied. {@code 0} disables the check. + */ + private volatile long totalNativeMemoryBytesThreshold; + public static final Setting SETTING_TOTAL_NATIVE_MEMORY_BYTES_THRESHOLD = Setting.longSetting( + "search_backpressure.search_task.total_native_memory_bytes_threshold", + Defaults.TOTAL_NATIVE_MEMORY_BYTES_THRESHOLD, + 0L, + Setting.Property.Dynamic, + Setting.Property.NodeScope + ); + + /** + * Defines the native-memory threshold (in bytes) for an individual search task before it is + * considered for cancellation. {@code 0} disables the check. + */ + private volatile long nativeMemoryBytesThreshold; + public static final Setting SETTING_NATIVE_MEMORY_BYTES_THRESHOLD = Setting.longSetting( + "search_backpressure.search_task.native_memory_bytes_threshold", + Defaults.NATIVE_MEMORY_BYTES_THRESHOLD, + 0L, + Setting.Property.Dynamic, + Setting.Property.NodeScope + ); + public SearchTaskSettings(Settings settings, ClusterSettings clusterSettings) { this.totalHeapPercentThreshold = SETTING_TOTAL_HEAP_PERCENT_THRESHOLD.get(settings); this.cpuTimeMillisThreshold = SETTING_CPU_TIME_MILLIS_THRESHOLD.get(settings); @@ -175,6 +206,8 @@ public SearchTaskSettings(Settings settings, ClusterSettings clusterSettings) { this.cancellationRatio = SETTING_CANCELLATION_RATIO.get(settings); this.cancellationRate = SETTING_CANCELLATION_RATE.get(settings); this.cancellationBurst = SETTING_CANCELLATION_BURST.get(settings); + this.totalNativeMemoryBytesThreshold = SETTING_TOTAL_NATIVE_MEMORY_BYTES_THRESHOLD.get(settings); + this.nativeMemoryBytesThreshold = SETTING_NATIVE_MEMORY_BYTES_THRESHOLD.get(settings); clusterSettings.addSettingsUpdateConsumer(SETTING_TOTAL_HEAP_PERCENT_THRESHOLD, this::setTotalHeapPercentThreshold); clusterSettings.addSettingsUpdateConsumer(SETTING_CPU_TIME_MILLIS_THRESHOLD, this::setCpuTimeMillisThreshold); @@ -185,6 +218,8 @@ public SearchTaskSettings(Settings settings, ClusterSettings clusterSettings) { clusterSettings.addSettingsUpdateConsumer(SETTING_CANCELLATION_RATIO, this::setCancellationRatio); clusterSettings.addSettingsUpdateConsumer(SETTING_CANCELLATION_RATE, this::setCancellationRate); clusterSettings.addSettingsUpdateConsumer(SETTING_CANCELLATION_BURST, this::setCancellationBurst); + clusterSettings.addSettingsUpdateConsumer(SETTING_TOTAL_NATIVE_MEMORY_BYTES_THRESHOLD, this::setTotalNativeMemoryBytesThreshold); + clusterSettings.addSettingsUpdateConsumer(SETTING_NATIVE_MEMORY_BYTES_THRESHOLD, this::setNativeMemoryBytesThreshold); } public double getTotalHeapPercentThreshold() { @@ -235,6 +270,22 @@ public void setHeapMovingAverageWindowSize(int heapMovingAverageWindowSize) { this.heapMovingAverageWindowSize = heapMovingAverageWindowSize; } + public long getTotalNativeMemoryBytesThreshold() { + return totalNativeMemoryBytesThreshold; + } + + private void setTotalNativeMemoryBytesThreshold(long totalNativeMemoryBytesThreshold) { + this.totalNativeMemoryBytesThreshold = totalNativeMemoryBytesThreshold; + } + + public long getNativeMemoryBytesThreshold() { + return nativeMemoryBytesThreshold; + } + + private void setNativeMemoryBytesThreshold(long nativeMemoryBytesThreshold) { + this.nativeMemoryBytesThreshold = nativeMemoryBytesThreshold; + } + public double getCancellationRatio() { return cancellationRatio; } diff --git a/server/src/main/java/org/opensearch/search/backpressure/stats/SearchShardTaskStats.java b/server/src/main/java/org/opensearch/search/backpressure/stats/SearchShardTaskStats.java index be714271c8919..64d60f65db60d 100644 --- a/server/src/main/java/org/opensearch/search/backpressure/stats/SearchShardTaskStats.java +++ b/server/src/main/java/org/opensearch/search/backpressure/stats/SearchShardTaskStats.java @@ -18,6 +18,7 @@ import org.opensearch.search.backpressure.trackers.CpuUsageTracker; import org.opensearch.search.backpressure.trackers.ElapsedTimeTracker; import org.opensearch.search.backpressure.trackers.HeapUsageTracker; +import org.opensearch.search.backpressure.trackers.NativeMemoryUsageTracker; import org.opensearch.search.backpressure.trackers.TaskResourceUsageTrackerType; import org.opensearch.search.backpressure.trackers.TaskResourceUsageTrackers.TaskResourceUsageTracker; @@ -59,6 +60,12 @@ public SearchShardTaskStats(StreamInput in) throws IOException { builder.put(TaskResourceUsageTrackerType.CPU_USAGE_TRACKER, in.readOptionalWriteable(CpuUsageTracker.Stats::new)); builder.put(TaskResourceUsageTrackerType.HEAP_USAGE_TRACKER, in.readOptionalWriteable(HeapUsageTracker.Stats::new)); builder.put(TaskResourceUsageTrackerType.ELAPSED_TIME_TRACKER, in.readOptionalWriteable(ElapsedTimeTracker.Stats::new)); + if (in.getVersion().onOrAfter(Version.V_3_7_0)) { + builder.put( + TaskResourceUsageTrackerType.NATIVE_MEMORY_USAGE_TRACKER, + in.readOptionalWriteable(NativeMemoryUsageTracker.Stats::new) + ); + } this.resourceUsageTrackerStats = builder.immutableMap(); } @@ -94,6 +101,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalWriteable(resourceUsageTrackerStats.get(TaskResourceUsageTrackerType.CPU_USAGE_TRACKER)); out.writeOptionalWriteable(resourceUsageTrackerStats.get(TaskResourceUsageTrackerType.HEAP_USAGE_TRACKER)); out.writeOptionalWriteable(resourceUsageTrackerStats.get(TaskResourceUsageTrackerType.ELAPSED_TIME_TRACKER)); + if (out.getVersion().onOrAfter(Version.V_3_7_0)) { + out.writeOptionalWriteable(resourceUsageTrackerStats.get(TaskResourceUsageTrackerType.NATIVE_MEMORY_USAGE_TRACKER)); + } } @Override diff --git a/server/src/main/java/org/opensearch/search/backpressure/stats/SearchTaskStats.java b/server/src/main/java/org/opensearch/search/backpressure/stats/SearchTaskStats.java index 0f5f409b15def..bf912eb0a7517 100644 --- a/server/src/main/java/org/opensearch/search/backpressure/stats/SearchTaskStats.java +++ b/server/src/main/java/org/opensearch/search/backpressure/stats/SearchTaskStats.java @@ -18,6 +18,7 @@ import org.opensearch.search.backpressure.trackers.CpuUsageTracker; import org.opensearch.search.backpressure.trackers.ElapsedTimeTracker; import org.opensearch.search.backpressure.trackers.HeapUsageTracker; +import org.opensearch.search.backpressure.trackers.NativeMemoryUsageTracker; import org.opensearch.search.backpressure.trackers.TaskResourceUsageTrackerType; import org.opensearch.search.backpressure.trackers.TaskResourceUsageTrackers.TaskResourceUsageTracker; @@ -60,6 +61,12 @@ public SearchTaskStats(StreamInput in) throws IOException { builder.put(TaskResourceUsageTrackerType.CPU_USAGE_TRACKER, in.readOptionalWriteable(CpuUsageTracker.Stats::new)); builder.put(TaskResourceUsageTrackerType.HEAP_USAGE_TRACKER, in.readOptionalWriteable(HeapUsageTracker.Stats::new)); builder.put(TaskResourceUsageTrackerType.ELAPSED_TIME_TRACKER, in.readOptionalWriteable(ElapsedTimeTracker.Stats::new)); + if (in.getVersion().onOrAfter(Version.V_3_7_0)) { + builder.put( + TaskResourceUsageTrackerType.NATIVE_MEMORY_USAGE_TRACKER, + in.readOptionalWriteable(NativeMemoryUsageTracker.Stats::new) + ); + } this.resourceUsageTrackerStats = builder.immutableMap(); } @@ -95,6 +102,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalWriteable(resourceUsageTrackerStats.get(TaskResourceUsageTrackerType.CPU_USAGE_TRACKER)); out.writeOptionalWriteable(resourceUsageTrackerStats.get(TaskResourceUsageTrackerType.HEAP_USAGE_TRACKER)); out.writeOptionalWriteable(resourceUsageTrackerStats.get(TaskResourceUsageTrackerType.ELAPSED_TIME_TRACKER)); + if (out.getVersion().onOrAfter(Version.V_3_7_0)) { + out.writeOptionalWriteable(resourceUsageTrackerStats.get(TaskResourceUsageTrackerType.NATIVE_MEMORY_USAGE_TRACKER)); + } } @Override diff --git a/server/src/main/java/org/opensearch/search/backpressure/trackers/NativeMemoryUsageTracker.java b/server/src/main/java/org/opensearch/search/backpressure/trackers/NativeMemoryUsageTracker.java new file mode 100644 index 0000000000000..b254f805435e7 --- /dev/null +++ b/server/src/main/java/org/opensearch/search/backpressure/trackers/NativeMemoryUsageTracker.java @@ -0,0 +1,248 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.backpressure.trackers; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.lucene.util.Constants; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.unit.ByteSizeValue; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.monitor.os.OsProbe; +import org.opensearch.search.backpressure.trackers.TaskResourceUsageTrackers.TaskResourceUsageTracker; +import org.opensearch.tasks.CancellableTask; +import org.opensearch.tasks.Task; +import org.opensearch.tasks.TaskCancellation; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.function.LongSupplier; +import java.util.function.Supplier; + +import static org.opensearch.search.backpressure.trackers.TaskResourceUsageTrackerType.NATIVE_MEMORY_USAGE_TRACKER; + +/** + * NativeMemoryUsageTracker cancels in-flight tasks that are holding more native (off-heap) + * memory than allowed. Unlike {@link HeapUsageTracker}, this tracker does not maintain a + * rolling average — native memory is a reservation that rises and falls over the query's + * lifetime, so an absolute byte threshold is the right shape once the node is already in + * native-memory duress (the {@link org.opensearch.search.backpressure.SearchBackpressureService} + * tracker-apply map gates on that). + * + *

Snapshot-per-tick model

+ *

Native-memory values come from a backend (e.g. DataFusion) over an FFI boundary. Per-task + * FFI reads don't scale: the backpressure service iterates every candidate task inside + * {@code doRun()}, and stats() iterates every live task twice (max + avg). Instead, the + * tracker holds a {@code Map} that is rebuilt in one shot by {@link #refresh()}. + * {@code SearchBackpressureService} calls {@code refresh()} once per cancellation-iteration + * (and once per {@code nodeStats()} call) before any per-task lookup runs. Between refreshes + * every task lookup is an O(1) hash probe. + * + *

The snapshot source is a {@link Supplier} installed by the backend plugin — typically + * calling the plugin's own {@code getActiveQueryMetrics()} equivalent and projecting the map + * down to {@code contextId -> currentBytes}. + * + * @opensearch.internal + */ +public class NativeMemoryUsageTracker extends TaskResourceUsageTracker { + private static final Logger logger = LogManager.getLogger(NativeMemoryUsageTracker.class); + + /** Empty map used when no supplier is installed or the supplier returns {@code null}. */ + private static volatile Supplier> snapshotSupplier = Collections::emptyMap; + + private final LongSupplier nativeMemoryBytesThresholdSupplier; + // Volatile so the map reference publishes safely from refresh() (called from the + // backpressure scheduler thread) to the per-task evaluate() path (same thread today, + // but also hit from nodeStats() which may run on a different thread). + private volatile Map bytesByTaskId = Collections.emptyMap(); + + public NativeMemoryUsageTracker(LongSupplier nativeMemoryBytesThresholdSupplier) { + this.nativeMemoryBytesThresholdSupplier = nativeMemoryBytesThresholdSupplier; + setDefaultResourceUsageBreachEvaluator(); + } + + /** + * Install the snapshot source. Called once from a backend plugin's {@code createComponents}. + * Last writer wins; backends that want to cooperate should compose around the previous + * value rather than overwriting. + */ + public static void setSnapshotSupplier(Supplier> supplier) { + if (supplier != null) { + snapshotSupplier = supplier; + logger.info("[nativemem-bp] tracker.setSnapshotSupplier: installed supplier [{}]", supplier.getClass().getName()); + } + } + + /** + * Cancellation rule: task's current native-memory reservation is at or above the threshold. + */ + private void setDefaultResourceUsageBreachEvaluator() { + this.resourceUsageBreachEvaluator = (task) -> { + if (task instanceof CancellableTask == false) { + return Optional.empty(); + } + long bytesThreshold = nativeMemoryBytesThresholdSupplier.getAsLong(); + if (bytesThreshold <= 0L) { + // Feature disabled — leave the tracker inert. + return Optional.empty(); + } + long currentUsage = bytesForTask(task); + if (currentUsage < bytesThreshold) { + return Optional.empty(); + } + int score = (int) Math.max(1L, currentUsage / Math.max(1L, bytesThreshold)); + logger.info( + "[nativemem-bp] evaluate: task={} exceeds threshold — currentBytes={}B, threshold={}B, score={}", + task.getId(), + currentUsage, + bytesThreshold, + score + ); + return Optional.of( + new TaskCancellation.Reason( + "native memory usage exceeded [" + + new ByteSizeValue(currentUsage) + + " >= " + + new ByteSizeValue(bytesThreshold) + + "]", + score + ) + ); + }; + } + + @Override + public String name() { + return NATIVE_MEMORY_USAGE_TRACKER.getName(); + } + + /** No-op: completion events carry no information we persist. */ + @Override + public void update(Task task) { + // intentionally empty + } + + /** + * Pull a fresh snapshot from the installed supplier and swap it in. Exactly one FFI + * call per invocation — callers (backpressure service) invoke this once per tick + * before per-task evaluation begins. + */ + @Override + public void refresh() { + Map snapshot = snapshotSupplier.get(); + bytesByTaskId = snapshot != null ? snapshot : Collections.emptyMap(); + logger.info( + "[nativemem-bp] tracker.refresh: snapshot loaded, size={} taskIds={}", + bytesByTaskId.size(), + bytesByTaskId.keySet() + ); + } + + /** Package-private for tests: look up a single task's bytes from the current snapshot. */ + long bytesForTask(Task task) { + if (task == null) { + return 0L; + } + Long bytes = bytesByTaskId.get(task.getId()); + return bytes == null ? 0L : bytes; + } + + /** + * Returns {@code true} when the node exposes the physical-memory signal this tracker + * relies on for duress. Symmetric with {@link HeapUsageTracker#isHeapTrackingSupported()}. + */ + public static boolean isNativeTrackingSupported() { + if (Constants.LINUX == false) { + return false; + } + return OsProbe.getInstance().getTotalPhysicalMemorySize() > 0L; + } + + /** + * Returns {@code true} if aggregate native-memory usage across {@code cancellableTasks} is + * at least {@code totalBytesThreshold}. Used by the backpressure service as a cheap + * upstream check before running the per-task cancellation pass. + * + *

Uses the latest snapshot installed by {@link #refresh}; callers should invoke + * {@code refresh()} before this method if they want a fresh reading. + */ + public boolean isNativeMemoryUsageDominatedBySearch(List cancellableTasks, long totalBytesThreshold) { + if (totalBytesThreshold <= 0L) { + return true; + } + long usage = cancellableTasks.stream().mapToLong(this::bytesForTask).sum(); + if (usage < totalBytesThreshold) { + logger.debug("native memory usage not dominated by search requests [{}/{}]", usage, totalBytesThreshold); + return false; + } + return true; + } + + @Override + public TaskResourceUsageTracker.Stats stats(List activeTasks) { + long currentMax = activeTasks.stream().mapToLong(this::bytesForTask).max().orElse(0); + long currentAvg = (long) activeTasks.stream().mapToLong(this::bytesForTask).average().orElse(0); + return new Stats(getCancellations(), currentMax, currentAvg); + } + + /** + * Stats for {@link NativeMemoryUsageTracker}. Serialization shape deliberately omits the + * rolling-average field that {@link HeapUsageTracker.Stats} carries: we don't maintain a + * rolling baseline for native memory, so exposing a zeroed field would be misleading. + */ + public static class Stats implements TaskResourceUsageTracker.Stats { + private final long cancellationCount; + private final long currentMax; + private final long currentAvg; + + public Stats(long cancellationCount, long currentMax, long currentAvg) { + this.cancellationCount = cancellationCount; + this.currentMax = currentMax; + this.currentAvg = currentAvg; + } + + public Stats(StreamInput in) throws IOException { + this(in.readVLong(), in.readVLong(), in.readVLong()); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return builder.startObject() + .field("cancellation_count", cancellationCount) + .humanReadableField("current_max_bytes", "current_max", new ByteSizeValue(currentMax)) + .humanReadableField("current_avg_bytes", "current_avg", new ByteSizeValue(currentAvg)) + .endObject(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVLong(cancellationCount); + out.writeVLong(currentMax); + out.writeVLong(currentAvg); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Stats stats = (Stats) o; + return cancellationCount == stats.cancellationCount && currentMax == stats.currentMax && currentAvg == stats.currentAvg; + } + + @Override + public int hashCode() { + return Objects.hash(cancellationCount, currentMax, currentAvg); + } + } +} diff --git a/server/src/main/java/org/opensearch/search/backpressure/trackers/NodeDuressTrackers.java b/server/src/main/java/org/opensearch/search/backpressure/trackers/NodeDuressTrackers.java index 2cf5f63144e9a..df64d51c18608 100644 --- a/server/src/main/java/org/opensearch/search/backpressure/trackers/NodeDuressTrackers.java +++ b/server/src/main/java/org/opensearch/search/backpressure/trackers/NodeDuressTrackers.java @@ -48,6 +48,15 @@ public boolean isResourceInDuress(ResourceType resourceType) { return resourceDuressCache.get(resourceType); } + /** + * Convenience accessor for native-memory duress. Equivalent to + * {@code isResourceInDuress(ResourceType.NATIVE_MEMORY)} but expressed as a method + * reference-friendly form used by the tracker-apply map in the backpressure service. + */ + public boolean isNativeMemoryInDuress() { + return isResourceInDuress(ResourceType.NATIVE_MEMORY); + } + /** * Method to evaluate whether the node is in duress or not * @return true if node is in duress because of either system resource @@ -63,8 +72,14 @@ public boolean isNodeInDuress() { private void updateCache() { if (nodeDuressCacheExpiryChecker.getAsBoolean()) { - for (ResourceType resourceType : ResourceType.values()) - resourceDuressCache.put(resourceType, duressTrackers.get(resourceType).test()); + // Callers may register duress trackers for only a subset of {@link ResourceType} + // values (e.g. WLM's WorkloadGroupService registers CPU + MEMORY only, leaving + // NATIVE_MEMORY unregistered). Treat absence as "not in duress" so unrelated + // call sites aren't forced to register a no-op tracker for every new enum value. + for (ResourceType resourceType : ResourceType.values()) { + NodeDuressTracker tracker = duressTrackers.get(resourceType); + resourceDuressCache.put(resourceType, tracker != null && tracker.test()); + } } } diff --git a/server/src/main/java/org/opensearch/search/backpressure/trackers/TaskResourceUsageTrackerType.java b/server/src/main/java/org/opensearch/search/backpressure/trackers/TaskResourceUsageTrackerType.java index 2211d28ad30c0..a598bc6eebb84 100644 --- a/server/src/main/java/org/opensearch/search/backpressure/trackers/TaskResourceUsageTrackerType.java +++ b/server/src/main/java/org/opensearch/search/backpressure/trackers/TaskResourceUsageTrackerType.java @@ -14,7 +14,8 @@ public enum TaskResourceUsageTrackerType { CPU_USAGE_TRACKER("cpu_usage_tracker"), HEAP_USAGE_TRACKER("heap_usage_tracker"), - ELAPSED_TIME_TRACKER("elapsed_time_tracker"); + ELAPSED_TIME_TRACKER("elapsed_time_tracker"), + NATIVE_MEMORY_USAGE_TRACKER("native_memory_usage_tracker"); private final String name; @@ -34,6 +35,8 @@ public static TaskResourceUsageTrackerType fromName(String name) { return HEAP_USAGE_TRACKER; case "elapsed_time_tracker": return ELAPSED_TIME_TRACKER; + case "native_memory_usage_tracker": + return NATIVE_MEMORY_USAGE_TRACKER; } throw new IllegalArgumentException("Invalid TaskResourceUsageTrackerType: " + name); diff --git a/server/src/main/java/org/opensearch/search/backpressure/trackers/TaskResourceUsageTrackers.java b/server/src/main/java/org/opensearch/search/backpressure/trackers/TaskResourceUsageTrackers.java index 3b0072288681c..1ac5c8e48d461 100644 --- a/server/src/main/java/org/opensearch/search/backpressure/trackers/TaskResourceUsageTrackers.java +++ b/server/src/main/java/org/opensearch/search/backpressure/trackers/TaskResourceUsageTrackers.java @@ -86,6 +86,17 @@ public long getCancellations() { return cancellations.get(); } + /** + * Rebuild any cached state this tracker maintains (e.g. a snapshot of per-task usage + * pulled across an FFI boundary). Called by the backpressure service once per + * cancellation-iteration per task class, immediately before {@link #getTaskCancellations} + * runs. Default is a no-op — override when the tracker wants to batch an expensive read + * instead of doing it per-task. + */ + public void refresh() { + // no-op by default + } + /** * Returns a unique name for this tracker. */ diff --git a/server/src/main/java/org/opensearch/wlm/ResourceType.java b/server/src/main/java/org/opensearch/wlm/ResourceType.java index a560268a66853..d73586cb3a750 100644 --- a/server/src/main/java/org/opensearch/wlm/ResourceType.java +++ b/server/src/main/java/org/opensearch/wlm/ResourceType.java @@ -12,6 +12,7 @@ import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.wlm.tracker.CpuUsageCalculator; import org.opensearch.wlm.tracker.MemoryUsageCalculator; +import org.opensearch.wlm.tracker.NativeMemoryUsageCalculator; import org.opensearch.wlm.tracker.ResourceUsageCalculator; import java.io.IOException; @@ -26,12 +27,20 @@ @PublicApi(since = "2.17.0") public enum ResourceType { CPU("cpu", true, CpuUsageCalculator.INSTANCE, WorkloadManagementSettings::getNodeLevelCpuCancellationThreshold), - MEMORY("memory", true, MemoryUsageCalculator.INSTANCE, WorkloadManagementSettings::getNodeLevelMemoryCancellationThreshold); + MEMORY("memory", true, MemoryUsageCalculator.INSTANCE, WorkloadManagementSettings::getNodeLevelMemoryCancellationThreshold), + // NATIVE_MEMORY is off-heap memory reported by native backends (e.g. DataFusion). It is + // NOT tracked per workload group — {@code statsEnabled=false} keeps WorkloadGroupState + // from allocating state slots for it, and its node-level WLM threshold is pinned at 1.0 + // so WLM cancellation never fires on this resource. The real consumer is search + // backpressure, which pulls the duress signal via NodeDuressTrackers#isResourceInDuress. + NATIVE_MEMORY("native_memory", false, NativeMemoryUsageCalculator.INSTANCE, settings -> 1.0); private final String name; private final boolean statsEnabled; private final ResourceUsageCalculator resourceUsageCalculator; private final Function nodeLevelThresholdSupplier; + // sortedValues intentionally excludes NATIVE_MEMORY — WLM XContent output and related + // consumers stayed at "CPU, MEMORY" before this enum grew, and we keep that stable. private static List sortedValues = List.of(CPU, MEMORY); ResourceType( diff --git a/server/src/main/java/org/opensearch/wlm/tracker/NativeMemoryUsageCalculator.java b/server/src/main/java/org/opensearch/wlm/tracker/NativeMemoryUsageCalculator.java new file mode 100644 index 0000000000000..09fd2cba95d9b --- /dev/null +++ b/server/src/main/java/org/opensearch/wlm/tracker/NativeMemoryUsageCalculator.java @@ -0,0 +1,40 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.wlm.tracker; + +import org.opensearch.wlm.WorkloadGroupTask; + +import java.util.List; + +/** + * Placeholder {@link ResourceUsageCalculator} for {@link org.opensearch.wlm.ResourceType#NATIVE_MEMORY}. + * + *

Workload Management does not account native (off-heap) memory per workload group — that + * live usage is reported by native backends (e.g. DataFusion) via the search backpressure + * {@code NativeMemoryUsageTracker}. The enum entry exists so {@link + * org.opensearch.search.backpressure.trackers.NodeDuressTrackers} can route native-memory + * duress through the same {@code ResourceType} keyed map as CPU and heap; this calculator + * simply returns zero for any task or task list so WLM surfaces that accidentally iterate it + * behave as "no usage". + */ +public final class NativeMemoryUsageCalculator extends ResourceUsageCalculator { + public static final NativeMemoryUsageCalculator INSTANCE = new NativeMemoryUsageCalculator(); + + private NativeMemoryUsageCalculator() {} + + @Override + public double calculateResourceUsage(List tasks) { + return 0.0; + } + + @Override + public double calculateTaskResourceUsage(WorkloadGroupTask task) { + return 0.0; + } +} diff --git a/server/src/main/java/org/opensearch/wlm/tracker/WorkloadGroupResourceUsageTrackerService.java b/server/src/main/java/org/opensearch/wlm/tracker/WorkloadGroupResourceUsageTrackerService.java index a0d39e28f77f6..b189e8442eeb2 100644 --- a/server/src/main/java/org/opensearch/wlm/tracker/WorkloadGroupResourceUsageTrackerService.java +++ b/server/src/main/java/org/opensearch/wlm/tracker/WorkloadGroupResourceUsageTrackerService.java @@ -24,7 +24,12 @@ * This class tracks resource usage per WorkloadGroup */ public class WorkloadGroupResourceUsageTrackerService { - public static final EnumSet TRACKED_RESOURCES = EnumSet.allOf(ResourceType.class); + // Explicit set (CPU + MEMORY) rather than {@code EnumSet.allOf(ResourceType.class)}. The + // NATIVE_MEMORY entry was added to {@link ResourceType} only to flow duress signals + // through NodeDuressTrackers for search backpressure; WLM does not account off-heap + // memory per workload group and iterating it here would run the no-op calculator, + // produce zero usage, and clutter stats output. + public static final EnumSet TRACKED_RESOURCES = EnumSet.of(ResourceType.CPU, ResourceType.MEMORY); private final TaskResourceTrackingService taskResourceTrackingService; /** diff --git a/server/src/test/java/org/opensearch/monitor/os/OsProbeTests.java b/server/src/test/java/org/opensearch/monitor/os/OsProbeTests.java index df4b263eb7984..b45e48e249026 100644 --- a/server/src/test/java/org/opensearch/monitor/os/OsProbeTests.java +++ b/server/src/test/java/org/opensearch/monitor/os/OsProbeTests.java @@ -511,4 +511,116 @@ boolean areCgroupStatsAvailable() { }; } + // ---- /proc/meminfo parsing ---- + + public void testParseMeminfoLineBytes_valid() { + assertEquals(123456L * 1024L, OsProbe.parseMeminfoLineBytes("MemAvailable: 123456 kB")); + assertEquals(0L, OsProbe.parseMeminfoLineBytes("MemFree: 0 kB")); + // No unit column — tolerate the trimmed form. + assertEquals(42L * 1024L, OsProbe.parseMeminfoLineBytes("MemAvailable: 42")); + // Unusual but well-formed whitespace. + assertEquals(7L * 1024L, OsProbe.parseMeminfoLineBytes("MemTotal:\t7 kB")); + } + + public void testParseMeminfoLineBytes_malformed() { + assertEquals(-1L, OsProbe.parseMeminfoLineBytes("")); + assertEquals(-1L, OsProbe.parseMeminfoLineBytes("MemAvailable")); + assertEquals(-1L, OsProbe.parseMeminfoLineBytes("MemAvailable:")); + assertEquals(-1L, OsProbe.parseMeminfoLineBytes("MemAvailable: not-a-number kB")); + assertEquals(-1L, OsProbe.parseMeminfoLineBytes("MemAvailable: -5 kB")); + } + + public void testGetFreePhysicalMemorySizeFromProcMeminfo_usesMemAvailableWhenPresent() { + assumeThat("requires Linux to exercise the /proc/meminfo path", Constants.LINUX, is(true)); + OsProbe probe = new OsProbe() { + @Override + List readProcMeminfo() { + return Arrays.asList( + "MemTotal: 16384000 kB", + "MemFree: 10000 kB", + // MemAvailable wins even when MemFree appears earlier in some layouts. + "MemAvailable: 5242880 kB", + "Buffers: 1234 kB" + ); + } + }; + assertEquals(5242880L * 1024L, probe.getFreePhysicalMemorySizeFromProcMeminfo()); + } + + public void testGetFreePhysicalMemorySizeFromProcMeminfo_fallsBackToMemFree() { + assumeThat("requires Linux to exercise the /proc/meminfo path", Constants.LINUX, is(true)); + OsProbe probe = new OsProbe() { + @Override + List readProcMeminfo() { + return Arrays.asList("MemTotal: 16384000 kB", "MemFree: 1048576 kB", "Buffers: 4096 kB"); + } + }; + assertEquals(1048576L * 1024L, probe.getFreePhysicalMemorySizeFromProcMeminfo()); + } + + public void testGetFreePhysicalMemorySizeFromProcMeminfo_returnsNegativeOnIoError() { + assumeThat("requires Linux to exercise the /proc/meminfo path", Constants.LINUX, is(true)); + OsProbe probe = new OsProbe() { + @Override + List readProcMeminfo() throws IOException { + throw new IOException("synthetic"); + } + }; + assertEquals(-1L, probe.getFreePhysicalMemorySizeFromProcMeminfo()); + } + + public void testGetFreePhysicalMemorySizeFromProcMeminfo_negativeOnNonLinux() { + assumeThat("only meaningful on non-Linux platforms", Constants.LINUX, is(false)); + assertEquals(-1L, OsProbe.getInstance().getFreePhysicalMemorySizeFromProcMeminfo()); + } + + // ---- /proc/self/status RssAnon parsing ---- + + public void testGetProcessRssAnonBytes_presentField() { + assumeThat("requires Linux to exercise the /proc/self/status path", Constants.LINUX, is(true)); + OsProbe probe = new OsProbe() { + @Override + List readProcSelfStatus() { + return Arrays.asList( + "Name:\topensearch", + "State:\tS (sleeping)", + "VmPeak:\t 12345 kB", + "VmRSS:\t 65536 kB", + "RssAnon:\t 32768 kB", + "RssFile:\t 20000 kB", + "RssShmem:\t 0 kB" + ); + } + }; + assertEquals(32768L * 1024L, probe.getProcessRssAnonBytes()); + } + + public void testGetProcessRssAnonBytes_missingFieldReturnsNegative() { + assumeThat("requires Linux to exercise the /proc/self/status path", Constants.LINUX, is(true)); + OsProbe probe = new OsProbe() { + @Override + List readProcSelfStatus() { + // RssAnon was added in kernel 4.5; pre-4.5 kernels expose only VmRSS. + return Arrays.asList("Name:\topensearch", "State:\tS (sleeping)", "VmRSS:\t 65536 kB"); + } + }; + assertEquals(-1L, probe.getProcessRssAnonBytes()); + } + + public void testGetProcessRssAnonBytes_returnsNegativeOnIoError() { + assumeThat("requires Linux to exercise the /proc/self/status path", Constants.LINUX, is(true)); + OsProbe probe = new OsProbe() { + @Override + List readProcSelfStatus() throws IOException { + throw new IOException("synthetic"); + } + }; + assertEquals(-1L, probe.getProcessRssAnonBytes()); + } + + public void testGetProcessRssAnonBytes_negativeOnNonLinux() { + assumeThat("only meaningful on non-Linux platforms", Constants.LINUX, is(false)); + assertEquals(-1L, OsProbe.getInstance().getProcessRssAnonBytes()); + } + } diff --git a/server/src/test/java/org/opensearch/search/backpressure/SearchBackpressureServiceTests.java b/server/src/test/java/org/opensearch/search/backpressure/SearchBackpressureServiceTests.java index 8e604824b73a6..3ae0f572ef094 100644 --- a/server/src/test/java/org/opensearch/search/backpressure/SearchBackpressureServiceTests.java +++ b/server/src/test/java/org/opensearch/search/backpressure/SearchBackpressureServiceTests.java @@ -607,6 +607,96 @@ private SearchBackpressureSettings getBackpressureSettings(String mode, double r ); } + public void testNativeMemoryDuressRunsOnlyNativeMemoryAndElapsedTrackers() { + // The native-memory tracker-apply condition is gated on isNativeTrackingSupported(), + // which today requires Linux + a readable /proc/meminfo. Non-Linux test hosts cannot + // exercise this short-circuit regardless of how the fixtures are wired. + assumeTrue("native memory tracking is only supported on Linux", org.apache.lucene.util.Constants.LINUX); + // When native-memory duress is active, heap/CPU trackers MUST NOT run. We install + // "always cancel" mocks for CPU and heap, and confirm nothing trips from them; the + // native-memory tracker is the only one that should contribute cancellations. + TaskManager mockTaskManager = spy(taskManager); + TaskResourceTrackingService mockTaskResourceTrackingService = mock(TaskResourceTrackingService.class); + AtomicLong mockTime = new AtomicLong(0); + LongSupplier mockTimeNanosSupplier = mockTime::get; + + // CPU/heap ResourceType duress both OFF — only the native-memory probe is firing. + EnumMap duressTrackers = new EnumMap<>(ResourceType.class) { + { + put(MEMORY, new NodeDuressTracker(() -> false, () -> 3)); + put(CPU, new NodeDuressTracker(() -> false, () -> 3)); + put(ResourceType.NATIVE_MEMORY, new NodeDuressTracker(() -> true, () -> 3)); + } + }; + NodeDuressTrackers nodeDuressTrackers = new NodeDuressTrackers(duressTrackers, resourceCacheExpiryChecker); + + // Unconditionally-cancelling CPU + heap trackers. These should never be consulted. + TaskResourceUsageTracker cpuUsageTracker = getMockedTaskResourceUsageTracker( + TaskResourceUsageTrackerType.CPU_USAGE_TRACKER, + (task) -> Optional.of(new TaskCancellation.Reason("cpu (should not fire)", 10)) + ); + TaskResourceUsageTracker heapUsageTracker = getMockedTaskResourceUsageTracker( + TaskResourceUsageTrackerType.HEAP_USAGE_TRACKER, + (task) -> Optional.of(new TaskCancellation.Reason("heap (should not fire)", 10)) + ); + // Native-memory tracker: cancels tasks whose memory bytes exceed a modest threshold. + TaskResourceUsageTracker nativeMemoryTracker = getMockedTaskResourceUsageTracker( + TaskResourceUsageTrackerType.NATIVE_MEMORY_USAGE_TRACKER, + (task) -> { + if (task.getTotalResourceStats().getMemoryInBytes() < 500) { + return Optional.empty(); + } + return Optional.of(new TaskCancellation.Reason("native memory exceeded", 5)); + } + ); + // Elapsed-time tracker left silent so the cancellation count is unambiguously from + // the native-memory tracker. + TaskResourceUsageTracker elapsedTimeTracker = getMockedTaskResourceUsageTracker( + TaskResourceUsageTrackerType.ELAPSED_TIME_TRACKER, + (task) -> Optional.empty() + ); + + TaskResourceUsageTrackers taskResourceUsageTrackers = new TaskResourceUsageTrackers(); + taskResourceUsageTrackers.addTracker(cpuUsageTracker, TaskResourceUsageTrackerType.CPU_USAGE_TRACKER); + taskResourceUsageTrackers.addTracker(heapUsageTracker, TaskResourceUsageTrackerType.HEAP_USAGE_TRACKER); + taskResourceUsageTrackers.addTracker(nativeMemoryTracker, TaskResourceUsageTrackerType.NATIVE_MEMORY_USAGE_TRACKER); + taskResourceUsageTrackers.addTracker(elapsedTimeTracker, TaskResourceUsageTrackerType.ELAPSED_TIME_TRACKER); + + SearchBackpressureSettings settings = getBackpressureSettings("enforced", 0.1, 0.003, 10.0); + SearchBackpressureService service = new SearchBackpressureService( + settings, + mockTaskResourceTrackingService, + threadPool, + mockTimeNanosSupplier, + nodeDuressTrackers, + taskResourceUsageTrackers, + new TaskResourceUsageTrackers(), + mockTaskManager, + workloadGroupService + ); + + when(workloadGroupService.shouldSBPHandle(any())).thenReturn(true); + + // Prime the native-memory duress streak (consecutive-breach window is 3). + service.doRun(); + service.doRun(); + + // Load 20 tasks; 4 are heavy (mem >= 500) so exactly 4 are native-memory cancellation + // candidates. Heap/CPU would want to cancel all 20 — verifying this test's whole point. + Map activeSearchTasks = new HashMap<>(); + for (long i = 0; i < 20; i++) { + long memoryBytes = (i % 5 == 0) ? 800 : 100; + activeSearchTasks.put(i, createMockTaskWithResourceStats(SearchTask.class, 100, memoryBytes, i)); + } + activeSearchTasks.values().forEach(task -> task.setWorkloadGroupId(threadPool.getThreadContext())); + doReturn(activeSearchTasks).when(mockTaskResourceTrackingService).getResourceAwareTasks(); + + service.doRun(); + // Exactly the heavy tasks were cancelled — no extras from the "always cancel" CPU/heap mocks. + verify(mockTaskManager, times(4)).cancelTaskAndDescendants(any(), anyString(), anyBoolean(), any()); + assertEquals(4, service.getSearchBackpressureState(SearchTask.class).getCancellationCount()); + } + private TaskResourceUsageTracker getMockedTaskResourceUsageTracker( TaskResourceUsageTrackerType type, TaskResourceUsageTracker.ResourceUsageBreachEvaluator evaluator diff --git a/server/src/test/java/org/opensearch/search/backpressure/stats/SearchShardTaskStatsTests.java b/server/src/test/java/org/opensearch/search/backpressure/stats/SearchShardTaskStatsTests.java index 45a44136d41f7..2fe026c0788b1 100644 --- a/server/src/test/java/org/opensearch/search/backpressure/stats/SearchShardTaskStatsTests.java +++ b/server/src/test/java/org/opensearch/search/backpressure/stats/SearchShardTaskStatsTests.java @@ -12,6 +12,7 @@ import org.opensearch.search.backpressure.trackers.CpuUsageTracker; import org.opensearch.search.backpressure.trackers.ElapsedTimeTracker; import org.opensearch.search.backpressure.trackers.HeapUsageTracker; +import org.opensearch.search.backpressure.trackers.NativeMemoryUsageTracker; import org.opensearch.search.backpressure.trackers.TaskResourceUsageTrackerType; import org.opensearch.search.backpressure.trackers.TaskResourceUsageTrackers.TaskResourceUsageTracker; import org.opensearch.test.AbstractWireSerializingTestCase; @@ -36,7 +37,9 @@ public static SearchShardTaskStats randomInstance() { TaskResourceUsageTrackerType.HEAP_USAGE_TRACKER, new HeapUsageTracker.Stats(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()), TaskResourceUsageTrackerType.ELAPSED_TIME_TRACKER, - new ElapsedTimeTracker.Stats(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()) + new ElapsedTimeTracker.Stats(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()), + TaskResourceUsageTrackerType.NATIVE_MEMORY_USAGE_TRACKER, + new NativeMemoryUsageTracker.Stats(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()) ); return new SearchShardTaskStats( diff --git a/server/src/test/java/org/opensearch/search/backpressure/stats/SearchTaskStatsTests.java b/server/src/test/java/org/opensearch/search/backpressure/stats/SearchTaskStatsTests.java index 3ac5cfd658fc3..31c90c19f9f1b 100644 --- a/server/src/test/java/org/opensearch/search/backpressure/stats/SearchTaskStatsTests.java +++ b/server/src/test/java/org/opensearch/search/backpressure/stats/SearchTaskStatsTests.java @@ -12,6 +12,7 @@ import org.opensearch.search.backpressure.trackers.CpuUsageTracker; import org.opensearch.search.backpressure.trackers.ElapsedTimeTracker; import org.opensearch.search.backpressure.trackers.HeapUsageTracker; +import org.opensearch.search.backpressure.trackers.NativeMemoryUsageTracker; import org.opensearch.search.backpressure.trackers.TaskResourceUsageTrackerType; import org.opensearch.search.backpressure.trackers.TaskResourceUsageTrackers.TaskResourceUsageTracker; import org.opensearch.test.AbstractWireSerializingTestCase; @@ -37,7 +38,9 @@ public static SearchTaskStats randomInstance() { TaskResourceUsageTrackerType.HEAP_USAGE_TRACKER, new HeapUsageTracker.Stats(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()), TaskResourceUsageTrackerType.ELAPSED_TIME_TRACKER, - new ElapsedTimeTracker.Stats(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()) + new ElapsedTimeTracker.Stats(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()), + TaskResourceUsageTrackerType.NATIVE_MEMORY_USAGE_TRACKER, + new NativeMemoryUsageTracker.Stats(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()) ); return new SearchTaskStats(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), resourceUsageTrackerStats); From 8ba35faf1e39cb6f2bb861ce615712b9f1063219 Mon Sep 17 00:00:00 2001 From: Pradeep L Date: Thu, 14 May 2026 02:34:02 +0530 Subject: [PATCH 02/10] Native memory search backpressure: bytes->percent rename and refactor Rename per-task native_memory_bytes_threshold to native_memory_percent_threshold to match its actual semantics. The setting was declared as Setting with a '_bytes_threshold' key but the value was being interpreted as a percent inside NativeMemoryUsageTracker. Switch to Setting bounded to [0.0, 1.0], mirroring heap_percent_threshold. Effective per-task byte threshold is now budget * fraction, where budget is the backend-installed native-memory budget. Also includes in-progress native-memory backpressure plumbing: NativeMemoryUsageTracker budget supplier, AnalyticsShardTask now extends SearchShardTask so SBP observes it, DataFusion plugin wires currentBytesByTaskId snapshot supplier, native registry top-N FFM call, and OsProbe getProcessNativeMemoryBytes helper. Several rough edges flagged in code review remain (see PR description). Signed-off-by: Pradeep L --- .../spi/AnalyticsSearchBackendPlugin.java | 4 + .../rust/src/ffm.rs | 40 +- .../rust/src/query_tracker.rs | 354 ++++++++++++------ .../be/datafusion/DataFusionPlugin.java | 48 +-- .../be/datafusion/nativelib/NativeBridge.java | 83 ++-- .../nativelib/QueryMemoryRegistry.java | 82 ---- .../nativelib/QueryMemoryUsage.java | 25 -- .../nativelib/QueryRegistryLayout.java | 79 ++-- .../exec/task/AnalyticsShardTask.java | 15 +- .../common/settings/ClusterSettings.java | 4 +- .../org/opensearch/monitor/os/OsProbe.java | 36 ++ .../SearchBackpressureService.java | 96 ++--- .../settings/SearchShardTaskSettings.java | 37 +- .../settings/SearchTaskSettings.java | 37 +- .../trackers/NativeMemoryUsageTracker.java | 110 ++++-- 15 files changed, 601 insertions(+), 449 deletions(-) delete mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/QueryMemoryRegistry.java delete mode 100644 sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/QueryMemoryUsage.java diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/AnalyticsSearchBackendPlugin.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/AnalyticsSearchBackendPlugin.java index 707284f93a5a1..15b92577508ac 100644 --- a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/AnalyticsSearchBackendPlugin.java +++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/AnalyticsSearchBackendPlugin.java @@ -121,6 +121,10 @@ default void configureFilterDelegation(FilterDelegationHandle handle, BackendExe * backend side. Implementations MUST return a non-null map (empty when nothing is tracked) * and SHOULD make it unmodifiable so callers cannot mutate backend state. * + *

Implementations MAY cap the result to a top-N subset by current memory usage to bound + * the FFI cost (the DataFusion backend caps at the heaviest 10 live queries). Callers that + * need a complete enumeration should not rely on this method. + * *

Default implementation returns an empty map so backends that do not track per-query * metrics don't have to opt in. */ diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs index e6f54e2db5799..452341af389b2 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs @@ -208,36 +208,36 @@ pub extern "C" fn df_cancel_query(context_id: i64) { } // --------------------------------------------------------------------------- -// Per-query registry snapshot (two-phase) +// Per-query registry top-N snapshot // -// Java calls `df_query_registry_len` to size a buffer, then `df_query_registry_snapshot` -// to populate it. See `query_tracker::WireQueryMetric` for the wire layout. +// One FFM call: Java allocates a buffer sized for `N` entries, Rust selects +// the heaviest live queries by `current_bytes` (bounded min-heap of size N) +// and writes them back-to-back. See `query_tracker::WireQueryMetric` for the +// wire layout. // --------------------------------------------------------------------------- -/// Returns the current number of entries in the query registry. -/// Value is racy — treat it as a sizing hint only. -#[no_mangle] -pub extern "C" fn df_query_registry_len() -> i64 { - let len = crate::query_tracker::query_registry_len(); - info!("[nativemem-bp] ffm.df_query_registry_len -> {}", len); - len as i64 -} - -/// Copies up to `cap_entries` `WireQueryMetric`s into the caller-provided buffer. +/// Copies up to `cap_entries` of the heaviest live queries (by +/// `current_bytes` desc) as `WireQueryMetric`s into the caller-provided buffer. /// Returns the number of entries actually written. /// -/// Safety: `out_ptr` must be non-null, 8-byte aligned, and point to storage for -/// at least `cap_entries * size_of::()` bytes. +/// Order of entries within the buffer is unspecified. Completed and zero-byte +/// trackers are filtered out. +/// +/// Safety: `out_ptr` must be non-null, 8-byte aligned, and point to storage +/// for at least `cap_entries * size_of::()` bytes. #[ffm_safe] #[no_mangle] -pub unsafe extern "C" fn df_query_registry_snapshot(out_ptr: *mut u8, cap_entries: i64) -> i64 { - use crate::query_tracker::{snapshot_query_registry, WireQueryMetric}; +pub unsafe extern "C" fn df_query_registry_top_n_by_current( + out_ptr: *mut u8, + cap_entries: i64, +) -> i64 { + use crate::query_tracker::{snapshot_top_n_by_current, WireQueryMetric}; if cap_entries < 0 { return Err(format!("negative capacity: {cap_entries}")); } if cap_entries == 0 { - info!("[nativemem-bp] ffm.df_query_registry_snapshot: capacity=0, nothing to write"); + info!("[nativemem-bp] ffm.df_query_registry_top_n_by_current: capacity=0, nothing to write"); return Ok(0); } if out_ptr.is_null() { @@ -245,9 +245,9 @@ pub unsafe extern "C" fn df_query_registry_snapshot(out_ptr: *mut u8, cap_entrie } let out: &mut [WireQueryMetric] = slice::from_raw_parts_mut(out_ptr as *mut WireQueryMetric, cap_entries as usize); - let written = snapshot_query_registry(out); + let written = snapshot_top_n_by_current(out); info!( - "[nativemem-bp] ffm.df_query_registry_snapshot: wrote {} entries (capacity {})", + "[nativemem-bp] ffm.df_query_registry_top_n_by_current: wrote {} entries (capacity {})", written, cap_entries ); Ok(written as i64) diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/query_tracker.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/query_tracker.rs index 9f5ece0bf578e..03333f25ceb58 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/query_tracker.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/query_tracker.rs @@ -174,10 +174,10 @@ pub fn drain_completed_query(context_id: i64) -> Option> { } // --------------------------------------------------------------------------- -// Registry snapshot — two-phase FFM export +// Registry snapshot — top-N FFM export // --------------------------------------------------------------------------- -/// Wire representation of a single query's tracker, used by the two-phase +/// Wire representation of a single query's tracker, used by the top-N /// snapshot API. Fields are `i64` so the struct crosses the FFM boundary with /// a stable, alignment-safe layout (8-byte aligned on every target we support). /// @@ -220,46 +220,112 @@ impl WireQueryMetric { } } -/// Phase 1 of the snapshot API: return the current number of entries in the -/// registry. Java uses this to allocate a buffer for phase 2. +/// Top-N snapshot: copy at most `out.len()` of the heaviest live queries +/// (ranked by `current_bytes` descending) into `out`. Returns the number of +/// entries actually written. /// -/// The value is racy by design — queries may register or be drained between -/// phase 1 and phase 2. Phase 2 handles both cases: if there are more entries -/// than the buffer holds, the extra entries are truncated; if there are -/// fewer, the unused tail of the buffer is left untouched and the actual -/// count is returned. -pub fn query_registry_len() -> usize { - let n = QUERY_REGISTRY.len(); - info!("[nativemem-bp] rust.query_registry_len = {}", n); - n -} - -/// Phase 2 of the snapshot API: copy up to `cap_entries` entries from the -/// registry into `out`. Returns the number of entries actually written. +/// Selection is done with a bounded min-heap of capacity `out.len()`, so a +/// single pass over the registry runs in `O(M log N)` where `M` is the +/// registry size and `N = out.len()`. The heap holds the *top N candidates* +/// seen so far; the smallest of those sits at the root, and any new entry +/// heavier than that root replaces it. /// -/// `out` may be a raw slice pointing at a Java-owned buffer; it must be -/// valid for writes of `cap_entries * size_of::()` bytes -/// and properly aligned for `WireQueryMetric` (8-byte aligned). Entries are -/// collected in the order DashMap's iterator returns them, which is not -/// otherwise specified. -pub fn snapshot_query_registry(out: &mut [WireQueryMetric]) -> usize { - let mut written = 0usize; +/// Order of entries written into `out` is unspecified — heap order, not +/// sorted. Callers that need a sorted view sort the prefix client-side. +/// +/// Filters applied on the Rust side, in this order: +/// - completed trackers are skipped (they retain `current_bytes == 0` until +/// `drain_completed_query` runs anyway, so this is mostly a fast path) +/// - `current_bytes == 0` is skipped (registered but un-allocated trackers +/// are not "heavy" candidates) +/// +/// `out` may be a raw slice pointing at a Java-owned buffer; it must be valid +/// for writes of `out.len() * size_of::()` bytes and properly +/// aligned for `WireQueryMetric` (8-byte aligned). +pub fn snapshot_top_n_by_current(out: &mut [WireQueryMetric]) -> usize { + use std::cmp::{Ordering, Reverse}; + use std::collections::BinaryHeap; + + /// Heap entry — orders purely by `bytes`. The tracker rides along but is + /// not part of the comparison, so `QueryTracker` doesn't need `Ord`. + struct HeapEntry { + bytes: i64, + tracker: Arc, + } + + impl Eq for HeapEntry {} + impl PartialEq for HeapEntry { + fn eq(&self, other: &Self) -> bool { + self.bytes == other.bytes + } + } + impl Ord for HeapEntry { + fn cmp(&self, other: &Self) -> Ordering { + self.bytes.cmp(&other.bytes) + } + } + impl PartialOrd for HeapEntry { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } + } + + let n = out.len(); + if n == 0 { + return 0; + } + + // BinaryHeap is a max-heap; Reverse flips it so the smallest current_bytes + // sits at the root. That root is the weakest member of the current top-N + // set — the one a heavier candidate displaces. + let mut heap: BinaryHeap> = BinaryHeap::with_capacity(n); + for entry in QUERY_REGISTRY.iter() { - if written >= out.len() { - break; + let tracker = entry.value(); + if tracker.is_completed() { + continue; } - let metric = WireQueryMetric::from_tracker(entry.value()); + let bytes = usize_to_i64_saturating(tracker.memory_pool.current_bytes()); + if bytes == 0 { + continue; + } + if heap.len() < n { + heap.push(Reverse(HeapEntry { + bytes, + tracker: Arc::clone(tracker), + })); + } else if let Some(Reverse(min)) = heap.peek() { + // Strictly greater: equal-byte ties keep the first-seen entry. + if bytes > min.bytes { + heap.pop(); + heap.push(Reverse(HeapEntry { + bytes, + tracker: Arc::clone(tracker), + })); + } + } + } + + let mut written = 0usize; + for Reverse(entry) in heap.into_iter() { + // Re-read the tracker fields here so the wire entry reflects the live + // values at materialization time, consistent with from_tracker. Ranking + // and final values are independent point-in-time samples. + out[written] = WireQueryMetric::from_tracker(&entry.tracker); info!( - "[nativemem-bp] rust.snapshot entry[{}]: ctx={}, current={}B, peak={}B, wall_ns={}, completed={}", - written, metric.context_id, metric.current_bytes, metric.peak_bytes, metric.wall_nanos, metric.completed + "[nativemem-bp] rust.snapshot_top_n entry[{}]: ctx={}, current={}B, peak={}B, wall_ns={}, completed={}", + written, + out[written].context_id, + out[written].current_bytes, + out[written].peak_bytes, + out[written].wall_nanos, + out[written].completed, ); - out[written] = metric; written += 1; } info!( - "[nativemem-bp] rust.snapshot_query_registry: wrote {} entries (buffer cap {})", - written, - out.len() + "[nativemem-bp] rust.snapshot_top_n_by_current: wrote {} entries (cap {})", + written, n ); written } @@ -596,35 +662,14 @@ mod tests { } // ----------------------------------------------------------------------- - // Two-phase snapshot tests + // Top-N snapshot tests // ----------------------------------------------------------------------- - #[test] - fn test_snapshot_captures_live_and_completed() { - let global = make_global_pool(1_000_000); - let live_id = 60_000; - let done_id = 60_001; - - let live_ctx = QueryTrackingContext::new(live_id, Arc::clone(&global)); - let live_pool = live_ctx.memory_pool().unwrap(); - let pool: Arc = live_pool.clone(); - let mut reservation = make_reservation(&pool, "live"); - reservation.try_grow(4096).unwrap(); - - let done_ctx = QueryTrackingContext::new(done_id, Arc::clone(&global)); - let done_pool = done_ctx.memory_pool().unwrap(); - let done_pool_dyn: Arc = done_pool.clone(); - let mut done_reservation = make_reservation(&done_pool_dyn, "done"); - done_reservation.try_grow(2048).unwrap(); - drop(done_reservation); - drop(done_ctx); - - // Phase 1 - let len = query_registry_len(); - assert!(len >= 2, "expected at least 2 entries, got {len}"); - - // Phase 2 with exact capacity - let mut buf = vec![ + /// Helper: drain all live tracker contexts so each top-N test starts from + /// a clean registry. Tests in this module are tagged with unique id ranges + /// but the registry is process-wide, so we only assert on contexts we own. + fn fresh_buf(n: usize) -> Vec { + vec![ WireQueryMetric { context_id: 0, current_bytes: 0, @@ -632,66 +677,117 @@ mod tests { wall_nanos: 0, completed: 0, }; - len - ]; - let written = snapshot_query_registry(&mut buf); - assert_eq!(written, len); - - let live = buf.iter().find(|m| m.context_id == live_id).expect("live not captured"); - assert_eq!(live.current_bytes, 4096); - assert_eq!(live.peak_bytes, 4096); - assert_eq!(live.completed, 0); - assert!(live.wall_nanos >= 0); - - let done = buf.iter().find(|m| m.context_id == done_id).expect("done not captured"); - assert_eq!(done.current_bytes, 0); - assert_eq!(done.peak_bytes, 2048); - assert_eq!(done.completed, 1); - assert!(done.wall_nanos > 0); + n + ] + } - drop(reservation); - drop(live_ctx); - QUERY_REGISTRY.remove(&live_id); - QUERY_REGISTRY.remove(&done_id); + #[test] + fn test_top_n_zero_capacity_returns_zero() { + let mut buf: Vec = Vec::new(); + let written = snapshot_top_n_by_current(&mut buf); + assert_eq!(written, 0); + } + + #[test] + fn test_top_n_picks_highest_current_bytes() { + let global = make_global_pool(10_000_000); + let id_lo = 70_000; + let id_md = 70_001; + let id_hi = 70_002; + + let ctx_lo = QueryTrackingContext::new(id_lo, Arc::clone(&global)); + let ctx_md = QueryTrackingContext::new(id_md, Arc::clone(&global)); + let ctx_hi = QueryTrackingContext::new(id_hi, Arc::clone(&global)); + + let pool_lo: Arc = ctx_lo.memory_pool().unwrap(); + let pool_md: Arc = ctx_md.memory_pool().unwrap(); + let pool_hi: Arc = ctx_hi.memory_pool().unwrap(); + + let mut r_lo = make_reservation(&pool_lo, "lo"); + let mut r_md = make_reservation(&pool_md, "md"); + let mut r_hi = make_reservation(&pool_hi, "hi"); + r_lo.try_grow(1_000).unwrap(); + r_md.try_grow(5_000).unwrap(); + r_hi.try_grow(9_000).unwrap(); + + // Top-2 must contain ids hi and md but not lo. Other parallel tests + // may push entries into the registry, so we filter to our id range. + let mut buf = fresh_buf(2); + let written = snapshot_top_n_by_current(&mut buf); + assert!(written >= 1 && written <= 2); + + let our: Vec<&WireQueryMetric> = buf[..written] + .iter() + .filter(|m| (id_lo..=id_hi).contains(&m.context_id)) + .collect(); + // At minimum, id_hi (the heaviest) must be in our slice if any of + // ours made it in. + if !our.is_empty() { + assert!(our.iter().any(|m| m.context_id == id_hi)); + assert!(!our.iter().any(|m| m.context_id == id_lo)); + } + + drop(r_lo); + drop(r_md); + drop(r_hi); + drop(ctx_lo); + drop(ctx_md); + drop(ctx_hi); + QUERY_REGISTRY.remove(&id_lo); + QUERY_REGISTRY.remove(&id_md); + QUERY_REGISTRY.remove(&id_hi); } #[test] - fn test_snapshot_truncates_when_buffer_is_smaller_than_registry() { + fn test_top_n_skips_completed_and_zero_byte_trackers() { let global = make_global_pool(1_000_000); - let ids: Vec = (60_100..60_105).collect(); - let contexts: Vec = ids + let live_id = 70_100; + let zero_id = 70_101; + let done_id = 70_102; + + let live_ctx = QueryTrackingContext::new(live_id, Arc::clone(&global)); + let live_pool: Arc = live_ctx.memory_pool().unwrap(); + let mut r_live = make_reservation(&live_pool, "live"); + r_live.try_grow(4_096).unwrap(); + + // Registered but never reserved — current_bytes stays 0. + let _zero_ctx = QueryTrackingContext::new(zero_id, Arc::clone(&global)); + + // Completed before snapshot. Drop reservation first so QueryMemoryPool + // is settled, then drop the context to flip the completed flag. + let done_ctx = QueryTrackingContext::new(done_id, Arc::clone(&global)); + let done_pool: Arc = done_ctx.memory_pool().unwrap(); + let mut r_done = make_reservation(&done_pool, "done"); + r_done.try_grow(8_192).unwrap(); + drop(r_done); + drop(done_ctx); + + let mut buf = fresh_buf(8); + let written = snapshot_top_n_by_current(&mut buf); + + let our: Vec<&WireQueryMetric> = buf[..written] .iter() - .map(|id| QueryTrackingContext::new(*id, Arc::clone(&global))) + .filter(|m| [live_id, zero_id, done_id].contains(&m.context_id)) .collect(); + assert!(our.iter().any(|m| m.context_id == live_id)); + assert!(!our.iter().any(|m| m.context_id == zero_id)); + assert!(!our.iter().any(|m| m.context_id == done_id)); - let mut buf = vec![ - WireQueryMetric { - context_id: 0, - current_bytes: 0, - peak_bytes: 0, - wall_nanos: 0, - completed: 0, - }; - 2 - ]; - let written = snapshot_query_registry(&mut buf); - assert_eq!(written, 2, "buffer should cap the write count"); - // QUERY_REGISTRY is process-wide. Other parallel tests may contribute entries to - // this 2-slot buffer, so the only invariant we can assert here is "no more than - // two entries were written". The more specific "every entry belongs to this - // test's id range" check is inherently racy; don't re-introduce it. - - drop(contexts); - for id in &ids { - QUERY_REGISTRY.remove(id); - } + drop(r_live); + drop(live_ctx); + QUERY_REGISTRY.remove(&live_id); + QUERY_REGISTRY.remove(&zero_id); + QUERY_REGISTRY.remove(&done_id); } #[test] - fn test_snapshot_into_oversized_buffer_leaves_tail_untouched() { + fn test_top_n_with_buffer_larger_than_live_set() { let global = make_global_pool(1_000_000); - let id = 60_200; + let id = 70_200; let ctx = QueryTrackingContext::new(id, global); + let pool: Arc = ctx.memory_pool().unwrap(); + let mut r = make_reservation(&pool, "only"); + r.try_grow(2_048).unwrap(); let sentinel = WireQueryMetric { context_id: -1, @@ -700,16 +796,48 @@ mod tests { wall_nanos: -1, completed: -1, }; - let mut buf = vec![sentinel; query_registry_len() + 4]; - let written = snapshot_query_registry(&mut buf); + let mut buf = vec![sentinel; 16]; + let written = snapshot_top_n_by_current(&mut buf); assert!(written >= 1); - assert!(written < buf.len(), "should not fill the oversized buffer"); - // Trailing slots keep the sentinel — verifies snapshot did not touch them. - for entry in &buf[written..] { - assert_eq!(entry.context_id, -1); + assert!(written < buf.len()); + // Tail past `written` keeps the sentinel — proves the snapshot did not + // touch it. + for slot in &buf[written..] { + assert_eq!(slot.context_id, -1); } + drop(r); drop(ctx); QUERY_REGISTRY.remove(&id); } + + #[test] + fn test_top_n_caps_at_buffer_capacity() { + let global = make_global_pool(10_000_000); + let ids: Vec = (70_300..70_310).collect(); + let mut contexts = Vec::with_capacity(ids.len()); + let mut reservations = Vec::with_capacity(ids.len()); + for (i, id) in ids.iter().enumerate() { + let ctx = QueryTrackingContext::new(*id, Arc::clone(&global)); + let pool: Arc = ctx.memory_pool().unwrap(); + let mut r = make_reservation(&pool, "cap"); + r.try_grow((i as usize + 1) * 1_000).unwrap(); + reservations.push(r); + contexts.push(ctx); + } + + let mut buf = fresh_buf(3); + let written = snapshot_top_n_by_current(&mut buf); + assert!(written <= 3, "must not exceed buffer capacity"); + + for r in reservations.drain(..) { + drop(r); + } + for ctx in contexts.drain(..) { + drop(ctx); + } + for id in &ids { + QUERY_REGISTRY.remove(id); + } + } } diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java index f3be8042fc32c..e99dafbabce9b 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java @@ -14,8 +14,7 @@ import org.opensearch.analytics.spi.QueryExecutionMetrics; import org.opensearch.be.datafusion.action.DataFusionStatsAction; import org.opensearch.be.datafusion.cache.CacheSettings; -import org.opensearch.be.datafusion.nativelib.QueryMemoryRegistry; -import org.opensearch.be.datafusion.nativelib.QueryMemoryUsage; +import org.opensearch.be.datafusion.nativelib.NativeBridge; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.service.ClusterService; @@ -48,7 +47,6 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.function.Supplier; @@ -113,6 +111,14 @@ public class DataFusionPlugin extends Plugin implements SearchBackEndPlugin createComponents( // (see ShardScanInstructionHandler / DatafusionSearchExecEngine), so the map is // already keyed by Task#getId on the consumer side. logger.info("[nativemem-bp] plugin: installing native-memory snapshot supplier for backpressure tracker"); - NativeMemoryUsageTracker.setSnapshotSupplier(this::snapshotNativeMemoryByTaskId); + NativeMemoryUsageTracker.setSnapshotSupplier(this::currentBytesByTaskId); + NativeMemoryUsageTracker.setNativeMemoryBudgetSupplier( + () -> DATAFUSION_MEMORY_POOL_LIMIT.get(clusterService.getSettings()) + ); this.substraitExtensions = loadSubstraitExtensions(); @@ -179,10 +188,12 @@ public Collection createComponents( /** * Project the active-query metrics map down to {@code taskId -> currentBytes} for the - * backpressure native-memory tracker. One FFM snapshot per call. Returns an empty map - * when the service isn't running, so startup/shutdown races don't surface bad data. + * backpressure native-memory tracker. One FFM snapshot per call, capped at + * {@link #ACTIVE_QUERY_METRICS_TOP_N} entries (the heaviest live queries). + * Returns an empty map when the service isn't running, so startup/shutdown races + * don't surface bad data. */ - private Map snapshotNativeMemoryByTaskId() { + private Map currentBytesByTaskId() { if (dataFusionService == null) { logger.info("[nativemem-bp] plugin.snapshot: service not running, returning empty map"); return Collections.emptyMap(); @@ -342,28 +353,21 @@ public void close() throws IOException { * *

Each entry mirrors one {@code QueryTracker} on the Rust side — current and peak memory * reservation, wall time, and whether the query has completed but not yet been drained. - * Ordering follows {@link QueryMemoryRegistry#snapshot()} (insertion-agnostic); a - * {@link LinkedHashMap} preserves that ordering for callers that want deterministic iteration - * in tests or logs. + * The map contains at most {@link #ACTIVE_QUERY_METRICS_TOP_N} entries — the heaviest live + * queries by {@code current_bytes}, selected on the Rust side. Iteration order matches the + * order Rust drained the bounded min-heap (unspecified but stable per snapshot). */ @Override public Map getActiveQueryMetrics() { if (dataFusionService == null) { return Collections.emptyMap(); } - List usages = QueryMemoryRegistry.snapshot(); - if (usages.isEmpty()) { + Map result = NativeBridge.queryRegistryTopN(ACTIVE_QUERY_METRICS_TOP_N); + if (result.isEmpty()) { logger.info("[nativemem-bp] plugin.getActiveQueryMetrics: native registry empty"); - return Collections.emptyMap(); - } - Map result = new LinkedHashMap<>(usages.size()); - for (QueryMemoryUsage u : usages) { - // DashMap keys are unique, but defensively keep the first snapshot — a duplicate - // would only occur if context ids were reused, which QueryTrackingContext::new - // already treats as a caller bug. - result.putIfAbsent(u.contextId(), new QueryExecutionMetrics(u.currentBytes(), u.peakBytes(), u.wallNanos(), u.completed())); + } else { + logger.info("[nativemem-bp] plugin.getActiveQueryMetrics: decoded {} entries from native registry", result.size()); } - logger.info("[nativemem-bp] plugin.getActiveQueryMetrics: decoded {} entries from native registry", result.size()); - return Collections.unmodifiableMap(result); + return result; } } diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java index 50f8161ae6e79..893e3a9432c52 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java @@ -9,6 +9,7 @@ package org.opensearch.be.datafusion.nativelib; import org.opensearch.analytics.backend.jni.NativeHandle; +import org.opensearch.analytics.spi.QueryExecutionMetrics; import org.opensearch.be.datafusion.stats.DataFusionStats; import org.opensearch.be.datafusion.stats.NativeExecutorsStats; import org.opensearch.be.datafusion.stats.TaskMonitorStats; @@ -19,12 +20,13 @@ import java.lang.foreign.FunctionDescriptor; import java.lang.foreign.Linker; -import java.lang.foreign.MemorySegment; import java.lang.foreign.SymbolLookup; import java.lang.foreign.ValueLayout; import java.lang.invoke.MethodHandle; +import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; /** * FFM bridge to native DataFusion library. @@ -85,6 +87,7 @@ public final class NativeBridge { private static final MethodHandle EXECUTE_WITH_CONTEXT; private static final MethodHandle CANCEL_QUERY; private static final MethodHandle STATS; + private static final MethodHandle QUERY_REGISTRY_TOP_N_BY_CURRENT; private static final MethodHandle PREPARE_PARTIAL_PLAN; private static final MethodHandle PREPARE_FINAL_PLAN; private static final MethodHandle EXECUTE_LOCAL_PREPARED_PLAN; @@ -410,6 +413,9 @@ public final class NativeBridge { FunctionDescriptor.of(ValueLayout.JAVA_LONG, ValueLayout.ADDRESS, ValueLayout.JAVA_LONG) ); + // i64 df_query_registry_top_n_by_current(out_ptr, cap_entries) + QUERY_REGISTRY_TOP_N_BY_CURRENT = linker.downcallHandle( + lib.find("df_query_registry_top_n_by_current").orElseThrow(), // ── Distributed aggregate: prepare partial/final plans ── // i64 df_prepare_partial_plan(handle_ptr, bytes_ptr, bytes_len) PREPARE_PARTIAL_PLAN = linker.downcallHandle( @@ -724,43 +730,58 @@ public static DataFusionStats stats() { } } - // ---- Per-query registry snapshot (two-phase) ---- + // ---- Per-query registry top-N snapshot ---- /** - * Phase 1: return the current size of the per-query registry on the native side. + * Snapshot the {@code n} heaviest live queries by {@code current_bytes}. * - *

The value is a sizing hint only — it can go stale the moment it's returned - * because queries may register or drain concurrently. {@link #queryRegistrySnapshot} - * handles both over- and under-sized buffers safely. - */ - public static int queryRegistryLen() { - try { - long result = NativeLibraryLoader.checkResult((long) QUERY_REGISTRY_LEN.invokeExact()); - if (result > Integer.MAX_VALUE) { - throw new IllegalStateException("Native registry length exceeds Integer.MAX_VALUE: " + result); - } - return (int) result; - } catch (Throwable t) { - throw t instanceof RuntimeException re ? re : new RuntimeException(t); - } - } - - /** - * Phase 2: copy up to {@code capacity} entries from the native registry into the - * given segment and return the number of entries actually written. + *

One FFM call: Java allocates an {@code n × ENTRY_BYTES} segment, Rust + * runs a bounded min-heap of size {@code n} over its registry and writes + * the survivors as {@link QueryRegistryLayout} entries. Completed and + * zero-byte trackers are filtered on the Rust side. Order within the + * buffer is unspecified. + * + *

Mirrors the {@link #stats()} pattern: the layout class decodes + * directly into the caller's final type ({@link QueryExecutionMetrics}) + * with no transport-only intermediate. * - *

{@code out} must be 8-byte aligned and large enough to hold - * {@code capacity * QueryRegistryLayout.ENTRY_BYTES} bytes. If the native - * registry is larger than {@code capacity}, the extra entries are silently - * truncated; if smaller, the unused tail of the buffer is left untouched. + * @param n maximum entries to return; must be non-negative. Zero short-circuits + * without crossing the FFM boundary. + * @return a non-null map of {@code contextId → metrics}, possibly empty, + * with at most {@code n} entries. + * @throws IllegalArgumentException if {@code n} is negative or implies a + * buffer larger than {@link Integer#MAX_VALUE} bytes */ - public static int queryRegistrySnapshot(MemorySegment out, int capacity) { - if (capacity < 0) { - throw new IllegalArgumentException("capacity must be non-negative: " + capacity); + public static Map queryRegistryTopN(int n) { + if (n < 0) { + throw new IllegalArgumentException("n must be non-negative: " + n); + } + if (n == 0) { + return Collections.emptyMap(); + } + long bytes = (long) n * QueryRegistryLayout.ENTRY_BYTES; + if (bytes > Integer.MAX_VALUE) { + throw new IllegalArgumentException("n too large for a single snapshot: " + n); } try (var call = new NativeCall()) { - long written = call.invoke(QUERY_REGISTRY_SNAPSHOT, out, (long) capacity); - return (int) written; + var seg = call.buf((int) bytes); + long written = call.invoke(QUERY_REGISTRY_TOP_N_BY_CURRENT, seg, (long) n); + if (written == 0L) { + return Collections.emptyMap(); + } + // Defensive: Rust contract is `written <= cap_entries`. A higher value + // would mean the buffer was overrun, so refuse to decode past `n`. + int rows = (int) Math.min(written, (long) n); + // LinkedHashMap preserves the heap-drain order Rust used so logs and + // tests see stable iteration; values are O(1) regardless. + Map out = new LinkedHashMap<>(rows); + for (int i = 0; i < rows; i++) { + long ctxId = QueryRegistryLayout.readContextId(seg, i); + // putIfAbsent guards against context_id reuse, which is a Rust-side + // caller bug but cheap to defend against here. + out.putIfAbsent(ctxId, QueryRegistryLayout.readMetrics(seg, i)); + } + return Collections.unmodifiableMap(out); } } diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/QueryMemoryRegistry.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/QueryMemoryRegistry.java deleted file mode 100644 index 82db18052ddff..0000000000000 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/QueryMemoryRegistry.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.be.datafusion.nativelib; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.opensearch.nativebridge.spi.NativeCall; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -/** - * Java-side view of the Rust {@code QUERY_REGISTRY}, exposing per-query memory - * tracking metrics (current + peak bytes, wall time, completion status). - * - *

Two-phase snapshot

- *
    - *
  1. Ask the native side how many entries the registry holds via - * {@link NativeBridge#queryRegistryLen()}.
    2. - *
  2. Allocate a buffer sized for that count and hand it to - * {@link NativeBridge#queryRegistrySnapshot(java.lang.foreign.MemorySegment, int)}, - * which writes entries back-to-back in the {@link QueryRegistryLayout} - * format and returns the actual number written.
    3. - *
- * - *

Between phase 1 and phase 2 the registry can grow — new queries register - * or complete. The native snapshot truncates silently when there are more - * entries than the buffer holds, and Java exposes the actual written count, so - * callers never see a partially-populated entry. If the registry is smaller - * than the buffer at snapshot time, only the written prefix is decoded and - * the remainder is ignored. - */ -public final class QueryMemoryRegistry { - - private static final Logger logger = LogManager.getLogger(QueryMemoryRegistry.class); - - private QueryMemoryRegistry() {} - - /** - * Snapshot the current per-query memory registry. - * - *

Returns an unmodifiable {@link List} of {@link QueryMemoryUsage} - * records — one per live or completed-but-undrained query. Ordering is - * unspecified (mirrors DashMap iteration order on the Rust side). - * - * @return zero-or-more entries; never {@code null}. - */ - public static List snapshot() { - int len = NativeBridge.queryRegistryLen(); - logger.info("[nativemem-bp] ffm.snapshot: phase-1 queryRegistryLen={}", len); - if (len <= 0) { - return Collections.emptyList(); - } - try (var call = new NativeCall()) { - // buf() returns an 8-byte aligned segment (Arena.allocate honours the - // alignment of the native layout we stride through in QueryRegistryLayout). - long bytes = (long) len * QueryRegistryLayout.ENTRY_BYTES; - if (bytes > Integer.MAX_VALUE) { - throw new IllegalStateException("Native registry too large for a single snapshot: " + len + " entries"); - } - var seg = call.buf((int) bytes); - int written = NativeBridge.queryRegistrySnapshot(seg, len); - logger.info( - "[nativemem-bp] ffm.snapshot: phase-2 queryRegistrySnapshot wrote={} entries into buffer of capacity={}", - written, - len - ); - List out = new ArrayList<>(written); - for (int i = 0; i < written; i++) { - out.add(QueryRegistryLayout.readEntry(seg, i)); - } - return Collections.unmodifiableList(out); - } - } -} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/QueryMemoryUsage.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/QueryMemoryUsage.java deleted file mode 100644 index e3dc0f5d3a3ce..0000000000000 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/QueryMemoryUsage.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.be.datafusion.nativelib; - -/** - * Per-query memory tracking snapshot, decoded from the native {@code QUERY_REGISTRY}. - * - *

A single record represents one query's tracker at the moment the native snapshot - * was taken. Completed queries stay in the registry until - * {@code query_tracker::drain_completed_query} removes them, so {@link #completed()} - * distinguishes live from finished queries. - * - * @param contextId the {@code context_id} passed to {@code QueryTrackingContext::new} - * @param currentBytes bytes currently reserved by this query (saturates at {@code Long.MAX_VALUE}) - * @param peakBytes peak bytes reserved over the query's lifetime - * @param wallNanos live or frozen wall-clock duration in nanoseconds - * @param completed {@code true} once the {@code QueryTrackingContext} has been dropped - */ -public record QueryMemoryUsage(long contextId, long currentBytes, long peakBytes, long wallNanos, boolean completed) {} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/QueryRegistryLayout.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/QueryRegistryLayout.java index 8584d72834807..5ed36d4ae13f9 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/QueryRegistryLayout.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/QueryRegistryLayout.java @@ -8,21 +8,32 @@ package org.opensearch.be.datafusion.nativelib; +import org.opensearch.analytics.spi.QueryExecutionMetrics; + import java.lang.foreign.MemoryLayout; import java.lang.foreign.MemoryLayout.PathElement; import java.lang.foreign.MemorySegment; +import java.lang.foreign.SequenceLayout; import java.lang.foreign.StructLayout; import java.lang.foreign.ValueLayout; +import java.lang.invoke.VarHandle; /** * Mirrors the Rust {@code query_tracker::WireQueryMetric} {@code #[repr(C)]} - * struct (5 × i64 = 40 bytes). A {@code df_query_registry_snapshot} call fills - * an array of these structs back-to-back into a caller-provided buffer. + * struct (5 × i64 = 40 bytes) and decodes a strided buffer of those structs + * directly into the SPI types consumed by + * {@link org.opensearch.analytics.spi.AnalyticsSearchBackendPlugin#getActiveQueryMetrics}. + * + *

This is the same idiom {@code StatsLayout} uses for the {@code df_stats} + * call: a {@link StructLayout} describes the wire shape, cached + * {@link VarHandle}s read individual fields, and the decoder produces the + * caller's final type with no transport-only intermediate. * - *

Provides a decoder that reads one entry at a time by striding over the - * buffer at multiples of {@link #ENTRY_BYTES}. Field offsets are derived from - * the {@link StructLayout} so they stay in sync with any future reshuffle of - * {@code WireQueryMetric}. + *

Buffer shape (populated by {@code df_query_registry_top_n_by_current}): + * 

+ *   [ entry 0 ][ entry 1 ] ... [ entry N-1 ]
+ *   each entry = { context_id, current_bytes, peak_bytes, wall_nanos, completed } (5 × i64)
+ *
*/ public final class QueryRegistryLayout { @@ -38,11 +49,18 @@ public final class QueryRegistryLayout { /** Byte size of one wire entry. Matches {@code size_of::()} on the Rust side. */ public static final long ENTRY_BYTES = ENTRY_LAYOUT.byteSize(); - private static final long OFF_CONTEXT_ID = ENTRY_LAYOUT.byteOffset(PathElement.groupElement("context_id")); - private static final long OFF_CURRENT_BYTES = ENTRY_LAYOUT.byteOffset(PathElement.groupElement("current_bytes")); - private static final long OFF_PEAK_BYTES = ENTRY_LAYOUT.byteOffset(PathElement.groupElement("peak_bytes")); - private static final long OFF_WALL_NANOS = ENTRY_LAYOUT.byteOffset(PathElement.groupElement("wall_nanos")); - private static final long OFF_COMPLETED = ENTRY_LAYOUT.byteOffset(PathElement.groupElement("completed")); + /** + * Sequence layout used to derive per-row {@link VarHandle}s. The first path + * element is a {@code sequenceElement()} (the array index), the second is + * the field name within {@code ENTRY_LAYOUT}. + */ + private static final SequenceLayout ROWS = MemoryLayout.sequenceLayout(Long.MAX_VALUE / ENTRY_BYTES, ENTRY_LAYOUT); + + private static final VarHandle CONTEXT_ID = rowHandle("context_id"); + private static final VarHandle CURRENT_BYTES = rowHandle("current_bytes"); + private static final VarHandle PEAK_BYTES = rowHandle("peak_bytes"); + private static final VarHandle WALL_NANOS = rowHandle("wall_nanos"); + private static final VarHandle COMPLETED = rowHandle("completed"); static { long expected = 5L * Long.BYTES; @@ -54,17 +72,36 @@ public final class QueryRegistryLayout { private QueryRegistryLayout() {} /** - * Decode a single entry at zero-based index {@code i} from a buffer populated - * by {@code df_query_registry_snapshot}. + * Read the {@code context_id} of the entry at zero-based index {@code i}. + * + * @param seg buffer populated by {@code df_query_registry_top_n_by_current} + * @param i row index, must be {@code < written} returned by the FFI call */ - public static QueryMemoryUsage readEntry(MemorySegment seg, int i) { - long base = i * ENTRY_BYTES; - return new QueryMemoryUsage( - seg.get(ValueLayout.JAVA_LONG, base + OFF_CONTEXT_ID), - seg.get(ValueLayout.JAVA_LONG, base + OFF_CURRENT_BYTES), - seg.get(ValueLayout.JAVA_LONG, base + OFF_PEAK_BYTES), - seg.get(ValueLayout.JAVA_LONG, base + OFF_WALL_NANOS), - seg.get(ValueLayout.JAVA_LONG, base + OFF_COMPLETED) != 0L + public static long readContextId(MemorySegment seg, int i) { + return (long) CONTEXT_ID.get(seg, 0L, (long) i); + } + + /** + * Read the metrics fields of the entry at zero-based index {@code i} into + * a fresh {@link QueryExecutionMetrics} record. + * + * @param seg buffer populated by {@code df_query_registry_top_n_by_current} + * @param i row index, must be {@code < written} returned by the FFI call + */ + public static QueryExecutionMetrics readMetrics(MemorySegment seg, int i) { + long row = (long) i; + return new QueryExecutionMetrics( + (long) CURRENT_BYTES.get(seg, 0L, row), + (long) PEAK_BYTES.get(seg, 0L, row), + (long) WALL_NANOS.get(seg, 0L, row), + (long) COMPLETED.get(seg, 0L, row) != 0L ); } + + private static VarHandle rowHandle(String field) { + // sequenceElement() leaves the row index as a free coordinate, so the + // returned VarHandle takes (segment, base, rowIndex) — we always pass + // base=0 because the FFM call writes at the start of the segment. + return ROWS.varHandle(PathElement.sequenceElement(), PathElement.groupElement(field)); + } } diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/task/AnalyticsShardTask.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/task/AnalyticsShardTask.java index 8bed052216aa2..8155091958a94 100644 --- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/task/AnalyticsShardTask.java +++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/task/AnalyticsShardTask.java @@ -8,19 +8,26 @@ package org.opensearch.analytics.exec.task; +import org.opensearch.action.search.SearchShardTask; import org.opensearch.core.tasks.TaskId; -import org.opensearch.tasks.CancellableTask; import java.util.Map; /** * Data-node shard task representing a single shard fragment execution. - * Analogous to {@link org.opensearch.action.search.SearchShardTask}. - * Cancelling this task does not cascade to children. + *

Extends {@link SearchShardTask} so that search backpressure and + * {@code TaskResourceTrackingService} observe and track analytics shard + * tasks alongside regular search shard tasks. The native-memory tracker + * in {@code SearchBackpressureService} filters by {@code SearchShardTask}, + * so inheriting from it is the integration point that exposes per-query + * DataFusion memory to cancellation. + * + *

Cancelling this task does not cascade to children (inherited behaviour + * from {@link SearchShardTask}). * * @opensearch.internal */ -public class AnalyticsShardTask extends CancellableTask { +public class AnalyticsShardTask extends SearchShardTask { public AnalyticsShardTask(long id, String type, String action, String description, TaskId parentTaskId, Map headers) { super(id, type, action, description, parentTaskId, headers); diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index 0d0c3646287f7..663669f59fe57 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -736,7 +736,7 @@ public void apply(Settings value, Settings current, Settings previous) { SearchTaskSettings.SETTING_ELAPSED_TIME_MILLIS_THRESHOLD, SearchTaskSettings.SETTING_TOTAL_HEAP_PERCENT_THRESHOLD, SearchTaskSettings.SETTING_TOTAL_NATIVE_MEMORY_BYTES_THRESHOLD, - SearchTaskSettings.SETTING_NATIVE_MEMORY_BYTES_THRESHOLD, + SearchTaskSettings.SETTING_NATIVE_MEMORY_PERCENT_THRESHOLD, SearchShardTaskSettings.SETTING_CANCELLATION_RATIO, SearchShardTaskSettings.SETTING_CANCELLATION_RATE, SearchShardTaskSettings.SETTING_CANCELLATION_BURST, @@ -747,7 +747,7 @@ public void apply(Settings value, Settings current, Settings previous) { SearchShardTaskSettings.SETTING_ELAPSED_TIME_MILLIS_THRESHOLD, SearchShardTaskSettings.SETTING_TOTAL_HEAP_PERCENT_THRESHOLD, SearchShardTaskSettings.SETTING_TOTAL_NATIVE_MEMORY_BYTES_THRESHOLD, - SearchShardTaskSettings.SETTING_NATIVE_MEMORY_BYTES_THRESHOLD, + SearchShardTaskSettings.SETTING_NATIVE_MEMORY_PERCENT_THRESHOLD, SearchBackpressureSettings.SETTING_CANCELLATION_RATIO, // deprecated SearchBackpressureSettings.SETTING_CANCELLATION_RATE, // deprecated SearchBackpressureSettings.SETTING_CANCELLATION_BURST, // deprecated diff --git a/server/src/main/java/org/opensearch/monitor/os/OsProbe.java b/server/src/main/java/org/opensearch/monitor/os/OsProbe.java index 39a2026af0ad6..159d7cdaf78a3 100644 --- a/server/src/main/java/org/opensearch/monitor/os/OsProbe.java +++ b/server/src/main/java/org/opensearch/monitor/os/OsProbe.java @@ -38,6 +38,7 @@ import org.opensearch.common.SuppressForbidden; import org.opensearch.common.io.PathUtils; import org.opensearch.monitor.Probes; +import org.opensearch.monitor.jvm.JvmStats; import java.io.IOException; import java.lang.management.ManagementFactory; @@ -301,6 +302,41 @@ long readRssAnonFromProcSelfStatus() throws IOException { return -1L; } + /** + * Estimates the process's native (off-heap) anonymous memory footprint in bytes — + * {@link #getProcessRssAnonBytes()} minus the JVM heap maximum, clamped at zero. + * + *

{@code RssAnon} on Linux includes both committed JVM heap pages and native + * allocator pages (malloc, Rust allocator, direct buffers, metaspace). Subtracting + * {@code -Xmx} gives a rough "everything outside the heap" view that's useful for + * detecting native-side pressure without watching mmapped index files. + * + *

The result is approximate by construction: + *

    + *
  • Early in a process's life, RssAnon can be lower than {@code -Xmx} because + * not every heap page has been touched / committed yet — the clamp avoids a + * negative reading.
    • + *
  • Heap pages that have been swapped out or unmapped also depress the + * reading, biasing the estimate low.
    • + *
+ * + *

On non-Linux platforms or older kernels where {@code RssAnon} isn't exposed, + * returns {@code -1} so callers can detect unsupported environments rather than + * treating the absence as zero pressure. + * + * @return estimated native (off-heap anonymous) bytes, or {@code -1} if the + * underlying RssAnon signal is unavailable on this platform / kernel + */ + public long getProcessNativeMemoryBytes() { + long rssAnon = getProcessRssAnonBytes(); + if (rssAnon < 0L) { + return -1L; + } + long jvmHeapMax = JvmStats.jvmStats().getMem().getHeapMax().getBytes(); + return Math.max(0L, rssAnon - jvmHeapMax); + } + + public short getSystemCpuPercent() { return Probes.getLoadAndScaleToPercent(getSystemCpuLoad, osMxBean); } diff --git a/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java b/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java index cb89a299c19fb..87b7620364a3c 100644 --- a/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java +++ b/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java @@ -92,10 +92,7 @@ public class SearchBackpressureService extends AbstractLifecycleComponent implem TaskResourceUsageTrackerType.NATIVE_MEMORY_USAGE_TRACKER, (nodeDuressTrackers) -> isNativeTrackingSupported() && nodeDuressTrackers.isNativeMemoryInDuress() ); - private static final Set NATIVE_MEMORY_DURESS_TRACKERS = EnumSet.of( - TaskResourceUsageTrackerType.NATIVE_MEMORY_USAGE_TRACKER, - TaskResourceUsageTrackerType.ELAPSED_TIME_TRACKER - ); + private volatile Scheduler.Cancellable scheduledFuture; private final SearchBackpressureSettings settings; @@ -135,48 +132,22 @@ public SearchBackpressureService( ) ); put(ResourceType.NATIVE_MEMORY, new NodeDuressTracker(() -> { - // POC native-memory duress probe. We want a "native-only" view — the - // off-heap footprint owned by DataFusion's pool + direct buffers + - // metaspace, without the JVM heap contribution. RssAnon on Linux is the - // private-anonymous portion of the process RSS, which includes BOTH the - // JVM heap pages AND the native allocator pages. Subtracting the - // configured heap-max gives an approximation of the non-heap anonymous - // footprint; clamp to zero because early-lifecycle RssAnon can be below - // heap-max (pages not yet touched / committed). - // - // nativeBytes = max(0, rssAnon - jvmHeapMax) - // usedFraction = nativeBytes / 10 GiB - // - // The denominator is a fixed 10 GiB budget for off-heap growth — a POC - // knob that makes the short-circuit path easy to exercise end-to-end. - // Replace with a real denominator (total physical memory, cgroup limit, - // etc.) before shipping. - // - // getProcessRssAnonBytes() returns -1 on non-Linux or older kernels where - // RssAnon isn't exposed; in that case we stay out of duress so we don't - // flip the cluster into cancellation on platforms where the signal isn't - // reliable. - OsProbe osProbe = OsProbe.getInstance(); - long rssAnon = osProbe.getProcessRssAnonBytes(); - if (rssAnon < 0L) { - logger.info( - "[nativemem-bp] duress-probe: RssAnon signal unavailable (rssAnon={}B, platform or older kernel)", - rssAnon - ); + // Native-memory duress probe. OsProbe.getProcessNativeMemoryFraction() + // returns nativeBytes / NATIVE_MEMORY_DENOMINATOR_BYTES on Linux, or -1.0 + // when the underlying RssAnon signal is unavailable (non-Linux or + // kernels < 4.5). The denominator is a POC constant in OsProbe — replace + // with total physical memory or a cgroup limit before shipping. + double used = OsProbe.getInstance().getProcessNativeMemoryBytes(); + double totalNative = settings.getSearchTaskSettings().getTotalNativeMemoryBytesThreshold(); + double usedFraction = used / totalNative; + if (usedFraction < 0.0d) { + logger.info("[nativemem-bp] duress-probe: native memory signal unavailable"); return false; } - long jvmHeapMax = JvmStats.jvmStats().getMem().getHeapMax().getBytes(); - long nativeBytes = Math.max(0L, rssAnon - jvmHeapMax); - final long nativeDenomBytes = 10L * 1024L * 1024L * 1024L; // 10 GiB - double usedFraction = (double) nativeBytes / (double) nativeDenomBytes; double threshold = settings.getNodeDuressSettings().getNativeMemoryThreshold(); boolean breached = usedFraction >= threshold; logger.info( - "[nativemem-bp] duress-probe: rssAnon={}B, jvmHeapMax={}B, nativeBytes={}B / denom=10GiB " - + "(usedFraction={}, threshold={}, breached={})", - rssAnon, - jvmHeapMax, - nativeBytes, + "[nativemem-bp] duress-probe: usedFraction={}, threshold={}, breached={}", String.format(java.util.Locale.ROOT, "%.4f", usedFraction), String.format(java.util.Locale.ROOT, "%.4f", threshold), breached @@ -191,7 +162,7 @@ public SearchBackpressureService( settings.getSearchTaskSettings()::getHeapPercentThreshold, settings.getSearchTaskSettings().getHeapMovingAverageWindowSize(), settings.getSearchTaskSettings()::getElapsedTimeNanosThreshold, - settings.getSearchTaskSettings()::getNativeMemoryBytesThreshold, + settings.getSearchTaskSettings()::getNativeMemoryPercentThreshold, settings.getClusterSettings(), SearchTaskSettings.SETTING_HEAP_MOVING_AVERAGE_WINDOW_SIZE ), @@ -201,7 +172,7 @@ public SearchBackpressureService( settings.getSearchShardTaskSettings()::getHeapPercentThreshold, settings.getSearchShardTaskSettings().getHeapMovingAverageWindowSize(), settings.getSearchShardTaskSettings()::getElapsedTimeNanosThreshold, - settings.getSearchShardTaskSettings()::getNativeMemoryBytesThreshold, + settings.getSearchShardTaskSettings()::getNativeMemoryPercentThreshold, settings.getClusterSettings(), SearchShardTaskSettings.SETTING_HEAP_MOVING_AVERAGE_WINDOW_SIZE ), @@ -286,10 +257,8 @@ void doRun() { ); } - final Map, List> cancellableTasks; - if (inNativeMemoryDuress) { - cancellableTasks = Map.of(SearchTask.class, searchTasks, SearchShardTask.class, searchShardTasks); - } else { + Map, List> cancellableTasks; + boolean isHeapUsageDominatedBySearchTasks = isHeapUsageDominatedBySearch( searchTasks, getSettings().getSearchTaskSettings().getTotalHeapPercentThreshold() @@ -304,6 +273,15 @@ void doRun() { SearchShardTask.class, isHeapUsageDominatedBySearchShardTasks ? searchShardTasks : Collections.emptyList() ); + + if (inNativeMemoryDuress) { + cancellableTasks = Map.of(SearchTask.class, searchTasks, SearchShardTask.class, searchShardTasks); + if (shouldApply(TaskResourceUsageTrackerType.NATIVE_MEMORY_USAGE_TRACKER)) { + logger.info("[nativemem-bp] doRun: refreshing tracker [{}]", TaskResourceUsageTrackerType.NATIVE_MEMORY_USAGE_TRACKER.getName()); + for (TaskResourceUsageTrackers trackers : taskTrackers.values()) { + trackers.getTracker(TaskResourceUsageTrackerType.NATIVE_MEMORY_USAGE_TRACKER).ifPresent(TaskResourceUsageTracker::refresh); + } + } } // Force-refresh usage stats of these tasks before making a cancellation decision. @@ -311,25 +289,7 @@ void doRun() { taskResourceTrackingService.refreshResourceStats(searchShardTasks.toArray(new Task[0])); List taskCancellations = new ArrayList<>(); - - // When native-memory duress is active, run ONLY the native-memory + elapsed-time - // trackers. Otherwise defer to the per-tracker duress conditions above. - Set activeTrackers = inNativeMemoryDuress - ? NATIVE_MEMORY_DURESS_TRACKERS - : EnumSet.allOf(TaskResourceUsageTrackerType.class); - // Refresh trackers that batch expensive cross-boundary reads (e.g. the - // native-memory tracker pulling a DataFusion snapshot via FFM). One refresh - // per active tracker per tick amortises the cost across every candidate task. - for (TaskResourceUsageTrackerType trackerType : activeTrackers) { - if (shouldApply(trackerType) == false) { - continue; - } - logger.info("[nativemem-bp] doRun: refreshing tracker [{}]", trackerType.getName()); - for (TaskResourceUsageTrackers trackers : taskTrackers.values()) { - trackers.getTracker(trackerType).ifPresent(TaskResourceUsageTracker::refresh); - } - } - for (TaskResourceUsageTrackerType trackerType : activeTrackers) { + for (TaskResourceUsageTrackerType trackerType : TaskResourceUsageTrackerType.values()) { if (shouldApply(trackerType)) { addResourceTrackerBasedCancellations(trackerType, taskCancellations, cancellableTasks); } @@ -486,7 +446,7 @@ public static TaskResourceUsageTrackers getTrackers( DoubleSupplier heapPercentThresholdSupplier, int heapMovingAverageWindowSize, LongSupplier ElapsedTimeNanosSupplier, - LongSupplier nativeMemoryBytesThresholdSupplier, + DoubleSupplier nativeMemoryPercentThresholdSupplier, ClusterSettings clusterSettings, Setting heapWindowSizeSetting ) { @@ -519,7 +479,7 @@ public static TaskResourceUsageTrackers getTrackers( // where the duress probe and total-physical-memory readings are meaningful. if (isNativeTrackingSupported()) { trackers.addTracker( - new NativeMemoryUsageTracker(nativeMemoryBytesThresholdSupplier), + new NativeMemoryUsageTracker(nativeMemoryPercentThresholdSupplier), TaskResourceUsageTrackerType.NATIVE_MEMORY_USAGE_TRACKER ); } else { diff --git a/server/src/main/java/org/opensearch/search/backpressure/settings/SearchShardTaskSettings.java b/server/src/main/java/org/opensearch/search/backpressure/settings/SearchShardTaskSettings.java index 7125369ff185d..c17c80e57120c 100644 --- a/server/src/main/java/org/opensearch/search/backpressure/settings/SearchShardTaskSettings.java +++ b/server/src/main/java/org/opensearch/search/backpressure/settings/SearchShardTaskSettings.java @@ -34,10 +34,12 @@ private static class Defaults { private static final double HEAP_PERCENT_THRESHOLD = 0.005; private static final double HEAP_VARIANCE_THRESHOLD = 2.0; private static final int HEAP_MOVING_AVERAGE_WINDOW_SIZE = 100; - // See SearchTaskSettings.Defaults for rationale: 0 keeps the native-memory tracker + // See SearchTaskSettings.Defaults for rationale: 0 / 0.0 keep the native-memory tracker // inert until an operator or backend plugin opts in by setting a non-zero threshold. + // The per-task threshold is a fraction of the backend-installed native memory budget, + // mirroring HEAP_PERCENT_THRESHOLD. private static final long TOTAL_NATIVE_MEMORY_BYTES_THRESHOLD = 0L; - private static final long NATIVE_MEMORY_BYTES_THRESHOLD = 0L; + private static final double NATIVE_MEMORY_PERCENT_THRESHOLD = 0.0; } /** @@ -180,14 +182,19 @@ private static class Defaults { ); /** - * Defines the native-memory threshold (in bytes) for an individual search shard task before - * it is considered for cancellation. {@code 0} disables the check. + * Defines the native-memory threshold (as a fraction of the backend-installed native-memory + * budget, in {@code [0.0, 1.0]}) for an individual search shard task before it is considered + * for cancellation. The effective per-task byte threshold is {@code budget * fraction}, where + * {@code budget} is the value installed via + * {@link org.opensearch.search.backpressure.trackers.NativeMemoryUsageTracker#setNativeMemoryBudgetSupplier}. + * {@code 0.0} disables the check. */ - private volatile long nativeMemoryBytesThreshold; - public static final Setting SETTING_NATIVE_MEMORY_BYTES_THRESHOLD = Setting.longSetting( - "search_backpressure.search_shard_task.native_memory_bytes_threshold", - Defaults.NATIVE_MEMORY_BYTES_THRESHOLD, - 0L, + private volatile double nativeMemoryPercentThreshold; + public static final Setting SETTING_NATIVE_MEMORY_PERCENT_THRESHOLD = Setting.doubleSetting( + "search_backpressure.search_shard_task.native_memory_percent_threshold", + Defaults.NATIVE_MEMORY_PERCENT_THRESHOLD, + 0.0, + 1.0, Setting.Property.Dynamic, Setting.Property.NodeScope ); @@ -203,7 +210,7 @@ public SearchShardTaskSettings(Settings settings, ClusterSettings clusterSetting this.cancellationRate = SETTING_CANCELLATION_RATE.get(settings); this.cancellationBurst = SETTING_CANCELLATION_BURST.get(settings); this.totalNativeMemoryBytesThreshold = SETTING_TOTAL_NATIVE_MEMORY_BYTES_THRESHOLD.get(settings); - this.nativeMemoryBytesThreshold = SETTING_NATIVE_MEMORY_BYTES_THRESHOLD.get(settings); + this.nativeMemoryPercentThreshold = SETTING_NATIVE_MEMORY_PERCENT_THRESHOLD.get(settings); clusterSettings.addSettingsUpdateConsumer(SETTING_TOTAL_HEAP_PERCENT_THRESHOLD, this::setTotalHeapPercentThreshold); clusterSettings.addSettingsUpdateConsumer(SETTING_CPU_TIME_MILLIS_THRESHOLD, this::setCpuTimeMillisThreshold); @@ -215,7 +222,7 @@ public SearchShardTaskSettings(Settings settings, ClusterSettings clusterSetting clusterSettings.addSettingsUpdateConsumer(SETTING_CANCELLATION_RATE, this::setCancellationRate); clusterSettings.addSettingsUpdateConsumer(SETTING_CANCELLATION_BURST, this::setCancellationBurst); clusterSettings.addSettingsUpdateConsumer(SETTING_TOTAL_NATIVE_MEMORY_BYTES_THRESHOLD, this::setTotalNativeMemoryBytesThreshold); - clusterSettings.addSettingsUpdateConsumer(SETTING_NATIVE_MEMORY_BYTES_THRESHOLD, this::setNativeMemoryBytesThreshold); + clusterSettings.addSettingsUpdateConsumer(SETTING_NATIVE_MEMORY_PERCENT_THRESHOLD, this::setNativeMemoryPercentThreshold); } public double getTotalHeapPercentThreshold() { @@ -274,12 +281,12 @@ private void setTotalNativeMemoryBytesThreshold(long totalNativeMemoryBytesThres this.totalNativeMemoryBytesThreshold = totalNativeMemoryBytesThreshold; } - public long getNativeMemoryBytesThreshold() { - return nativeMemoryBytesThreshold; + public double getNativeMemoryPercentThreshold() { + return nativeMemoryPercentThreshold; } - private void setNativeMemoryBytesThreshold(long nativeMemoryBytesThreshold) { - this.nativeMemoryBytesThreshold = nativeMemoryBytesThreshold; + private void setNativeMemoryPercentThreshold(double nativeMemoryPercentThreshold) { + this.nativeMemoryPercentThreshold = nativeMemoryPercentThreshold; } public double getCancellationRatio() { diff --git a/server/src/main/java/org/opensearch/search/backpressure/settings/SearchTaskSettings.java b/server/src/main/java/org/opensearch/search/backpressure/settings/SearchTaskSettings.java index 62bc8c0e1e017..251bb69bf477f 100644 --- a/server/src/main/java/org/opensearch/search/backpressure/settings/SearchTaskSettings.java +++ b/server/src/main/java/org/opensearch/search/backpressure/settings/SearchTaskSettings.java @@ -40,9 +40,11 @@ private static class Defaults { private static final int HEAP_MOVING_AVERAGE_WINDOW_SIZE = 100; // Native-memory tracking is opt-in. Zero keeps the tracker inert (see // NativeMemoryUsageTracker); raise either threshold to engage cancellation for - // native-memory-heavy tasks while the node is in native-memory duress. + // native-memory-heavy tasks while the node is in native-memory duress. The + // per-task threshold is expressed as a fraction of the backend-installed native + // memory budget (e.g. DataFusion's pool limit), mirroring HEAP_PERCENT_THRESHOLD. private static final long TOTAL_NATIVE_MEMORY_BYTES_THRESHOLD = 0L; - private static final long NATIVE_MEMORY_BYTES_THRESHOLD = 0L; + private static final double NATIVE_MEMORY_PERCENT_THRESHOLD = 0.0; } /** @@ -184,14 +186,19 @@ private static class Defaults { ); /** - * Defines the native-memory threshold (in bytes) for an individual search task before it is - * considered for cancellation. {@code 0} disables the check. + * Defines the native-memory threshold (as a fraction of the backend-installed native-memory + * budget, in {@code [0.0, 1.0]}) for an individual search task before it is considered for + * cancellation. The effective per-task byte threshold is {@code budget * fraction}, where + * {@code budget} is the value installed via + * {@link org.opensearch.search.backpressure.trackers.NativeMemoryUsageTracker#setNativeMemoryBudgetSupplier}. + * {@code 0.0} disables the check. */ - private volatile long nativeMemoryBytesThreshold; - public static final Setting SETTING_NATIVE_MEMORY_BYTES_THRESHOLD = Setting.longSetting( - "search_backpressure.search_task.native_memory_bytes_threshold", - Defaults.NATIVE_MEMORY_BYTES_THRESHOLD, - 0L, + private volatile double nativeMemoryPercentThreshold; + public static final Setting SETTING_NATIVE_MEMORY_PERCENT_THRESHOLD = Setting.doubleSetting( + "search_backpressure.search_task.native_memory_percent_threshold", + Defaults.NATIVE_MEMORY_PERCENT_THRESHOLD, + 0.0, + 1.0, Setting.Property.Dynamic, Setting.Property.NodeScope ); @@ -207,7 +214,7 @@ public SearchTaskSettings(Settings settings, ClusterSettings clusterSettings) { this.cancellationRate = SETTING_CANCELLATION_RATE.get(settings); this.cancellationBurst = SETTING_CANCELLATION_BURST.get(settings); this.totalNativeMemoryBytesThreshold = SETTING_TOTAL_NATIVE_MEMORY_BYTES_THRESHOLD.get(settings); - this.nativeMemoryBytesThreshold = SETTING_NATIVE_MEMORY_BYTES_THRESHOLD.get(settings); + this.nativeMemoryPercentThreshold = SETTING_NATIVE_MEMORY_PERCENT_THRESHOLD.get(settings); clusterSettings.addSettingsUpdateConsumer(SETTING_TOTAL_HEAP_PERCENT_THRESHOLD, this::setTotalHeapPercentThreshold); clusterSettings.addSettingsUpdateConsumer(SETTING_CPU_TIME_MILLIS_THRESHOLD, this::setCpuTimeMillisThreshold); @@ -219,7 +226,7 @@ public SearchTaskSettings(Settings settings, ClusterSettings clusterSettings) { clusterSettings.addSettingsUpdateConsumer(SETTING_CANCELLATION_RATE, this::setCancellationRate); clusterSettings.addSettingsUpdateConsumer(SETTING_CANCELLATION_BURST, this::setCancellationBurst); clusterSettings.addSettingsUpdateConsumer(SETTING_TOTAL_NATIVE_MEMORY_BYTES_THRESHOLD, this::setTotalNativeMemoryBytesThreshold); - clusterSettings.addSettingsUpdateConsumer(SETTING_NATIVE_MEMORY_BYTES_THRESHOLD, this::setNativeMemoryBytesThreshold); + clusterSettings.addSettingsUpdateConsumer(SETTING_NATIVE_MEMORY_PERCENT_THRESHOLD, this::setNativeMemoryPercentThreshold); } public double getTotalHeapPercentThreshold() { @@ -278,12 +285,12 @@ private void setTotalNativeMemoryBytesThreshold(long totalNativeMemoryBytesThres this.totalNativeMemoryBytesThreshold = totalNativeMemoryBytesThreshold; } - public long getNativeMemoryBytesThreshold() { - return nativeMemoryBytesThreshold; + public double getNativeMemoryPercentThreshold() { + return nativeMemoryPercentThreshold; } - private void setNativeMemoryBytesThreshold(long nativeMemoryBytesThreshold) { - this.nativeMemoryBytesThreshold = nativeMemoryBytesThreshold; + private void setNativeMemoryPercentThreshold(double nativeMemoryPercentThreshold) { + this.nativeMemoryPercentThreshold = nativeMemoryPercentThreshold; } public double getCancellationRatio() { diff --git a/server/src/main/java/org/opensearch/search/backpressure/trackers/NativeMemoryUsageTracker.java b/server/src/main/java/org/opensearch/search/backpressure/trackers/NativeMemoryUsageTracker.java index b254f805435e7..419c71d96d34c 100644 --- a/server/src/main/java/org/opensearch/search/backpressure/trackers/NativeMemoryUsageTracker.java +++ b/server/src/main/java/org/opensearch/search/backpressure/trackers/NativeMemoryUsageTracker.java @@ -27,6 +27,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.function.DoubleSupplier; import java.util.function.LongSupplier; import java.util.function.Supplier; @@ -36,9 +37,15 @@ * NativeMemoryUsageTracker cancels in-flight tasks that are holding more native (off-heap) * memory than allowed. Unlike {@link HeapUsageTracker}, this tracker does not maintain a * rolling average — native memory is a reservation that rises and falls over the query's - * lifetime, so an absolute byte threshold is the right shape once the node is already in - * native-memory duress (the {@link org.opensearch.search.backpressure.SearchBackpressureService} - * tracker-apply map gates on that). + * lifetime, so a fraction-of-budget threshold is the right shape once the node is already + * in native-memory duress (the + * {@link org.opensearch.search.backpressure.SearchBackpressureService} tracker-apply map + * gates on that). + * + *

The per-task threshold is expressed as a fraction of the backend-installed native-memory + * budget (see {@link #setNativeMemoryBudgetSupplier}), mirroring the heap tracker's + * {@code heap_percent_threshold}. Effective per-task byte threshold is + * {@code budget * fraction}. * *

Snapshot-per-tick model

*

Native-memory values come from a backend (e.g. DataFusion) over an FFI boundary. Per-task @@ -59,16 +66,33 @@ public class NativeMemoryUsageTracker extends TaskResourceUsageTracker { private static final Logger logger = LogManager.getLogger(NativeMemoryUsageTracker.class); /** Empty map used when no supplier is installed or the supplier returns {@code null}. */ - private static volatile Supplier> snapshotSupplier = Collections::emptyMap; + private static final Supplier> DEFAULT_EMPTY_SUPPLIER = Collections::emptyMap; + private static volatile Supplier> snapshotSupplier = DEFAULT_EMPTY_SUPPLIER; + + /** + * Source of the total native-memory budget in bytes (e.g. DataFusion's + * configured pool limit). Installed by a backend plugin via + * {@link #setNativeMemoryBudgetSupplier(LongSupplier)}; defaults to {@code 0} + * which keeps the tracker inert until a backend wires up a real budget. + */ + private static volatile LongSupplier nativeMemoryBudgetSupplier = () -> 0L; - private final LongSupplier nativeMemoryBytesThresholdSupplier; + /** + * Per-task threshold expressed as a fraction of the installed native-memory budget, + * mirroring {@code heap_percent_threshold} on {@link HeapUsageTracker}. Range is + * {@code [0.0, 1.0]} — the {@link org.opensearch.common.settings.Setting} validator + * already enforces those bounds; this class clamps defensively. + * + *

Effective per-task byte threshold = {@code budget * fraction}. + */ + private final DoubleSupplier nativeMemoryPercentThresholdSupplier; // Volatile so the map reference publishes safely from refresh() (called from the // backpressure scheduler thread) to the per-task evaluate() path (same thread today, // but also hit from nodeStats() which may run on a different thread). private volatile Map bytesByTaskId = Collections.emptyMap(); - public NativeMemoryUsageTracker(LongSupplier nativeMemoryBytesThresholdSupplier) { - this.nativeMemoryBytesThresholdSupplier = nativeMemoryBytesThresholdSupplier; + public NativeMemoryUsageTracker(DoubleSupplier nativeMemoryPercentThresholdSupplier) { + this.nativeMemoryPercentThresholdSupplier = nativeMemoryPercentThresholdSupplier; setDefaultResourceUsageBreachEvaluator(); } @@ -85,16 +109,45 @@ public static void setSnapshotSupplier(Supplier> supplier) { } /** - * Cancellation rule: task's current native-memory reservation is at or above the threshold. + * Install the native-memory budget source. A backend plugin (e.g. DataFusion) calls + * this from {@code createComponents} with a supplier reading its configured pool + * limit. Last writer wins. Pass a supplier returning {@code 0} (or never call this) + * to keep the tracker inert. + */ + public static void setNativeMemoryBudgetSupplier(LongSupplier supplier) { + if (supplier != null) { + nativeMemoryBudgetSupplier = supplier; + logger.info("[nativemem-bp] tracker.setNativeMemoryBudgetSupplier: installed"); + } + } + + /** Current native-memory budget in bytes; {@code 0} when no backend has installed a supplier. */ + public static long getNativeMemoryBudgetBytes() { + return Math.max(0L, nativeMemoryBudgetSupplier.getAsLong()); + } + + /** + * Cancellation rule: task's current native-memory reservation is at or above + * {@code budget * fraction}, where {@code budget} is the installed native-memory + * budget (e.g. DataFusion's pool limit) and {@code fraction} comes from the + * per-task threshold setting (range {@code [0.0, 1.0]}). */ private void setDefaultResourceUsageBreachEvaluator() { this.resourceUsageBreachEvaluator = (task) -> { if (task instanceof CancellableTask == false) { return Optional.empty(); } - long bytesThreshold = nativeMemoryBytesThresholdSupplier.getAsLong(); + double fraction = nativeMemoryPercentThresholdSupplier.getAsDouble(); + long budget = getNativeMemoryBudgetBytes(); + if (fraction <= 0.0d || budget <= 0L) { + // Feature disabled — either the operator hasn't set a threshold, or no + // backend has installed a budget supplier. Leave the tracker inert. + return Optional.empty(); + } + // Defensive clamp; the Setting validator already enforces [0.0, 1.0]. + double boundedFraction = Math.min(1.0d, fraction); + long bytesThreshold = (long) (budget * boundedFraction); if (bytesThreshold <= 0L) { - // Feature disabled — leave the tracker inert. return Optional.empty(); } long currentUsage = bytesForTask(task); @@ -102,11 +155,14 @@ private void setDefaultResourceUsageBreachEvaluator() { return Optional.empty(); } int score = (int) Math.max(1L, currentUsage / Math.max(1L, bytesThreshold)); - logger.info( - "[nativemem-bp] evaluate: task={} exceeds threshold — currentBytes={}B, threshold={}B, score={}", + logger.debug( + "[nativemem-bp] evaluate: task={} exceeds threshold — currentBytes={}B, " + + "threshold={}B (fraction={} of budget={}B), score={}", task.getId(), currentUsage, bytesThreshold, + boundedFraction, + budget, score ); return Optional.of( @@ -158,6 +214,15 @@ long bytesForTask(Task task) { return bytes == null ? 0L : bytes; } + /** + * Returns true if a backend plugin has installed a snapshot supplier. False if no + * backend with native-memory tracking is loaded, in which case registering this + * tracker would just produce empty refreshes every tick. + */ + public static boolean hasSnapshotProvider() { + return snapshotSupplier != DEFAULT_EMPTY_SUPPLIER; + } + /** * Returns {@code true} when the node exposes the physical-memory signal this tracker * relies on for duress. Symmetric with {@link HeapUsageTracker#isHeapTrackingSupported()}. @@ -166,27 +231,10 @@ public static boolean isNativeTrackingSupported() { if (Constants.LINUX == false) { return false; } - return OsProbe.getInstance().getTotalPhysicalMemorySize() > 0L; - } - - /** - * Returns {@code true} if aggregate native-memory usage across {@code cancellableTasks} is - * at least {@code totalBytesThreshold}. Used by the backpressure service as a cheap - * upstream check before running the per-task cancellation pass. - * - *

Uses the latest snapshot installed by {@link #refresh}; callers should invoke - * {@code refresh()} before this method if they want a fresh reading. - */ - public boolean isNativeMemoryUsageDominatedBySearch(List cancellableTasks, long totalBytesThreshold) { - if (totalBytesThreshold <= 0L) { - return true; - } - long usage = cancellableTasks.stream().mapToLong(this::bytesForTask).sum(); - if (usage < totalBytesThreshold) { - logger.debug("native memory usage not dominated by search requests [{}/{}]", usage, totalBytesThreshold); + if(hasSnapshotProvider() == false) { return false; } - return true; + return OsProbe.getInstance().getTotalPhysicalMemorySize() > 0L; } @Override From b4d81c19bf5e813e1ce3378db192b1e189574166 Mon Sep 17 00:00:00 2001 From: Pradeep L Date: Thu, 14 May 2026 16:35:03 +0530 Subject: [PATCH 03/10] Add debug logs for native-memory backpressure Tracing for the FFM rust path and the Java BP tick. Rust ffm.rs: log enter/ok-exit/err-exit for df_execute_query and df_execute_with_context. Rust query_tracker.rs: log cancel_query found/not-found and drain_completed_query drained/not-drained. NativeMemoryUsageTracker.evaluate now logs every code path (skip, inert, below-threshold, exceeds). NativeMemoryUsageTracker.refresh distinguishes null vs empty supplier and logs heaviest 5 task IDs. SearchBackpressureService.doRun logs per-tracker reasons-produced count, merge result, limiter outcome. Signed-off-by: Pradeep L --- .../rust/src/query_tracker.rs | 31 ++++++++++- .../SearchBackpressureService.java | 35 ++++++++++++- .../trackers/NativeMemoryUsageTracker.java | 52 +++++++++++++++++-- 3 files changed, 110 insertions(+), 8 deletions(-) diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/query_tracker.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/query_tracker.rs index 03333f25ceb58..f867f30ccc0eb 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/query_tracker.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/query_tracker.rs @@ -168,9 +168,24 @@ static QUERY_REGISTRY: Lazy>> = Lazy::new(DashMap /// Remove a completed tracker from the registry and return it. /// Called from JNI after Java has consumed the metrics. pub fn drain_completed_query(context_id: i64) -> Option> { - QUERY_REGISTRY + let result = QUERY_REGISTRY .remove_if(&context_id, |_, t| t.is_completed()) - .map(|(_, t)| t) + .map(|(_, t)| t); + match &result { + Some(t) => info!( + "[nativemem-bp] rust.drain_completed_query: drained ctx={} (peak_bytes={}, wall_secs={:.3}, registry_size_after={})", + context_id, + t.memory_pool.peak_bytes(), + t.wall_secs(), + QUERY_REGISTRY.len() + ), + None => info!( + "[nativemem-bp] rust.drain_completed_query: ctx={} not drained (absent or not completed) (registry_size={})", + context_id, + QUERY_REGISTRY.len() + ), + } + result } // --------------------------------------------------------------------------- @@ -334,7 +349,19 @@ pub fn snapshot_top_n_by_current(out: &mut [WireQueryMetric]) -> usize { /// No-op for unknown or already-completed queries. pub fn cancel_query(context_id: i64) { if let Some(tracker) = QUERY_REGISTRY.get(&context_id) { + info!( + "[nativemem-bp] rust.cancel_query: firing token for ctx={} (completed={}, current_bytes={})", + context_id, + tracker.is_completed(), + tracker.memory_pool.current_bytes() + ); tracker.cancellation_token.cancel(); + } else { + info!( + "[nativemem-bp] rust.cancel_query: ctx={} not in registry (registry_size={}) — no-op", + context_id, + QUERY_REGISTRY.len() + ); } } diff --git a/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java b/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java index 87b7620364a3c..b6b1d15dab833 100644 --- a/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java +++ b/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java @@ -291,16 +291,31 @@ void doRun() { List taskCancellations = new ArrayList<>(); for (TaskResourceUsageTrackerType trackerType : TaskResourceUsageTrackerType.values()) { if (shouldApply(trackerType)) { + int before = taskCancellations.size(); addResourceTrackerBasedCancellations(trackerType, taskCancellations, cancellableTasks); + int added = taskCancellations.size() - before; + logger.info( + "[nativemem-bp] doRun: tracker [{}] applied — produced {} cancellation reason(s)", + trackerType.getName(), + added + ); + } else { + logger.info("[nativemem-bp] doRun: tracker [{}] not applied (shouldApply=false)", trackerType.getName()); } } // Since these cancellations might be duplicate due to multiple trackers causing cancellation for same task // We need to merge them + int beforeMerge = taskCancellations.size(); taskCancellations = mergeTaskCancellations(taskCancellations).stream() .map(this::addSBPStateUpdateCallback) .filter(TaskCancellation::isEligibleForCancellation) .collect(Collectors.toList()); + logger.info( + "[nativemem-bp] doRun: merged {} reasons -> {} eligible cancellations", + beforeMerge, + taskCancellations.size() + ); for (TaskCancellation taskCancellation : taskCancellations) { logger.warn( @@ -311,6 +326,11 @@ void doRun() { ); if (mode != SearchBackpressureMode.ENFORCED) { + logger.info( + "[nativemem-bp] doRun: mode={} (not ENFORCED) — would-cancel task={} only logged, not actioned", + mode.getName(), + taskCancellation.getTask().getId() + ); continue; } @@ -323,11 +343,24 @@ void doRun() { // Stop cancelling tasks if there are no tokens in either of the two token buckets. if (rateLimitReached && ratioLimitReached) { - logger.debug("task cancellation limit reached"); + logger.warn( + "[nativemem-bp] doRun: cancellation limit reached for {} — task={} not cancelled (rateLimit={} ratioLimit={})", + taskType.getSimpleName(), + taskCancellation.getTask().getId(), + rateLimitReached, + ratioLimitReached + ); searchBackpressureState.incrementLimitReachedCount(); break; } + logger.info( + "[nativemem-bp] doRun: actioning cancellation — task={} type={} rateLimitOk={} ratioLimitOk={}", + taskCancellation.getTask().getId(), + taskType.getSimpleName(), + rateLimitReached == false, + ratioLimitReached == false + ); taskCancellation.cancelTaskAndDescendants(taskManager); } } diff --git a/server/src/main/java/org/opensearch/search/backpressure/trackers/NativeMemoryUsageTracker.java b/server/src/main/java/org/opensearch/search/backpressure/trackers/NativeMemoryUsageTracker.java index 419c71d96d34c..f7a83bedfd5f2 100644 --- a/server/src/main/java/org/opensearch/search/backpressure/trackers/NativeMemoryUsageTracker.java +++ b/server/src/main/java/org/opensearch/search/backpressure/trackers/NativeMemoryUsageTracker.java @@ -135,6 +135,11 @@ public static long getNativeMemoryBudgetBytes() { private void setDefaultResourceUsageBreachEvaluator() { this.resourceUsageBreachEvaluator = (task) -> { if (task instanceof CancellableTask == false) { + logger.info( + "[nativemem-bp] evaluate: task={} type={} not CancellableTask — skipping", + task.getId(), + task.getClass().getSimpleName() + ); return Optional.empty(); } double fraction = nativeMemoryPercentThresholdSupplier.getAsDouble(); @@ -142,21 +147,42 @@ private void setDefaultResourceUsageBreachEvaluator() { if (fraction <= 0.0d || budget <= 0L) { // Feature disabled — either the operator hasn't set a threshold, or no // backend has installed a budget supplier. Leave the tracker inert. + logger.info( + "[nativemem-bp] evaluate: task={} inert — fraction={} budget={}B (one is 0; tracker disabled)", + task.getId(), + fraction, + budget + ); return Optional.empty(); } // Defensive clamp; the Setting validator already enforces [0.0, 1.0]. double boundedFraction = Math.min(1.0d, fraction); long bytesThreshold = (long) (budget * boundedFraction); if (bytesThreshold <= 0L) { + logger.info( + "[nativemem-bp] evaluate: task={} bytesThreshold=0 (budget={}B fraction={}) — skipping", + task.getId(), + budget, + boundedFraction + ); return Optional.empty(); } long currentUsage = bytesForTask(task); if (currentUsage < bytesThreshold) { + logger.info( + "[nativemem-bp] evaluate: task={} below threshold — currentBytes={}B threshold={}B " + + "(fraction={} budget={}B) — no cancellation", + task.getId(), + currentUsage, + bytesThreshold, + boundedFraction, + budget + ); return Optional.empty(); } int score = (int) Math.max(1L, currentUsage / Math.max(1L, bytesThreshold)); - logger.debug( - "[nativemem-bp] evaluate: task={} exceeds threshold — currentBytes={}B, " + logger.warn( + "[nativemem-bp] evaluate: task={} EXCEEDS threshold — currentBytes={}B, " + "threshold={}B (fraction={} of budget={}B), score={}", task.getId(), currentUsage, @@ -197,12 +223,28 @@ public void update(Task task) { @Override public void refresh() { Map snapshot = snapshotSupplier.get(); - bytesByTaskId = snapshot != null ? snapshot : Collections.emptyMap(); + boolean nullSnapshot = (snapshot == null); + bytesByTaskId = nullSnapshot ? Collections.emptyMap() : snapshot; + if (nullSnapshot) { + logger.warn("[nativemem-bp] tracker.refresh: supplier returned null — using empty map"); + } logger.info( - "[nativemem-bp] tracker.refresh: snapshot loaded, size={} taskIds={}", + "[nativemem-bp] tracker.refresh: snapshot loaded, size={} taskIds={} budget={}B", bytesByTaskId.size(), - bytesByTaskId.keySet() + bytesByTaskId.keySet(), + getNativeMemoryBudgetBytes() ); + if (bytesByTaskId.size() > 0) { + // log the heaviest few so we can see whether registry has anything substantive + bytesByTaskId.entrySet().stream() + .sorted((a, b) -> Long.compare(b.getValue(), a.getValue())) + .limit(5) + .forEach(e -> logger.info( + "[nativemem-bp] tracker.refresh: taskId={} currentBytes={}", + e.getKey(), + e.getValue() + )); + } } /** Package-private for tests: look up a single task's bytes from the current snapshot. */ From 67cee63d61b8c1b80c3fd179f87ae713bd2140dc Mon Sep 17 00:00:00 2001 From: Pradeep L Date: Fri, 15 May 2026 02:57:46 +0530 Subject: [PATCH 04/10] Fix the SBP flow Signed-off-by: Pradeep L --- .../spi/AnalyticsSearchBackendPlugin.java | 2 + .../rust/src/lib.rs | 8 + .../rust/src/query_tracker.rs | 45 ++- .../be/datafusion/nativelib/NativeBridge.java | 17 +- .../nativelib/QueryRegistryLayout.java | 52 ++- .../exec/task/AnalyticsQueryTask.java | 7 +- .../org/opensearch/monitor/os/OsProbe.java | 1 - .../SearchBackpressureService.java | 62 ++-- .../trackers/NativeMemoryUsageTracker.java | 17 +- .../opensearch/monitor/os/OsProbeTests.java | 39 +++ .../settings/NodeDuressSettingsTests.java | 59 ++++ .../SearchShardTaskSettingsTests.java | 79 +++++ .../settings/SearchTaskSettingsTests.java | 77 +++++ .../NativeMemoryUsageTrackerTests.java | 326 ++++++++++++++++++ .../trackers/NodeDuressTrackersTests.java | 53 +++ .../TaskResourceUsageTrackerTypeTests.java | 41 +++ .../org/opensearch/wlm/ResourceTypeTests.java | 22 ++ .../NativeMemoryUsageCalculatorTests.java | 43 +++ ...GroupResourceUsageTrackerServiceTests.java | 36 ++ 19 files changed, 879 insertions(+), 107 deletions(-) create mode 100644 server/src/test/java/org/opensearch/search/backpressure/settings/NodeDuressSettingsTests.java create mode 100644 server/src/test/java/org/opensearch/search/backpressure/settings/SearchShardTaskSettingsTests.java create mode 100644 server/src/test/java/org/opensearch/search/backpressure/settings/SearchTaskSettingsTests.java create mode 100644 server/src/test/java/org/opensearch/search/backpressure/trackers/NativeMemoryUsageTrackerTests.java create mode 100644 server/src/test/java/org/opensearch/search/backpressure/trackers/TaskResourceUsageTrackerTypeTests.java create mode 100644 server/src/test/java/org/opensearch/wlm/tracker/NativeMemoryUsageCalculatorTests.java create mode 100644 server/src/test/java/org/opensearch/wlm/tracker/WorkloadGroupResourceUsageTrackerServiceTests.java diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/AnalyticsSearchBackendPlugin.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/AnalyticsSearchBackendPlugin.java index 15b92577508ac..5da01da075766 100644 --- a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/AnalyticsSearchBackendPlugin.java +++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/AnalyticsSearchBackendPlugin.java @@ -8,7 +8,9 @@ package org.opensearch.analytics.spi; +import java.util.Collections; import java.util.List; +import java.util.Map; /** * SPI extension point for backend query engine plugins. diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/lib.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/lib.rs index fb522332de9a4..b1201c56c195e 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/lib.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/lib.rs @@ -13,6 +13,14 @@ pub(crate) mod agg_mode; pub mod api; + +// Re-export the native logging bridge macros so this crate's modules can emit logs +// that flow through `RustLoggerBridge` into the Java/Log4j sink. The standard +// `log` crate macros (`log::info!`, `log::debug!`, …) are no-ops here because no +// global `log` dispatcher is registered — using these macros instead is the same +// pattern the `parquet-data-format` crate uses. +pub use native_bridge_common::{log_debug, log_error, log_info}; + pub mod cache; pub mod cancellation; pub mod cross_rt_stream; diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/query_tracker.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/query_tracker.rs index f867f30ccc0eb..4ef7d783e3f14 100644 --- a/sandbox/plugins/analytics-backend-datafusion/rust/src/query_tracker.rs +++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/query_tracker.rs @@ -22,13 +22,14 @@ use std::sync::Arc; use std::time::Instant; use dashmap::DashMap; -use log::{debug, info}; use once_cell::sync::Lazy; use tokio_util::sync::CancellationToken; use datafusion::common::DataFusionError; use datafusion::execution::memory_pool::{MemoryConsumer, MemoryPool, MemoryReservation}; +use crate::{log_debug, log_info}; + // --------------------------------------------------------------------------- // Per-query memory pool // --------------------------------------------------------------------------- @@ -172,14 +173,14 @@ pub fn drain_completed_query(context_id: i64) -> Option> { .remove_if(&context_id, |_, t| t.is_completed()) .map(|(_, t)| t); match &result { - Some(t) => info!( + Some(t) => log_info!( "[nativemem-bp] rust.drain_completed_query: drained ctx={} (peak_bytes={}, wall_secs={:.3}, registry_size_after={})", context_id, t.memory_pool.peak_bytes(), t.wall_secs(), QUERY_REGISTRY.len() ), - None => info!( + None => log_info!( "[nativemem-bp] rust.drain_completed_query: ctx={} not drained (absent or not completed) (registry_size={})", context_id, QUERY_REGISTRY.len() @@ -295,12 +296,32 @@ pub fn snapshot_top_n_by_current(out: &mut [WireQueryMetric]) -> usize { // set — the one a heavier candidate displaces. let mut heap: BinaryHeap> = BinaryHeap::with_capacity(n); + // Diagnostic: log registry size at entry, plus a sample of what's in there. + // Helps distinguish "registry empty" from "registry has entries but all are + // completed or have current_bytes == 0". + let registry_size = QUERY_REGISTRY.len(); + let mut sample_logged = 0usize; + log_info!( + "[nativemem-bp] rust.snapshot_top_n_by_current: enter cap={} registry_size={}", + n, registry_size + ); + for entry in QUERY_REGISTRY.iter() { let tracker = entry.value(); - if tracker.is_completed() { + let bytes_raw = tracker.memory_pool.current_bytes(); + let peak_raw = tracker.memory_pool.peak_bytes(); + let completed = tracker.is_completed(); + if sample_logged < 5 { + log_info!( + "[nativemem-bp] rust.snapshot_top_n_by_current: sample ctx={} current_bytes={} peak_bytes={} completed={}", + tracker.context_id, bytes_raw, peak_raw, completed + ); + sample_logged += 1; + } + if completed { continue; } - let bytes = usize_to_i64_saturating(tracker.memory_pool.current_bytes()); + let bytes = usize_to_i64_saturating(bytes_raw); if bytes == 0 { continue; } @@ -327,7 +348,7 @@ pub fn snapshot_top_n_by_current(out: &mut [WireQueryMetric]) -> usize { // values at materialization time, consistent with from_tracker. Ranking // and final values are independent point-in-time samples. out[written] = WireQueryMetric::from_tracker(&entry.tracker); - info!( + log_info!( "[nativemem-bp] rust.snapshot_top_n entry[{}]: ctx={}, current={}B, peak={}B, wall_ns={}, completed={}", written, out[written].context_id, @@ -338,7 +359,7 @@ pub fn snapshot_top_n_by_current(out: &mut [WireQueryMetric]) -> usize { ); written += 1; } - info!( + log_info!( "[nativemem-bp] rust.snapshot_top_n_by_current: wrote {} entries (cap {})", written, n ); @@ -349,7 +370,7 @@ pub fn snapshot_top_n_by_current(out: &mut [WireQueryMetric]) -> usize { /// No-op for unknown or already-completed queries. pub fn cancel_query(context_id: i64) { if let Some(tracker) = QUERY_REGISTRY.get(&context_id) { - info!( + log_info!( "[nativemem-bp] rust.cancel_query: firing token for ctx={} (completed={}, current_bytes={})", context_id, tracker.is_completed(), @@ -357,7 +378,7 @@ pub fn cancel_query(context_id: i64) { ); tracker.cancellation_token.cancel(); } else { - info!( + log_info!( "[nativemem-bp] rust.cancel_query: ctx={} not in registry (registry_size={}) — no-op", context_id, QUERY_REGISTRY.len() @@ -406,7 +427,7 @@ impl QueryTrackingContext { wall_nanos: std::sync::atomic::AtomicU64::new(0), }); QUERY_REGISTRY.insert(context_id, Arc::clone(&tracker)); - info!( + log_info!( "[nativemem-bp] rust.QueryTrackingContext::new: registered ctx={} (registry_size={})", context_id, QUERY_REGISTRY.len() @@ -432,7 +453,7 @@ impl Drop for QueryTrackingContext { fn drop(&mut self) { if let Some(tracker) = &self.tracker { tracker.mark_completed(); - info!( + log_info!( "[nativemem-bp] rust.QueryTrackingContext::drop: ctx={} completed (wall={:.3}s, mem_current={}B, mem_peak={}B)", tracker.context_id, tracker.wall_secs(), @@ -440,7 +461,7 @@ impl Drop for QueryTrackingContext { tracker.memory_pool.peak_bytes(), ); // Keep the debug line for operators who already tail the Rust debug log. - debug!( + log_debug!( "Query completed ctx={}: wall={:.3}s, mem_current={}B, mem_peak={}B", tracker.context_id, tracker.wall_secs(), diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java index 893e3a9432c52..f5c66c5a863f6 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java @@ -91,8 +91,6 @@ public final class NativeBridge { private static final MethodHandle PREPARE_PARTIAL_PLAN; private static final MethodHandle PREPARE_FINAL_PLAN; private static final MethodHandle EXECUTE_LOCAL_PREPARED_PLAN; - private static final MethodHandle QUERY_REGISTRY_LEN; - private static final MethodHandle QUERY_REGISTRY_SNAPSHOT; static { SymbolLookup lib = NativeLibraryLoader.symbolLookup(); @@ -416,6 +414,9 @@ public final class NativeBridge { // i64 df_query_registry_top_n_by_current(out_ptr, cap_entries) QUERY_REGISTRY_TOP_N_BY_CURRENT = linker.downcallHandle( lib.find("df_query_registry_top_n_by_current").orElseThrow(), + FunctionDescriptor.of(ValueLayout.JAVA_LONG, ValueLayout.ADDRESS, ValueLayout.JAVA_LONG) + ); + // ── Distributed aggregate: prepare partial/final plans ── // i64 df_prepare_partial_plan(handle_ptr, bytes_ptr, bytes_len) PREPARE_PARTIAL_PLAN = linker.downcallHandle( @@ -434,18 +435,6 @@ public final class NativeBridge { lib.find("df_execute_local_prepared_plan").orElseThrow(), FunctionDescriptor.of(ValueLayout.JAVA_LONG, ValueLayout.JAVA_LONG) ); - - // i64 df_query_registry_len() - QUERY_REGISTRY_LEN = linker.downcallHandle( - lib.find("df_query_registry_len").orElseThrow(), - FunctionDescriptor.of(ValueLayout.JAVA_LONG) - ); - - // i64 df_query_registry_snapshot(out_ptr, cap_entries) - QUERY_REGISTRY_SNAPSHOT = linker.downcallHandle( - lib.find("df_query_registry_snapshot").orElseThrow(), - FunctionDescriptor.of(ValueLayout.JAVA_LONG, ValueLayout.ADDRESS, ValueLayout.JAVA_LONG) - ); } private NativeBridge() {} diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/QueryRegistryLayout.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/QueryRegistryLayout.java index 5ed36d4ae13f9..6f9ae0e217ccb 100644 --- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/QueryRegistryLayout.java +++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/QueryRegistryLayout.java @@ -11,12 +11,9 @@ import org.opensearch.analytics.spi.QueryExecutionMetrics; import java.lang.foreign.MemoryLayout; -import java.lang.foreign.MemoryLayout.PathElement; import java.lang.foreign.MemorySegment; -import java.lang.foreign.SequenceLayout; import java.lang.foreign.StructLayout; import java.lang.foreign.ValueLayout; -import java.lang.invoke.VarHandle; /** * Mirrors the Rust {@code query_tracker::WireQueryMetric} {@code #[repr(C)]} @@ -24,10 +21,12 @@ * directly into the SPI types consumed by * {@link org.opensearch.analytics.spi.AnalyticsSearchBackendPlugin#getActiveQueryMetrics}. * - *

This is the same idiom {@code StatsLayout} uses for the {@code df_stats} - * call: a {@link StructLayout} describes the wire shape, cached - * {@link VarHandle}s read individual fields, and the decoder produces the - * caller's final type with no transport-only intermediate. + *

Decoder reads each row by slicing the segment to that row's 40-byte + * window. We deliberately avoid {@code SequenceLayout}-derived + * {@code VarHandle}s because their bounds check enforces the segment span + * the entire sequence layout, not just up to the requested row. With a + * generic {@code Long.MAX_VALUE / ENTRY_BYTES} sequence layout the read + * always fails an {@link IndexOutOfBoundsException}. * *

Buffer shape (populated by {@code df_query_registry_top_n_by_current}): * 

@@ -49,18 +48,13 @@ public final class QueryRegistryLayout {
     /** Byte size of one wire entry. Matches {@code size_of::()} on the Rust side. */
     public static final long ENTRY_BYTES = ENTRY_LAYOUT.byteSize();
 
-    /**
-     * Sequence layout used to derive per-row {@link VarHandle}s. The first path
-     * element is a {@code sequenceElement()} (the array index), the second is
-     * the field name within {@code ENTRY_LAYOUT}.
-     */
-    private static final SequenceLayout ROWS = MemoryLayout.sequenceLayout(Long.MAX_VALUE / ENTRY_BYTES, ENTRY_LAYOUT);
-
-    private static final VarHandle CONTEXT_ID = rowHandle("context_id");
-    private static final VarHandle CURRENT_BYTES = rowHandle("current_bytes");
-    private static final VarHandle PEAK_BYTES = rowHandle("peak_bytes");
-    private static final VarHandle WALL_NANOS = rowHandle("wall_nanos");
-    private static final VarHandle COMPLETED = rowHandle("completed");
+    // Byte offsets within a single entry. Field order must match
+    // ENTRY_LAYOUT and the Rust #[repr(C)] WireQueryMetric struct.
+    private static final long OFF_CONTEXT_ID    = 0L;
+    private static final long OFF_CURRENT_BYTES = 8L;
+    private static final long OFF_PEAK_BYTES    = 16L;
+    private static final long OFF_WALL_NANOS    = 24L;
+    private static final long OFF_COMPLETED     = 32L;
 
     static {
         long expected = 5L * Long.BYTES;
@@ -78,7 +72,8 @@ private QueryRegistryLayout() {}
      * @param i   row index, must be {@code < written} returned by the FFI call
      */
     public static long readContextId(MemorySegment seg, int i) {
-        return (long) CONTEXT_ID.get(seg, 0L, (long) i);
+        long rowOffset = (long) i * ENTRY_BYTES;
+        return seg.get(ValueLayout.JAVA_LONG, rowOffset + OFF_CONTEXT_ID);
     }
 
     /**
@@ -89,19 +84,12 @@ public static long readContextId(MemorySegment seg, int i) {
      * @param i   row index, must be {@code < written} returned by the FFI call
      */
     public static QueryExecutionMetrics readMetrics(MemorySegment seg, int i) {
-        long row = (long) i;
+        long rowOffset = (long) i * ENTRY_BYTES;
         return new QueryExecutionMetrics(
-            (long) CURRENT_BYTES.get(seg, 0L, row),
-            (long) PEAK_BYTES.get(seg, 0L, row),
-            (long) WALL_NANOS.get(seg, 0L, row),
-            (long) COMPLETED.get(seg, 0L, row) != 0L
+            seg.get(ValueLayout.JAVA_LONG, rowOffset + OFF_CURRENT_BYTES),
+            seg.get(ValueLayout.JAVA_LONG, rowOffset + OFF_PEAK_BYTES),
+            seg.get(ValueLayout.JAVA_LONG, rowOffset + OFF_WALL_NANOS),
+            seg.get(ValueLayout.JAVA_LONG, rowOffset + OFF_COMPLETED) != 0L
         );
     }
-
-    private static VarHandle rowHandle(String field) {
-        // sequenceElement() leaves the row index as a free coordinate, so the
-        // returned VarHandle takes (segment, base, rowIndex) — we always pass
-        // base=0 because the FFM call writes at the start of the segment.
-        return ROWS.varHandle(PathElement.sequenceElement(), PathElement.groupElement(field));
-    }
 }
diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/task/AnalyticsQueryTask.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/task/AnalyticsQueryTask.java
index 8fcb1b911048f..37d9695aab5c3 100644
--- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/task/AnalyticsQueryTask.java
+++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/task/AnalyticsQueryTask.java
@@ -15,11 +15,10 @@
 import org.opensearch.common.Nullable;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.tasks.TaskId;
-import org.opensearch.tasks.CancellableTask;
-import org.opensearch.tasks.SearchBackpressureTask;
 
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Supplier;
 
 /**
  * Coordinator-level cancellable task representing a running analytics query.
@@ -28,7 +27,7 @@
  *
  * @opensearch.internal
  */
-public class AnalyticsQueryTask extends CancellableTask implements SearchBackpressureTask {
+public class AnalyticsQueryTask extends SearchTask {
 
     private static final Logger logger = LogManager.getLogger(AnalyticsQueryTask.class);
 
@@ -49,7 +48,7 @@ public AnalyticsQueryTask(
             id,
             type,
             action,
-            "queryId[" + queryId + "]",
+            (Supplier) () -> "queryId[" + queryId + "]",
             parentTaskId,
             headers,
             cancelAfterTimeInterval != null ? cancelAfterTimeInterval : TimeValue.MINUS_ONE
diff --git a/server/src/main/java/org/opensearch/monitor/os/OsProbe.java b/server/src/main/java/org/opensearch/monitor/os/OsProbe.java
index 159d7cdaf78a3..a26bfc7361200 100644
--- a/server/src/main/java/org/opensearch/monitor/os/OsProbe.java
+++ b/server/src/main/java/org/opensearch/monitor/os/OsProbe.java
@@ -336,7 +336,6 @@ public long getProcessNativeMemoryBytes() {
         return Math.max(0L, rssAnon - jvmHeapMax);
     }
 
-
     public short getSystemCpuPercent() {
         return Probes.getLoadAndScaleToPercent(getSystemCpuLoad, osMxBean);
     }
diff --git a/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java b/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java
index b6b1d15dab833..109ef5f42fcde 100644
--- a/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java
+++ b/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java
@@ -52,12 +52,10 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.EnumMap;
-import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
-import java.util.Set;
 import java.util.function.DoubleSupplier;
 import java.util.function.Function;
 import java.util.function.LongSupplier;
@@ -75,13 +73,13 @@
 public class SearchBackpressureService extends AbstractLifecycleComponent implements TaskCompletionListener {
     private static final Logger logger = LogManager.getLogger(SearchBackpressureService.class);
     // Tracker-apply rules:
-    //   - When native-memory duress is active, we run ONLY the native-memory and elapsed-time
-    //     trackers. CPU and heap trackers are intentionally skipped so that a runaway native
-    //     allocator doesn't trigger unrelated cancellation paths (heap is still low, CPU may
-    //     not be pegged). See the dedicated short-circuit in doRun().
-    //   - Otherwise, CPU and heap trackers run when their corresponding resource is in duress,
-    //     elapsed-time always runs, and the native-memory tracker runs only under its own
-    //     native-memory duress condition.
+    // - When native-memory duress is active, we run ONLY the native-memory and elapsed-time
+    // trackers. CPU and heap trackers are intentionally skipped so that a runaway native
+    // allocator doesn't trigger unrelated cancellation paths (heap is still low, CPU may
+    // not be pegged). See the dedicated short-circuit in doRun().
+    // - Otherwise, CPU and heap trackers run when their corresponding resource is in duress,
+    // elapsed-time always runs, and the native-memory tracker runs only under its own
+    // native-memory duress condition.
     private static final Map> trackerApplyConditions = Map.of(
         TaskResourceUsageTrackerType.CPU_USAGE_TRACKER,
         (nodeDuressTrackers) -> nodeDuressTrackers.isResourceInDuress(ResourceType.CPU),
@@ -257,29 +255,33 @@ void doRun() {
             );
         }
 
-         Map, List> cancellableTasks;
+        Map, List> cancellableTasks;
 
-            boolean isHeapUsageDominatedBySearchTasks = isHeapUsageDominatedBySearch(
-                searchTasks,
-                getSettings().getSearchTaskSettings().getTotalHeapPercentThreshold()
-            );
-            boolean isHeapUsageDominatedBySearchShardTasks = isHeapUsageDominatedBySearch(
-                searchShardTasks,
-                getSettings().getSearchShardTaskSettings().getTotalHeapPercentThreshold()
-            );
-            cancellableTasks = Map.of(
-                SearchTask.class,
-                isHeapUsageDominatedBySearchTasks ? searchTasks : Collections.emptyList(),
-                SearchShardTask.class,
-                isHeapUsageDominatedBySearchShardTasks ? searchShardTasks : Collections.emptyList()
-            );
+        boolean isHeapUsageDominatedBySearchTasks = isHeapUsageDominatedBySearch(
+            searchTasks,
+            getSettings().getSearchTaskSettings().getTotalHeapPercentThreshold()
+        );
+        boolean isHeapUsageDominatedBySearchShardTasks = isHeapUsageDominatedBySearch(
+            searchShardTasks,
+            getSettings().getSearchShardTaskSettings().getTotalHeapPercentThreshold()
+        );
+        cancellableTasks = Map.of(
+            SearchTask.class,
+            isHeapUsageDominatedBySearchTasks ? searchTasks : Collections.emptyList(),
+            SearchShardTask.class,
+            isHeapUsageDominatedBySearchShardTasks ? searchShardTasks : Collections.emptyList()
+        );
 
         if (inNativeMemoryDuress) {
-             cancellableTasks = Map.of(SearchTask.class, searchTasks, SearchShardTask.class, searchShardTasks);
+            cancellableTasks = Map.of(SearchTask.class, searchTasks, SearchShardTask.class, searchShardTasks);
             if (shouldApply(TaskResourceUsageTrackerType.NATIVE_MEMORY_USAGE_TRACKER)) {
-                logger.info("[nativemem-bp] doRun: refreshing tracker [{}]", TaskResourceUsageTrackerType.NATIVE_MEMORY_USAGE_TRACKER.getName());
+                logger.info(
+                    "[nativemem-bp] doRun: refreshing tracker [{}]",
+                    TaskResourceUsageTrackerType.NATIVE_MEMORY_USAGE_TRACKER.getName()
+                );
                 for (TaskResourceUsageTrackers trackers : taskTrackers.values()) {
-                    trackers.getTracker(TaskResourceUsageTrackerType.NATIVE_MEMORY_USAGE_TRACKER).ifPresent(TaskResourceUsageTracker::refresh);
+                    trackers.getTracker(TaskResourceUsageTrackerType.NATIVE_MEMORY_USAGE_TRACKER)
+                        .ifPresent(TaskResourceUsageTracker::refresh);
                 }
             }
         }
@@ -311,11 +313,7 @@ void doRun() {
             .map(this::addSBPStateUpdateCallback)
             .filter(TaskCancellation::isEligibleForCancellation)
             .collect(Collectors.toList());
-        logger.info(
-            "[nativemem-bp] doRun: merged {} reasons -> {} eligible cancellations",
-            beforeMerge,
-            taskCancellations.size()
-        );
+        logger.info("[nativemem-bp] doRun: merged {} reasons -> {} eligible cancellations", beforeMerge, taskCancellations.size());
 
         for (TaskCancellation taskCancellation : taskCancellations) {
             logger.warn(
diff --git a/server/src/main/java/org/opensearch/search/backpressure/trackers/NativeMemoryUsageTracker.java b/server/src/main/java/org/opensearch/search/backpressure/trackers/NativeMemoryUsageTracker.java
index f7a83bedfd5f2..8626226c34b6b 100644
--- a/server/src/main/java/org/opensearch/search/backpressure/trackers/NativeMemoryUsageTracker.java
+++ b/server/src/main/java/org/opensearch/search/backpressure/trackers/NativeMemoryUsageTracker.java
@@ -193,11 +193,7 @@ private void setDefaultResourceUsageBreachEvaluator() {
             );
             return Optional.of(
                 new TaskCancellation.Reason(
-                    "native memory usage exceeded ["
-                        + new ByteSizeValue(currentUsage)
-                        + " >= "
-                        + new ByteSizeValue(bytesThreshold)
-                        + "]",
+                    "native memory usage exceeded [" + new ByteSizeValue(currentUsage) + " >= " + new ByteSizeValue(bytesThreshold) + "]",
                     score
                 )
             );
@@ -236,14 +232,11 @@ public void refresh() {
         );
         if (bytesByTaskId.size() > 0) {
             // log the heaviest few so we can see whether registry has anything substantive
-            bytesByTaskId.entrySet().stream()
+            bytesByTaskId.entrySet()
+                .stream()
                 .sorted((a, b) -> Long.compare(b.getValue(), a.getValue()))
                 .limit(5)
-                .forEach(e -> logger.info(
-                    "[nativemem-bp] tracker.refresh:   taskId={} currentBytes={}",
-                    e.getKey(),
-                    e.getValue()
-                ));
+                .forEach(e -> logger.info("[nativemem-bp] tracker.refresh:   taskId={} currentBytes={}", e.getKey(), e.getValue()));
         }
     }
 
@@ -273,7 +266,7 @@ public static boolean isNativeTrackingSupported() {
         if (Constants.LINUX == false) {
             return false;
         }
-        if(hasSnapshotProvider() == false) {
+        if (hasSnapshotProvider() == false) {
             return false;
         }
         return OsProbe.getInstance().getTotalPhysicalMemorySize() > 0L;
diff --git a/server/src/test/java/org/opensearch/monitor/os/OsProbeTests.java b/server/src/test/java/org/opensearch/monitor/os/OsProbeTests.java
index b45e48e249026..ce144a4184f40 100644
--- a/server/src/test/java/org/opensearch/monitor/os/OsProbeTests.java
+++ b/server/src/test/java/org/opensearch/monitor/os/OsProbeTests.java
@@ -623,4 +623,43 @@ public void testGetProcessRssAnonBytes_negativeOnNonLinux() {
         assertEquals(-1L, OsProbe.getInstance().getProcessRssAnonBytes());
     }
 
+    // ---- getProcessNativeMemoryBytes (RssAnon - heapMax, clamped) ----
+
+    public void testGetProcessNativeMemoryBytes_returnsNegativeWhenRssAnonUnavailable() {
+        // Override getProcessRssAnonBytes to return -1 (the "not supported" sentinel). The
+        // method must propagate that signal upward without subtracting from -1.
+        OsProbe probe = new OsProbe() {
+            @Override
+            public long getProcessRssAnonBytes() {
+                return -1L;
+            }
+        };
+        assertEquals(-1L, probe.getProcessNativeMemoryBytes());
+    }
+
+    public void testGetProcessNativeMemoryBytes_subtractsHeapMaxAndClampsAtZero() {
+        // RssAnon below heapMax (early process lifetime, before heap pages are committed).
+        // The method must clamp at 0 instead of returning a negative.
+        OsProbe probe = new OsProbe() {
+            @Override
+            public long getProcessRssAnonBytes() {
+                return 1L; // way below any real -Xmx setting
+            }
+        };
+        assertEquals(0L, probe.getProcessNativeMemoryBytes());
+    }
+
+    public void testGetProcessNativeMemoryBytes_returnsDifferenceWhenRssAnonExceedsHeap() {
+        // RssAnon clearly larger than the JVM heap max — the difference is reported back.
+        long heapMax = org.opensearch.monitor.jvm.JvmStats.jvmStats().getMem().getHeapMax().getBytes();
+        long rssAnon = heapMax + 64L * 1024L * 1024L;
+        OsProbe probe = new OsProbe() {
+            @Override
+            public long getProcessRssAnonBytes() {
+                return rssAnon;
+            }
+        };
+        assertEquals(64L * 1024L * 1024L, probe.getProcessNativeMemoryBytes());
+    }
+
 }
diff --git a/server/src/test/java/org/opensearch/search/backpressure/settings/NodeDuressSettingsTests.java b/server/src/test/java/org/opensearch/search/backpressure/settings/NodeDuressSettingsTests.java
new file mode 100644
index 0000000000000..574d8aa11c935
--- /dev/null
+++ b/server/src/test/java/org/opensearch/search/backpressure/settings/NodeDuressSettingsTests.java
@@ -0,0 +1,59 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.search.backpressure.settings;
+
+import org.opensearch.common.settings.ClusterSettings;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.test.OpenSearchTestCase;
+
+/**
+ * Unit tests covering the native-memory threshold added to {@link NodeDuressSettings}.
+ */
+public class NodeDuressSettingsTests extends OpenSearchTestCase {
+
+    public void testDefaultNativeMemoryThreshold() {
+        NodeDuressSettings settings = new NodeDuressSettings(
+            Settings.EMPTY,
+            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)
+        );
+        // Default per NodeDuressSettings.Defaults.NATIVE_MEMORY_THRESHOLD.
+        assertEquals(0.85d, settings.getNativeMemoryThreshold(), 0.0d);
+    }
+
+    public void testInitialNativeMemoryThresholdRespectsSetting() {
+        Settings raw = Settings.builder().put(NodeDuressSettings.SETTING_NATIVE_MEMORY_THRESHOLD.getKey(), 0.42d).build();
+        NodeDuressSettings settings = new NodeDuressSettings(raw, new ClusterSettings(raw, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
+        assertEquals(0.42d, settings.getNativeMemoryThreshold(), 0.0d);
+    }
+
+    public void testNativeMemoryThresholdIsDynamic() {
+        ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+        NodeDuressSettings settings = new NodeDuressSettings(Settings.EMPTY, clusterSettings);
+        assertEquals(0.85d, settings.getNativeMemoryThreshold(), 0.0d);
+
+        // Apply a runtime update — the consumer should propagate to the field.
+        clusterSettings.applySettings(Settings.builder().put(NodeDuressSettings.SETTING_NATIVE_MEMORY_THRESHOLD.getKey(), 0.55d).build());
+        assertEquals(0.55d, settings.getNativeMemoryThreshold(), 0.0d);
+    }
+
+    public void testNativeMemoryThresholdRejectsOutOfRange() {
+        // Range [0.0, 1.0] enforced by Setting.doubleSetting(min, max).
+        Settings raw = Settings.builder().put(NodeDuressSettings.SETTING_NATIVE_MEMORY_THRESHOLD.getKey(), 1.5d).build();
+        expectThrows(
+            IllegalArgumentException.class,
+            () -> new NodeDuressSettings(raw, new ClusterSettings(raw, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS))
+        );
+
+        Settings raw2 = Settings.builder().put(NodeDuressSettings.SETTING_NATIVE_MEMORY_THRESHOLD.getKey(), -0.1d).build();
+        expectThrows(
+            IllegalArgumentException.class,
+            () -> new NodeDuressSettings(raw2, new ClusterSettings(raw2, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS))
+        );
+    }
+}
diff --git a/server/src/test/java/org/opensearch/search/backpressure/settings/SearchShardTaskSettingsTests.java b/server/src/test/java/org/opensearch/search/backpressure/settings/SearchShardTaskSettingsTests.java
new file mode 100644
index 0000000000000..176712a9ad333
--- /dev/null
+++ b/server/src/test/java/org/opensearch/search/backpressure/settings/SearchShardTaskSettingsTests.java
@@ -0,0 +1,79 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.search.backpressure.settings;
+
+import org.opensearch.common.settings.ClusterSettings;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.test.OpenSearchTestCase;
+
+/**
+ * Unit tests covering the native-memory threshold settings added to
+ * {@link SearchShardTaskSettings}.
+ */
+public class SearchShardTaskSettingsTests extends OpenSearchTestCase {
+
+    public void testDefaultNativeMemoryThresholdsKeepTrackerInert() {
+        SearchShardTaskSettings settings = new SearchShardTaskSettings(
+            Settings.EMPTY,
+            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)
+        );
+        assertEquals(0L, settings.getTotalNativeMemoryBytesThreshold());
+        assertEquals(0.0d, settings.getNativeMemoryPercentThreshold(), 0.0d);
+    }
+
+    public void testInitialNativeMemoryThresholdsRespectSettings() {
+        long bytesThreshold = 512L * 1024L * 1024L;
+        double fractionThreshold = 0.6d;
+        Settings raw = Settings.builder()
+            .put(SearchShardTaskSettings.SETTING_TOTAL_NATIVE_MEMORY_BYTES_THRESHOLD.getKey(), bytesThreshold)
+            .put(SearchShardTaskSettings.SETTING_NATIVE_MEMORY_PERCENT_THRESHOLD.getKey(), fractionThreshold)
+            .build();
+        SearchShardTaskSettings settings = new SearchShardTaskSettings(
+            raw,
+            new ClusterSettings(raw, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)
+        );
+        assertEquals(bytesThreshold, settings.getTotalNativeMemoryBytesThreshold());
+        assertEquals(fractionThreshold, settings.getNativeMemoryPercentThreshold(), 0.0d);
+    }
+
+    public void testNativeMemoryThresholdsAreDynamic() {
+        ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+        SearchShardTaskSettings settings = new SearchShardTaskSettings(Settings.EMPTY, clusterSettings);
+        clusterSettings.applySettings(
+            Settings.builder()
+                .put(SearchShardTaskSettings.SETTING_TOTAL_NATIVE_MEMORY_BYTES_THRESHOLD.getKey(), 8192L)
+                .put(SearchShardTaskSettings.SETTING_NATIVE_MEMORY_PERCENT_THRESHOLD.getKey(), 0.25d)
+                .build()
+        );
+        assertEquals(8192L, settings.getTotalNativeMemoryBytesThreshold());
+        assertEquals(0.25d, settings.getNativeMemoryPercentThreshold(), 0.0d);
+    }
+
+    public void testNativeMemoryPercentThresholdRejectsOutOfRange() {
+        Settings raw = Settings.builder().put(SearchShardTaskSettings.SETTING_NATIVE_MEMORY_PERCENT_THRESHOLD.getKey(), 2.0d).build();
+        expectThrows(
+            IllegalArgumentException.class,
+            () -> new SearchShardTaskSettings(raw, new ClusterSettings(raw, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS))
+        );
+
+        Settings raw2 = Settings.builder().put(SearchShardTaskSettings.SETTING_NATIVE_MEMORY_PERCENT_THRESHOLD.getKey(), -0.5d).build();
+        expectThrows(
+            IllegalArgumentException.class,
+            () -> new SearchShardTaskSettings(raw2, new ClusterSettings(raw2, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS))
+        );
+    }
+
+    public void testTotalNativeMemoryBytesThresholdRejectsNegative() {
+        Settings raw = Settings.builder().put(SearchShardTaskSettings.SETTING_TOTAL_NATIVE_MEMORY_BYTES_THRESHOLD.getKey(), -1L).build();
+        expectThrows(
+            IllegalArgumentException.class,
+            () -> new SearchShardTaskSettings(raw, new ClusterSettings(raw, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS))
+        );
+    }
+}
diff --git a/server/src/test/java/org/opensearch/search/backpressure/settings/SearchTaskSettingsTests.java b/server/src/test/java/org/opensearch/search/backpressure/settings/SearchTaskSettingsTests.java
new file mode 100644
index 0000000000000..7d6c329e14088
--- /dev/null
+++ b/server/src/test/java/org/opensearch/search/backpressure/settings/SearchTaskSettingsTests.java
@@ -0,0 +1,77 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.search.backpressure.settings;
+
+import org.opensearch.common.settings.ClusterSettings;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.test.OpenSearchTestCase;
+
+/**
+ * Unit tests covering the native-memory threshold settings added to {@link SearchTaskSettings}.
+ */
+public class SearchTaskSettingsTests extends OpenSearchTestCase {
+
+    private SearchTaskSettings buildDefault() {
+        return new SearchTaskSettings(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
+    }
+
+    public void testDefaultNativeMemoryThresholdsKeepTrackerInert() {
+        SearchTaskSettings settings = buildDefault();
+        // Defaults are 0 / 0.0 — the tracker treats both as "feature disabled".
+        assertEquals(0L, settings.getTotalNativeMemoryBytesThreshold());
+        assertEquals(0.0d, settings.getNativeMemoryPercentThreshold(), 0.0d);
+    }
+
+    public void testInitialNativeMemoryThresholdsRespectSettings() {
+        long bytesThreshold = 2L * 1024L * 1024L * 1024L;
+        double fractionThreshold = 0.75d;
+        Settings raw = Settings.builder()
+            .put(SearchTaskSettings.SETTING_TOTAL_NATIVE_MEMORY_BYTES_THRESHOLD.getKey(), bytesThreshold)
+            .put(SearchTaskSettings.SETTING_NATIVE_MEMORY_PERCENT_THRESHOLD.getKey(), fractionThreshold)
+            .build();
+        SearchTaskSettings settings = new SearchTaskSettings(raw, new ClusterSettings(raw, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
+        assertEquals(bytesThreshold, settings.getTotalNativeMemoryBytesThreshold());
+        assertEquals(fractionThreshold, settings.getNativeMemoryPercentThreshold(), 0.0d);
+    }
+
+    public void testNativeMemoryThresholdsAreDynamic() {
+        ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+        SearchTaskSettings settings = new SearchTaskSettings(Settings.EMPTY, clusterSettings);
+        clusterSettings.applySettings(
+            Settings.builder()
+                .put(SearchTaskSettings.SETTING_TOTAL_NATIVE_MEMORY_BYTES_THRESHOLD.getKey(), 1024L)
+                .put(SearchTaskSettings.SETTING_NATIVE_MEMORY_PERCENT_THRESHOLD.getKey(), 0.33d)
+                .build()
+        );
+        assertEquals(1024L, settings.getTotalNativeMemoryBytesThreshold());
+        assertEquals(0.33d, settings.getNativeMemoryPercentThreshold(), 0.0d);
+    }
+
+    public void testNativeMemoryPercentThresholdRejectsOutOfRange() {
+        Settings raw = Settings.builder().put(SearchTaskSettings.SETTING_NATIVE_MEMORY_PERCENT_THRESHOLD.getKey(), 1.5d).build();
+        expectThrows(
+            IllegalArgumentException.class,
+            () -> new SearchTaskSettings(raw, new ClusterSettings(raw, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS))
+        );
+
+        Settings raw2 = Settings.builder().put(SearchTaskSettings.SETTING_NATIVE_MEMORY_PERCENT_THRESHOLD.getKey(), -0.1d).build();
+        expectThrows(
+            IllegalArgumentException.class,
+            () -> new SearchTaskSettings(raw2, new ClusterSettings(raw2, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS))
+        );
+    }
+
+    public void testTotalNativeMemoryBytesThresholdRejectsNegative() {
+        Settings raw = Settings.builder().put(SearchTaskSettings.SETTING_TOTAL_NATIVE_MEMORY_BYTES_THRESHOLD.getKey(), -1L).build();
+        expectThrows(
+            IllegalArgumentException.class,
+            () -> new SearchTaskSettings(raw, new ClusterSettings(raw, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS))
+        );
+    }
+}
diff --git a/server/src/test/java/org/opensearch/search/backpressure/trackers/NativeMemoryUsageTrackerTests.java b/server/src/test/java/org/opensearch/search/backpressure/trackers/NativeMemoryUsageTrackerTests.java
new file mode 100644
index 0000000000000..13d1a0ac6d5df
--- /dev/null
+++ b/server/src/test/java/org/opensearch/search/backpressure/trackers/NativeMemoryUsageTrackerTests.java
@@ -0,0 +1,326 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.search.backpressure.trackers;
+
+import org.opensearch.action.search.SearchShardTask;
+import org.opensearch.action.search.SearchTask;
+import org.opensearch.common.io.stream.BytesStreamOutput;
+import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.tasks.TaskId;
+import org.opensearch.tasks.CancellableTask;
+import org.opensearch.tasks.Task;
+import org.opensearch.tasks.TaskCancellation;
+import org.opensearch.test.OpenSearchTestCase;
+import org.junit.After;
+import org.junit.Before;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.function.Supplier;
+
+import static org.opensearch.search.backpressure.SearchBackpressureTestHelpers.createMockTaskWithResourceStats;
+
+/**
+ * Unit tests for {@link NativeMemoryUsageTracker}.
+ *
+ * 
The tracker maintains static suppliers (snapshot + budget) that survive across tests
+ * because they're installed by backend plugins at boot time. {@link #setUp()} and
+ * {@link #tearDown()} reset them to their defaults so every test starts from a known
+ * "no backend installed" state.
+ */
+public class NativeMemoryUsageTrackerTests extends OpenSearchTestCase {
+
+    @Before
+    public void resetStaticsBefore() {
+        // Static suppliers are package-shared with the production wiring; clear before every test
+        // so each test owns its own supplier surface and doesn't see leftover state from a prior
+        // test (or from production code paths exercised by another suite).
+        NativeMemoryUsageTracker.setSnapshotSupplier(java.util.Collections::emptyMap);
+        NativeMemoryUsageTracker.setNativeMemoryBudgetSupplier(() -> 0L);
+    }
+
+    @After
+    public void resetStaticsAfter() {
+        NativeMemoryUsageTracker.setSnapshotSupplier(java.util.Collections::emptyMap);
+        NativeMemoryUsageTracker.setNativeMemoryBudgetSupplier(() -> 0L);
+    }
+
+    public void testNameMatchesEnum() {
+        NativeMemoryUsageTracker tracker = new NativeMemoryUsageTracker(() -> 0.5);
+        assertEquals(TaskResourceUsageTrackerType.NATIVE_MEMORY_USAGE_TRACKER.getName(), tracker.name());
+    }
+
+    public void testRefreshEmptySnapshotIsSafe() {
+        NativeMemoryUsageTracker tracker = new NativeMemoryUsageTracker(() -> 0.5);
+        // No supplier installed — default empty map should be picked up.
+        tracker.refresh();
+        Task task = createMockTaskWithResourceStats(SearchTask.class, 1, 1, randomNonNegativeLong());
+        // bytesForTask must be 0 when no entry is in the snapshot.
+        assertEquals(0L, tracker.bytesForTask(task));
+    }
+
+    public void testRefreshNullSnapshotFallsBackToEmpty() {
+        // A misbehaving backend that returns null must not propagate NPE through the refresh path.
+        NativeMemoryUsageTracker.setSnapshotSupplier(() -> null);
+        NativeMemoryUsageTracker tracker = new NativeMemoryUsageTracker(() -> 0.5);
+        tracker.refresh();
+        Task task = createMockTaskWithResourceStats(SearchTask.class, 1, 1, 42L);
+        assertEquals(0L, tracker.bytesForTask(task));
+    }
+
+    public void testBytesForTaskAfterRefreshReadsSnapshot() {
+        long taskId = 12345L;
+        long bytes = 8L * 1024 * 1024;
+        Map snapshot = new HashMap<>();
+        snapshot.put(taskId, bytes);
+        NativeMemoryUsageTracker.setSnapshotSupplier(() -> snapshot);
+        NativeMemoryUsageTracker tracker = new NativeMemoryUsageTracker(() -> 0.5);
+        tracker.refresh();
+        Task task = mockTaskWithId(SearchTask.class, taskId);
+        assertEquals(bytes, tracker.bytesForTask(task));
+    }
+
+    public void testBytesForTaskNullTaskReturnsZero() {
+        NativeMemoryUsageTracker tracker = new NativeMemoryUsageTracker(() -> 0.5);
+        assertEquals(0L, tracker.bytesForTask(null));
+    }
+
+    public void testHasSnapshotProviderReflectsInstallation() {
+        // Once a backend installs a snapshot supplier, hasSnapshotProvider() must report true
+        // even when the supplier itself returns an empty map. The contract is "is something
+        // wired up", not "does the wired thing currently have data".
+        Supplier> supplier = java.util.Collections::emptyMap;
+        NativeMemoryUsageTracker.setSnapshotSupplier(supplier);
+        assertTrue(NativeMemoryUsageTracker.hasSnapshotProvider());
+    }
+
+    public void testNullSnapshotSupplierIgnored() {
+        // Existing supplier should be retained when caller passes null.
+        Supplier> good = java.util.Collections::emptyMap;
+        NativeMemoryUsageTracker.setSnapshotSupplier(good);
+        assertTrue(NativeMemoryUsageTracker.hasSnapshotProvider());
+        NativeMemoryUsageTracker.setSnapshotSupplier(null);
+        // Still installed because null is a no-op.
+        assertTrue(NativeMemoryUsageTracker.hasSnapshotProvider());
+    }
+
+    public void testGetNativeMemoryBudgetBytesClampsNegative() {
+        NativeMemoryUsageTracker.setNativeMemoryBudgetSupplier(() -> -1L);
+        assertEquals(0L, NativeMemoryUsageTracker.getNativeMemoryBudgetBytes());
+    }
+
+    public void testGetNativeMemoryBudgetBytesPropagatesValue() {
+        NativeMemoryUsageTracker.setNativeMemoryBudgetSupplier(() -> 1024L * 1024L * 1024L);
+        assertEquals(1024L * 1024L * 1024L, NativeMemoryUsageTracker.getNativeMemoryBudgetBytes());
+    }
+
+    public void testNullBudgetSupplierIgnored() {
+        NativeMemoryUsageTracker.setNativeMemoryBudgetSupplier(() -> 99L);
+        NativeMemoryUsageTracker.setNativeMemoryBudgetSupplier(null);
+        assertEquals(99L, NativeMemoryUsageTracker.getNativeMemoryBudgetBytes());
+    }
+
+    public void testEvaluateInertWhenFractionZero() {
+        // Operator hasn't opted in (fraction=0): tracker MUST NOT cancel even with budget + heavy task.
+        NativeMemoryUsageTracker.setNativeMemoryBudgetSupplier(() -> 1_000L);
+        Map snapshot = new HashMap<>();
+        long taskId = 1L;
+        snapshot.put(taskId, 999L);
+        NativeMemoryUsageTracker.setSnapshotSupplier(() -> snapshot);
+        NativeMemoryUsageTracker tracker = new NativeMemoryUsageTracker(() -> 0.0);
+        tracker.refresh();
+        Task task = mockTaskWithId(SearchTask.class, taskId);
+        Optional reason = tracker.checkAndMaybeGetCancellationReason(task);
+        assertFalse(reason.isPresent());
+    }
+
+    public void testEvaluateInertWhenBudgetZero() {
+        // No backend installed budget: tracker MUST NOT cancel even with a fraction set.
+        Map snapshot = new HashMap<>();
+        long taskId = 1L;
+        snapshot.put(taskId, 999L);
+        NativeMemoryUsageTracker.setSnapshotSupplier(() -> snapshot);
+        NativeMemoryUsageTracker tracker = new NativeMemoryUsageTracker(() -> 0.5);
+        tracker.refresh();
+        Task task = mockTaskWithId(SearchTask.class, taskId);
+        Optional reason = tracker.checkAndMaybeGetCancellationReason(task);
+        assertFalse(reason.isPresent());
+    }
+
+    public void testEvaluateNonCancellableTaskSkipped() {
+        NativeMemoryUsageTracker.setNativeMemoryBudgetSupplier(() -> 1_000L);
+        NativeMemoryUsageTracker.setSnapshotSupplier(() -> Map.of(1L, 999L));
+        NativeMemoryUsageTracker tracker = new NativeMemoryUsageTracker(() -> 0.5);
+        tracker.refresh();
+        // Plain Task (not a CancellableTask) should be ignored.
+        Task task = new Task(1L, "type", "action", "description", TaskId.EMPTY_TASK_ID, java.util.Collections.emptyMap());
+        Optional reason = tracker.checkAndMaybeGetCancellationReason(task);
+        assertFalse(reason.isPresent());
+    }
+
+    public void testEvaluateBelowThreshold() {
+        long budget = 1_000L;
+        double fraction = 0.5;
+        long taskId = 7L;
+        // 400 < 500 (budget*fraction)
+        NativeMemoryUsageTracker.setNativeMemoryBudgetSupplier(() -> budget);
+        NativeMemoryUsageTracker.setSnapshotSupplier(() -> Map.of(taskId, 400L));
+        NativeMemoryUsageTracker tracker = new NativeMemoryUsageTracker(() -> fraction);
+        tracker.refresh();
+        Task task = mockTaskWithId(SearchTask.class, taskId);
+        Optional reason = tracker.checkAndMaybeGetCancellationReason(task);
+        assertFalse(reason.isPresent());
+    }
+
+    public void testEvaluateAtAndAboveThresholdReturnsReason() {
+        long budget = 1_000L;
+        double fraction = 0.5;
+        long thresholdBytes = (long) (budget * fraction);
+        long taskId = 7L;
+        long usage = thresholdBytes + 1L; // strictly above to keep score > 1
+        NativeMemoryUsageTracker.setNativeMemoryBudgetSupplier(() -> budget);
+        NativeMemoryUsageTracker.setSnapshotSupplier(() -> Map.of(taskId, usage));
+        NativeMemoryUsageTracker tracker = new NativeMemoryUsageTracker(() -> fraction);
+        tracker.refresh();
+        Task task = mockTaskWithId(SearchShardTask.class, taskId);
+        Optional reason = tracker.checkAndMaybeGetCancellationReason(task);
+        assertTrue(reason.isPresent());
+        assertTrue("score must be at least 1", reason.get().getCancellationScore() >= 1);
+        assertTrue(
+            "reason message should mention native memory usage",
+            reason.get().getMessage().toLowerCase(java.util.Locale.ROOT).contains("native memory")
+        );
+    }
+
+    public void testEvaluateScoreScalesWithUsage() {
+        // Score is floor(usage / threshold) per tracker's evaluator.
+        long budget = 1_000L;
+        double fraction = 0.1; // threshold = 100
+        long taskId = 9L;
+        NativeMemoryUsageTracker.setNativeMemoryBudgetSupplier(() -> budget);
+        NativeMemoryUsageTracker.setSnapshotSupplier(() -> Map.of(taskId, 350L)); // 3x threshold
+        NativeMemoryUsageTracker tracker = new NativeMemoryUsageTracker(() -> fraction);
+        tracker.refresh();
+        Task task = mockTaskWithId(SearchTask.class, taskId);
+        Optional reason = tracker.checkAndMaybeGetCancellationReason(task);
+        assertTrue(reason.isPresent());
+        assertEquals(3, reason.get().getCancellationScore());
+    }
+
+    public void testEvaluateFractionClampedToOne() {
+        // Fraction > 1.0 should be clamped to 1.0 — only tasks at or above the full budget cancel.
+        long budget = 1_000L;
+        long taskId = 1L;
+        NativeMemoryUsageTracker.setNativeMemoryBudgetSupplier(() -> budget);
+        NativeMemoryUsageTracker.setSnapshotSupplier(() -> Map.of(taskId, 999L));
+        NativeMemoryUsageTracker tracker = new NativeMemoryUsageTracker(() -> 5.0);
+        tracker.refresh();
+        Task task = mockTaskWithId(SearchTask.class, taskId);
+        // 999 < 1000 (clamped threshold) — must NOT cancel
+        assertFalse(tracker.checkAndMaybeGetCancellationReason(task).isPresent());
+
+        NativeMemoryUsageTracker.setSnapshotSupplier(() -> Map.of(taskId, 1_000L));
+        tracker.refresh();
+        // 1000 >= 1000 — cancel
+        assertTrue(tracker.checkAndMaybeGetCancellationReason(task).isPresent());
+    }
+
+    public void testStatsMaxAndAvgOverActiveTasks() {
+        Map snapshot = new HashMap<>();
+        snapshot.put(1L, 100L);
+        snapshot.put(2L, 200L);
+        snapshot.put(3L, 300L);
+        NativeMemoryUsageTracker.setSnapshotSupplier(() -> snapshot);
+        NativeMemoryUsageTracker tracker = new NativeMemoryUsageTracker(() -> 0.5);
+        tracker.refresh();
+
+        List tasks = List.of(
+            mockTaskWithId(SearchTask.class, 1L),
+            mockTaskWithId(SearchTask.class, 2L),
+            mockTaskWithId(SearchTask.class, 3L)
+        );
+        NativeMemoryUsageTracker.Stats stats = (NativeMemoryUsageTracker.Stats) tracker.stats(tasks);
+        // Cancellation count is zero — no checkAndMaybeGetCancellationReason invocations counted yet.
+        NativeMemoryUsageTracker.Stats expected = new NativeMemoryUsageTracker.Stats(0L, 300L, 200L);
+        assertEquals(expected, stats);
+    }
+
+    public void testStatsEmptyTaskList() {
+        NativeMemoryUsageTracker tracker = new NativeMemoryUsageTracker(() -> 0.5);
+        tracker.refresh();
+        NativeMemoryUsageTracker.Stats stats = (NativeMemoryUsageTracker.Stats) tracker.stats(List.of());
+        assertEquals(new NativeMemoryUsageTracker.Stats(0L, 0L, 0L), stats);
+    }
+
+    public void testStatsWireSerializationRoundTrip() throws Exception {
+        NativeMemoryUsageTracker.Stats original = new NativeMemoryUsageTracker.Stats(7L, 1024L, 512L);
+        try (BytesStreamOutput out = new BytesStreamOutput()) {
+            original.writeTo(out);
+            try (StreamInput in = out.bytes().streamInput()) {
+                NativeMemoryUsageTracker.Stats deserialized = new NativeMemoryUsageTracker.Stats(in);
+                assertEquals(original, deserialized);
+                assertEquals(original.hashCode(), deserialized.hashCode());
+            }
+        }
+    }
+
+    public void testStatsEqualsAndHashCode() {
+        NativeMemoryUsageTracker.Stats a = new NativeMemoryUsageTracker.Stats(1L, 2L, 3L);
+        NativeMemoryUsageTracker.Stats b = new NativeMemoryUsageTracker.Stats(1L, 2L, 3L);
+        NativeMemoryUsageTracker.Stats c = new NativeMemoryUsageTracker.Stats(1L, 2L, 4L);
+        assertEquals(a, a);
+        assertEquals(a, b);
+        assertNotEquals(a, c);
+        assertNotEquals(a, null);
+        assertNotEquals(a, "not a stats object");
+        assertEquals(a.hashCode(), b.hashCode());
+    }
+
+    public void testIsNativeTrackingSupportedRequiresProvider() {
+        // isNativeTrackingSupported() returns false when no backend has installed a real
+        // supplier. Because the static supplier is a process-wide singleton, this assertion
+        // is best-effort: on Linux with prior installation it may be true, but on every
+        // platform it should agree with hasSnapshotProvider() AND a positive total memory
+        // reading. We verify the predicate is consistent with its inputs rather than
+        // hard-coding a value.
+        boolean supported = NativeMemoryUsageTracker.isNativeTrackingSupported();
+        if (supported) {
+            assertTrue(NativeMemoryUsageTracker.hasSnapshotProvider());
+            assertTrue(org.apache.lucene.util.Constants.LINUX);
+        }
+        // The negative direction always holds: non-Linux MUST report unsupported.
+        if (org.apache.lucene.util.Constants.LINUX == false) {
+            assertFalse(supported);
+        }
+    }
+
+    public void testUpdateIsNoOp() {
+        // update() is documented as a no-op; calling it must not throw or change snapshot state.
+        NativeMemoryUsageTracker.setSnapshotSupplier(() -> Map.of(1L, 100L));
+        NativeMemoryUsageTracker tracker = new NativeMemoryUsageTracker(() -> 0.5);
+        tracker.refresh();
+        Task task = mockTaskWithId(SearchTask.class, 1L);
+        tracker.update(task);
+        assertEquals(100L, tracker.bytesForTask(task));
+    }
+
+    /**
+     * Build a {@link CancellableTask} mock whose {@code getId()} returns the given id.
+     * The test-framework helper randomizes ids, which doesn't fit a snapshot-keyed test.
+     */
+    private  T mockTaskWithId(Class type, long id) {
+        T task = org.mockito.Mockito.mock(type);
+        org.mockito.Mockito.when(task.getId()).thenReturn(id);
+        org.mockito.Mockito.when(task.getTotalResourceStats())
+            .thenReturn(new org.opensearch.core.tasks.resourcetracker.TaskResourceUsage(0L, 0L));
+        return task;
+    }
+}
diff --git a/server/src/test/java/org/opensearch/search/backpressure/trackers/NodeDuressTrackersTests.java b/server/src/test/java/org/opensearch/search/backpressure/trackers/NodeDuressTrackersTests.java
index f46d84e1034a2..2610d4f468ce6 100644
--- a/server/src/test/java/org/opensearch/search/backpressure/trackers/NodeDuressTrackersTests.java
+++ b/server/src/test/java/org/opensearch/search/backpressure/trackers/NodeDuressTrackersTests.java
@@ -84,4 +84,57 @@ public void testNodeInDuressWhenCPUAndHeapInDuress() {
         // for the third time it should be in duress
         assertTrue(nodeDuressTrackers.isNodeInDuress());
     }
+
+    public void testNodeInDuressWhenNativeMemoryInDuress() {
+        // Only NATIVE_MEMORY tracker registered. With breach threshold 9, isNodeInDuress
+        // crosses the threshold on the third call (each isNodeInDuress invocation triggers
+        // updateCache once per ResourceType value, so each call increments the streak by
+        // ResourceType.values().length).
+        EnumMap map = new EnumMap<>(ResourceType.class) {
+            {
+                put(ResourceType.NATIVE_MEMORY, new NodeDuressTracker(() -> true, () -> 9));
+            }
+        };
+
+        NodeDuressTrackers nodeDuressTrackers = new NodeDuressTrackers(map, resourceCacheExpiryChecker);
+
+        assertFalse(nodeDuressTrackers.isNodeInDuress());
+        assertFalse(nodeDuressTrackers.isNodeInDuress());
+        assertTrue(nodeDuressTrackers.isNodeInDuress());
+        assertTrue(nodeDuressTrackers.isNativeMemoryInDuress());
+    }
+
+    public void testIsNativeMemoryInDuressFalseWhenUnregistered() {
+        // CPU + MEMORY only — NATIVE_MEMORY missing entirely. Per
+        // NodeDuressTrackers#updateCache, missing trackers must be treated as "not in duress"
+        // rather than throwing on the absent enum value.
+        EnumMap map = new EnumMap<>(ResourceType.class) {
+            {
+                put(ResourceType.CPU, new NodeDuressTracker(() -> true, () -> 1));
+                put(ResourceType.MEMORY, new NodeDuressTracker(() -> false, () -> 1));
+            }
+        };
+
+        NodeDuressTrackers nodeDuressTrackers = new NodeDuressTrackers(map, resourceCacheExpiryChecker);
+        // Drive isResourceInDuress to populate the cache for all enum values.
+        assertTrue(nodeDuressTrackers.isResourceInDuress(ResourceType.CPU));
+        assertFalse(nodeDuressTrackers.isResourceInDuress(ResourceType.MEMORY));
+        // NATIVE_MEMORY was not registered — must be reported as not in duress.
+        assertFalse(nodeDuressTrackers.isResourceInDuress(ResourceType.NATIVE_MEMORY));
+        assertFalse(nodeDuressTrackers.isNativeMemoryInDuress());
+    }
+
+    public void testIsNativeMemoryInDuressFalseWhenStreakNotMet() {
+        // Tracker is registered but breach threshold not reached. With threshold 100 the
+        // streak never crosses regardless of how many calls we make in this test loop.
+        EnumMap map = new EnumMap<>(ResourceType.class) {
+            {
+                put(ResourceType.NATIVE_MEMORY, new NodeDuressTracker(() -> true, () -> 100));
+            }
+        };
+        NodeDuressTrackers nodeDuressTrackers = new NodeDuressTrackers(map, resourceCacheExpiryChecker);
+        for (int i = 0; i < 10; i++) {
+            assertFalse(nodeDuressTrackers.isNativeMemoryInDuress());
+        }
+    }
 }
diff --git a/server/src/test/java/org/opensearch/search/backpressure/trackers/TaskResourceUsageTrackerTypeTests.java b/server/src/test/java/org/opensearch/search/backpressure/trackers/TaskResourceUsageTrackerTypeTests.java
new file mode 100644
index 0000000000000..c7dac71c970c4
--- /dev/null
+++ b/server/src/test/java/org/opensearch/search/backpressure/trackers/TaskResourceUsageTrackerTypeTests.java
@@ -0,0 +1,41 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.search.backpressure.trackers;
+
+import org.opensearch.test.OpenSearchTestCase;
+
+/**
+ * Unit tests for {@link TaskResourceUsageTrackerType} ensuring the new
+ * {@code NATIVE_MEMORY_USAGE_TRACKER} entry round-trips through {@code fromName}.
+ */
+public class TaskResourceUsageTrackerTypeTests extends OpenSearchTestCase {
+
+    public void testNativeMemoryUsageTrackerName() {
+        assertEquals("native_memory_usage_tracker", TaskResourceUsageTrackerType.NATIVE_MEMORY_USAGE_TRACKER.getName());
+    }
+
+    public void testFromNameRoundTrip() {
+        for (TaskResourceUsageTrackerType type : TaskResourceUsageTrackerType.values()) {
+            assertSame(type, TaskResourceUsageTrackerType.fromName(type.getName()));
+        }
+    }
+
+    public void testFromNameUnknownThrows() {
+        IllegalArgumentException ex = expectThrows(
+            IllegalArgumentException.class,
+            () -> TaskResourceUsageTrackerType.fromName("not_a_real_tracker")
+        );
+        assertTrue(ex.getMessage().contains("not_a_real_tracker"));
+    }
+
+    public void testFromNameIsCaseSensitive() {
+        // Mirrors the strictness of ResourceType.fromName — uppercase is not accepted.
+        expectThrows(IllegalArgumentException.class, () -> TaskResourceUsageTrackerType.fromName("NATIVE_MEMORY_USAGE_TRACKER"));
+    }
+}
diff --git a/server/src/test/java/org/opensearch/wlm/ResourceTypeTests.java b/server/src/test/java/org/opensearch/wlm/ResourceTypeTests.java
index 16bd8b7e66266..ade5bb3d6f861 100644
--- a/server/src/test/java/org/opensearch/wlm/ResourceTypeTests.java
+++ b/server/src/test/java/org/opensearch/wlm/ResourceTypeTests.java
@@ -23,10 +23,32 @@ public void testFromName() {
         assertThrows(IllegalArgumentException.class, () -> { ResourceType.fromName("JVM"); });
         assertThrows(IllegalArgumentException.class, () -> { ResourceType.fromName("Heap"); });
         assertThrows(IllegalArgumentException.class, () -> { ResourceType.fromName("Disk"); });
+
+        assertSame(ResourceType.NATIVE_MEMORY, ResourceType.fromName("native_memory"));
+        assertThrows(IllegalArgumentException.class, () -> { ResourceType.fromName("NATIVE_MEMORY"); });
+        assertThrows(IllegalArgumentException.class, () -> { ResourceType.fromName("Native_Memory"); });
     }
 
     public void testGetName() {
         assertEquals("cpu", ResourceType.CPU.getName());
         assertEquals("memory", ResourceType.MEMORY.getName());
+        assertEquals("native_memory", ResourceType.NATIVE_MEMORY.getName());
+    }
+
+    public void testNativeMemoryStatsDisabled() {
+        // statsEnabled=false ensures WorkloadGroupState skips allocating slots for NATIVE_MEMORY.
+        assertFalse(ResourceType.NATIVE_MEMORY.hasStatsEnabled());
+        // CPU and MEMORY remain stats-enabled; this guards against accidental refactors.
+        assertTrue(ResourceType.CPU.hasStatsEnabled());
+        assertTrue(ResourceType.MEMORY.hasStatsEnabled());
+    }
+
+    public void testNativeMemoryUsesPlaceholderCalculator() {
+        // Calculator is a no-op (returns 0.0) so accidental iteration over NATIVE_MEMORY by
+        // a WLM stats path does not produce spurious numbers.
+        assertSame(
+            org.opensearch.wlm.tracker.NativeMemoryUsageCalculator.INSTANCE,
+            ResourceType.NATIVE_MEMORY.getResourceUsageCalculator()
+        );
     }
 }
diff --git a/server/src/test/java/org/opensearch/wlm/tracker/NativeMemoryUsageCalculatorTests.java b/server/src/test/java/org/opensearch/wlm/tracker/NativeMemoryUsageCalculatorTests.java
new file mode 100644
index 0000000000000..621b699042af3
--- /dev/null
+++ b/server/src/test/java/org/opensearch/wlm/tracker/NativeMemoryUsageCalculatorTests.java
@@ -0,0 +1,43 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.wlm.tracker;
+
+import org.opensearch.test.OpenSearchTestCase;
+import org.opensearch.wlm.WorkloadGroupTask;
+
+import java.util.List;
+
+import static org.mockito.Mockito.mock;
+
+/**
+ * Unit tests for {@link NativeMemoryUsageCalculator}, the placeholder calculator wired to
+ * {@link org.opensearch.wlm.ResourceType#NATIVE_MEMORY}. WLM does not account off-heap
+ * memory per workload group, so this calculator MUST return zero unconditionally.
+ */
+public class NativeMemoryUsageCalculatorTests extends OpenSearchTestCase {
+
+    public void testCalculatorIsSingleton() {
+        assertSame(NativeMemoryUsageCalculator.INSTANCE, NativeMemoryUsageCalculator.INSTANCE);
+    }
+
+    public void testCalculateResourceUsageReturnsZero() {
+        // Multi-task input with mocks that have no behavior — calculator must not consult them.
+        List tasks = List.of(mock(WorkloadGroupTask.class), mock(WorkloadGroupTask.class));
+        assertEquals(0.0d, NativeMemoryUsageCalculator.INSTANCE.calculateResourceUsage(tasks), 0.0d);
+    }
+
+    public void testCalculateResourceUsageEmptyList() {
+        assertEquals(0.0d, NativeMemoryUsageCalculator.INSTANCE.calculateResourceUsage(List.of()), 0.0d);
+    }
+
+    public void testCalculateTaskResourceUsageReturnsZero() {
+        WorkloadGroupTask task = mock(WorkloadGroupTask.class);
+        assertEquals(0.0d, NativeMemoryUsageCalculator.INSTANCE.calculateTaskResourceUsage(task), 0.0d);
+    }
+}
diff --git a/server/src/test/java/org/opensearch/wlm/tracker/WorkloadGroupResourceUsageTrackerServiceTests.java b/server/src/test/java/org/opensearch/wlm/tracker/WorkloadGroupResourceUsageTrackerServiceTests.java
new file mode 100644
index 0000000000000..954e4cb8f011f
--- /dev/null
+++ b/server/src/test/java/org/opensearch/wlm/tracker/WorkloadGroupResourceUsageTrackerServiceTests.java
@@ -0,0 +1,36 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.wlm.tracker;
+
+import org.opensearch.test.OpenSearchTestCase;
+import org.opensearch.wlm.ResourceType;
+
+/**
+ * Unit tests for {@link WorkloadGroupResourceUsageTrackerService}'s {@code TRACKED_RESOURCES}
+ * set. After NATIVE_MEMORY was added to {@link ResourceType} for search-backpressure duress
+ * routing, this set MUST stay restricted to CPU + MEMORY so per-group accounting is unchanged.
+ */
+public class WorkloadGroupResourceUsageTrackerServiceTests extends OpenSearchTestCase {
+
+    public void testTrackedResourcesContainsCpuAndMemory() {
+        assertTrue(WorkloadGroupResourceUsageTrackerService.TRACKED_RESOURCES.contains(ResourceType.CPU));
+        assertTrue(WorkloadGroupResourceUsageTrackerService.TRACKED_RESOURCES.contains(ResourceType.MEMORY));
+    }
+
+    public void testTrackedResourcesExcludesNativeMemory() {
+        // Per the explicit comment on TRACKED_RESOURCES — WLM intentionally does not iterate
+        // NATIVE_MEMORY because the calculator is a no-op and the duress signal is consumed
+        // by SearchBackpressureService instead.
+        assertFalse(WorkloadGroupResourceUsageTrackerService.TRACKED_RESOURCES.contains(ResourceType.NATIVE_MEMORY));
+    }
+
+    public void testTrackedResourcesHasExactlyTwoEntries() {
+        assertEquals(2, WorkloadGroupResourceUsageTrackerService.TRACKED_RESOURCES.size());
+    }
+}

From 0dc3cc3994c5bd21f74c2d669cba48dd2b494aea Mon Sep 17 00:00:00 2001
From: Pradeep L 
Date: Sat, 16 May 2026 12:08:14 +0530
Subject: [PATCH 05/10] removing completed field

Signed-off-by: Pradeep L 
---
 .../analytics/spi/QueryExecutionMetrics.java  |  8 +--
 .../rust/src/query_tracker.rs                 | 58 ++-----------------
 .../nativelib/QueryRegistryLayout.java        | 18 ++----
 3 files changed, 12 insertions(+), 72 deletions(-)

diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/QueryExecutionMetrics.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/QueryExecutionMetrics.java
index 8a246d74c86dc..216f56a1494b7 100644
--- a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/QueryExecutionMetrics.java
+++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/QueryExecutionMetrics.java
@@ -12,15 +12,11 @@
  * Snapshot of per-query execution metrics reported by an analytics backend.
  *
  * 
The context id (the map key in {@link AnalyticsSearchBackendPlugin#getActiveQueryMetrics()})
- * is not repeated here — this record holds only the remaining parameters: memory accounting,
- * wall time, and whether the query has completed but not yet been drained.
+ * is not repeated here — this record holds only the memory accounting fields.
  *
  * @param currentBytes bytes currently reserved by the query's memory pool
  * @param peakBytes    high-water mark of bytes reserved during the query's lifetime
- * @param wallNanos    live or frozen wall-clock duration in nanoseconds
- * @param completed    {@code true} when the query has finished executing but its entry has not
- *                     yet been drained from the backend's internal registry
  *
  * @opensearch.internal
  */
-public record QueryExecutionMetrics(long currentBytes, long peakBytes, long wallNanos, boolean completed) {}
+public record QueryExecutionMetrics(long currentBytes, long peakBytes) {}
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/query_tracker.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/query_tracker.rs
index 4ef7d783e3f14..c4c85b82ea489 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/query_tracker.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/query_tracker.rs
@@ -133,21 +133,6 @@ impl QueryTracker {
         }
     }
 
-    /// Wall-clock duration in nanoseconds, as an `i64` for FFM transport.
-    /// Returns the frozen snapshot if completed, otherwise live elapsed time.
-    /// Elapsed nanos is `u128` internally; saturates at `i64::MAX` (~292 years)
-    /// so it can always be represented as an `i64`.
-    pub fn elapsed_nanos(&self) -> i64 {
-        let frozen = self.wall_nanos.load(Ordering::Acquire);
-        if frozen > 0 {
-            // `AtomicU64` → `i64`: `frozen` was produced from `elapsed().as_nanos() as u64`
-            // so its high bit is effectively clear. Still clamp defensively.
-            frozen.min(i64::MAX as u64) as i64
-        } else {
-            self.start_time.elapsed().as_nanos().min(i64::MAX as u128) as i64
-        }
-    }
-
     pub fn is_completed(&self) -> bool {
         self.completed.load(Ordering::Acquire)
     }
@@ -166,29 +151,6 @@ impl QueryTracker {
 
 static QUERY_REGISTRY: Lazy>> = Lazy::new(DashMap::new);
 
-/// Remove a completed tracker from the registry and return it.
-/// Called from JNI after Java has consumed the metrics.
-pub fn drain_completed_query(context_id: i64) -> Option> {
-    let result = QUERY_REGISTRY
-        .remove_if(&context_id, |_, t| t.is_completed())
-        .map(|(_, t)| t);
-    match &result {
-        Some(t) => log_info!(
-            "[nativemem-bp] rust.drain_completed_query: drained ctx={} (peak_bytes={}, wall_secs={:.3}, registry_size_after={})",
-            context_id,
-            t.memory_pool.peak_bytes(),
-            t.wall_secs(),
-            QUERY_REGISTRY.len()
-        ),
-        None => log_info!(
-            "[nativemem-bp] rust.drain_completed_query: ctx={} not drained (absent or not completed) (registry_size={})",
-            context_id,
-            QUERY_REGISTRY.len()
-        ),
-    }
-    result
-}
-
 // ---------------------------------------------------------------------------
 // Registry snapshot — top-N FFM export
 // ---------------------------------------------------------------------------
@@ -202,19 +164,15 @@ pub fn drain_completed_query(context_id: i64) -> Option> {
 /// | context_id    | `QueryTracker::context_id`                                |
 /// | current_bytes | `QueryMemoryPool::current_bytes`, clamped to `i64::MAX`   |
 /// | peak_bytes    | `QueryMemoryPool::peak_bytes`, clamped to `i64::MAX`      |
-/// | wall_nanos    | live elapsed or frozen wall time (see `elapsed_nanos()`)  |
-/// | completed     | 1 if the tracker has been marked completed, else 0        |
 #[repr(C)]
 #[derive(Debug, Clone, Copy)]
 pub struct WireQueryMetric {
     pub context_id: i64,
     pub current_bytes: i64,
     pub peak_bytes: i64,
-    pub wall_nanos: i64,
-    pub completed: i64,
 }
 
-const _: () = assert!(std::mem::size_of::() == 5 * 8);
+const _: () = assert!(std::mem::size_of::() == 3 * 8);
 
 fn usize_to_i64_saturating(value: usize) -> i64 {
     if value > i64::MAX as usize {
@@ -230,8 +188,6 @@ impl WireQueryMetric {
             context_id: tracker.context_id,
             current_bytes: usize_to_i64_saturating(tracker.memory_pool.current_bytes()),
             peak_bytes: usize_to_i64_saturating(tracker.memory_pool.peak_bytes()),
-            wall_nanos: tracker.elapsed_nanos(),
-            completed: if tracker.is_completed() { 1 } else { 0 },
         }
     }
 }
@@ -250,8 +206,8 @@ impl WireQueryMetric {
 /// sorted. Callers that need a sorted view sort the prefix client-side.
 ///
 /// Filters applied on the Rust side, in this order:
-/// - completed trackers are skipped (they retain `current_bytes == 0` until
-///   `drain_completed_query` runs anyway, so this is mostly a fast path)
+/// - completed trackers are skipped (a completed query's `current_bytes`
+///   drops to zero on `Drop` anyway, so this is mostly a fast path)
 /// - `current_bytes == 0` is skipped (registered but un-allocated trackers
 ///   are not "heavy" candidates)
 ///
@@ -349,13 +305,11 @@ pub fn snapshot_top_n_by_current(out: &mut [WireQueryMetric]) -> usize {
         // and final values are independent point-in-time samples.
         out[written] = WireQueryMetric::from_tracker(&entry.tracker);
         log_info!(
-            "[nativemem-bp] rust.snapshot_top_n entry[{}]: ctx={}, current={}B, peak={}B, wall_ns={}, completed={}",
+            "[nativemem-bp] rust.snapshot_top_n entry[{}]: ctx={}, current={}B, peak={}B",
             written,
             out[written].context_id,
             out[written].current_bytes,
             out[written].peak_bytes,
-            out[written].wall_nanos,
-            out[written].completed,
         );
         written += 1;
     }
@@ -722,8 +676,6 @@ mod tests {
                 context_id: 0,
                 current_bytes: 0,
                 peak_bytes: 0,
-                wall_nanos: 0,
-                completed: 0,
             };
             n
         ]
@@ -841,8 +793,6 @@ mod tests {
             context_id: -1,
             current_bytes: -1,
             peak_bytes: -1,
-            wall_nanos: -1,
-            completed: -1,
         };
         let mut buf = vec![sentinel; 16];
         let written = snapshot_top_n_by_current(&mut buf);
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/QueryRegistryLayout.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/QueryRegistryLayout.java
index 6f9ae0e217ccb..0f3564b108c07 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/QueryRegistryLayout.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/QueryRegistryLayout.java
@@ -17,11 +17,11 @@
 
 /**
  * Mirrors the Rust {@code query_tracker::WireQueryMetric} {@code #[repr(C)]}
- * struct (5 × i64 = 40 bytes) and decodes a strided buffer of those structs
+ * struct (3 × i64 = 24 bytes) and decodes a strided buffer of those structs
  * directly into the SPI types consumed by
  * {@link org.opensearch.analytics.spi.AnalyticsSearchBackendPlugin#getActiveQueryMetrics}.
  *
- * 
Decoder reads each row by slicing the segment to that row's 40-byte
+ * 
Decoder reads each row by slicing the segment to that row's 24-byte
  * window. We deliberately avoid {@code SequenceLayout}-derived
  * {@code VarHandle}s because their bounds check enforces the segment span
  * the entire sequence layout, not just up to the requested row. With a
@@ -31,7 +31,7 @@
  * 
Buffer shape (populated by {@code df_query_registry_top_n_by_current}):
  * 
  *   [ entry 0 ][ entry 1 ] ... [ entry N-1 ]
- *   each entry = { context_id, current_bytes, peak_bytes, wall_nanos, completed } (5 × i64)
+ *   each entry = { context_id, current_bytes, peak_bytes } (3 × i64)
  * 

  */
 public final class QueryRegistryLayout {
@@ -40,9 +40,7 @@ public final class QueryRegistryLayout {
     public static final StructLayout ENTRY_LAYOUT = MemoryLayout.structLayout(
         ValueLayout.JAVA_LONG.withName("context_id"),
         ValueLayout.JAVA_LONG.withName("current_bytes"),
-        ValueLayout.JAVA_LONG.withName("peak_bytes"),
-        ValueLayout.JAVA_LONG.withName("wall_nanos"),
-        ValueLayout.JAVA_LONG.withName("completed")
+        ValueLayout.JAVA_LONG.withName("peak_bytes")
     );
 
     /** Byte size of one wire entry. Matches {@code size_of::()} on the Rust side. */
@@ -53,11 +51,9 @@ public final class QueryRegistryLayout {
     private static final long OFF_CONTEXT_ID    = 0L;
     private static final long OFF_CURRENT_BYTES = 8L;
     private static final long OFF_PEAK_BYTES    = 16L;
-    private static final long OFF_WALL_NANOS    = 24L;
-    private static final long OFF_COMPLETED     = 32L;
 
     static {
-        long expected = 5L * Long.BYTES;
+        long expected = 3L * Long.BYTES;
         if (ENTRY_BYTES != expected) {
             throw new AssertionError("QueryRegistryLayout entry size mismatch: expected " + expected + " but got " + ENTRY_BYTES);
         }
@@ -87,9 +83,7 @@ public static QueryExecutionMetrics readMetrics(MemorySegment seg, int i) {
         long rowOffset = (long) i * ENTRY_BYTES;
         return new QueryExecutionMetrics(
             seg.get(ValueLayout.JAVA_LONG, rowOffset + OFF_CURRENT_BYTES),
-            seg.get(ValueLayout.JAVA_LONG, rowOffset + OFF_PEAK_BYTES),
-            seg.get(ValueLayout.JAVA_LONG, rowOffset + OFF_WALL_NANOS),
-            seg.get(ValueLayout.JAVA_LONG, rowOffset + OFF_COMPLETED) != 0L
+            seg.get(ValueLayout.JAVA_LONG, rowOffset + OFF_PEAK_BYTES)
         );
     }
 }

From e2da33fefda7d8b633bc4e8cbbd17184cf93749d Mon Sep 17 00:00:00 2001
From: Pradeep L 
Date: Mon, 18 May 2026 00:34:52 +0530
Subject: [PATCH 06/10] native memory SBP refactoring

Signed-off-by: Pradeep L 
---
 .../spi/AnalyticsSearchBackendPlugin.java     |   2 +-
 .../analytics/spi/QueryExecutionMetrics.java  |   5 +-
 .../libs/dataformat-native/rust/Cargo.toml    |   2 +-
 .../rust/src/ffm.rs                           |   5 -
 .../rust/src/query_tracker.rs                 | 129 ++----
 .../DataFusionAnalyticsBackendPlugin.java     |   4 +-
 .../be/datafusion/DataFusionPlugin.java       |  30 +-
 .../be/datafusion/nativelib/NativeBridge.java |   2 +-
 .../nativelib/QueryRegistryLayout.java        |  21 +-
 .../analytics/exec/DefaultPlanExecutor.java   |  43 +-
 .../exec/task/AnalyticsQueryTask.java         |  14 +-
 .../NativeMemorySearchBackpressureIT.java     | 404 ++++++++++++++++++
 .../common/settings/ClusterSettings.java      |   3 +-
 .../org/opensearch/monitor/os/OsProbe.java    |  51 +--
 .../NativeMemoryUsageService.java             | 188 ++++++++
 .../SearchBackpressureService.java            | 180 +++-----
 .../settings/NodeDuressSettings.java          |  37 +-
 .../settings/SearchShardTaskSettings.java     |  29 +-
 .../settings/SearchTaskSettings.java          |  28 +-
 .../trackers/NativeMemoryUsageTracker.java    | 187 +++-----
 .../trackers/NodeDuressTrackers.java          |   9 +-
 .../trackers/TaskResourceUsageTrackers.java   |  11 -
 .../opensearch/monitor/os/OsProbeTests.java   |  87 +---
 .../NativeMemoryUsageServiceTests.java        | 263 ++++++++++++
 .../SearchBackpressureServiceTests.java       | 166 +++----
 .../settings/NodeDuressSettingsTests.java     |  48 ++-
 .../SearchShardTaskSettingsTests.java         |  29 +-
 .../settings/SearchTaskSettingsTests.java     |  30 +-
 .../stats/SearchShardTaskStatsTests.java      |  62 +++
 .../stats/SearchTaskStatsTests.java           |  62 +++
 .../NativeMemoryUsageTrackerTests.java        | 193 ++++-----
 31 files changed, 1519 insertions(+), 805 deletions(-)
 create mode 100644 server/src/internalClusterTest/java/org/opensearch/search/backpressure/NativeMemorySearchBackpressureIT.java
 create mode 100644 server/src/main/java/org/opensearch/search/backpressure/NativeMemoryUsageService.java
 create mode 100644 server/src/test/java/org/opensearch/search/backpressure/NativeMemoryUsageServiceTests.java

diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/AnalyticsSearchBackendPlugin.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/AnalyticsSearchBackendPlugin.java
index 5da01da075766..42d5fbe386ed5 100644
--- a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/AnalyticsSearchBackendPlugin.java
+++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/AnalyticsSearchBackendPlugin.java
@@ -130,7 +130,7 @@ default void configureFilterDelegation(FilterDelegationHandle handle, BackendExe
      * 
Default implementation returns an empty map so backends that do not track per-query
      * metrics don't have to opt in.
      */
-    default Map getActiveQueryMetrics() {
+    default Map getTopQueriesByMemory() {
         return Collections.emptyMap();
     }
 
diff --git a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/QueryExecutionMetrics.java b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/QueryExecutionMetrics.java
index 216f56a1494b7..0e03287d3ead7 100644
--- a/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/QueryExecutionMetrics.java
+++ b/sandbox/libs/analytics-framework/src/main/java/org/opensearch/analytics/spi/QueryExecutionMetrics.java
@@ -11,12 +11,11 @@
 /**
  * Snapshot of per-query execution metrics reported by an analytics backend.
  *
- * 
The context id (the map key in {@link AnalyticsSearchBackendPlugin#getActiveQueryMetrics()})
+ * 
The context id (the map key in {@link AnalyticsSearchBackendPlugin#getTopQueriesByMemory()})
  * is not repeated here — this record holds only the memory accounting fields.
  *
  * @param currentBytes bytes currently reserved by the query's memory pool
- * @param peakBytes    high-water mark of bytes reserved during the query's lifetime
  *
  * @opensearch.internal
  */
-public record QueryExecutionMetrics(long currentBytes, long peakBytes) {}
+public record QueryExecutionMetrics(long currentBytes) {}
diff --git a/sandbox/libs/dataformat-native/rust/Cargo.toml b/sandbox/libs/dataformat-native/rust/Cargo.toml
index 0c890cc18177e..4f391153203eb 100644
--- a/sandbox/libs/dataformat-native/rust/Cargo.toml
+++ b/sandbox/libs/dataformat-native/rust/Cargo.toml
@@ -81,7 +81,7 @@ opensearch-repository-azure = { path = "../../../plugins/native-repository-azure
 opensearch-repository-fs = { path = "../../../plugins/native-repository-fs/src/main/rust" }
 
 [profile.release]
-lto = "thin"
+lto = true
 codegen-units = 1
 incremental = true
 debug = "line-tables-only"
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs
index 452341af389b2..d5b75de6b7aaa 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs
@@ -237,7 +237,6 @@ pub unsafe extern "C" fn df_query_registry_top_n_by_current(
         return Err(format!("negative capacity: {cap_entries}"));
     }
     if cap_entries == 0 {
-        info!("[nativemem-bp] ffm.df_query_registry_top_n_by_current: capacity=0, nothing to write");
         return Ok(0);
     }
     if out_ptr.is_null() {
@@ -246,10 +245,6 @@ pub unsafe extern "C" fn df_query_registry_top_n_by_current(
     let out: &mut [WireQueryMetric] =
         slice::from_raw_parts_mut(out_ptr as *mut WireQueryMetric, cap_entries as usize);
     let written = snapshot_top_n_by_current(out);
-    info!(
-        "[nativemem-bp] ffm.df_query_registry_top_n_by_current: wrote {} entries (capacity {})",
-        written, cap_entries
-    );
     Ok(written as i64)
 }
 
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/query_tracker.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/query_tracker.rs
index c4c85b82ea489..900fbedf6f58f 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/query_tracker.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/query_tracker.rs
@@ -28,7 +28,7 @@ use tokio_util::sync::CancellationToken;
 use datafusion::common::DataFusionError;
 use datafusion::execution::memory_pool::{MemoryConsumer, MemoryPool, MemoryReservation};
 
-use crate::{log_debug, log_info};
+use crate::log_debug;
 
 // ---------------------------------------------------------------------------
 // Per-query memory pool
@@ -163,16 +163,14 @@ static QUERY_REGISTRY: Lazy>> = Lazy::new(DashMap
 /// |---------------|-----------------------------------------------------------|
 /// | context_id    | `QueryTracker::context_id`                                |
 /// | current_bytes | `QueryMemoryPool::current_bytes`, clamped to `i64::MAX`   |
-/// | peak_bytes    | `QueryMemoryPool::peak_bytes`, clamped to `i64::MAX`      |
 #[repr(C)]
 #[derive(Debug, Clone, Copy)]
 pub struct WireQueryMetric {
     pub context_id: i64,
     pub current_bytes: i64,
-    pub peak_bytes: i64,
 }
 
-const _: () = assert!(std::mem::size_of::() == 3 * 8);
+const _: () = assert!(std::mem::size_of::() == 2 * 8);
 
 fn usize_to_i64_saturating(value: usize) -> i64 {
     if value > i64::MAX as usize {
@@ -182,16 +180,6 @@ fn usize_to_i64_saturating(value: usize) -> i64 {
     }
 }
 
-impl WireQueryMetric {
-    fn from_tracker(tracker: &QueryTracker) -> Self {
-        Self {
-            context_id: tracker.context_id,
-            current_bytes: usize_to_i64_saturating(tracker.memory_pool.current_bytes()),
-            peak_bytes: usize_to_i64_saturating(tracker.memory_pool.peak_bytes()),
-        }
-    }
-}
-
 /// Top-N snapshot: copy at most `out.len()` of the heaviest live queries
 /// (ranked by `current_bytes` descending) into `out`. Returns the number of
 /// entries actually written.
@@ -218,22 +206,30 @@ pub fn snapshot_top_n_by_current(out: &mut [WireQueryMetric]) -> usize {
     use std::cmp::{Ordering, Reverse};
     use std::collections::BinaryHeap;
 
-    /// Heap entry — orders purely by `bytes`. The tracker rides along but is
-    /// not part of the comparison, so `QueryTracker` doesn't need `Ord`.
+    /// Heap entry — POD copy of the tracker fields sampled at iteration
+    /// time. We deliberately avoid carrying an `Arc` here:
+    /// re-reading the live tracker at materialization time would let the
+    /// reported `current_bytes` drift away from the value the entry was
+    /// ranked on (e.g. a query finishes between rank and report and ships
+    /// out as zero), and would also defeat the `current_bytes == 0` filter
+    /// applied during iteration. Sampling once keeps rank and report
+    /// consistent and removes the per-survivor `Arc::clone` from the hot
+    /// path.
+    #[derive(Clone, Copy)]
     struct HeapEntry {
-        bytes: i64,
-        tracker: Arc,
+        context_id: i64,
+        current_bytes: i64,
     }
 
     impl Eq for HeapEntry {}
     impl PartialEq for HeapEntry {
         fn eq(&self, other: &Self) -> bool {
-            self.bytes == other.bytes
+            self.current_bytes == other.current_bytes
         }
     }
     impl Ord for HeapEntry {
         fn cmp(&self, other: &Self) -> Ordering {
-            self.bytes.cmp(&other.bytes)
+            self.current_bytes.cmp(&other.current_bytes)
         }
     }
     impl PartialOrd for HeapEntry {
@@ -252,71 +248,38 @@ pub fn snapshot_top_n_by_current(out: &mut [WireQueryMetric]) -> usize {
     // set — the one a heavier candidate displaces.
     let mut heap: BinaryHeap> = BinaryHeap::with_capacity(n);
 
-    // Diagnostic: log registry size at entry, plus a sample of what's in there.
-    // Helps distinguish "registry empty" from "registry has entries but all are
-    // completed or have current_bytes == 0".
-    let registry_size = QUERY_REGISTRY.len();
-    let mut sample_logged = 0usize;
-    log_info!(
-        "[nativemem-bp] rust.snapshot_top_n_by_current: enter cap={} registry_size={}",
-        n, registry_size
-    );
-
     for entry in QUERY_REGISTRY.iter() {
         let tracker = entry.value();
-        let bytes_raw = tracker.memory_pool.current_bytes();
-        let peak_raw = tracker.memory_pool.peak_bytes();
-        let completed = tracker.is_completed();
-        if sample_logged < 5 {
-            log_info!(
-                "[nativemem-bp] rust.snapshot_top_n_by_current: sample ctx={} current_bytes={} peak_bytes={} completed={}",
-                tracker.context_id, bytes_raw, peak_raw, completed
-            );
-            sample_logged += 1;
-        }
-        if completed {
+        if tracker.is_completed() {
             continue;
         }
-        let bytes = usize_to_i64_saturating(bytes_raw);
-        if bytes == 0 {
+        let current_raw = tracker.memory_pool.current_bytes();
+        let current_bytes = usize_to_i64_saturating(current_raw);
+        if current_bytes == 0 {
             continue;
         }
+        let candidate = HeapEntry {
+            context_id: tracker.context_id,
+            current_bytes,
+        };
         if heap.len() < n {
-            heap.push(Reverse(HeapEntry {
-                bytes,
-                tracker: Arc::clone(tracker),
-            }));
+            heap.push(Reverse(candidate));
         } else if let Some(Reverse(min)) = heap.peek() {
-            // Strictly greater: equal-byte ties keep the first-seen entry.
-            if bytes > min.bytes {
+            if candidate.current_bytes > min.current_bytes {
                 heap.pop();
-                heap.push(Reverse(HeapEntry {
-                    bytes,
-                    tracker: Arc::clone(tracker),
-                }));
+                heap.push(Reverse(candidate));
             }
         }
     }
 
     let mut written = 0usize;
     for Reverse(entry) in heap.into_iter() {
-        // Re-read the tracker fields here so the wire entry reflects the live
-        // values at materialization time, consistent with from_tracker. Ranking
-        // and final values are independent point-in-time samples.
-        out[written] = WireQueryMetric::from_tracker(&entry.tracker);
-        log_info!(
-            "[nativemem-bp] rust.snapshot_top_n entry[{}]: ctx={}, current={}B, peak={}B",
-            written,
-            out[written].context_id,
-            out[written].current_bytes,
-            out[written].peak_bytes,
-        );
+        out[written] = WireQueryMetric {
+            context_id: entry.context_id,
+            current_bytes: entry.current_bytes,
+        };
         written += 1;
     }
-    log_info!(
-        "[nativemem-bp] rust.snapshot_top_n_by_current: wrote {} entries (cap {})",
-        written, n
-    );
     written
 }
 
@@ -324,19 +287,7 @@ pub fn snapshot_top_n_by_current(out: &mut [WireQueryMetric]) -> usize {
 /// No-op for unknown or already-completed queries.
 pub fn cancel_query(context_id: i64) {
     if let Some(tracker) = QUERY_REGISTRY.get(&context_id) {
-        log_info!(
-            "[nativemem-bp] rust.cancel_query: firing token for ctx={} (completed={}, current_bytes={})",
-            context_id,
-            tracker.is_completed(),
-            tracker.memory_pool.current_bytes()
-        );
         tracker.cancellation_token.cancel();
-    } else {
-        log_info!(
-            "[nativemem-bp] rust.cancel_query: ctx={} not in registry (registry_size={}) — no-op",
-            context_id,
-            QUERY_REGISTRY.len()
-        );
     }
 }
 
@@ -381,11 +332,6 @@ impl QueryTrackingContext {
             wall_nanos: std::sync::atomic::AtomicU64::new(0),
         });
         QUERY_REGISTRY.insert(context_id, Arc::clone(&tracker));
-        log_info!(
-            "[nativemem-bp] rust.QueryTrackingContext::new: registered ctx={} (registry_size={})",
-            context_id,
-            QUERY_REGISTRY.len()
-        );
         Self {
             tracker: Some(tracker),
         }
@@ -407,19 +353,10 @@ impl Drop for QueryTrackingContext {
     fn drop(&mut self) {
         if let Some(tracker) = &self.tracker {
             tracker.mark_completed();
-            log_info!(
-                "[nativemem-bp] rust.QueryTrackingContext::drop: ctx={} completed (wall={:.3}s, mem_current={}B, mem_peak={}B)",
-                tracker.context_id,
-                tracker.wall_secs(),
-                tracker.memory_pool.current_bytes(),
-                tracker.memory_pool.peak_bytes(),
-            );
-            // Keep the debug line for operators who already tail the Rust debug log.
             log_debug!(
-                "Query completed ctx={}: wall={:.3}s, mem_current={}B, mem_peak={}B",
+                "query completed ctx={}: wall={:.3}s, peak={}B",
                 tracker.context_id,
                 tracker.wall_secs(),
-                tracker.memory_pool.current_bytes(),
                 tracker.memory_pool.peak_bytes(),
             );
             QUERY_REGISTRY.remove(&tracker.context_id);
@@ -675,7 +612,6 @@ mod tests {
             WireQueryMetric {
                 context_id: 0,
                 current_bytes: 0,
-                peak_bytes: 0,
             };
             n
         ]
@@ -792,7 +728,6 @@ mod tests {
         let sentinel = WireQueryMetric {
             context_id: -1,
             current_bytes: -1,
-            peak_bytes: -1,
         };
         let mut buf = vec![sentinel; 16];
         let written = snapshot_top_n_by_current(&mut buf);
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java
index bb1e24ec57114..7112c8246dac8 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionAnalyticsBackendPlugin.java
@@ -706,11 +706,11 @@ public void configureFilterDelegation(FilterDelegationHandle handle, BackendExec
     }
 
     @Override
-    public Map getActiveQueryMetrics() {
+    public Map getTopQueriesByMemory() {
         // Delegate to the plugin that owns the DataFusionService and native runtime.
         // Keeping the real implementation on DataFusionPlugin lets operators call it
         // directly (e.g., from a REST action) without going through the SPI.
-        return plugin.getActiveQueryMetrics();
+        return plugin.getTopQueriesByMemory();
     }
 
     @Override
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java
index e99dafbabce9b..bd1cb0da19707 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java
@@ -112,7 +112,7 @@ public class DataFusionPlugin extends Plugin implements SearchBackEndPlugin createComponents(
         // The OpenSearch task id is used as the DataFusion context_id at query launch
         // (see ShardScanInstructionHandler / DatafusionSearchExecEngine), so the map is
         // already keyed by Task#getId on the consumer side.
-        logger.info("[nativemem-bp] plugin: installing native-memory snapshot supplier for backpressure tracker");
+        logger.info("installing native-memory snapshot supplier for search backpressure");
         NativeMemoryUsageTracker.setSnapshotSupplier(this::currentBytesByTaskId);
         NativeMemoryUsageTracker.setNativeMemoryBudgetSupplier(
             () -> DATAFUSION_MEMORY_POOL_LIMIT.get(clusterService.getSettings())
@@ -195,27 +195,19 @@ public Collection createComponents(
      */
     private Map currentBytesByTaskId() {
         if (dataFusionService == null) {
-            logger.info("[nativemem-bp] plugin.snapshot: service not running, returning empty map");
             return Collections.emptyMap();
         }
-        long t0 = System.nanoTime();
-        Map metrics = getActiveQueryMetrics();
+        Map metrics = getTopQueriesByMemory();
         if (metrics.isEmpty()) {
-            logger.info(
-                "[nativemem-bp] plugin.snapshot: empty registry (elapsedMicros={})",
-                (System.nanoTime() - t0) / 1000L
-            );
             return Collections.emptyMap();
         }
         Map out = new HashMap<>(metrics.size());
         for (Map.Entry e : metrics.entrySet()) {
             out.put(e.getKey(), e.getValue().currentBytes());
         }
-        logger.info(
-            "[nativemem-bp] plugin.snapshot: built taskId->currentBytes map, size={}, elapsedMicros={}",
-            out.size(),
-            (System.nanoTime() - t0) / 1000L
-        );
+        if (logger.isDebugEnabled()) {
+            logger.debug("native memory snapshot: {} active queries", out.size());
+        }
         return out;
     }
 
@@ -358,15 +350,13 @@ public void close() throws IOException {
      * order Rust drained the bounded min-heap (unspecified but stable per snapshot).
      */
     @Override
-    public Map getActiveQueryMetrics() {
+    public Map getTopQueriesByMemory() {
         if (dataFusionService == null) {
             return Collections.emptyMap();
         }
-        Map result = NativeBridge.queryRegistryTopN(ACTIVE_QUERY_METRICS_TOP_N);
-        if (result.isEmpty()) {
-            logger.info("[nativemem-bp] plugin.getActiveQueryMetrics: native registry empty");
-        } else {
-            logger.info("[nativemem-bp] plugin.getActiveQueryMetrics: decoded {} entries from native registry", result.size());
+        Map result = NativeBridge.getTopNQueriesByMemory(ACTIVE_QUERY_METRICS_TOP_N);
+        if (logger.isDebugEnabled()) {
+            logger.debug("getTopQueriesByMemory: {} entries from native registry", result.size());
         }
         return result;
     }
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java
index f5c66c5a863f6..631face32a91f 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java
@@ -741,7 +741,7 @@ public static DataFusionStats stats() {
      * @throws IllegalArgumentException if {@code n} is negative or implies a
      *         buffer larger than {@link Integer#MAX_VALUE} bytes
      */
-    public static Map queryRegistryTopN(int n) {
+    public static Map getTopNQueriesByMemory(int n) {
         if (n < 0) {
             throw new IllegalArgumentException("n must be non-negative: " + n);
         }
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/QueryRegistryLayout.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/QueryRegistryLayout.java
index 0f3564b108c07..6cda9dc1cdb72 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/QueryRegistryLayout.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/QueryRegistryLayout.java
@@ -1,4 +1,4 @@
-/*
+    /*
  * SPDX-License-Identifier: Apache-2.0
  *
  * The OpenSearch Contributors require contributions made to
@@ -17,11 +17,11 @@
 
 /**
  * Mirrors the Rust {@code query_tracker::WireQueryMetric} {@code #[repr(C)]}
- * struct (3 × i64 = 24 bytes) and decodes a strided buffer of those structs
+ * struct (2 × i64 = 16 bytes) and decodes a strided buffer of those structs
  * directly into the SPI types consumed by
- * {@link org.opensearch.analytics.spi.AnalyticsSearchBackendPlugin#getActiveQueryMetrics}.
+ * {@link org.opensearch.analytics.spi.AnalyticsSearchBackendPlugin#getTopQueriesByMemory}.
  *
- * 
Decoder reads each row by slicing the segment to that row's 24-byte
+ * 
Decoder reads each row by slicing the segment to that row's 16-byte
  * window. We deliberately avoid {@code SequenceLayout}-derived
  * {@code VarHandle}s because their bounds check enforces the segment span
  * the entire sequence layout, not just up to the requested row. With a
@@ -31,7 +31,7 @@
  * 
Buffer shape (populated by {@code df_query_registry_top_n_by_current}):
  * 
  *   [ entry 0 ][ entry 1 ] ... [ entry N-1 ]
- *   each entry = { context_id, current_bytes, peak_bytes } (3 × i64)
+ *   each entry = { context_id, current_bytes } (2 × i64)
  * 

  */
 public final class QueryRegistryLayout {
@@ -39,8 +39,7 @@ public final class QueryRegistryLayout {
     /** Layout of a single wire entry. */
     public static final StructLayout ENTRY_LAYOUT = MemoryLayout.structLayout(
         ValueLayout.JAVA_LONG.withName("context_id"),
-        ValueLayout.JAVA_LONG.withName("current_bytes"),
-        ValueLayout.JAVA_LONG.withName("peak_bytes")
+        ValueLayout.JAVA_LONG.withName("current_bytes")
     );
 
     /** Byte size of one wire entry. Matches {@code size_of::()} on the Rust side. */
@@ -50,10 +49,9 @@ public final class QueryRegistryLayout {
     // ENTRY_LAYOUT and the Rust #[repr(C)] WireQueryMetric struct.
     private static final long OFF_CONTEXT_ID    = 0L;
     private static final long OFF_CURRENT_BYTES = 8L;
-    private static final long OFF_PEAK_BYTES    = 16L;
 
     static {
-        long expected = 3L * Long.BYTES;
+        long expected = 2L * Long.BYTES;
         if (ENTRY_BYTES != expected) {
             throw new AssertionError("QueryRegistryLayout entry size mismatch: expected " + expected + " but got " + ENTRY_BYTES);
         }
@@ -81,9 +79,6 @@ public static long readContextId(MemorySegment seg, int i) {
      */
     public static QueryExecutionMetrics readMetrics(MemorySegment seg, int i) {
         long rowOffset = (long) i * ENTRY_BYTES;
-        return new QueryExecutionMetrics(
-            seg.get(ValueLayout.JAVA_LONG, rowOffset + OFF_CURRENT_BYTES),
-            seg.get(ValueLayout.JAVA_LONG, rowOffset + OFF_PEAK_BYTES)
-        );
+        return new QueryExecutionMetrics(seg.get(ValueLayout.JAVA_LONG, rowOffset + OFF_CURRENT_BYTES));
     }
 }
diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/DefaultPlanExecutor.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/DefaultPlanExecutor.java
index a37914bbe769b..ae7004f0d7e26 100644
--- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/DefaultPlanExecutor.java
+++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/DefaultPlanExecutor.java
@@ -30,6 +30,8 @@
 import org.opensearch.analytics.planner.dag.PlanForker;
 import org.opensearch.analytics.planner.dag.QueryDAG;
 import org.opensearch.arrow.memory.ArrowAllocatorService;
+import org.opensearch.analytics.planner.dag.Stage;
+import org.opensearch.analytics.planner.rel.OpenSearchTableScan;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.Nullable;
 import org.opensearch.common.inject.Inject;
@@ -46,8 +48,10 @@
 import org.opensearch.transport.client.node.NodeClient;
 
 import java.util.ArrayList;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.Executor;
 
 import static org.opensearch.action.search.TransportSearchAction.SEARCH_CANCEL_AFTER_TIME_INTERVAL_SETTING;
@@ -142,10 +146,11 @@ private void executeInternal(RelNode logicalFragment, ActionListener indices = collectIndices(dag);
         final AnalyticsQueryTask queryTask = (AnalyticsQueryTask) taskManager.register(
             "transport",
             "analytics_query",
-            new AnalyticsQueryTaskRequest(dag.queryId(), null)
+            new AnalyticsQueryTaskRequest(dag.queryId(), indices, null)
         );
         final QueryContext config = new QueryContext(dag, searchExecutor, queryTask, allocatorService);
 
@@ -191,11 +196,13 @@ protected void doExecute(Task task, ActionRequest request, ActionListener indices;
         private final TimeValue cancelAfterTimeInterval;
         private TaskId parentTaskId = TaskId.EMPTY_TASK_ID;
 
-        AnalyticsQueryTaskRequest(String queryId, @Nullable TimeValue cancelAfterTimeInterval) {
+        AnalyticsQueryTaskRequest(String queryId, List indices, @Nullable TimeValue cancelAfterTimeInterval) {
             this.queryId = queryId;
+            this.indices = List.copyOf(indices);
             this.cancelAfterTimeInterval = cancelAfterTimeInterval;
         }
 
@@ -211,7 +218,37 @@ public TaskId getParentTask() {
 
         @Override
         public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) {
-            return new AnalyticsQueryTask(id, type, action, queryId, parentTaskId, headers, cancelAfterTimeInterval);
+            return new AnalyticsQueryTask(id, type, action, queryId, indices, parentTaskId, headers, cancelAfterTimeInterval);
+        }
+    }
+
+    /**
+     * Walks the DAG and returns the distinct index names referenced by all
+     * {@link OpenSearchTableScan} leaves, preserving discovery order. Used to
+     * populate the coordinator task's description for the Tasks API and slow logs.
+     */
+    private static List collectIndices(QueryDAG dag) {
+        Set indices = new LinkedHashSet<>();
+        collectIndices(dag.rootStage(), indices);
+        return new ArrayList<>(indices);
+    }
+
+    private static void collectIndices(Stage stage, Set indices) {
+        if (stage.getFragment() != null) {
+            collectIndicesFromFragment(stage.getFragment(), indices);
+        }
+        for (Stage child : stage.getChildStages()) {
+            collectIndices(child, indices);
+        }
+    }
+
+    private static void collectIndicesFromFragment(RelNode node, Set indices) {
+        if (node instanceof OpenSearchTableScan scan) {
+            indices.add(scan.getTable().getQualifiedName().getLast());
+            return;
+        }
+        for (RelNode input : node.getInputs()) {
+            collectIndicesFromFragment(input, indices);
         }
     }
 
diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/task/AnalyticsQueryTask.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/task/AnalyticsQueryTask.java
index 37d9695aab5c3..7414d11f1871c 100644
--- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/task/AnalyticsQueryTask.java
+++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/task/AnalyticsQueryTask.java
@@ -16,6 +16,7 @@
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.tasks.TaskId;
 
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Supplier;
@@ -32,6 +33,7 @@ public class AnalyticsQueryTask extends SearchTask {
     private static final Logger logger = LogManager.getLogger(AnalyticsQueryTask.class);
 
     private final String queryId;
+    private final List indices;
     private final TimeValue cancelAfterTimeInterval;
     private final AtomicReference onCancelCallback = new AtomicReference<>();
 
@@ -40,6 +42,7 @@ public AnalyticsQueryTask(
         String type,
         String action,
         String queryId,
+        List indices,
         TaskId parentTaskId,
         Map headers,
         @Nullable TimeValue cancelAfterTimeInterval
@@ -48,19 +51,16 @@ public AnalyticsQueryTask(
             id,
             type,
             action,
-            (Supplier) () -> "queryId[" + queryId + "]",
+            (Supplier) () -> "queryId[" + queryId + "] indices[" + String.join(",", indices) + "]",
             parentTaskId,
             headers,
             cancelAfterTimeInterval != null ? cancelAfterTimeInterval : TimeValue.MINUS_ONE
         );
         this.queryId = queryId;
+        this.indices = List.copyOf(indices);
         this.cancelAfterTimeInterval = cancelAfterTimeInterval;
     }
 
-    public AnalyticsQueryTask(long id, String type, String action, String queryId, TaskId parentTaskId, Map headers) {
-        this(id, type, action, queryId, parentTaskId, headers, null);
-    }
-
     @Override
     public boolean shouldCancelChildrenOnCancellation() {
         return true;
@@ -70,6 +70,10 @@ public String getQueryId() {
         return queryId;
     }
 
+    public List getIndices() {
+        return indices;
+    }
+
     @Nullable
     public TimeValue getCancelAfterTimeInterval() {
         return cancelAfterTimeInterval;
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/backpressure/NativeMemorySearchBackpressureIT.java b/server/src/internalClusterTest/java/org/opensearch/search/backpressure/NativeMemorySearchBackpressureIT.java
new file mode 100644
index 0000000000000..1e6f3a08ea724
--- /dev/null
+++ b/server/src/internalClusterTest/java/org/opensearch/search/backpressure/NativeMemorySearchBackpressureIT.java
@@ -0,0 +1,404 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.search.backpressure;
+
+import org.apache.lucene.util.Constants;
+import org.opensearch.action.ActionRequest;
+import org.opensearch.action.ActionRequestValidationException;
+import org.opensearch.action.ActionType;
+import org.opensearch.action.search.SearchShardTask;
+import org.opensearch.action.support.ActionFilters;
+import org.opensearch.action.support.HandledTransportAction;
+import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
+import org.opensearch.cluster.service.ClusterService;
+import org.opensearch.common.SuppressForbidden;
+import org.opensearch.common.inject.Inject;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.unit.TimeValue;
+import org.opensearch.core.action.ActionListener;
+import org.opensearch.core.action.ActionResponse;
+import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.common.io.stream.StreamOutput;
+import org.opensearch.core.tasks.TaskCancelledException;
+import org.opensearch.core.tasks.TaskId;
+import org.opensearch.core.xcontent.NamedXContentRegistry;
+import org.opensearch.env.Environment;
+import org.opensearch.env.NodeEnvironment;
+import org.opensearch.plugins.ActionPlugin;
+import org.opensearch.plugins.Plugin;
+import org.opensearch.repositories.RepositoriesService;
+import org.opensearch.script.ScriptService;
+import org.opensearch.search.backpressure.settings.NodeDuressSettings;
+import org.opensearch.search.backpressure.settings.SearchBackpressureSettings;
+import org.opensearch.search.backpressure.settings.SearchShardTaskSettings;
+import org.opensearch.search.backpressure.trackers.NativeMemoryUsageTracker;
+import org.opensearch.tasks.CancellableTask;
+import org.opensearch.tasks.Task;
+import org.opensearch.test.OpenSearchIntegTestCase;
+import org.opensearch.threadpool.ThreadPool;
+import org.opensearch.transport.TransportService;
+import org.opensearch.transport.client.Client;
+import org.opensearch.watcher.ResourceWatcherService;
+import org.opensearch.wlm.WorkloadGroupTask;
+import org.hamcrest.MatcherAssert;
+import org.junit.After;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Supplier;
+
+import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.junit.Assume.assumeTrue;
+
+/**
+ * Integration tests for the native-memory path of {@link SearchBackpressureService}.
+ *
+ * 
Native-memory tracking depends on three things being true at SBP-construction time:
+ * 
    
+ *   
  1. The host is Linux (so {@code OsProbe.getProcessNativeMemoryBytes()} is non-negative).
    2. 
+ *   
  2. A backend has installed a snapshot supplier on
+ *       {@link org.opensearch.search.backpressure.NativeMemoryUsageService} before SBP is built.
    3. 
+ *   
  3. {@code OsProbe.getInstance().getTotalPhysicalMemorySize() > 0}.
    4. 
+ * 

+ *
+ * 
{@link NativeMemoryTestPlugin} satisfies the second condition by installing fake snapshot
+ * and budget suppliers from {@code createComponents}, mirroring the production wiring path
+ * {@code DataFusionPlugin} uses. The first and third are checked via {@code assumeTrue} so the
+ * suite is skipped on macOS/Windows CI rather than failing.
+ */
+@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 1)
+public class NativeMemorySearchBackpressureIT extends OpenSearchIntegTestCase {
+
+    private static final TimeValue TIMEOUT = new TimeValue(10, TimeUnit.SECONDS);
+    private static final long DEFAULT_BUDGET_BYTES = 1024L * 1024L * 1024L; // 1 GiB
+
+    /**
+     * Per-task-id native bytes published into {@link NativeMemoryUsageTracker}'s snapshot
+     * supplier. Tests mutate this map between requests; the supplier installed by
+     * {@link NativeMemoryTestPlugin} reads the live reference on every refresh.
+     */
+    private static final Map SNAPSHOT = new ConcurrentHashMap<>();
+    private static final AtomicLong BUDGET_BYTES = new AtomicLong(DEFAULT_BUDGET_BYTES);
+
+    @Override
+    protected Collection> nodePlugins() {
+        final List> plugins = new ArrayList<>(super.nodePlugins());
+        plugins.add(NativeMemoryTestPlugin.class);
+        return plugins;
+    }
+
+    @Before
+    public final void setupNodeSettings() {
+        // Native-memory duress + tracking only engages on Linux; skip otherwise.
+        assumeTrue("native-memory backpressure path is Linux-only (OsProbe.getProcessNativeMemoryBytes)", Constants.LINUX);
+
+        // Force native-memory duress to trip after a single observation by setting the threshold
+        // to 0.0 — any usedFraction above zero counts as a breach. Combined with a tiny
+        // numSuccessiveBreaches the duress flips on the next tick deterministically.
+        Settings request = Settings.builder()
+            .put(NodeDuressSettings.SETTING_CPU_THRESHOLD.getKey(), 1.0) // disable CPU duress
+            .put(NodeDuressSettings.SETTING_HEAP_THRESHOLD.getKey(), 1.0) // disable heap duress
+            .put(NodeDuressSettings.SETTING_NATIVE_MEMORY_THRESHOLD.getKey(), 0.0)
+            .put(NodeDuressSettings.SETTING_NUM_SUCCESSIVE_BREACHES.getKey(), 1)
+            // node.native_memory.limit must be > 0 or the duress lambda short-circuits
+            // ("totalNative is zero"). 1 byte is enough — the probe reports a non-zero RssAnon.
+            .put(NodeDuressSettings.NODE_NATIVE_MEMORY_LIMIT_SETTING.getKey(), "1b")
+            .build();
+        assertAcked(client().admin().cluster().prepareUpdateSettings().setPersistentSettings(request).get());
+
+        SNAPSHOT.clear();
+        BUDGET_BYTES.set(DEFAULT_BUDGET_BYTES);
+    }
+
+    @After
+    public final void cleanupNodeSettings() {
+        SNAPSHOT.clear();
+        BUDGET_BYTES.set(DEFAULT_BUDGET_BYTES);
+        assertAcked(
+            client().admin()
+                .cluster()
+                .prepareUpdateSettings()
+                .setPersistentSettings(Settings.builder().putNull("*"))
+                .setTransientSettings(Settings.builder().putNull("*"))
+        );
+    }
+
+    /**
+     * The shard task's reported native bytes exceed {@code budget * fraction}; SBP must cancel
+     * with a "native memory usage exceeded" reason when running in {@code enforced} mode.
+     */
+    public void testSearchShardTaskCancellationWithHighNativeMemory() throws InterruptedException {
+        Settings request = Settings.builder()
+            .put(SearchBackpressureSettings.SETTING_MODE.getKey(), "enforced")
+            // Threshold = 0.5 of 1 GiB = 512 MiB. Snapshot publishes 768 MiB for the task —
+            // strictly above, so the evaluator must produce a cancellation reason.
+            .put(SearchShardTaskSettings.SETTING_NATIVE_MEMORY_PERCENT_THRESHOLD.getKey(), 0.5)
+            .build();
+        assertAcked(client().admin().cluster().prepareUpdateSettings().setPersistentSettings(request).get());
+
+        ExceptionCatchingListener listener = new ExceptionCatchingListener();
+        client().execute(TestTransportAction.ACTION, new TestRequest(taskId -> SNAPSHOT.put(taskId, 768L * 1024L * 1024L)), listener);
+        assertTrue("task should have been cancelled within " + TIMEOUT, listener.latch.await(TIMEOUT.getSeconds() * 2, TimeUnit.SECONDS));
+
+        Exception caughtException = listener.getException();
+        assertNotNull("SearchShardTask should have been cancelled with TaskCancelledException", caughtException);
+        MatcherAssert.assertThat(caughtException, instanceOf(TaskCancelledException.class));
+        MatcherAssert.assertThat(caughtException.getMessage(), containsString("native memory usage exceeded"));
+    }
+
+    /**
+     * Per-task threshold is set, but the published bytes are below the evaluator's cutoff. SBP
+     * must NOT cancel even though native-memory duress is active. This pins the contract that
+     * duress alone doesn't kill tasks — the evaluator decides per task.
+     */
+    public void testSearchShardTaskNotCancelledBelowThreshold() throws InterruptedException {
+        Settings request = Settings.builder()
+            .put(SearchBackpressureSettings.SETTING_MODE.getKey(), "enforced")
+            // Threshold = 0.9 of 1 GiB ≈ 922 MiB. Snapshot publishes 100 MiB — well under.
+            .put(SearchShardTaskSettings.SETTING_NATIVE_MEMORY_PERCENT_THRESHOLD.getKey(), 0.9)
+            .build();
+        assertAcked(client().admin().cluster().prepareUpdateSettings().setPersistentSettings(request).get());
+
+        ExceptionCatchingListener listener = new ExceptionCatchingListener();
+        client().execute(TestTransportAction.ACTION, new TestRequest(taskId -> SNAPSHOT.put(taskId, 100L * 1024L * 1024L)), listener);
+        // Task self-completes after TIMEOUT; allow extra room.
+        assertTrue(
+            "task should have completed within " + (TIMEOUT.getSeconds() * 3) + "s",
+            listener.latch.await(TIMEOUT.getSeconds() * 3, TimeUnit.SECONDS)
+        );
+        assertNull("SearchShardTask under threshold must not be cancelled by the native-memory tracker", listener.getException());
+    }
+
+    /**
+     * Native-memory threshold setting explicitly set to {@code 0.0} (the rollout-gate inert state).
+     * Even with heavy reported bytes the tracker must stay silent, because {@code fraction == 0}
+     * disables evaluation. Pins the spec's "operator can opt out" contract.
+     */
+    public void testTrackerInertWhenPerTaskFractionIsZero() throws InterruptedException {
+        Settings request = Settings.builder()
+            .put(SearchBackpressureSettings.SETTING_MODE.getKey(), "enforced")
+            // Explicitly set to 0.0 to override the production default and force the inert path.
+            .put(SearchShardTaskSettings.SETTING_NATIVE_MEMORY_PERCENT_THRESHOLD.getKey(), 0.0)
+            .build();
+        assertAcked(client().admin().cluster().prepareUpdateSettings().setPersistentSettings(request).get());
+
+        ExceptionCatchingListener listener = new ExceptionCatchingListener();
+        client().execute(
+            TestTransportAction.ACTION,
+            new TestRequest(taskId -> SNAPSHOT.put(taskId, 4L * 1024L * 1024L * 1024L)), // 4 GiB
+            listener
+        );
+        assertTrue("task should have completed without cancellation", listener.latch.await(TIMEOUT.getSeconds() * 3, TimeUnit.SECONDS));
+        assertNull("Per-task fraction explicitly set to 0 — the tracker must not cancel", listener.getException());
+    }
+
+    /**
+     * In {@code monitor_only} mode, SBP must log breaches but never invoke
+     * {@code TaskManager.cancelTaskAndDescendants}. The task therefore self-completes.
+     */
+    public void testMonitorOnlyModeDoesNotCancel() throws InterruptedException {
+        Settings request = Settings.builder()
+            .put(SearchBackpressureSettings.SETTING_MODE.getKey(), "monitor_only")
+            .put(SearchShardTaskSettings.SETTING_NATIVE_MEMORY_PERCENT_THRESHOLD.getKey(), 0.5)
+            .build();
+        assertAcked(client().admin().cluster().prepareUpdateSettings().setPersistentSettings(request).get());
+
+        ExceptionCatchingListener listener = new ExceptionCatchingListener();
+        client().execute(TestTransportAction.ACTION, new TestRequest(taskId -> SNAPSHOT.put(taskId, 768L * 1024L * 1024L)), listener);
+        assertTrue(
+            "task should have completed without cancellation in monitor_only",
+            listener.latch.await(TIMEOUT.getSeconds() * 3, TimeUnit.SECONDS)
+        );
+        assertNull("monitor_only must not cancel", listener.getException());
+    }
+
+    /**
+     * {@code search_backpressure.mode = disabled} short-circuits the entire {@code doRun} loop
+     * before any tracker is consulted. The task must complete normally even with a heavy
+     * snapshot reading.
+     */
+    public void testDisabledModeShortCircuits() throws InterruptedException {
+        Settings request = Settings.builder()
+            .put(SearchBackpressureSettings.SETTING_MODE.getKey(), "disabled")
+            .put(SearchShardTaskSettings.SETTING_NATIVE_MEMORY_PERCENT_THRESHOLD.getKey(), 0.5)
+            .build();
+        assertAcked(client().admin().cluster().prepareUpdateSettings().setPersistentSettings(request).get());
+
+        ExceptionCatchingListener listener = new ExceptionCatchingListener();
+        client().execute(TestTransportAction.ACTION, new TestRequest(taskId -> SNAPSHOT.put(taskId, 999L * 1024L * 1024L)), listener);
+        assertTrue(
+            "task should have completed without cancellation in disabled mode",
+            listener.latch.await(TIMEOUT.getSeconds() * 3, TimeUnit.SECONDS)
+        );
+        assertNull("disabled mode must not cancel", listener.getException());
+    }
+
+    // -----------------------------------------------------------------------
+    // Test plugin + transport action
+    // -----------------------------------------------------------------------
+
+    /**
+     * Installs the snapshot + budget suppliers on {@link NativeMemoryUsageTracker} during
+     * {@code createComponents} — same boot-time hook the production DataFusion plugin uses.
+     * SBP is constructed in {@code Node} after {@code createComponents} runs, so the
+     * suppliers are visible when {@code isNativeTrackingSupported()} is evaluated.
+     */
+    public static class NativeMemoryTestPlugin extends Plugin implements ActionPlugin {
+        @Override
+        public Collection createComponents(
+            Client client,
+            ClusterService clusterService,
+            ThreadPool threadPool,
+            ResourceWatcherService resourceWatcherService,
+            ScriptService scriptService,
+            NamedXContentRegistry xContentRegistry,
+            Environment environment,
+            NodeEnvironment nodeEnvironment,
+            org.opensearch.core.common.io.stream.NamedWriteableRegistry namedWriteableRegistry,
+            IndexNameExpressionResolver indexNameExpressionResolver,
+            Supplier repositoriesServiceSupplier
+        ) {
+            // Live-reading the static maps so test methods can mutate them between requests.
+            NativeMemoryUsageTracker.setSnapshotSupplier(() -> new HashMap<>(SNAPSHOT));
+            NativeMemoryUsageTracker.setNativeMemoryBudgetSupplier(BUDGET_BYTES::get);
+            return Collections.emptyList();
+        }
+
+        @Override
+        public List> getActions() {
+            return Collections.singletonList(new ActionHandler<>(TestTransportAction.ACTION, TestTransportAction.class));
+        }
+
+        @Override
+        public List> getClientActions() {
+            return Collections.singletonList(TestTransportAction.ACTION);
+        }
+    }
+
+    /**
+     * A request whose handler-side hook lets each test publish a per-task snapshot entry
+     * before busy-waiting for cancellation. The hook receives the task id assigned by
+     * {@code TaskManager.register}, which is the same id SBP looks up in the snapshot.
+     */
+    public static class TestRequest extends ActionRequest {
+        private final transient SnapshotPublisher publisher;
+
+        public TestRequest(SnapshotPublisher publisher) {
+            this.publisher = publisher;
+        }
+
+        public TestRequest(StreamInput in) throws IOException {
+            super(in);
+            this.publisher = id -> {};
+        }
+
+        @Override
+        public ActionRequestValidationException validate() {
+            return null;
+        }
+
+        @Override
+        public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) {
+            // Use SearchShardTask so SBP picks it up via getTaskByType(SearchShardTask.class).
+            SearchShardTask task = new SearchShardTask(id, type, action, "native-memory-it", parentTaskId, headers);
+            publisher.publish(id);
+            return task;
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+        }
+    }
+
+    /** Side-channel for tests to inject the per-task snapshot reading at task-creation time. */
+    @FunctionalInterface
+    public interface SnapshotPublisher {
+        void publish(long taskId);
+    }
+
+    public static class TestResponse extends ActionResponse {
+        public TestResponse() {}
+
+        public TestResponse(StreamInput in) {}
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {}
+    }
+
+    public static class TestTransportAction extends HandledTransportAction {
+        public static final ActionType ACTION = new ActionType<>("internal::native_memory_bp_test_action", TestResponse::new);
+        private final ThreadPool threadPool;
+
+        @Inject
+        public TestTransportAction(TransportService transportService, ThreadPool threadPool, ActionFilters actionFilters) {
+            super(ACTION.name(), transportService, actionFilters, TestRequest::new);
+            this.threadPool = threadPool;
+        }
+
+        @Override
+        @SuppressForbidden(reason = "Simulating a busy task by sleeping")
+        protected void doExecute(Task task, TestRequest request, ActionListener listener) {
+            threadPool.executor(ThreadPool.Names.SEARCH).execute(() -> {
+                try {
+                    CancellableTask cancellableTask = (CancellableTask) task;
+                    ((WorkloadGroupTask) task).setWorkloadGroupId(threadPool.getThreadContext());
+                    long startTime = System.nanoTime();
+                    // Busy-wait for cancellation or timeout. SBP runs every interval_millis
+                    // (default 1s); the loop must outlive at least two ticks for the duress
+                    // streak to trip and the cancel decision to fire.
+                    do {
+                        Thread.sleep(50);
+                    } while (cancellableTask.isCancelled() == false && (System.nanoTime() - startTime) < TIMEOUT.getNanos());
+
+                    if (cancellableTask.isCancelled()) {
+                        throw new TaskCancelledException(cancellableTask.getReasonCancelled());
+                    } else {
+                        listener.onResponse(new TestResponse());
+                    }
+                } catch (Exception e) {
+                    listener.onFailure(e);
+                }
+            });
+        }
+    }
+
+    /** Reused in-suite instead of importing from {@link SearchBackpressureIT}. */
+    public static class ExceptionCatchingListener implements ActionListener {
+        private final CountDownLatch latch = new CountDownLatch(1);
+        private volatile Exception exception = null;
+
+        @Override
+        public void onResponse(TestResponse r) {
+            latch.countDown();
+        }
+
+        @Override
+        public void onFailure(Exception e) {
+            this.exception = e;
+            latch.countDown();
+        }
+
+        public Exception getException() {
+            return exception;
+        }
+    }
+}
diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
index 663669f59fe57..35825cd10157c 100644
--- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
+++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
@@ -725,6 +725,7 @@ public void apply(Settings value, Settings current, Settings previous) {
                 NodeDuressSettings.SETTING_NUM_SUCCESSIVE_BREACHES,
                 NodeDuressSettings.SETTING_CPU_THRESHOLD,
                 NodeDuressSettings.SETTING_HEAP_THRESHOLD,
+                NodeDuressSettings.NODE_NATIVE_MEMORY_LIMIT_SETTING,
                 NodeDuressSettings.SETTING_NATIVE_MEMORY_THRESHOLD,
                 SearchTaskSettings.SETTING_CANCELLATION_RATIO,
                 SearchTaskSettings.SETTING_CANCELLATION_RATE,
@@ -735,7 +736,6 @@ public void apply(Settings value, Settings current, Settings previous) {
                 SearchTaskSettings.SETTING_CPU_TIME_MILLIS_THRESHOLD,
                 SearchTaskSettings.SETTING_ELAPSED_TIME_MILLIS_THRESHOLD,
                 SearchTaskSettings.SETTING_TOTAL_HEAP_PERCENT_THRESHOLD,
-                SearchTaskSettings.SETTING_TOTAL_NATIVE_MEMORY_BYTES_THRESHOLD,
                 SearchTaskSettings.SETTING_NATIVE_MEMORY_PERCENT_THRESHOLD,
                 SearchShardTaskSettings.SETTING_CANCELLATION_RATIO,
                 SearchShardTaskSettings.SETTING_CANCELLATION_RATE,
@@ -746,7 +746,6 @@ public void apply(Settings value, Settings current, Settings previous) {
                 SearchShardTaskSettings.SETTING_CPU_TIME_MILLIS_THRESHOLD,
                 SearchShardTaskSettings.SETTING_ELAPSED_TIME_MILLIS_THRESHOLD,
                 SearchShardTaskSettings.SETTING_TOTAL_HEAP_PERCENT_THRESHOLD,
-                SearchShardTaskSettings.SETTING_TOTAL_NATIVE_MEMORY_BYTES_THRESHOLD,
                 SearchShardTaskSettings.SETTING_NATIVE_MEMORY_PERCENT_THRESHOLD,
                 SearchBackpressureSettings.SETTING_CANCELLATION_RATIO,  // deprecated
                 SearchBackpressureSettings.SETTING_CANCELLATION_RATE,   // deprecated
diff --git a/server/src/main/java/org/opensearch/monitor/os/OsProbe.java b/server/src/main/java/org/opensearch/monitor/os/OsProbe.java
index a26bfc7361200..fb74bff50921b 100644
--- a/server/src/main/java/org/opensearch/monitor/os/OsProbe.java
+++ b/server/src/main/java/org/opensearch/monitor/os/OsProbe.java
@@ -237,6 +237,18 @@ String readProcLoadavg() throws IOException {
         return readSingleLine(PathUtils.get("/proc/loadavg"));
     }
 
+    /**
+     * Reads the contents of {@code /proc/self/status} as a list of lines, one per key.
+     * Package-private so tests can override with a synthetic file layout.
+     *
+     * @return the lines from {@code /proc/self/status}
+     * @throws IOException if an I/O exception occurs reading the file
+     */
+    @SuppressForbidden(reason = "access /proc/self/status")
+    List readProcSelfStatus() throws IOException {
+        return Files.readAllLines(PathUtils.get("/proc/self/status"));
+    }
+
     /**
      * Reads the {@code RssAnon} field (anonymous resident memory) of the current process from
      * {@code /proc/self/status} and returns it in bytes. Returns {@code -1L} when the host is
@@ -258,18 +270,6 @@ public long getProcessRssAnon() {
         }
     }
 
-    /**
-     * Reads the lines of {@code /proc/self/status}. Package-private so tests can override it
-     * with canned file contents.
-     *
-     * @return the lines of {@code /proc/self/status}
-     * @throws IOException if the file cannot be opened or read
-     */
-    @SuppressForbidden(reason = "access /proc/self/status")
-    List readProcSelfStatus() throws IOException {
-        return Files.readAllLines(PathUtils.get("/proc/self/status"));
-    }
-
     /**
      * Reads the {@code RssAnon} field from {@code /proc/self/status}.
      *
@@ -302,33 +302,8 @@ long readRssAnonFromProcSelfStatus() throws IOException {
         return -1L;
     }
 
-    /**
-     * Estimates the process's native (off-heap) anonymous memory footprint in bytes —
-     * {@link #getProcessRssAnonBytes()} minus the JVM heap maximum, clamped at zero.
-     *
-     * 
{@code RssAnon} on Linux includes both committed JVM heap pages and native
-     * allocator pages (malloc, Rust allocator, direct buffers, metaspace). Subtracting
-     * {@code -Xmx} gives a rough "everything outside the heap" view that's useful for
-     * detecting native-side pressure without watching mmapped index files.
-     *
-     * 
The result is approximate by construction:
-     * 
    
-     *   
  • Early in a process's life, RssAnon can be lower than {@code -Xmx} because
-     *       not every heap page has been touched / committed yet — the clamp avoids a
-     *       negative reading.
    • 
-     *   
  • Heap pages that have been swapped out or unmapped also depress the
-     *       reading, biasing the estimate low.
    • 
-     * 

-     *
-     * 
On non-Linux platforms or older kernels where {@code RssAnon} isn't exposed,
-     * returns {@code -1} so callers can detect unsupported environments rather than
-     * treating the absence as zero pressure.
-     *
-     * @return estimated native (off-heap anonymous) bytes, or {@code -1} if the
-     *         underlying RssAnon signal is unavailable on this platform / kernel
-     */
     public long getProcessNativeMemoryBytes() {
-        long rssAnon = getProcessRssAnonBytes();
+        long rssAnon = getProcessRssAnon();
         if (rssAnon < 0L) {
             return -1L;
         }
diff --git a/server/src/main/java/org/opensearch/search/backpressure/NativeMemoryUsageService.java b/server/src/main/java/org/opensearch/search/backpressure/NativeMemoryUsageService.java
new file mode 100644
index 0000000000000..07f9475d5f821
--- /dev/null
+++ b/server/src/main/java/org/opensearch/search/backpressure/NativeMemoryUsageService.java
@@ -0,0 +1,188 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.search.backpressure;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.Collections;
+import java.util.Map;
+import java.util.function.LongSupplier;
+import java.util.function.Supplier;
+
+/**
+ * Owns the cross-tick snapshot of per-query native (off-heap) memory usage that
+ * {@link org.opensearch.search.backpressure.trackers.NativeMemoryUsageTracker}
+ * consults. Keeps a single source of truth so all per-task-class trackers share one
+ * snapshot rather than each holding its own copy.
+ *
+ * 
Why a separate service

+ * 
The tracker is per task class (today: {@code SearchTask} and
+ * {@code SearchShardTask}), so without a shared service there are two parallel
+ * snapshot maps that {@code SearchBackpressureService.doRun()} refreshes
+ * independently — two FFM calls for the same data. Moving the state into a single
+ * service collapses that to one refresh per tick and gives the deferred
+ * production-readiness work (state machine, observability counters, cancel-to-release
+ * propagation) a natural home.
+ *
+ * 
Lifecycle

+ * 
The service is a process-wide singleton accessed via {@link #getInstance()}.
+ * That mirrors the previous static-supplier wiring on {@code NativeMemoryUsageTracker}
+ * and lets a backend plugin call {@link #setSnapshotSupplier} from
+ * {@code createComponents} without threading a service reference through plugin SPI.
+ * A future spec can promote this to an injected component once this service
+ * graduates to a proper injected component.
+ *
+ * 
Thread safety

+ * 
    
+ *   
  • Suppliers are {@code volatile} — single-writer (plugin {@code createComponents}),
+ *       multi-reader (tracker refresh path, stats path).
    • 
+ *   
  • The snapshot reference is {@code volatile} — published once per
+ *       {@link #refresh()} call.
    • 
+ *   
  • Per-task lookups via {@link #currentBytes(long)} read a stable
+ *       map reference; concurrent {@link #refresh()} swaps the reference, never
+ *       mutates the in-flight map.
    • 
+ * 

+ *
+ * @opensearch.internal
+ */
+public final class NativeMemoryUsageService {
+
+    private static final Logger logger = LogManager.getLogger(NativeMemoryUsageService.class);
+
+    private static final Supplier> EMPTY_SNAPSHOT_SUPPLIER = Collections::emptyMap;
+    private static final LongSupplier ZERO_BUDGET_SUPPLIER = () -> 0L;
+
+    private static final NativeMemoryUsageService INSTANCE = new NativeMemoryUsageService();
+
+    /**
+     * Source of the per-query native-memory snapshot. Installed by a backend plugin
+     * (e.g. DataFusion) at boot; defaults to an empty-map supplier so the tracker
+     * stays inert until a backend opts in.
+     */
+    private volatile Supplier> snapshotSupplier = EMPTY_SNAPSHOT_SUPPLIER;
+
+    /**
+     * Source of the total native-memory budget in bytes (e.g. DataFusion's configured
+     * pool limit). Defaults to {@code 0} — the tracker treats {@code budget == 0} as
+     * "feature disabled" and skips evaluation.
+     */
+    private volatile LongSupplier budgetSupplier = ZERO_BUDGET_SUPPLIER;
+
+    /**
+     * Last snapshot loaded by {@link #refresh()}. {@code volatile} so the publish from
+     * the SBP scheduler thread is visible to the per-task lookup path. Read paths must
+     * not mutate this map; producers always swap in a new immutable map.
+     */
+    private volatile Map bytesByTaskId = Collections.emptyMap();
+
+    private NativeMemoryUsageService() {}
+
+    /** Process-wide singleton. */
+    public static NativeMemoryUsageService getInstance() {
+        return INSTANCE;
+    }
+
+    /**
+     * Install the snapshot supplier. Called once from a backend plugin's
+     * {@code createComponents}. Last writer wins. Passing {@code null} is a no-op so
+     * a reload-style call site can't accidentally clear a working installation.
+     */
+    public void setSnapshotSupplier(Supplier> supplier) {
+        if (supplier == null) {
+            return;
+        }
+        this.snapshotSupplier = supplier;
+        logger.info("native memory snapshot supplier installed [{}]", supplier.getClass().getName());
+    }
+
+    /**
+     * Install the native-memory budget supplier. Last writer wins. Passing {@code null}
+     * is a no-op (same rationale as {@link #setSnapshotSupplier}).
+     */
+    public void setBudgetSupplier(LongSupplier supplier) {
+        if (supplier == null) {
+            return;
+        }
+        this.budgetSupplier = supplier;
+        logger.info("native memory budget supplier installed");
+    }
+
+    /** {@code true} when a backend has installed a non-default snapshot supplier. */
+    public boolean hasSnapshotProvider() {
+        return snapshotSupplier != EMPTY_SNAPSHOT_SUPPLIER;
+    }
+
+    /**
+     * Current native-memory budget in bytes. Clamped to {@code >= 0} so a misbehaving
+     * supplier returning a negative value can't flip the threshold math.
+     */
+    public long getBudgetBytes() {
+        return Math.max(0L, budgetSupplier.getAsLong());
+    }
+
+    /**
+     * Pull a fresh snapshot from the installed supplier and publish it. Exactly one
+     * supplier call per invocation — for the DataFusion backend that maps to one FFM
+     * round-trip. Callers (SBP service) invoke this once per tick before any per-task
+     * lookup.
+     *
+     * 
A {@code null} return from the supplier is treated as "empty snapshot" and
+     * logged at warn level — the tracker must never NPE because of a misbehaving
+     * backend.
+     */
+    public void refresh() {
+        Map snapshot;
+        try {
+            snapshot = snapshotSupplier.get();
+        } catch (RuntimeException e) {
+            logger.warn("native memory snapshot supplier threw, using empty snapshot", e);
+            snapshot = null;
+        }
+        if (snapshot == null) {
+            logger.warn("native memory snapshot supplier returned null, using empty snapshot");
+            snapshot = Collections.emptyMap();
+        }
+        this.bytesByTaskId = snapshot;
+        if (logger.isDebugEnabled()) {
+            logger.debug("native memory snapshot refreshed: size={} budget={}B", snapshot.size(), getBudgetBytes());
+        }
+    }
+
+    /**
+     * Look up a single task's current native-memory reservation. Returns {@code 0}
+     * when the task is not in the snapshot (either it's not an analytics query, or
+     * the registry hasn't yet recorded any reservations for it).
+     */
+    public long currentBytes(long taskId) {
+        Long bytes = bytesByTaskId.get(taskId);
+        return bytes == null ? 0L : bytes;
+    }
+
+    /**
+     * Visible-for-testing snapshot view. Returns the live reference; callers MUST NOT
+     * mutate. Provided so tests can assert on what {@link #refresh()} loaded without
+     * needing to reach into the tracker.
+     */
+    Map snapshotView() {
+        return bytesByTaskId;
+    }
+
+    /**
+     * Visible-for-testing reset. Restores defaults so a test that installs a supplier
+     * doesn't leak state into the next test. Production code never calls this. Public
+     * so tests in sibling packages (e.g. {@code .trackers}) can reach it without
+     * reflection.
+     */
+    public void resetForTesting() {
+        this.snapshotSupplier = EMPTY_SNAPSHOT_SUPPLIER;
+        this.budgetSupplier = ZERO_BUDGET_SUPPLIER;
+        this.bytesByTaskId = Collections.emptyMap();
+    }
+}
diff --git a/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java b/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java
index 109ef5f42fcde..dcf8abae1b042 100644
--- a/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java
+++ b/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java
@@ -72,14 +72,17 @@
  */
 public class SearchBackpressureService extends AbstractLifecycleComponent implements TaskCompletionListener {
     private static final Logger logger = LogManager.getLogger(SearchBackpressureService.class);
-    // Tracker-apply rules:
-    // - When native-memory duress is active, we run ONLY the native-memory and elapsed-time
-    // trackers. CPU and heap trackers are intentionally skipped so that a runaway native
-    // allocator doesn't trigger unrelated cancellation paths (heap is still low, CPU may
-    // not be pegged). See the dedicated short-circuit in doRun().
-    // - Otherwise, CPU and heap trackers run when their corresponding resource is in duress,
-    // elapsed-time always runs, and the native-memory tracker runs only under its own
-    // native-memory duress condition.
+    // Tracker-apply rules (each tracker decides independently via this map):
+    // - CPU tracker fires when CPU is in duress.
+    // - Heap tracker fires when heap is in duress (and heap tracking is supported).
+    // - Elapsed-time tracker always fires.
+    // - Native-memory tracker fires when native memory is in duress (and tracking is supported).
+    //
+    // When native-memory duress is active, doRun() bypasses the heap-dominance gate so all
+    // in-flight tasks become cancellation candidates (off-heap pressure is invisible to heap
+    // metrics). If CPU or heap are ALSO in duress simultaneously, their trackers will fire
+    // too — that is intentional: multiple resource pressures compound, and each tracker
+    // independently contributes cancellation reasons that are merged before execution.
     private static final Map> trackerApplyConditions = Map.of(
         TaskResourceUsageTrackerType.CPU_USAGE_TRACKER,
         (nodeDuressTrackers) -> nodeDuressTrackers.isResourceInDuress(ResourceType.CPU),
@@ -88,7 +91,7 @@ public class SearchBackpressureService extends AbstractLifecycleComponent implem
         TaskResourceUsageTrackerType.ELAPSED_TIME_TRACKER,
         (nodeDuressTrackers) -> true,
         TaskResourceUsageTrackerType.NATIVE_MEMORY_USAGE_TRACKER,
-        (nodeDuressTrackers) -> isNativeTrackingSupported() && nodeDuressTrackers.isNativeMemoryInDuress()
+        (nodeDuressTrackers) -> isNativeTrackingSupported() && nodeDuressTrackers.isResourceInDuress(ResourceType.NATIVE_MEMORY)
     );
 
     private volatile Scheduler.Cancellable scheduledFuture;
@@ -96,6 +99,7 @@ public class SearchBackpressureService extends AbstractLifecycleComponent implem
     private final SearchBackpressureSettings settings;
     private final TaskResourceTrackingService taskResourceTrackingService;
     private final ThreadPool threadPool;
+    private final NativeMemoryUsageService nativeMemoryUsageService = NativeMemoryUsageService.getInstance();
 
     private final NodeDuressTrackers nodeDuressTrackers;
     private final Map, TaskResourceUsageTrackers> taskTrackers;
@@ -130,26 +134,26 @@ public SearchBackpressureService(
                     )
                 );
                 put(ResourceType.NATIVE_MEMORY, new NodeDuressTracker(() -> {
-                    // Native-memory duress probe. OsProbe.getProcessNativeMemoryFraction()
-                    // returns nativeBytes / NATIVE_MEMORY_DENOMINATOR_BYTES on Linux, or -1.0
-                    // when the underlying RssAnon signal is unavailable (non-Linux or
-                    // kernels < 4.5). The denominator is a POC constant in OsProbe — replace
-                    // with total physical memory or a cgroup limit before shipping.
                     double used = OsProbe.getInstance().getProcessNativeMemoryBytes();
-                    double totalNative = settings.getSearchTaskSettings().getTotalNativeMemoryBytesThreshold();
+                    double totalNative = settings.getNodeDuressSettings().getNodeNativeMemory();
+                    if (totalNative == 0) {
+                        return false;
+                    }
                     double usedFraction = used / totalNative;
                     if (usedFraction < 0.0d) {
-                        logger.info("[nativemem-bp] duress-probe: native memory signal unavailable");
+                        logger.debug("native memory duress probe: signal unavailable (usedBytes={})", used);
                         return false;
                     }
                     double threshold = settings.getNodeDuressSettings().getNativeMemoryThreshold();
                     boolean breached = usedFraction >= threshold;
-                    logger.info(
-                        "[nativemem-bp] duress-probe: usedFraction={}, threshold={}, breached={}",
-                        String.format(java.util.Locale.ROOT, "%.4f", usedFraction),
-                        String.format(java.util.Locale.ROOT, "%.4f", threshold),
-                        breached
-                    );
+                    if (logger.isDebugEnabled()) {
+                        logger.debug(
+                            "native memory duress probe: usedFraction={}, threshold={}, breached={}",
+                            String.format(java.util.Locale.ROOT, "%.4f", usedFraction),
+                            String.format(java.util.Locale.ROOT, "%.4f", threshold),
+                            breached
+                        );
+                    }
                     return breached;
                 }, () -> settings.getNodeDuressSettings().getNumSuccessiveBreaches()));
             }
@@ -235,55 +239,42 @@ void doRun() {
         List searchTasks = getTaskByType(SearchTask.class);
         List searchShardTasks = getTaskByType(SearchShardTask.class);
 
-        // Native-memory duress takes precedence: it's a symptom of off-heap pressure the heap
-        // share check isn't designed to diagnose, so we skip the heap-dominance gate and only
-        // run the native-memory + elapsed-time trackers. Heap/CPU trackers would target the
-        // wrong workload here.
-        final boolean inNativeMemoryDuress = nodeDuressTrackers.isNativeMemoryInDuress();
-        if (inNativeMemoryDuress) {
-            logger.info(
-                "[nativemem-bp] doRun: native-memory duress active, short-circuiting to "
-                    + "[NATIVE_MEMORY_USAGE_TRACKER, ELAPSED_TIME_TRACKER] — searchTasks={}, searchShardTasks={}",
-                searchTasks.size(),
-                searchShardTasks.size()
-            );
-        } else {
-            logger.info(
-                "[nativemem-bp] doRun: node in duress (non-native-memory) — searchTasks={}, searchShardTasks={}",
-                searchTasks.size(),
-                searchShardTasks.size()
-            );
+        final boolean isNativeMemoryInDuress = nodeDuressTrackers.isNativeMemoryInDuress();
+        if (logger.isDebugEnabled()) {
+            if (isNativeMemoryInDuress) {
+                logger.debug(
+                    "native memory duress active, bypassing heap-dominance gate — searchTasks={}, searchShardTasks={}",
+                    searchTasks.size(),
+                    searchShardTasks.size()
+                );
+            } else {
+                logger.debug(
+                    "node in duress (non-native-memory) — searchTasks={}, searchShardTasks={}",
+                    searchTasks.size(),
+                    searchShardTasks.size()
+                );
+            }
         }
 
-        Map, List> cancellableTasks;
-
-        boolean isHeapUsageDominatedBySearchTasks = isHeapUsageDominatedBySearch(
-            searchTasks,
-            getSettings().getSearchTaskSettings().getTotalHeapPercentThreshold()
-        );
-        boolean isHeapUsageDominatedBySearchShardTasks = isHeapUsageDominatedBySearch(
-            searchShardTasks,
-            getSettings().getSearchShardTaskSettings().getTotalHeapPercentThreshold()
-        );
-        cancellableTasks = Map.of(
-            SearchTask.class,
-            isHeapUsageDominatedBySearchTasks ? searchTasks : Collections.emptyList(),
-            SearchShardTask.class,
-            isHeapUsageDominatedBySearchShardTasks ? searchShardTasks : Collections.emptyList()
-        );
-
-        if (inNativeMemoryDuress) {
+        final Map, List> cancellableTasks;
+        if (isNativeMemoryInDuress) {
             cancellableTasks = Map.of(SearchTask.class, searchTasks, SearchShardTask.class, searchShardTasks);
-            if (shouldApply(TaskResourceUsageTrackerType.NATIVE_MEMORY_USAGE_TRACKER)) {
-                logger.info(
-                    "[nativemem-bp] doRun: refreshing tracker [{}]",
-                    TaskResourceUsageTrackerType.NATIVE_MEMORY_USAGE_TRACKER.getName()
-                );
-                for (TaskResourceUsageTrackers trackers : taskTrackers.values()) {
-                    trackers.getTracker(TaskResourceUsageTrackerType.NATIVE_MEMORY_USAGE_TRACKER)
-                        .ifPresent(TaskResourceUsageTracker::refresh);
-                }
-            }
+            nativeMemoryUsageService.refresh();
+        } else {
+            boolean searchHeapDominated = isHeapUsageDominatedBySearch(
+                searchTasks,
+                getSettings().getSearchTaskSettings().getTotalHeapPercentThreshold()
+            );
+            boolean searchShardHeapDominated = isHeapUsageDominatedBySearch(
+                searchShardTasks,
+                getSettings().getSearchShardTaskSettings().getTotalHeapPercentThreshold()
+            );
+            cancellableTasks = Map.of(
+                SearchTask.class,
+                searchHeapDominated ? searchTasks : Collections.emptyList(),
+                SearchShardTask.class,
+                searchShardHeapDominated ? searchShardTasks : Collections.emptyList()
+            );
         }
 
         // Force-refresh usage stats of these tasks before making a cancellation decision.
@@ -293,27 +284,16 @@ void doRun() {
         List taskCancellations = new ArrayList<>();
         for (TaskResourceUsageTrackerType trackerType : TaskResourceUsageTrackerType.values()) {
             if (shouldApply(trackerType)) {
-                int before = taskCancellations.size();
                 addResourceTrackerBasedCancellations(trackerType, taskCancellations, cancellableTasks);
-                int added = taskCancellations.size() - before;
-                logger.info(
-                    "[nativemem-bp] doRun: tracker [{}] applied — produced {} cancellation reason(s)",
-                    trackerType.getName(),
-                    added
-                );
-            } else {
-                logger.info("[nativemem-bp] doRun: tracker [{}] not applied (shouldApply=false)", trackerType.getName());
             }
         }
 
         // Since these cancellations might be duplicate due to multiple trackers causing cancellation for same task
         // We need to merge them
-        int beforeMerge = taskCancellations.size();
         taskCancellations = mergeTaskCancellations(taskCancellations).stream()
             .map(this::addSBPStateUpdateCallback)
             .filter(TaskCancellation::isEligibleForCancellation)
             .collect(Collectors.toList());
-        logger.info("[nativemem-bp] doRun: merged {} reasons -> {} eligible cancellations", beforeMerge, taskCancellations.size());
 
         for (TaskCancellation taskCancellation : taskCancellations) {
             logger.warn(
@@ -324,11 +304,6 @@ void doRun() {
             );
 
             if (mode != SearchBackpressureMode.ENFORCED) {
-                logger.info(
-                    "[nativemem-bp] doRun: mode={} (not ENFORCED) — would-cancel task={} only logged, not actioned",
-                    mode.getName(),
-                    taskCancellation.getTask().getId()
-                );
                 continue;
             }
 
@@ -342,23 +317,14 @@ void doRun() {
             // Stop cancelling tasks if there are no tokens in either of the two token buckets.
             if (rateLimitReached && ratioLimitReached) {
                 logger.warn(
-                    "[nativemem-bp] doRun: cancellation limit reached for {} — task={} not cancelled (rateLimit={} ratioLimit={})",
+                    "cancellation limit reached for [{}], not cancelling task [{}]",
                     taskType.getSimpleName(),
-                    taskCancellation.getTask().getId(),
-                    rateLimitReached,
-                    ratioLimitReached
+                    taskCancellation.getTask().getId()
                 );
                 searchBackpressureState.incrementLimitReachedCount();
                 break;
             }
 
-            logger.info(
-                "[nativemem-bp] doRun: actioning cancellation — task={} type={} rateLimitOk={} ratioLimitOk={}",
-                taskCancellation.getTask().getId(),
-                taskType.getSimpleName(),
-                rateLimitReached == false,
-                ratioLimitReached == false
-            );
             taskCancellation.cancelTaskAndDescendants(taskManager);
         }
     }
@@ -443,10 +409,6 @@ boolean isNodeInDuress() {
         return nodeDuressTrackers.isNodeInDuress();
     }
 
-    /*
-      Returns true if the increase in heap usage is due to search requests.
-     */
-
     /**
      * Filters and returns the list of currently running tasks of specified type.
      */
@@ -501,13 +463,6 @@ public static TaskResourceUsageTrackers getTrackers(
             new ElapsedTimeTracker(ElapsedTimeNanosSupplier, System::nanoTime),
             TaskResourceUsageTrackerType.ELAPSED_TIME_TRACKER
         );
-        // Native-memory tracker pulls per-task bytes from a snapshot installed via
-        // NativeMemoryUsageTracker#setSnapshotSupplier (typically by a backend plugin).
-        // Per-task evaluation reads from the snapshot map — no FFI call per task. The
-        // service calls tracker.refresh() once per tick to rebuild the snapshot.
-        //
-        // Gate on isNativeTrackingSupported() so we only register the tracker on platforms
-        // where the duress probe and total-physical-memory readings are meaningful.
         if (isNativeTrackingSupported()) {
             trackers.addTracker(
                 new NativeMemoryUsageTracker(nativeMemoryPercentThresholdSupplier),
@@ -572,19 +527,6 @@ protected void doClose() throws IOException {}
     public SearchBackpressureStats nodeStats() {
         List searchTasks = getTaskByType(SearchTask.class);
         List searchShardTasks = getTaskByType(SearchShardTask.class);
-        // One refresh per tracker before stats() iterates activeTasks twice (max + avg).
-        // Mirrors the refresh pass in doRun() so both paths stay consistent — a tracker
-        // that caches an expensive cross-boundary read never has to re-read per task.
-        logger.info(
-            "[nativemem-bp] nodeStats: refreshing all trackers — searchTasks={}, searchShardTasks={}",
-            searchTasks.size(),
-            searchShardTasks.size()
-        );
-        for (TaskResourceUsageTrackers trackers : taskTrackers.values()) {
-            for (TaskResourceUsageTracker tracker : trackers.all()) {
-                tracker.refresh();
-            }
-        }
         SearchTaskStats searchTaskStats = new SearchTaskStats(
             searchBackpressureStates.get(SearchTask.class).getCancellationCount(),
             searchBackpressureStates.get(SearchTask.class).getLimitReachedCount(),
diff --git a/server/src/main/java/org/opensearch/search/backpressure/settings/NodeDuressSettings.java b/server/src/main/java/org/opensearch/search/backpressure/settings/NodeDuressSettings.java
index 72acdb0eaeb87..3779fbc22d06d 100644
--- a/server/src/main/java/org/opensearch/search/backpressure/settings/NodeDuressSettings.java
+++ b/server/src/main/java/org/opensearch/search/backpressure/settings/NodeDuressSettings.java
@@ -11,6 +11,7 @@
 import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Setting;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.core.common.unit.ByteSizeValue;
 
 /**
  * Defines the settings for a node to be considered in duress.
@@ -22,7 +23,7 @@ private static class Defaults {
         private static final int NUM_SUCCESSIVE_BREACHES = 3;
         private static final double CPU_THRESHOLD = 0.9;
         private static final double HEAP_THRESHOLD = 0.7;
-        // Trip native-memory duress when (totalPhysical - memAvailable) / totalPhysical
+        // Trip native-memory duress when processNativeBytes / nodeNativeMemoryLimit
         // exceeds this fraction. Default 0.85 — tighter than HEAP_THRESHOLD because
         // native allocations bypass GC and hit the OS scheduler directly.
         private static final double NATIVE_MEMORY_THRESHOLD = 0.85;
@@ -67,14 +68,14 @@ private static class Defaults {
     );
 
     /**
-     * Defines the physical-memory usage threshold (as a fraction of total physical memory) above
-     * which a node is considered "in duress" for native memory. The probe computes used bytes as
-     * {@code totalPhysicalMemory - memAvailableFromProcMeminfo} and compares
-     * {@code used / totalPhysicalMemory} against this threshold; when the comparison holds for
+     * Defines the native-memory usage threshold (as a fraction) above which a node is considered
+     * "in duress" for native memory. The duress probe computes
+     * {@code usedFraction = OsProbe.getProcessNativeMemoryBytes() / nodeNativeMemoryLimit} and
+     * compares against this threshold; when the comparison holds for
      * {@link #numSuccessiveBreaches} consecutive observations the node is marked in duress.
      *
      * 
This gate is independent of heap duress: backends that manage memory outside the JVM
-     * heap (e.g. DataFusion's memory pool) can cancel search tasks when physical RAM is tight
+     * heap (e.g. DataFusion's memory pool) can cancel search tasks when native memory is tight
      * even if the heap itself is nowhere near its threshold.
      */
     private volatile double nativeMemoryThreshold;
@@ -87,6 +88,18 @@ private static class Defaults {
         Setting.Property.NodeScope
     );
 
+    /**
+     * Absolute native-memory budget for this node, in bytes. When the value is {@link ByteSizeValue#ZERO}
+     * (default) the tracker treats the budget as unconfigured and reports {@code 0%}.
+     */
+    private volatile ByteSizeValue nodeNativeMemory;
+    public static final Setting NODE_NATIVE_MEMORY_LIMIT_SETTING = Setting.byteSizeSetting(
+        "node.native_memory.limit",
+        ByteSizeValue.ZERO,
+        Setting.Property.Dynamic,
+        Setting.Property.NodeScope
+    );
+
     public NodeDuressSettings(Settings settings, ClusterSettings clusterSettings) {
         numSuccessiveBreaches = SETTING_NUM_SUCCESSIVE_BREACHES.get(settings);
         clusterSettings.addSettingsUpdateConsumer(SETTING_NUM_SUCCESSIVE_BREACHES, this::setNumSuccessiveBreaches);
@@ -99,6 +112,9 @@ public NodeDuressSettings(Settings settings, ClusterSettings clusterSettings) {
 
         nativeMemoryThreshold = SETTING_NATIVE_MEMORY_THRESHOLD.get(settings);
         clusterSettings.addSettingsUpdateConsumer(SETTING_NATIVE_MEMORY_THRESHOLD, this::setNativeMemoryThreshold);
+
+        nodeNativeMemory = NODE_NATIVE_MEMORY_LIMIT_SETTING.get(settings);
+        clusterSettings.addSettingsUpdateConsumer(NODE_NATIVE_MEMORY_LIMIT_SETTING, this::setNodeNativeMemory);
     }
 
     public int getNumSuccessiveBreaches() {
@@ -132,4 +148,13 @@ public double getNativeMemoryThreshold() {
     private void setNativeMemoryThreshold(double nativeMemoryThreshold) {
         this.nativeMemoryThreshold = nativeMemoryThreshold;
     }
+
+    public long getNodeNativeMemory() {
+        return nodeNativeMemory.getBytes();
+    }
+
+    public void setNodeNativeMemory(ByteSizeValue nativeMemoryLimitBytes) {
+        this.nodeNativeMemory = nativeMemoryLimitBytes;
+    }
+
 }
diff --git a/server/src/main/java/org/opensearch/search/backpressure/settings/SearchShardTaskSettings.java b/server/src/main/java/org/opensearch/search/backpressure/settings/SearchShardTaskSettings.java
index c17c80e57120c..7e4b8b4740662 100644
--- a/server/src/main/java/org/opensearch/search/backpressure/settings/SearchShardTaskSettings.java
+++ b/server/src/main/java/org/opensearch/search/backpressure/settings/SearchShardTaskSettings.java
@@ -38,8 +38,7 @@ private static class Defaults {
         // inert until an operator or backend plugin opts in by setting a non-zero threshold.
         // The per-task threshold is a fraction of the backend-installed native memory budget,
         // mirroring HEAP_PERCENT_THRESHOLD.
-        private static final long TOTAL_NATIVE_MEMORY_BYTES_THRESHOLD = 0L;
-        private static final double NATIVE_MEMORY_PERCENT_THRESHOLD = 0.0;
+        private static final double NATIVE_MEMORY_PERCENT_THRESHOLD = 0.05;
     }
 
     /**
@@ -167,26 +166,12 @@ private static class Defaults {
         Setting.Property.NodeScope
     );
 
-    /**
-     * Defines the native-memory threshold (in bytes) for the sum of native-memory usages across
-     * all search shard tasks before in-flight cancellation is applied. {@code 0} disables the
-     * check.
-     */
-    private volatile long totalNativeMemoryBytesThreshold;
-    public static final Setting SETTING_TOTAL_NATIVE_MEMORY_BYTES_THRESHOLD = Setting.longSetting(
-        "search_backpressure.search_shard_task.total_native_memory_bytes_threshold",
-        Defaults.TOTAL_NATIVE_MEMORY_BYTES_THRESHOLD,
-        0L,
-        Setting.Property.Dynamic,
-        Setting.Property.NodeScope
-    );
-
     /**
      * Defines the native-memory threshold (as a fraction of the backend-installed native-memory
      * budget, in {@code [0.0, 1.0]}) for an individual search shard task before it is considered
      * for cancellation. The effective per-task byte threshold is {@code budget * fraction}, where
      * {@code budget} is the value installed via
-     * {@link org.opensearch.search.backpressure.trackers.NativeMemoryUsageTracker#setNativeMemoryBudgetSupplier}.
+     * {@link org.opensearch.search.backpressure.NativeMemoryUsageService#setBudgetSupplier}.
      * {@code 0.0} disables the check.
      */
     private volatile double nativeMemoryPercentThreshold;
@@ -209,7 +194,6 @@ public SearchShardTaskSettings(Settings settings, ClusterSettings clusterSetting
         this.cancellationRatio = SETTING_CANCELLATION_RATIO.get(settings);
         this.cancellationRate = SETTING_CANCELLATION_RATE.get(settings);
         this.cancellationBurst = SETTING_CANCELLATION_BURST.get(settings);
-        this.totalNativeMemoryBytesThreshold = SETTING_TOTAL_NATIVE_MEMORY_BYTES_THRESHOLD.get(settings);
         this.nativeMemoryPercentThreshold = SETTING_NATIVE_MEMORY_PERCENT_THRESHOLD.get(settings);
 
         clusterSettings.addSettingsUpdateConsumer(SETTING_TOTAL_HEAP_PERCENT_THRESHOLD, this::setTotalHeapPercentThreshold);
@@ -221,7 +205,6 @@ public SearchShardTaskSettings(Settings settings, ClusterSettings clusterSetting
         clusterSettings.addSettingsUpdateConsumer(SETTING_CANCELLATION_RATIO, this::setCancellationRatio);
         clusterSettings.addSettingsUpdateConsumer(SETTING_CANCELLATION_RATE, this::setCancellationRate);
         clusterSettings.addSettingsUpdateConsumer(SETTING_CANCELLATION_BURST, this::setCancellationBurst);
-        clusterSettings.addSettingsUpdateConsumer(SETTING_TOTAL_NATIVE_MEMORY_BYTES_THRESHOLD, this::setTotalNativeMemoryBytesThreshold);
         clusterSettings.addSettingsUpdateConsumer(SETTING_NATIVE_MEMORY_PERCENT_THRESHOLD, this::setNativeMemoryPercentThreshold);
     }
 
@@ -273,14 +256,6 @@ public void setHeapMovingAverageWindowSize(int heapMovingAverageWindowSize) {
         this.heapMovingAverageWindowSize = heapMovingAverageWindowSize;
     }
 
-    public long getTotalNativeMemoryBytesThreshold() {
-        return totalNativeMemoryBytesThreshold;
-    }
-
-    private void setTotalNativeMemoryBytesThreshold(long totalNativeMemoryBytesThreshold) {
-        this.totalNativeMemoryBytesThreshold = totalNativeMemoryBytesThreshold;
-    }
-
     public double getNativeMemoryPercentThreshold() {
         return nativeMemoryPercentThreshold;
     }
diff --git a/server/src/main/java/org/opensearch/search/backpressure/settings/SearchTaskSettings.java b/server/src/main/java/org/opensearch/search/backpressure/settings/SearchTaskSettings.java
index 251bb69bf477f..72f137a974c8a 100644
--- a/server/src/main/java/org/opensearch/search/backpressure/settings/SearchTaskSettings.java
+++ b/server/src/main/java/org/opensearch/search/backpressure/settings/SearchTaskSettings.java
@@ -43,8 +43,7 @@ private static class Defaults {
         // native-memory-heavy tasks while the node is in native-memory duress. The
         // per-task threshold is expressed as a fraction of the backend-installed native
         // memory budget (e.g. DataFusion's pool limit), mirroring HEAP_PERCENT_THRESHOLD.
-        private static final long TOTAL_NATIVE_MEMORY_BYTES_THRESHOLD = 0L;
-        private static final double NATIVE_MEMORY_PERCENT_THRESHOLD = 0.0;
+        private static final double NATIVE_MEMORY_PERCENT_THRESHOLD = 0.05;
     }
 
     /**
@@ -172,25 +171,12 @@ private static class Defaults {
         Setting.Property.NodeScope
     );
 
-    /**
-     * Defines the native-memory threshold (in bytes) for the sum of native-memory usages across all
-     * search tasks before in-flight cancellation is applied. {@code 0} disables the check.
-     */
-    private volatile long totalNativeMemoryBytesThreshold;
-    public static final Setting SETTING_TOTAL_NATIVE_MEMORY_BYTES_THRESHOLD = Setting.longSetting(
-        "search_backpressure.search_task.total_native_memory_bytes_threshold",
-        Defaults.TOTAL_NATIVE_MEMORY_BYTES_THRESHOLD,
-        0L,
-        Setting.Property.Dynamic,
-        Setting.Property.NodeScope
-    );
-
     /**
      * Defines the native-memory threshold (as a fraction of the backend-installed native-memory
      * budget, in {@code [0.0, 1.0]}) for an individual search task before it is considered for
      * cancellation. The effective per-task byte threshold is {@code budget * fraction}, where
      * {@code budget} is the value installed via
-     * {@link org.opensearch.search.backpressure.trackers.NativeMemoryUsageTracker#setNativeMemoryBudgetSupplier}.
+     * {@link org.opensearch.search.backpressure.NativeMemoryUsageService#setBudgetSupplier}.
      * {@code 0.0} disables the check.
      */
     private volatile double nativeMemoryPercentThreshold;
@@ -213,7 +199,6 @@ public SearchTaskSettings(Settings settings, ClusterSettings clusterSettings) {
         this.cancellationRatio = SETTING_CANCELLATION_RATIO.get(settings);
         this.cancellationRate = SETTING_CANCELLATION_RATE.get(settings);
         this.cancellationBurst = SETTING_CANCELLATION_BURST.get(settings);
-        this.totalNativeMemoryBytesThreshold = SETTING_TOTAL_NATIVE_MEMORY_BYTES_THRESHOLD.get(settings);
         this.nativeMemoryPercentThreshold = SETTING_NATIVE_MEMORY_PERCENT_THRESHOLD.get(settings);
 
         clusterSettings.addSettingsUpdateConsumer(SETTING_TOTAL_HEAP_PERCENT_THRESHOLD, this::setTotalHeapPercentThreshold);
@@ -225,7 +210,6 @@ public SearchTaskSettings(Settings settings, ClusterSettings clusterSettings) {
         clusterSettings.addSettingsUpdateConsumer(SETTING_CANCELLATION_RATIO, this::setCancellationRatio);
         clusterSettings.addSettingsUpdateConsumer(SETTING_CANCELLATION_RATE, this::setCancellationRate);
         clusterSettings.addSettingsUpdateConsumer(SETTING_CANCELLATION_BURST, this::setCancellationBurst);
-        clusterSettings.addSettingsUpdateConsumer(SETTING_TOTAL_NATIVE_MEMORY_BYTES_THRESHOLD, this::setTotalNativeMemoryBytesThreshold);
         clusterSettings.addSettingsUpdateConsumer(SETTING_NATIVE_MEMORY_PERCENT_THRESHOLD, this::setNativeMemoryPercentThreshold);
     }
 
@@ -277,14 +261,6 @@ public void setHeapMovingAverageWindowSize(int heapMovingAverageWindowSize) {
         this.heapMovingAverageWindowSize = heapMovingAverageWindowSize;
     }
 
-    public long getTotalNativeMemoryBytesThreshold() {
-        return totalNativeMemoryBytesThreshold;
-    }
-
-    private void setTotalNativeMemoryBytesThreshold(long totalNativeMemoryBytesThreshold) {
-        this.totalNativeMemoryBytesThreshold = totalNativeMemoryBytesThreshold;
-    }
-
     public double getNativeMemoryPercentThreshold() {
         return nativeMemoryPercentThreshold;
     }
diff --git a/server/src/main/java/org/opensearch/search/backpressure/trackers/NativeMemoryUsageTracker.java b/server/src/main/java/org/opensearch/search/backpressure/trackers/NativeMemoryUsageTracker.java
index 8626226c34b6b..fac1bcc56656e 100644
--- a/server/src/main/java/org/opensearch/search/backpressure/trackers/NativeMemoryUsageTracker.java
+++ b/server/src/main/java/org/opensearch/search/backpressure/trackers/NativeMemoryUsageTracker.java
@@ -16,13 +16,13 @@
 import org.opensearch.core.common.unit.ByteSizeValue;
 import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.monitor.os.OsProbe;
+import org.opensearch.search.backpressure.NativeMemoryUsageService;
 import org.opensearch.search.backpressure.trackers.TaskResourceUsageTrackers.TaskResourceUsageTracker;
 import org.opensearch.tasks.CancellableTask;
 import org.opensearch.tasks.Task;
 import org.opensearch.tasks.TaskCancellation;
 
 import java.io.IOException;
-import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -43,40 +43,30 @@
  * gates on that).
  *
  * 
The per-task threshold is expressed as a fraction of the backend-installed native-memory
- * budget (see {@link #setNativeMemoryBudgetSupplier}), mirroring the heap tracker's
- * {@code heap_percent_threshold}. Effective per-task byte threshold is
- * {@code budget * fraction}.
+ * budget, mirroring the heap tracker's {@code heap_percent_threshold}. Effective per-task
+ * byte threshold is {@code budget * fraction}.
  *
  * 
Snapshot-per-tick model

  * 
Native-memory values come from a backend (e.g. DataFusion) over an FFI boundary. Per-task
  * FFI reads don't scale: the backpressure service iterates every candidate task inside
- * {@code doRun()}, and stats() iterates every live task twice (max + avg). Instead, the
- * tracker holds a {@code Map} that is rebuilt in one shot by {@link #refresh()}.
- * {@code SearchBackpressureService} calls {@code refresh()} once per cancellation-iteration
- * (and once per {@code nodeStats()} call) before any per-task lookup runs. Between refreshes
- * every task lookup is an O(1) hash probe.
+ * {@code doRun()}, and stats() iterates every live task twice (max + avg). The shared
+ * {@link NativeMemoryUsageService} holds a single {@code Map} that the SBP
+ * service refreshes once per tick. Per-task lookups are then O(1) hash probes into a stable
+ * map reference.
  *
- * 
The snapshot source is a {@link Supplier} installed by the backend plugin — typically
- * calling the plugin's own {@code getActiveQueryMetrics()} equivalent and projecting the map
- * down to {@code contextId -> currentBytes}.
+ * 
This tracker is a thin consumer of the service: {@code bytesForTask(Task)} delegates to
+ * {@code NativeMemoryUsageService#currentBytes(long)}. The service's {@code refresh()} is
+ * called by {@code SearchBackpressureService.doRun()} once per tick before per-task evaluation
+ * begins — this tracker does not own the refresh lifecycle. The static setters retained on
+ * this class are kept as delegators so existing call sites (backend plugin
+ * {@code createComponents}, tracker tests) don't have to migrate at the same time as the
+ * service extraction.
  *
  * @opensearch.internal
  */
 public class NativeMemoryUsageTracker extends TaskResourceUsageTracker {
     private static final Logger logger = LogManager.getLogger(NativeMemoryUsageTracker.class);
 
-    /** Empty map used when no supplier is installed or the supplier returns {@code null}. */
-    private static final Supplier> DEFAULT_EMPTY_SUPPLIER = Collections::emptyMap;
-    private static volatile Supplier> snapshotSupplier = DEFAULT_EMPTY_SUPPLIER;
-
-    /**
-     * Source of the total native-memory budget in bytes (e.g. DataFusion's
-     * configured pool limit). Installed by a backend plugin via
-     * {@link #setNativeMemoryBudgetSupplier(LongSupplier)}; defaults to {@code 0}
-     * which keeps the tracker inert until a backend wires up a real budget.
-     */
-    private static volatile LongSupplier nativeMemoryBudgetSupplier = () -> 0L;
-
     /**
      * Per-task threshold expressed as a fraction of the installed native-memory budget,
      * mirroring {@code heap_percent_threshold} on {@link HeapUsageTracker}. Range is
@@ -86,44 +76,49 @@ public class NativeMemoryUsageTracker extends TaskResourceUsageTracker {
      * 
Effective per-task byte threshold = {@code budget * fraction}.
      */
     private final DoubleSupplier nativeMemoryPercentThresholdSupplier;
-    // Volatile so the map reference publishes safely from refresh() (called from the
-    // backpressure scheduler thread) to the per-task evaluate() path (same thread today,
-    // but also hit from nodeStats() which may run on a different thread).
-    private volatile Map bytesByTaskId = Collections.emptyMap();
+
+    /**
+     * Singleton service that owns the snapshot map and the budget supplier. Held as a
+     * field rather than read via {@link NativeMemoryUsageService#getInstance()} on every
+     * call so tests can substitute (via reflection or future package-private constructor).
+     */
+    private final NativeMemoryUsageService service;
 
     public NativeMemoryUsageTracker(DoubleSupplier nativeMemoryPercentThresholdSupplier) {
+        this(nativeMemoryPercentThresholdSupplier, NativeMemoryUsageService.getInstance());
+    }
+
+    /** Package-private for tests that want to inject a service instance. */
+    NativeMemoryUsageTracker(DoubleSupplier nativeMemoryPercentThresholdSupplier, NativeMemoryUsageService service) {
         this.nativeMemoryPercentThresholdSupplier = nativeMemoryPercentThresholdSupplier;
+        this.service = service;
         setDefaultResourceUsageBreachEvaluator();
     }
 
-    /**
-     * Install the snapshot source. Called once from a backend plugin's {@code createComponents}.
-     * Last writer wins; backends that want to cooperate should compose around the previous
-     * value rather than overwriting.
-     */
+    // ---------------------------------------------------------------------
+    // Static delegators — preserved so existing callers (backend plugin
+    // wiring, NativeMemoryUsageTrackerTests) keep compiling. New code should
+    // prefer NativeMemoryUsageService directly.
+    // ---------------------------------------------------------------------
+
+    /** @see NativeMemoryUsageService#setSnapshotSupplier(Supplier) */
     public static void setSnapshotSupplier(Supplier> supplier) {
-        if (supplier != null) {
-            snapshotSupplier = supplier;
-            logger.info("[nativemem-bp] tracker.setSnapshotSupplier: installed supplier [{}]", supplier.getClass().getName());
-        }
+        NativeMemoryUsageService.getInstance().setSnapshotSupplier(supplier);
     }
 
-    /**
-     * Install the native-memory budget source. A backend plugin (e.g. DataFusion) calls
-     * this from {@code createComponents} with a supplier reading its configured pool
-     * limit. Last writer wins. Pass a supplier returning {@code 0} (or never call this)
-     * to keep the tracker inert.
-     */
+    /** @see NativeMemoryUsageService#setBudgetSupplier(LongSupplier) */
     public static void setNativeMemoryBudgetSupplier(LongSupplier supplier) {
-        if (supplier != null) {
-            nativeMemoryBudgetSupplier = supplier;
-            logger.info("[nativemem-bp] tracker.setNativeMemoryBudgetSupplier: installed");
-        }
+        NativeMemoryUsageService.getInstance().setBudgetSupplier(supplier);
     }
 
-    /** Current native-memory budget in bytes; {@code 0} when no backend has installed a supplier. */
+    /** @see NativeMemoryUsageService#getBudgetBytes() */
     public static long getNativeMemoryBudgetBytes() {
-        return Math.max(0L, nativeMemoryBudgetSupplier.getAsLong());
+        return NativeMemoryUsageService.getInstance().getBudgetBytes();
+    }
+
+    /** @see NativeMemoryUsageService#hasSnapshotProvider() */
+    public static boolean hasSnapshotProvider() {
+        return NativeMemoryUsageService.getInstance().hasSnapshotProvider();
     }
 
     /**
@@ -135,62 +130,36 @@ public static long getNativeMemoryBudgetBytes() {
     private void setDefaultResourceUsageBreachEvaluator() {
         this.resourceUsageBreachEvaluator = (task) -> {
             if (task instanceof CancellableTask == false) {
-                logger.info(
-                    "[nativemem-bp] evaluate: task={} type={} not CancellableTask — skipping",
-                    task.getId(),
-                    task.getClass().getSimpleName()
-                );
                 return Optional.empty();
             }
             double fraction = nativeMemoryPercentThresholdSupplier.getAsDouble();
-            long budget = getNativeMemoryBudgetBytes();
+            long budget = service.getBudgetBytes();
             if (fraction <= 0.0d || budget <= 0L) {
                 // Feature disabled — either the operator hasn't set a threshold, or no
                 // backend has installed a budget supplier. Leave the tracker inert.
-                logger.info(
-                    "[nativemem-bp] evaluate: task={} inert — fraction={} budget={}B (one is 0; tracker disabled)",
-                    task.getId(),
-                    fraction,
-                    budget
-                );
                 return Optional.empty();
             }
             // Defensive clamp; the Setting validator already enforces [0.0, 1.0].
             double boundedFraction = Math.min(1.0d, fraction);
             long bytesThreshold = (long) (budget * boundedFraction);
             if (bytesThreshold <= 0L) {
-                logger.info(
-                    "[nativemem-bp] evaluate: task={} bytesThreshold=0 (budget={}B fraction={}) — skipping",
-                    task.getId(),
-                    budget,
-                    boundedFraction
-                );
                 return Optional.empty();
             }
             long currentUsage = bytesForTask(task);
             if (currentUsage < bytesThreshold) {
-                logger.info(
-                    "[nativemem-bp] evaluate: task={} below threshold — currentBytes={}B threshold={}B "
-                        + "(fraction={} budget={}B) — no cancellation",
+                return Optional.empty();
+            }
+            int score = (int) Math.max(1L, currentUsage / Math.max(1L, bytesThreshold));
+            if (logger.isDebugEnabled()) {
+                logger.debug(
+                    "native memory threshold exceeded for task [{}]: currentBytes={}B, threshold={}B ({}% of budget={}B)",
                     task.getId(),
                     currentUsage,
                     bytesThreshold,
-                    boundedFraction,
+                    (int) (boundedFraction * 100),
                     budget
                 );
-                return Optional.empty();
             }
-            int score = (int) Math.max(1L, currentUsage / Math.max(1L, bytesThreshold));
-            logger.warn(
-                "[nativemem-bp] evaluate: task={} EXCEEDS threshold — currentBytes={}B, "
-                    + "threshold={}B (fraction={} of budget={}B), score={}",
-                task.getId(),
-                currentUsage,
-                bytesThreshold,
-                boundedFraction,
-                budget,
-                score
-            );
             return Optional.of(
                 new TaskCancellation.Reason(
                     "native memory usage exceeded [" + new ByteSizeValue(currentUsage) + " >= " + new ByteSizeValue(bytesThreshold) + "]",
@@ -211,51 +180,12 @@ public void update(Task task) {
         // intentionally empty
     }
 
-    /**
-     * Pull a fresh snapshot from the installed supplier and swap it in. Exactly one FFI
-     * call per invocation — callers (backpressure service) invoke this once per tick
-     * before per-task evaluation begins.
-     */
-    @Override
-    public void refresh() {
-        Map snapshot = snapshotSupplier.get();
-        boolean nullSnapshot = (snapshot == null);
-        bytesByTaskId = nullSnapshot ? Collections.emptyMap() : snapshot;
-        if (nullSnapshot) {
-            logger.warn("[nativemem-bp] tracker.refresh: supplier returned null — using empty map");
-        }
-        logger.info(
-            "[nativemem-bp] tracker.refresh: snapshot loaded, size={} taskIds={} budget={}B",
-            bytesByTaskId.size(),
-            bytesByTaskId.keySet(),
-            getNativeMemoryBudgetBytes()
-        );
-        if (bytesByTaskId.size() > 0) {
-            // log the heaviest few so we can see whether registry has anything substantive
-            bytesByTaskId.entrySet()
-                .stream()
-                .sorted((a, b) -> Long.compare(b.getValue(), a.getValue()))
-                .limit(5)
-                .forEach(e -> logger.info("[nativemem-bp] tracker.refresh:   taskId={} currentBytes={}", e.getKey(), e.getValue()));
-        }
-    }
-
     /** Package-private for tests: look up a single task's bytes from the current snapshot. */
     long bytesForTask(Task task) {
         if (task == null) {
             return 0L;
         }
-        Long bytes = bytesByTaskId.get(task.getId());
-        return bytes == null ? 0L : bytes;
-    }
-
-    /**
-     * Returns true if a backend plugin has installed a snapshot supplier. False if no
-     * backend with native-memory tracking is loaded, in which case registering this
-     * tracker would just produce empty refreshes every tick.
-     */
-    public static boolean hasSnapshotProvider() {
-        return snapshotSupplier != DEFAULT_EMPTY_SUPPLIER;
+        return service.currentBytes(task.getId());
     }
 
     /**
@@ -327,5 +257,16 @@ public boolean equals(Object o) {
         public int hashCode() {
             return Objects.hash(cancellationCount, currentMax, currentAvg);
         }
+
+        @Override
+        public String toString() {
+            return "NativeMemoryUsageTracker.Stats{cancellationCount="
+                + cancellationCount
+                + ", currentMax="
+                + currentMax
+                + ", currentAvg="
+                + currentAvg
+                + "}";
+        }
     }
 }
diff --git a/server/src/main/java/org/opensearch/search/backpressure/trackers/NodeDuressTrackers.java b/server/src/main/java/org/opensearch/search/backpressure/trackers/NodeDuressTrackers.java
index df64d51c18608..c3812fbf352cf 100644
--- a/server/src/main/java/org/opensearch/search/backpressure/trackers/NodeDuressTrackers.java
+++ b/server/src/main/java/org/opensearch/search/backpressure/trackers/NodeDuressTrackers.java
@@ -50,8 +50,9 @@ public boolean isResourceInDuress(ResourceType resourceType) {
 
     /**
      * Convenience accessor for native-memory duress. Equivalent to
-     * {@code isResourceInDuress(ResourceType.NATIVE_MEMORY)} but expressed as a method
-     * reference-friendly form used by the tracker-apply map in the backpressure service.
+     * {@code isResourceInDuress(ResourceType.NATIVE_MEMORY)}. Used by
+     * {@code SearchBackpressureService.doRun()} to decide whether to bypass the
+     * heap-dominance gate and include all tasks as cancellation candidates.
      */
     public boolean isNativeMemoryInDuress() {
         return isResourceInDuress(ResourceType.NATIVE_MEMORY);
@@ -72,10 +73,6 @@ public boolean isNodeInDuress() {
 
     private void updateCache() {
         if (nodeDuressCacheExpiryChecker.getAsBoolean()) {
-            // Callers may register duress trackers for only a subset of {@link ResourceType}
-            // values (e.g. WLM's WorkloadGroupService registers CPU + MEMORY only, leaving
-            // NATIVE_MEMORY unregistered). Treat absence as "not in duress" so unrelated
-            // call sites aren't forced to register a no-op tracker for every new enum value.
             for (ResourceType resourceType : ResourceType.values()) {
                 NodeDuressTracker tracker = duressTrackers.get(resourceType);
                 resourceDuressCache.put(resourceType, tracker != null && tracker.test());
diff --git a/server/src/main/java/org/opensearch/search/backpressure/trackers/TaskResourceUsageTrackers.java b/server/src/main/java/org/opensearch/search/backpressure/trackers/TaskResourceUsageTrackers.java
index 1ac5c8e48d461..3b0072288681c 100644
--- a/server/src/main/java/org/opensearch/search/backpressure/trackers/TaskResourceUsageTrackers.java
+++ b/server/src/main/java/org/opensearch/search/backpressure/trackers/TaskResourceUsageTrackers.java
@@ -86,17 +86,6 @@ public long getCancellations() {
             return cancellations.get();
         }
 
-        /**
-         * Rebuild any cached state this tracker maintains (e.g. a snapshot of per-task usage
-         * pulled across an FFI boundary). Called by the backpressure service once per
-         * cancellation-iteration per task class, immediately before {@link #getTaskCancellations}
-         * runs. Default is a no-op — override when the tracker wants to batch an expensive read
-         * instead of doing it per-task.
-         */
-        public void refresh() {
-            // no-op by default
-        }
-
         /**
          * Returns a unique name for this tracker.
          */
diff --git a/server/src/test/java/org/opensearch/monitor/os/OsProbeTests.java b/server/src/test/java/org/opensearch/monitor/os/OsProbeTests.java
index ce144a4184f40..05fb0ba84f6e9 100644
--- a/server/src/test/java/org/opensearch/monitor/os/OsProbeTests.java
+++ b/server/src/test/java/org/opensearch/monitor/os/OsProbeTests.java
@@ -511,72 +511,9 @@ boolean areCgroupStatsAvailable() {
         };
     }
 
-    // ---- /proc/meminfo parsing ----
-
-    public void testParseMeminfoLineBytes_valid() {
-        assertEquals(123456L * 1024L, OsProbe.parseMeminfoLineBytes("MemAvailable:   123456 kB"));
-        assertEquals(0L, OsProbe.parseMeminfoLineBytes("MemFree:        0 kB"));
-        // No unit column — tolerate the trimmed form.
-        assertEquals(42L * 1024L, OsProbe.parseMeminfoLineBytes("MemAvailable: 42"));
-        // Unusual but well-formed whitespace.
-        assertEquals(7L * 1024L, OsProbe.parseMeminfoLineBytes("MemTotal:\t7 kB"));
-    }
-
-    public void testParseMeminfoLineBytes_malformed() {
-        assertEquals(-1L, OsProbe.parseMeminfoLineBytes(""));
-        assertEquals(-1L, OsProbe.parseMeminfoLineBytes("MemAvailable"));
-        assertEquals(-1L, OsProbe.parseMeminfoLineBytes("MemAvailable:"));
-        assertEquals(-1L, OsProbe.parseMeminfoLineBytes("MemAvailable: not-a-number kB"));
-        assertEquals(-1L, OsProbe.parseMeminfoLineBytes("MemAvailable: -5 kB"));
-    }
-
-    public void testGetFreePhysicalMemorySizeFromProcMeminfo_usesMemAvailableWhenPresent() {
-        assumeThat("requires Linux to exercise the /proc/meminfo path", Constants.LINUX, is(true));
-        OsProbe probe = new OsProbe() {
-            @Override
-            List readProcMeminfo() {
-                return Arrays.asList(
-                    "MemTotal:        16384000 kB",
-                    "MemFree:            10000 kB",
-                    // MemAvailable wins even when MemFree appears earlier in some layouts.
-                    "MemAvailable:     5242880 kB",
-                    "Buffers:             1234 kB"
-                );
-            }
-        };
-        assertEquals(5242880L * 1024L, probe.getFreePhysicalMemorySizeFromProcMeminfo());
-    }
-
-    public void testGetFreePhysicalMemorySizeFromProcMeminfo_fallsBackToMemFree() {
-        assumeThat("requires Linux to exercise the /proc/meminfo path", Constants.LINUX, is(true));
-        OsProbe probe = new OsProbe() {
-            @Override
-            List readProcMeminfo() {
-                return Arrays.asList("MemTotal:        16384000 kB", "MemFree:          1048576 kB", "Buffers:             4096 kB");
-            }
-        };
-        assertEquals(1048576L * 1024L, probe.getFreePhysicalMemorySizeFromProcMeminfo());
-    }
-
-    public void testGetFreePhysicalMemorySizeFromProcMeminfo_returnsNegativeOnIoError() {
-        assumeThat("requires Linux to exercise the /proc/meminfo path", Constants.LINUX, is(true));
-        OsProbe probe = new OsProbe() {
-            @Override
-            List readProcMeminfo() throws IOException {
-                throw new IOException("synthetic");
-            }
-        };
-        assertEquals(-1L, probe.getFreePhysicalMemorySizeFromProcMeminfo());
-    }
-
-    public void testGetFreePhysicalMemorySizeFromProcMeminfo_negativeOnNonLinux() {
-        assumeThat("only meaningful on non-Linux platforms", Constants.LINUX, is(false));
-        assertEquals(-1L, OsProbe.getInstance().getFreePhysicalMemorySizeFromProcMeminfo());
-    }
-
     // ---- /proc/self/status RssAnon parsing ----
 
-    public void testGetProcessRssAnonBytes_presentField() {
+    public void testGetProcessRssAnon_presentField() {
         assumeThat("requires Linux to exercise the /proc/self/status path", Constants.LINUX, is(true));
         OsProbe probe = new OsProbe() {
             @Override
@@ -592,10 +529,10 @@ List readProcSelfStatus() {
                 );
             }
         };
-        assertEquals(32768L * 1024L, probe.getProcessRssAnonBytes());
+        assertEquals(32768L * 1024L, probe.getProcessRssAnon());
     }
 
-    public void testGetProcessRssAnonBytes_missingFieldReturnsNegative() {
+    public void testGetProcessRssAnon_missingFieldReturnsNegative() {
         assumeThat("requires Linux to exercise the /proc/self/status path", Constants.LINUX, is(true));
         OsProbe probe = new OsProbe() {
             @Override
@@ -604,10 +541,10 @@ List readProcSelfStatus() {
                 return Arrays.asList("Name:\topensearch", "State:\tS (sleeping)", "VmRSS:\t 65536 kB");
             }
         };
-        assertEquals(-1L, probe.getProcessRssAnonBytes());
+        assertEquals(-1L, probe.getProcessRssAnon());
     }
 
-    public void testGetProcessRssAnonBytes_returnsNegativeOnIoError() {
+    public void testGetProcessRssAnon_returnsNegativeOnIoError() {
         assumeThat("requires Linux to exercise the /proc/self/status path", Constants.LINUX, is(true));
         OsProbe probe = new OsProbe() {
             @Override
@@ -615,22 +552,22 @@ List readProcSelfStatus() throws IOException {
                 throw new IOException("synthetic");
             }
         };
-        assertEquals(-1L, probe.getProcessRssAnonBytes());
+        assertEquals(-1L, probe.getProcessRssAnon());
     }
 
-    public void testGetProcessRssAnonBytes_negativeOnNonLinux() {
+    public void testGetProcessRssAnon_negativeOnNonLinux() {
         assumeThat("only meaningful on non-Linux platforms", Constants.LINUX, is(false));
-        assertEquals(-1L, OsProbe.getInstance().getProcessRssAnonBytes());
+        assertEquals(-1L, OsProbe.getInstance().getProcessRssAnon());
     }
 
     // ---- getProcessNativeMemoryBytes (RssAnon - heapMax, clamped) ----
 
     public void testGetProcessNativeMemoryBytes_returnsNegativeWhenRssAnonUnavailable() {
-        // Override getProcessRssAnonBytes to return -1 (the "not supported" sentinel). The
+        // Override getProcessRssAnon to return -1 (the "not supported" sentinel). The
         // method must propagate that signal upward without subtracting from -1.
         OsProbe probe = new OsProbe() {
             @Override
-            public long getProcessRssAnonBytes() {
+            public long getProcessRssAnon() {
                 return -1L;
             }
         };
@@ -642,7 +579,7 @@ public void testGetProcessNativeMemoryBytes_subtractsHeapMaxAndClampsAtZero() {
         // The method must clamp at 0 instead of returning a negative.
         OsProbe probe = new OsProbe() {
             @Override
-            public long getProcessRssAnonBytes() {
+            public long getProcessRssAnon() {
                 return 1L; // way below any real -Xmx setting
             }
         };
@@ -655,7 +592,7 @@ public void testGetProcessNativeMemoryBytes_returnsDifferenceWhenRssAnonExceedsH
         long rssAnon = heapMax + 64L * 1024L * 1024L;
         OsProbe probe = new OsProbe() {
             @Override
-            public long getProcessRssAnonBytes() {
+            public long getProcessRssAnon() {
                 return rssAnon;
             }
         };
diff --git a/server/src/test/java/org/opensearch/search/backpressure/NativeMemoryUsageServiceTests.java b/server/src/test/java/org/opensearch/search/backpressure/NativeMemoryUsageServiceTests.java
new file mode 100644
index 0000000000000..0cd96d21b04d0
--- /dev/null
+++ b/server/src/test/java/org/opensearch/search/backpressure/NativeMemoryUsageServiceTests.java
@@ -0,0 +1,263 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.search.backpressure;
+
+import org.opensearch.test.OpenSearchTestCase;
+import org.junit.After;
+import org.junit.Before;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.LongSupplier;
+import java.util.function.Supplier;
+
+/**
+ * Unit tests for {@link NativeMemoryUsageService}.
+ *
+ * 
The service is a process-wide singleton. Each test must reset state in
+ * {@link #setUp()}/{@link #tearDown()} so suppliers installed by a prior test
+ * don't leak into the next.
+ */
+public class NativeMemoryUsageServiceTests extends OpenSearchTestCase {
+
+    private NativeMemoryUsageService service;
+
+    @Before
+    public void resetServiceBefore() {
+        service = NativeMemoryUsageService.getInstance();
+        service.resetForTesting();
+    }
+
+    @After
+    public void resetServiceAfter() {
+        service.resetForTesting();
+    }
+
+    public void testGetInstanceReturnsSameSingleton() {
+        // Singleton contract: every call returns the same instance.
+        assertSame(NativeMemoryUsageService.getInstance(), NativeMemoryUsageService.getInstance());
+    }
+
+    public void testDefaultBudgetIsZero() {
+        assertEquals(0L, service.getBudgetBytes());
+    }
+
+    public void testDefaultHasSnapshotProviderIsFalse() {
+        // Until a backend opts in, the predicate must report false.
+        assertFalse(service.hasSnapshotProvider());
+    }
+
+    public void testDefaultSnapshotIsEmpty() {
+        // currentBytes for any id with the default supplier must be 0.
+        assertEquals(0L, service.currentBytes(1L));
+        assertEquals(0L, service.currentBytes(randomNonNegativeLong()));
+    }
+
+    public void testSetSnapshotSupplierFlipsHasSnapshotProvider() {
+        Supplier> supplier = Collections::emptyMap;
+        service.setSnapshotSupplier(supplier);
+        assertTrue(service.hasSnapshotProvider());
+    }
+
+    public void testSetSnapshotSupplierWithNullIsNoOp() {
+        Supplier> supplier = Collections::emptyMap;
+        service.setSnapshotSupplier(supplier);
+        assertTrue(service.hasSnapshotProvider());
+        // null must NOT erase a working installation.
+        service.setSnapshotSupplier(null);
+        assertTrue(service.hasSnapshotProvider());
+    }
+
+    public void testSetBudgetSupplierWithNullIsNoOp() {
+        service.setBudgetSupplier(() -> 1024L);
+        assertEquals(1024L, service.getBudgetBytes());
+        service.setBudgetSupplier(null);
+        // Previous supplier must still be active.
+        assertEquals(1024L, service.getBudgetBytes());
+    }
+
+    public void testGetBudgetBytesClampsNegative() {
+        service.setBudgetSupplier(() -> -42L);
+        // Negative values from a misbehaving supplier must NOT flip threshold math; clamp to 0.
+        assertEquals(0L, service.getBudgetBytes());
+    }
+
+    public void testGetBudgetBytesPropagatesPositive() {
+        long pool = 4L * 1024L * 1024L * 1024L;
+        service.setBudgetSupplier(() -> pool);
+        assertEquals(pool, service.getBudgetBytes());
+    }
+
+    public void testRefreshLoadsSnapshot() {
+        Map snapshot = new HashMap<>();
+        snapshot.put(10L, 100L);
+        snapshot.put(20L, 200L);
+        service.setSnapshotSupplier(() -> snapshot);
+
+        service.refresh();
+        assertEquals(100L, service.currentBytes(10L));
+        assertEquals(200L, service.currentBytes(20L));
+        assertEquals(0L, service.currentBytes(30L));
+        assertEquals(2, service.snapshotView().size());
+    }
+
+    public void testRefreshNullSnapshotFallsBackToEmpty() {
+        // A misbehaving backend that returns null must NOT propagate NPE through the read path.
+        service.setSnapshotSupplier(() -> null);
+        service.refresh();
+        assertEquals(0L, service.currentBytes(1L));
+        assertTrue(service.snapshotView().isEmpty());
+    }
+
+    public void testRefreshSwallowsRuntimeExceptionFromSupplier() {
+        // A throwing supplier (e.g. transient FFM error) must not break the SBP scheduler tick.
+        service.setSnapshotSupplier(() -> { throw new RuntimeException("boom"); });
+        service.refresh();
+        assertEquals(0L, service.currentBytes(1L));
+        assertTrue(service.snapshotView().isEmpty());
+    }
+
+    public void testRefreshReplacesPreviousSnapshot() {
+        // First refresh loads {1: 100}; second refresh loads {2: 200}. The map reference must
+        // swap atomically — entry 1 must no longer be present after the second refresh.
+        Map first = Map.of(1L, 100L);
+        Map second = Map.of(2L, 200L);
+        AtomicInteger calls = new AtomicInteger();
+        service.setSnapshotSupplier(() -> calls.getAndIncrement() == 0 ? first : second);
+
+        service.refresh();
+        assertEquals(100L, service.currentBytes(1L));
+        assertEquals(0L, service.currentBytes(2L));
+
+        service.refresh();
+        assertEquals(0L, service.currentBytes(1L));
+        assertEquals(200L, service.currentBytes(2L));
+    }
+
+    public void testRefreshCallsSupplierExactlyOnce() {
+        // The SBP service refreshes once per tick. This must translate to exactly one supplier
+        // invocation per refresh — for the DataFusion backend that maps to one FFM round-trip.
+        AtomicInteger calls = new AtomicInteger();
+        service.setSnapshotSupplier(() -> {
+            calls.incrementAndGet();
+            return Collections.emptyMap();
+        });
+        service.refresh();
+        assertEquals(1, calls.get());
+
+        // A subsequent currentBytes call must NOT trigger another supplier invocation.
+        service.currentBytes(1L);
+        service.currentBytes(2L);
+        assertEquals(1, calls.get());
+
+        service.refresh();
+        assertEquals(2, calls.get());
+    }
+
+    public void testCurrentBytesReturnsZeroForUnknownTaskId() {
+        service.setSnapshotSupplier(() -> Map.of(1L, 100L));
+        service.refresh();
+        assertEquals(100L, service.currentBytes(1L));
+        assertEquals(0L, service.currentBytes(99L));
+    }
+
+    public void testResetForTestingClearsState() {
+        service.setSnapshotSupplier(() -> Map.of(1L, 100L));
+        service.setBudgetSupplier(() -> 1024L);
+        service.refresh();
+        assertTrue(service.hasSnapshotProvider());
+        assertEquals(1024L, service.getBudgetBytes());
+        assertEquals(100L, service.currentBytes(1L));
+
+        service.resetForTesting();
+
+        assertFalse(service.hasSnapshotProvider());
+        assertEquals(0L, service.getBudgetBytes());
+        assertEquals(0L, service.currentBytes(1L));
+        assertTrue(service.snapshotView().isEmpty());
+    }
+
+    public void testConcurrentRefreshAndRead() throws Exception {
+        // Reader threads must always see a consistent snapshot — never an in-flight mutation.
+        // We alternate between two non-overlapping snapshots and assert that every read either
+        // finds the value from snapshot A OR from snapshot B, never something that would imply
+        // a partially-published map (e.g. a NullPointerException from an internal HashMap).
+        Map snapA = Map.of(1L, 100L);
+        Map snapB = Map.of(1L, 200L);
+        AtomicInteger toggle = new AtomicInteger();
+        service.setSnapshotSupplier(() -> toggle.getAndIncrement() % 2 == 0 ? snapA : snapB);
+
+        int readerCount = 4;
+        int iterationsPerReader = 1_000;
+        CountDownLatch start = new CountDownLatch(1);
+        CountDownLatch done = new CountDownLatch(readerCount + 1);
+
+        Thread refresher = new Thread(() -> {
+            try {
+                start.await();
+                for (int i = 0; i < iterationsPerReader; i++) {
+                    service.refresh();
+                }
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+            } finally {
+                done.countDown();
+            }
+        }, "nm-service-refresher");
+        refresher.start();
+
+        Thread[] readers = new Thread[readerCount];
+        for (int r = 0; r < readerCount; r++) {
+            readers[r] = new Thread(() -> {
+                try {
+                    start.await();
+                    for (int i = 0; i < iterationsPerReader; i++) {
+                        long bytes = service.currentBytes(1L);
+                        // Every observation must be 0 (between refreshes, transient), 100 (snapA),
+                        // or 200 (snapB). Anything else implies inconsistent publication.
+                        if (bytes != 0L && bytes != 100L && bytes != 200L) {
+                            throw new AssertionError("inconsistent snapshot value: " + bytes);
+                        }
+                    }
+                } catch (InterruptedException e) {
+                    Thread.currentThread().interrupt();
+                } finally {
+                    done.countDown();
+                }
+            }, "nm-service-reader-" + r);
+            readers[r].start();
+        }
+
+        start.countDown();
+        assertTrue("threads did not finish in time", done.await(30, TimeUnit.SECONDS));
+        for (Thread reader : readers) {
+            reader.join();
+        }
+        refresher.join();
+    }
+
+    public void testBudgetSupplierInvokedEachCall() {
+        // The service does NOT cache the budget — every getBudgetBytes call routes through the
+        // supplier. This matters because the operator may flip the backend pool size at runtime.
+        AtomicInteger calls = new AtomicInteger();
+        LongSupplier supplier = () -> {
+            calls.incrementAndGet();
+            return 64L;
+        };
+        service.setBudgetSupplier(supplier);
+        service.getBudgetBytes();
+        service.getBudgetBytes();
+        service.getBudgetBytes();
+        assertEquals(3, calls.get());
+    }
+}
diff --git a/server/src/test/java/org/opensearch/search/backpressure/SearchBackpressureServiceTests.java b/server/src/test/java/org/opensearch/search/backpressure/SearchBackpressureServiceTests.java
index 3ae0f572ef094..7531c9527e29f 100644
--- a/server/src/test/java/org/opensearch/search/backpressure/SearchBackpressureServiceTests.java
+++ b/server/src/test/java/org/opensearch/search/backpressure/SearchBackpressureServiceTests.java
@@ -612,89 +612,99 @@ public void testNativeMemoryDuressRunsOnlyNativeMemoryAndElapsedTrackers() {
         // which today requires Linux + a readable /proc/meminfo. Non-Linux test hosts cannot
         // exercise this short-circuit regardless of how the fixtures are wired.
         assumeTrue("native memory tracking is only supported on Linux", org.apache.lucene.util.Constants.LINUX);
-        // When native-memory duress is active, heap/CPU trackers MUST NOT run. We install
-        // "always cancel" mocks for CPU and heap, and confirm nothing trips from them; the
-        // native-memory tracker is the only one that should contribute cancellations.
-        TaskManager mockTaskManager = spy(taskManager);
-        TaskResourceTrackingService mockTaskResourceTrackingService = mock(TaskResourceTrackingService.class);
-        AtomicLong mockTime = new AtomicLong(0);
-        LongSupplier mockTimeNanosSupplier = mockTime::get;
-
-        // CPU/heap ResourceType duress both OFF — only the native-memory probe is firing.
-        EnumMap duressTrackers = new EnumMap<>(ResourceType.class) {
-            {
-                put(MEMORY, new NodeDuressTracker(() -> false, () -> 3));
-                put(CPU, new NodeDuressTracker(() -> false, () -> 3));
-                put(ResourceType.NATIVE_MEMORY, new NodeDuressTracker(() -> true, () -> 3));
-            }
-        };
-        NodeDuressTrackers nodeDuressTrackers = new NodeDuressTrackers(duressTrackers, resourceCacheExpiryChecker);
-
-        // Unconditionally-cancelling CPU + heap trackers. These should never be consulted.
-        TaskResourceUsageTracker cpuUsageTracker = getMockedTaskResourceUsageTracker(
-            TaskResourceUsageTrackerType.CPU_USAGE_TRACKER,
-            (task) -> Optional.of(new TaskCancellation.Reason("cpu (should not fire)", 10))
-        );
-        TaskResourceUsageTracker heapUsageTracker = getMockedTaskResourceUsageTracker(
-            TaskResourceUsageTrackerType.HEAP_USAGE_TRACKER,
-            (task) -> Optional.of(new TaskCancellation.Reason("heap (should not fire)", 10))
-        );
-        // Native-memory tracker: cancels tasks whose memory bytes exceed a modest threshold.
-        TaskResourceUsageTracker nativeMemoryTracker = getMockedTaskResourceUsageTracker(
-            TaskResourceUsageTrackerType.NATIVE_MEMORY_USAGE_TRACKER,
-            (task) -> {
-                if (task.getTotalResourceStats().getMemoryInBytes() < 500) {
-                    return Optional.empty();
+        // isNativeTrackingSupported() also gates on whether a backend has installed a
+        // snapshot provider on NativeMemoryUsageService. The mock tracker in this test
+        // doesn't go through that path, so we install a sentinel non-empty supplier here
+        // and reset it in finally so other tests don't see leftover state.
+        org.opensearch.search.backpressure.NativeMemoryUsageService.getInstance().setSnapshotSupplier(java.util.Collections::emptyMap);
+        try {
+            // When native-memory duress is active, heap/CPU trackers MUST NOT run. We install
+            // "always cancel" mocks for CPU and heap, and confirm nothing trips from them; the
+            // native-memory tracker is the only one that should contribute cancellations.
+            TaskManager mockTaskManager = spy(taskManager);
+            TaskResourceTrackingService mockTaskResourceTrackingService = mock(TaskResourceTrackingService.class);
+            AtomicLong mockTime = new AtomicLong(0);
+            LongSupplier mockTimeNanosSupplier = mockTime::get;
+
+            // CPU/heap ResourceType duress both OFF — only the native-memory probe is firing.
+            EnumMap duressTrackers = new EnumMap<>(ResourceType.class) {
+                {
+                    put(MEMORY, new NodeDuressTracker(() -> false, () -> 3));
+                    put(CPU, new NodeDuressTracker(() -> false, () -> 3));
+                    put(ResourceType.NATIVE_MEMORY, new NodeDuressTracker(() -> true, () -> 3));
                 }
-                return Optional.of(new TaskCancellation.Reason("native memory exceeded", 5));
-            }
-        );
-        // Elapsed-time tracker left silent so the cancellation count is unambiguously from
-        // the native-memory tracker.
-        TaskResourceUsageTracker elapsedTimeTracker = getMockedTaskResourceUsageTracker(
-            TaskResourceUsageTrackerType.ELAPSED_TIME_TRACKER,
-            (task) -> Optional.empty()
-        );
-
-        TaskResourceUsageTrackers taskResourceUsageTrackers = new TaskResourceUsageTrackers();
-        taskResourceUsageTrackers.addTracker(cpuUsageTracker, TaskResourceUsageTrackerType.CPU_USAGE_TRACKER);
-        taskResourceUsageTrackers.addTracker(heapUsageTracker, TaskResourceUsageTrackerType.HEAP_USAGE_TRACKER);
-        taskResourceUsageTrackers.addTracker(nativeMemoryTracker, TaskResourceUsageTrackerType.NATIVE_MEMORY_USAGE_TRACKER);
-        taskResourceUsageTrackers.addTracker(elapsedTimeTracker, TaskResourceUsageTrackerType.ELAPSED_TIME_TRACKER);
-
-        SearchBackpressureSettings settings = getBackpressureSettings("enforced", 0.1, 0.003, 10.0);
-        SearchBackpressureService service = new SearchBackpressureService(
-            settings,
-            mockTaskResourceTrackingService,
-            threadPool,
-            mockTimeNanosSupplier,
-            nodeDuressTrackers,
-            taskResourceUsageTrackers,
-            new TaskResourceUsageTrackers(),
-            mockTaskManager,
-            workloadGroupService
-        );
+            };
+            NodeDuressTrackers nodeDuressTrackers = new NodeDuressTrackers(duressTrackers, resourceCacheExpiryChecker);
+
+            // Unconditionally-cancelling CPU + heap trackers. These should never be consulted.
+            TaskResourceUsageTracker cpuUsageTracker = getMockedTaskResourceUsageTracker(
+                TaskResourceUsageTrackerType.CPU_USAGE_TRACKER,
+                (task) -> Optional.of(new TaskCancellation.Reason("cpu (should not fire)", 10))
+            );
+            TaskResourceUsageTracker heapUsageTracker = getMockedTaskResourceUsageTracker(
+                TaskResourceUsageTrackerType.HEAP_USAGE_TRACKER,
+                (task) -> Optional.of(new TaskCancellation.Reason("heap (should not fire)", 10))
+            );
+            // Native-memory tracker: cancels tasks whose memory bytes exceed a modest threshold.
+            TaskResourceUsageTracker nativeMemoryTracker = getMockedTaskResourceUsageTracker(
+                TaskResourceUsageTrackerType.NATIVE_MEMORY_USAGE_TRACKER,
+                (task) -> {
+                    if (task.getTotalResourceStats().getMemoryInBytes() < 500) {
+                        return Optional.empty();
+                    }
+                    return Optional.of(new TaskCancellation.Reason("native memory exceeded", 5));
+                }
+            );
+            // Elapsed-time tracker left silent so the cancellation count is unambiguously from
+            // the native-memory tracker.
+            TaskResourceUsageTracker elapsedTimeTracker = getMockedTaskResourceUsageTracker(
+                TaskResourceUsageTrackerType.ELAPSED_TIME_TRACKER,
+                (task) -> Optional.empty()
+            );
+
+            TaskResourceUsageTrackers taskResourceUsageTrackers = new TaskResourceUsageTrackers();
+            taskResourceUsageTrackers.addTracker(cpuUsageTracker, TaskResourceUsageTrackerType.CPU_USAGE_TRACKER);
+            taskResourceUsageTrackers.addTracker(heapUsageTracker, TaskResourceUsageTrackerType.HEAP_USAGE_TRACKER);
+            taskResourceUsageTrackers.addTracker(nativeMemoryTracker, TaskResourceUsageTrackerType.NATIVE_MEMORY_USAGE_TRACKER);
+            taskResourceUsageTrackers.addTracker(elapsedTimeTracker, TaskResourceUsageTrackerType.ELAPSED_TIME_TRACKER);
+
+            SearchBackpressureSettings settings = getBackpressureSettings("enforced", 0.1, 0.003, 10.0);
+            SearchBackpressureService service = new SearchBackpressureService(
+                settings,
+                mockTaskResourceTrackingService,
+                threadPool,
+                mockTimeNanosSupplier,
+                nodeDuressTrackers,
+                taskResourceUsageTrackers,
+                new TaskResourceUsageTrackers(),
+                mockTaskManager,
+                workloadGroupService
+            );
 
-        when(workloadGroupService.shouldSBPHandle(any())).thenReturn(true);
+            when(workloadGroupService.shouldSBPHandle(any())).thenReturn(true);
 
-        // Prime the native-memory duress streak (consecutive-breach window is 3).
-        service.doRun();
-        service.doRun();
+            // Prime the native-memory duress streak (consecutive-breach window is 3).
+            service.doRun();
+            service.doRun();
 
-        // Load 20 tasks; 4 are heavy (mem >= 500) so exactly 4 are native-memory cancellation
-        // candidates. Heap/CPU would want to cancel all 20 — verifying this test's whole point.
-        Map activeSearchTasks = new HashMap<>();
-        for (long i = 0; i < 20; i++) {
-            long memoryBytes = (i % 5 == 0) ? 800 : 100;
-            activeSearchTasks.put(i, createMockTaskWithResourceStats(SearchTask.class, 100, memoryBytes, i));
+            // Load 20 tasks; 4 are heavy (mem >= 500) so exactly 4 are native-memory cancellation
+            // candidates. Heap/CPU would want to cancel all 20 — verifying this test's whole point.
+            Map activeSearchTasks = new HashMap<>();
+            for (long i = 0; i < 20; i++) {
+                long memoryBytes = (i % 5 == 0) ? 800 : 100;
+                activeSearchTasks.put(i, createMockTaskWithResourceStats(SearchTask.class, 100, memoryBytes, i));
+            }
+            activeSearchTasks.values().forEach(task -> task.setWorkloadGroupId(threadPool.getThreadContext()));
+            doReturn(activeSearchTasks).when(mockTaskResourceTrackingService).getResourceAwareTasks();
+
+            service.doRun();
+            // Exactly the heavy tasks were cancelled — no extras from the "always cancel" CPU/heap mocks.
+            verify(mockTaskManager, times(4)).cancelTaskAndDescendants(any(), anyString(), anyBoolean(), any());
+            assertEquals(4, service.getSearchBackpressureState(SearchTask.class).getCancellationCount());
+        } finally {
+            // Don't leak the sentinel supplier into other tests in this suite.
+            org.opensearch.search.backpressure.NativeMemoryUsageService.getInstance().resetForTesting();
         }
-        activeSearchTasks.values().forEach(task -> task.setWorkloadGroupId(threadPool.getThreadContext()));
-        doReturn(activeSearchTasks).when(mockTaskResourceTrackingService).getResourceAwareTasks();
-
-        service.doRun();
-        // Exactly the heavy tasks were cancelled — no extras from the "always cancel" CPU/heap mocks.
-        verify(mockTaskManager, times(4)).cancelTaskAndDescendants(any(), anyString(), anyBoolean(), any());
-        assertEquals(4, service.getSearchBackpressureState(SearchTask.class).getCancellationCount());
     }
 
     private TaskResourceUsageTracker getMockedTaskResourceUsageTracker(
diff --git a/server/src/test/java/org/opensearch/search/backpressure/settings/NodeDuressSettingsTests.java b/server/src/test/java/org/opensearch/search/backpressure/settings/NodeDuressSettingsTests.java
index 574d8aa11c935..8850096547ca9 100644
--- a/server/src/test/java/org/opensearch/search/backpressure/settings/NodeDuressSettingsTests.java
+++ b/server/src/test/java/org/opensearch/search/backpressure/settings/NodeDuressSettingsTests.java
@@ -10,10 +10,12 @@
 
 import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.core.common.unit.ByteSizeValue;
 import org.opensearch.test.OpenSearchTestCase;
 
 /**
- * Unit tests covering the native-memory threshold added to {@link NodeDuressSettings}.
+ * Unit tests covering the native-memory threshold and node native-memory limit
+ * settings on {@link NodeDuressSettings}.
  */
 public class NodeDuressSettingsTests extends OpenSearchTestCase {
 
@@ -56,4 +58,48 @@ public void testNativeMemoryThresholdRejectsOutOfRange() {
             () -> new NodeDuressSettings(raw2, new ClusterSettings(raw2, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS))
         );
     }
+
+    public void testDefaultNodeNativeMemoryLimitIsZero() {
+        // Default ByteSizeValue.ZERO — operator hasn't configured a native pool, so the
+        // duress probe must self-report "unconfigured" and the SBP path treats it as 0.
+        NodeDuressSettings settings = new NodeDuressSettings(
+            Settings.EMPTY,
+            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)
+        );
+        assertEquals(0L, settings.getNodeNativeMemory());
+    }
+
+    public void testInitialNodeNativeMemoryLimitRespectsSetting() {
+        // ByteSizeValue parses unit-suffixed strings; verify the configured value reaches the field.
+        Settings raw = Settings.builder().put(NodeDuressSettings.NODE_NATIVE_MEMORY_LIMIT_SETTING.getKey(), "2gb").build();
+        NodeDuressSettings settings = new NodeDuressSettings(raw, new ClusterSettings(raw, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
+        assertEquals(2L * 1024 * 1024 * 1024, settings.getNodeNativeMemory());
+    }
+
+    public void testNodeNativeMemoryLimitIsDynamic() {
+        // Operator must be able to flip the limit at runtime without a node restart.
+        ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+        NodeDuressSettings settings = new NodeDuressSettings(Settings.EMPTY, clusterSettings);
+        assertEquals(0L, settings.getNodeNativeMemory());
+
+        clusterSettings.applySettings(
+            Settings.builder().put(NodeDuressSettings.NODE_NATIVE_MEMORY_LIMIT_SETTING.getKey(), "512mb").build()
+        );
+        assertEquals(512L * 1024 * 1024, settings.getNodeNativeMemory());
+
+        clusterSettings.applySettings(Settings.builder().put(NodeDuressSettings.NODE_NATIVE_MEMORY_LIMIT_SETTING.getKey(), "1gb").build());
+        assertEquals(1024L * 1024 * 1024, settings.getNodeNativeMemory());
+    }
+
+    public void testNodeNativeMemoryLimitSetterAcceptsByteSizeValue() {
+        // Direct setter contract — the cluster-settings consumer routes through this setter,
+        // so assert the contract directly to catch regressions in the consumer wiring here
+        // rather than only in an integration test.
+        NodeDuressSettings settings = new NodeDuressSettings(
+            Settings.EMPTY,
+            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)
+        );
+        settings.setNodeNativeMemory(new ByteSizeValue(4096L));
+        assertEquals(4096L, settings.getNodeNativeMemory());
+    }
 }
diff --git a/server/src/test/java/org/opensearch/search/backpressure/settings/SearchShardTaskSettingsTests.java b/server/src/test/java/org/opensearch/search/backpressure/settings/SearchShardTaskSettingsTests.java
index 176712a9ad333..285398d7323eb 100644
--- a/server/src/test/java/org/opensearch/search/backpressure/settings/SearchShardTaskSettingsTests.java
+++ b/server/src/test/java/org/opensearch/search/backpressure/settings/SearchShardTaskSettingsTests.java
@@ -18,40 +18,35 @@
  */
 public class SearchShardTaskSettingsTests extends OpenSearchTestCase {
 
-    public void testDefaultNativeMemoryThresholdsKeepTrackerInert() {
+    public void testDefaultNativeMemoryPercentThresholdMatchesDefaults() {
+        // Production default: a small non-zero fraction so the tracker engages once a backend
+        // installs a non-zero budget. Test pinned to the same value as Defaults.NATIVE_MEMORY_PERCENT_THRESHOLD
+        // so a default change here trips this assertion deliberately.
         SearchShardTaskSettings settings = new SearchShardTaskSettings(
             Settings.EMPTY,
             new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)
         );
-        assertEquals(0L, settings.getTotalNativeMemoryBytesThreshold());
-        assertEquals(0.0d, settings.getNativeMemoryPercentThreshold(), 0.0d);
+        assertEquals(0.05d, settings.getNativeMemoryPercentThreshold(), 0.0d);
     }
 
-    public void testInitialNativeMemoryThresholdsRespectSettings() {
-        long bytesThreshold = 512L * 1024L * 1024L;
+    public void testInitialNativeMemoryPercentThresholdRespectsSetting() {
         double fractionThreshold = 0.6d;
         Settings raw = Settings.builder()
-            .put(SearchShardTaskSettings.SETTING_TOTAL_NATIVE_MEMORY_BYTES_THRESHOLD.getKey(), bytesThreshold)
             .put(SearchShardTaskSettings.SETTING_NATIVE_MEMORY_PERCENT_THRESHOLD.getKey(), fractionThreshold)
             .build();
         SearchShardTaskSettings settings = new SearchShardTaskSettings(
             raw,
             new ClusterSettings(raw, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)
         );
-        assertEquals(bytesThreshold, settings.getTotalNativeMemoryBytesThreshold());
         assertEquals(fractionThreshold, settings.getNativeMemoryPercentThreshold(), 0.0d);
     }
 
-    public void testNativeMemoryThresholdsAreDynamic() {
+    public void testNativeMemoryPercentThresholdIsDynamic() {
         ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
         SearchShardTaskSettings settings = new SearchShardTaskSettings(Settings.EMPTY, clusterSettings);
         clusterSettings.applySettings(
-            Settings.builder()
-                .put(SearchShardTaskSettings.SETTING_TOTAL_NATIVE_MEMORY_BYTES_THRESHOLD.getKey(), 8192L)
-                .put(SearchShardTaskSettings.SETTING_NATIVE_MEMORY_PERCENT_THRESHOLD.getKey(), 0.25d)
-                .build()
+            Settings.builder().put(SearchShardTaskSettings.SETTING_NATIVE_MEMORY_PERCENT_THRESHOLD.getKey(), 0.25d).build()
         );
-        assertEquals(8192L, settings.getTotalNativeMemoryBytesThreshold());
         assertEquals(0.25d, settings.getNativeMemoryPercentThreshold(), 0.0d);
     }
 
@@ -68,12 +63,4 @@ public void testNativeMemoryPercentThresholdRejectsOutOfRange() {
             () -> new SearchShardTaskSettings(raw2, new ClusterSettings(raw2, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS))
         );
     }
-
-    public void testTotalNativeMemoryBytesThresholdRejectsNegative() {
-        Settings raw = Settings.builder().put(SearchShardTaskSettings.SETTING_TOTAL_NATIVE_MEMORY_BYTES_THRESHOLD.getKey(), -1L).build();
-        expectThrows(
-            IllegalArgumentException.class,
-            () -> new SearchShardTaskSettings(raw, new ClusterSettings(raw, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS))
-        );
-    }
 }
diff --git a/server/src/test/java/org/opensearch/search/backpressure/settings/SearchTaskSettingsTests.java b/server/src/test/java/org/opensearch/search/backpressure/settings/SearchTaskSettingsTests.java
index 7d6c329e14088..1f0b78bd90f32 100644
--- a/server/src/test/java/org/opensearch/search/backpressure/settings/SearchTaskSettingsTests.java
+++ b/server/src/test/java/org/opensearch/search/backpressure/settings/SearchTaskSettingsTests.java
@@ -21,35 +21,29 @@ private SearchTaskSettings buildDefault() {
         return new SearchTaskSettings(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
     }
 
-    public void testDefaultNativeMemoryThresholdsKeepTrackerInert() {
+    public void testDefaultNativeMemoryPercentThresholdMatchesDefaults() {
+        // Production default: a small non-zero fraction so the tracker engages once a backend
+        // installs a non-zero budget. Test pinned to the same value as Defaults.NATIVE_MEMORY_PERCENT_THRESHOLD
+        // so a default change here trips this assertion deliberately.
         SearchTaskSettings settings = buildDefault();
-        // Defaults are 0 / 0.0 — the tracker treats both as "feature disabled".
-        assertEquals(0L, settings.getTotalNativeMemoryBytesThreshold());
-        assertEquals(0.0d, settings.getNativeMemoryPercentThreshold(), 0.0d);
+        assertEquals(0.05d, settings.getNativeMemoryPercentThreshold(), 0.0d);
     }
 
-    public void testInitialNativeMemoryThresholdsRespectSettings() {
-        long bytesThreshold = 2L * 1024L * 1024L * 1024L;
+    public void testInitialNativeMemoryPercentThresholdRespectsSetting() {
         double fractionThreshold = 0.75d;
         Settings raw = Settings.builder()
-            .put(SearchTaskSettings.SETTING_TOTAL_NATIVE_MEMORY_BYTES_THRESHOLD.getKey(), bytesThreshold)
             .put(SearchTaskSettings.SETTING_NATIVE_MEMORY_PERCENT_THRESHOLD.getKey(), fractionThreshold)
             .build();
         SearchTaskSettings settings = new SearchTaskSettings(raw, new ClusterSettings(raw, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
-        assertEquals(bytesThreshold, settings.getTotalNativeMemoryBytesThreshold());
         assertEquals(fractionThreshold, settings.getNativeMemoryPercentThreshold(), 0.0d);
     }
 
-    public void testNativeMemoryThresholdsAreDynamic() {
+    public void testNativeMemoryPercentThresholdIsDynamic() {
         ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
         SearchTaskSettings settings = new SearchTaskSettings(Settings.EMPTY, clusterSettings);
         clusterSettings.applySettings(
-            Settings.builder()
-                .put(SearchTaskSettings.SETTING_TOTAL_NATIVE_MEMORY_BYTES_THRESHOLD.getKey(), 1024L)
-                .put(SearchTaskSettings.SETTING_NATIVE_MEMORY_PERCENT_THRESHOLD.getKey(), 0.33d)
-                .build()
+            Settings.builder().put(SearchTaskSettings.SETTING_NATIVE_MEMORY_PERCENT_THRESHOLD.getKey(), 0.33d).build()
         );
-        assertEquals(1024L, settings.getTotalNativeMemoryBytesThreshold());
         assertEquals(0.33d, settings.getNativeMemoryPercentThreshold(), 0.0d);
     }
 
@@ -66,12 +60,4 @@ public void testNativeMemoryPercentThresholdRejectsOutOfRange() {
             () -> new SearchTaskSettings(raw2, new ClusterSettings(raw2, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS))
         );
     }
-
-    public void testTotalNativeMemoryBytesThresholdRejectsNegative() {
-        Settings raw = Settings.builder().put(SearchTaskSettings.SETTING_TOTAL_NATIVE_MEMORY_BYTES_THRESHOLD.getKey(), -1L).build();
-        expectThrows(
-            IllegalArgumentException.class,
-            () -> new SearchTaskSettings(raw, new ClusterSettings(raw, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS))
-        );
-    }
 }
diff --git a/server/src/test/java/org/opensearch/search/backpressure/stats/SearchShardTaskStatsTests.java b/server/src/test/java/org/opensearch/search/backpressure/stats/SearchShardTaskStatsTests.java
index 2fe026c0788b1..fb8870d127b2a 100644
--- a/server/src/test/java/org/opensearch/search/backpressure/stats/SearchShardTaskStatsTests.java
+++ b/server/src/test/java/org/opensearch/search/backpressure/stats/SearchShardTaskStatsTests.java
@@ -8,6 +8,9 @@
 
 package org.opensearch.search.backpressure.stats;
 
+import org.opensearch.Version;
+import org.opensearch.common.io.stream.BytesStreamOutput;
+import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.Writeable;
 import org.opensearch.search.backpressure.trackers.CpuUsageTracker;
 import org.opensearch.search.backpressure.trackers.ElapsedTimeTracker;
@@ -17,6 +20,7 @@
 import org.opensearch.search.backpressure.trackers.TaskResourceUsageTrackers.TaskResourceUsageTracker;
 import org.opensearch.test.AbstractWireSerializingTestCase;
 
+import java.util.HashMap;
 import java.util.Map;
 
 public class SearchShardTaskStatsTests extends AbstractWireSerializingTestCase {
@@ -49,4 +53,62 @@ public static SearchShardTaskStats randomInstance() {
             resourceUsageTrackerStats
         );
     }
+
+    /**
+     * Pre-V_3_7_0 nodes don't recognize the {@code NATIVE_MEMORY_USAGE_TRACKER} entry. A round
+     * trip via the older wire version MUST drop that entry rather than corrupting the stream;
+     * the remaining trackers MUST round-trip unchanged.
+     *
+     * 
To assert this without reaching into private fields, we construct a deterministic
+     * input plus a parallel "expected" instance that omits the native-memory entry. Equality
+     * via {@link SearchShardTaskStats#equals} then exercises the same invariants.
+     */
+    public void testNativeMemoryEntryDroppedOnPreV37Wire() throws Exception {
+        long cancellations = 7L;
+        long limitReached = 3L;
+        long completions = 11L;
+
+        TaskResourceUsageTracker.Stats cpu = new CpuUsageTracker.Stats(1L, 2L, 3L);
+        TaskResourceUsageTracker.Stats heap = new HeapUsageTracker.Stats(4L, 5L, 6L, 7L);
+        TaskResourceUsageTracker.Stats elapsed = new ElapsedTimeTracker.Stats(8L, 9L, 10L);
+        TaskResourceUsageTracker.Stats nativeMem = new NativeMemoryUsageTracker.Stats(11L, 12L, 13L);
+
+        Map withNative = new HashMap<>();
+        withNative.put(TaskResourceUsageTrackerType.CPU_USAGE_TRACKER, cpu);
+        withNative.put(TaskResourceUsageTrackerType.HEAP_USAGE_TRACKER, heap);
+        withNative.put(TaskResourceUsageTrackerType.ELAPSED_TIME_TRACKER, elapsed);
+        withNative.put(TaskResourceUsageTrackerType.NATIVE_MEMORY_USAGE_TRACKER, nativeMem);
+
+        Map withoutNative = new HashMap<>(withNative);
+        withoutNative.remove(TaskResourceUsageTrackerType.NATIVE_MEMORY_USAGE_TRACKER);
+
+        SearchShardTaskStats original = new SearchShardTaskStats(cancellations, limitReached, completions, withNative);
+        SearchShardTaskStats expected = new SearchShardTaskStats(cancellations, limitReached, completions, withoutNative);
+
+        try (BytesStreamOutput out = new BytesStreamOutput()) {
+            out.setVersion(Version.V_3_6_0);
+            original.writeTo(out);
+            try (StreamInput in = out.bytes().streamInput()) {
+                in.setVersion(Version.V_3_6_0);
+                SearchShardTaskStats deserialized = new SearchShardTaskStats(in);
+                assertEquals("pre-V_3_7_0 wire must drop the native-memory tracker entry but preserve the others", expected, deserialized);
+            }
+        }
+    }
+
+    /**
+     * V_3_7_0 round-trip must preserve the native-memory entry verbatim.
+     */
+    public void testNativeMemoryEntryRoundTripsOnV37Wire() throws Exception {
+        SearchShardTaskStats original = randomInstance();
+        try (BytesStreamOutput out = new BytesStreamOutput()) {
+            out.setVersion(Version.V_3_7_0);
+            original.writeTo(out);
+            try (StreamInput in = out.bytes().streamInput()) {
+                in.setVersion(Version.V_3_7_0);
+                SearchShardTaskStats deserialized = new SearchShardTaskStats(in);
+                assertEquals(original, deserialized);
+            }
+        }
+    }
 }
diff --git a/server/src/test/java/org/opensearch/search/backpressure/stats/SearchTaskStatsTests.java b/server/src/test/java/org/opensearch/search/backpressure/stats/SearchTaskStatsTests.java
index 31c90c19f9f1b..aeb5c69b84d6c 100644
--- a/server/src/test/java/org/opensearch/search/backpressure/stats/SearchTaskStatsTests.java
+++ b/server/src/test/java/org/opensearch/search/backpressure/stats/SearchTaskStatsTests.java
@@ -8,6 +8,9 @@
 
 package org.opensearch.search.backpressure.stats;
 
+import org.opensearch.Version;
+import org.opensearch.common.io.stream.BytesStreamOutput;
+import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.Writeable;
 import org.opensearch.search.backpressure.trackers.CpuUsageTracker;
 import org.opensearch.search.backpressure.trackers.ElapsedTimeTracker;
@@ -17,6 +20,7 @@
 import org.opensearch.search.backpressure.trackers.TaskResourceUsageTrackers.TaskResourceUsageTracker;
 import org.opensearch.test.AbstractWireSerializingTestCase;
 
+import java.util.HashMap;
 import java.util.Map;
 
 public class SearchTaskStatsTests extends AbstractWireSerializingTestCase {
@@ -45,4 +49,62 @@ public static SearchTaskStats randomInstance() {
 
         return new SearchTaskStats(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), resourceUsageTrackerStats);
     }
+
+    /**
+     * Pre-V_3_7_0 nodes don't recognize the {@code NATIVE_MEMORY_USAGE_TRACKER} entry. A round
+     * trip via the older wire version MUST drop that entry rather than corrupting the stream;
+     * the remaining trackers MUST round-trip unchanged.
+     *
+     * 
To assert this without reaching into private fields, we construct a deterministic
+     * input plus a parallel "expected" instance that omits the native-memory entry. Equality
+     * via {@link SearchTaskStats#equals} then exercises the same invariants.
+     */
+    public void testNativeMemoryEntryDroppedOnPreV37Wire() throws Exception {
+        long cancellations = 7L;
+        long limitReached = 3L;
+        long completions = 11L;
+
+        TaskResourceUsageTracker.Stats cpu = new CpuUsageTracker.Stats(1L, 2L, 3L);
+        TaskResourceUsageTracker.Stats heap = new HeapUsageTracker.Stats(4L, 5L, 6L, 7L);
+        TaskResourceUsageTracker.Stats elapsed = new ElapsedTimeTracker.Stats(8L, 9L, 10L);
+        TaskResourceUsageTracker.Stats nativeMem = new NativeMemoryUsageTracker.Stats(11L, 12L, 13L);
+
+        Map withNative = new HashMap<>();
+        withNative.put(TaskResourceUsageTrackerType.CPU_USAGE_TRACKER, cpu);
+        withNative.put(TaskResourceUsageTrackerType.HEAP_USAGE_TRACKER, heap);
+        withNative.put(TaskResourceUsageTrackerType.ELAPSED_TIME_TRACKER, elapsed);
+        withNative.put(TaskResourceUsageTrackerType.NATIVE_MEMORY_USAGE_TRACKER, nativeMem);
+
+        Map withoutNative = new HashMap<>(withNative);
+        withoutNative.remove(TaskResourceUsageTrackerType.NATIVE_MEMORY_USAGE_TRACKER);
+
+        SearchTaskStats original = new SearchTaskStats(cancellations, limitReached, completions, withNative);
+        SearchTaskStats expected = new SearchTaskStats(cancellations, limitReached, completions, withoutNative);
+
+        try (BytesStreamOutput out = new BytesStreamOutput()) {
+            out.setVersion(Version.V_3_6_0);
+            original.writeTo(out);
+            try (StreamInput in = out.bytes().streamInput()) {
+                in.setVersion(Version.V_3_6_0);
+                SearchTaskStats deserialized = new SearchTaskStats(in);
+                assertEquals("pre-V_3_7_0 wire must drop the native-memory tracker entry but preserve the others", expected, deserialized);
+            }
+        }
+    }
+
+    /**
+     * V_3_7_0 round-trip must preserve the native-memory entry verbatim.
+     */
+    public void testNativeMemoryEntryRoundTripsOnV37Wire() throws Exception {
+        SearchTaskStats original = randomInstance();
+        try (BytesStreamOutput out = new BytesStreamOutput()) {
+            out.setVersion(Version.V_3_7_0);
+            original.writeTo(out);
+            try (StreamInput in = out.bytes().streamInput()) {
+                in.setVersion(Version.V_3_7_0);
+                SearchTaskStats deserialized = new SearchTaskStats(in);
+                assertEquals(original, deserialized);
+            }
+        }
+    }
 }
diff --git a/server/src/test/java/org/opensearch/search/backpressure/trackers/NativeMemoryUsageTrackerTests.java b/server/src/test/java/org/opensearch/search/backpressure/trackers/NativeMemoryUsageTrackerTests.java
index 13d1a0ac6d5df..f670203eac2c4 100644
--- a/server/src/test/java/org/opensearch/search/backpressure/trackers/NativeMemoryUsageTrackerTests.java
+++ b/server/src/test/java/org/opensearch/search/backpressure/trackers/NativeMemoryUsageTrackerTests.java
@@ -13,6 +13,7 @@
 import org.opensearch.common.io.stream.BytesStreamOutput;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.tasks.TaskId;
+import org.opensearch.search.backpressure.NativeMemoryUsageService;
 import org.opensearch.tasks.CancellableTask;
 import org.opensearch.tasks.Task;
 import org.opensearch.tasks.TaskCancellation;
@@ -20,37 +21,32 @@
 import org.junit.After;
 import org.junit.Before;
 
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.function.Supplier;
 
-import static org.opensearch.search.backpressure.SearchBackpressureTestHelpers.createMockTaskWithResourceStats;
-
 /**
  * Unit tests for {@link NativeMemoryUsageTracker}.
  *
- * 
The tracker maintains static suppliers (snapshot + budget) that survive across tests
- * because they're installed by backend plugins at boot time. {@link #setUp()} and
- * {@link #tearDown()} reset them to their defaults so every test starts from a known
- * "no backend installed" state.
+ * 
The tracker is a thin wrapper over {@link NativeMemoryUsageService}, which is a
+ * process-wide singleton. The service holds the snapshot supplier and budget supplier
+ * that backend plugins install at boot. Each test must reset the singleton in
+ * {@link #setUp()}/{@link #tearDown()} so leftover suppliers from a prior test (or from
+ * production code paths exercised by another suite) don't leak.
  */
 public class NativeMemoryUsageTrackerTests extends OpenSearchTestCase {
 
     @Before
-    public void resetStaticsBefore() {
-        // Static suppliers are package-shared with the production wiring; clear before every test
-        // so each test owns its own supplier surface and doesn't see leftover state from a prior
-        // test (or from production code paths exercised by another suite).
-        NativeMemoryUsageTracker.setSnapshotSupplier(java.util.Collections::emptyMap);
-        NativeMemoryUsageTracker.setNativeMemoryBudgetSupplier(() -> 0L);
+    public void resetServiceBefore() {
+        NativeMemoryUsageService.getInstance().resetForTesting();
     }
 
     @After
-    public void resetStaticsAfter() {
-        NativeMemoryUsageTracker.setSnapshotSupplier(java.util.Collections::emptyMap);
-        NativeMemoryUsageTracker.setNativeMemoryBudgetSupplier(() -> 0L);
+    public void resetServiceAfter() {
+        NativeMemoryUsageService.getInstance().resetForTesting();
     }
 
     public void testNameMatchesEnum() {
@@ -58,33 +54,25 @@ public void testNameMatchesEnum() {
         assertEquals(TaskResourceUsageTrackerType.NATIVE_MEMORY_USAGE_TRACKER.getName(), tracker.name());
     }
 
-    public void testRefreshEmptySnapshotIsSafe() {
-        NativeMemoryUsageTracker tracker = new NativeMemoryUsageTracker(() -> 0.5);
-        // No supplier installed — default empty map should be picked up.
-        tracker.refresh();
-        Task task = createMockTaskWithResourceStats(SearchTask.class, 1, 1, randomNonNegativeLong());
-        // bytesForTask must be 0 when no entry is in the snapshot.
-        assertEquals(0L, tracker.bytesForTask(task));
-    }
-
-    public void testRefreshNullSnapshotFallsBackToEmpty() {
-        // A misbehaving backend that returns null must not propagate NPE through the refresh path.
-        NativeMemoryUsageTracker.setSnapshotSupplier(() -> null);
+    public void testBytesForTaskWithEmptySnapshot() {
+        // No supplier installed — the service's default empty snapshot must yield 0 for any task id.
         NativeMemoryUsageTracker tracker = new NativeMemoryUsageTracker(() -> 0.5);
-        tracker.refresh();
-        Task task = createMockTaskWithResourceStats(SearchTask.class, 1, 1, 42L);
+        Task task = createMockTask(SearchTask.class, randomNonNegativeLong());
         assertEquals(0L, tracker.bytesForTask(task));
     }
 
-    public void testBytesForTaskAfterRefreshReadsSnapshot() {
+    public void testBytesForTaskAfterServiceRefresh() {
         long taskId = 12345L;
         long bytes = 8L * 1024 * 1024;
         Map snapshot = new HashMap<>();
         snapshot.put(taskId, bytes);
         NativeMemoryUsageTracker.setSnapshotSupplier(() -> snapshot);
+
         NativeMemoryUsageTracker tracker = new NativeMemoryUsageTracker(() -> 0.5);
-        tracker.refresh();
-        Task task = mockTaskWithId(SearchTask.class, taskId);
+        // Refresh is now owned by the service; the tracker reads what the service publishes.
+        NativeMemoryUsageService.getInstance().refresh();
+
+        Task task = createMockTask(SearchTask.class, taskId);
         assertEquals(bytes, tracker.bytesForTask(task));
     }
 
@@ -94,21 +82,21 @@ public void testBytesForTaskNullTaskReturnsZero() {
     }
 
     public void testHasSnapshotProviderReflectsInstallation() {
-        // Once a backend installs a snapshot supplier, hasSnapshotProvider() must report true
-        // even when the supplier itself returns an empty map. The contract is "is something
-        // wired up", not "does the wired thing currently have data".
-        Supplier> supplier = java.util.Collections::emptyMap;
-        NativeMemoryUsageTracker.setSnapshotSupplier(supplier);
+        // Before any backend installs a supplier, the predicate is false.
+        assertFalse(NativeMemoryUsageTracker.hasSnapshotProvider());
+
+        // Once installed, hasSnapshotProvider must return true even when the supplier itself
+        // returns an empty map. Contract is "is something wired up?", not "does it have data?".
+        NativeMemoryUsageTracker.setSnapshotSupplier(Collections::emptyMap);
         assertTrue(NativeMemoryUsageTracker.hasSnapshotProvider());
     }
 
     public void testNullSnapshotSupplierIgnored() {
-        // Existing supplier should be retained when caller passes null.
-        Supplier> good = java.util.Collections::emptyMap;
+        Supplier> good = Collections::emptyMap;
         NativeMemoryUsageTracker.setSnapshotSupplier(good);
         assertTrue(NativeMemoryUsageTracker.hasSnapshotProvider());
+        // Passing null must not erase a working installation.
         NativeMemoryUsageTracker.setSnapshotSupplier(null);
-        // Still installed because null is a no-op.
         assertTrue(NativeMemoryUsageTracker.hasSnapshotProvider());
     }
 
@@ -131,26 +119,24 @@ public void testNullBudgetSupplierIgnored() {
     public void testEvaluateInertWhenFractionZero() {
         // Operator hasn't opted in (fraction=0): tracker MUST NOT cancel even with budget + heavy task.
         NativeMemoryUsageTracker.setNativeMemoryBudgetSupplier(() -> 1_000L);
-        Map snapshot = new HashMap<>();
         long taskId = 1L;
-        snapshot.put(taskId, 999L);
-        NativeMemoryUsageTracker.setSnapshotSupplier(() -> snapshot);
+        NativeMemoryUsageTracker.setSnapshotSupplier(() -> Map.of(taskId, 999L));
+        NativeMemoryUsageService.getInstance().refresh();
+
         NativeMemoryUsageTracker tracker = new NativeMemoryUsageTracker(() -> 0.0);
-        tracker.refresh();
-        Task task = mockTaskWithId(SearchTask.class, taskId);
+        Task task = createMockTask(SearchTask.class, taskId);
         Optional reason = tracker.checkAndMaybeGetCancellationReason(task);
         assertFalse(reason.isPresent());
     }
 
     public void testEvaluateInertWhenBudgetZero() {
         // No backend installed budget: tracker MUST NOT cancel even with a fraction set.
-        Map snapshot = new HashMap<>();
         long taskId = 1L;
-        snapshot.put(taskId, 999L);
-        NativeMemoryUsageTracker.setSnapshotSupplier(() -> snapshot);
+        NativeMemoryUsageTracker.setSnapshotSupplier(() -> Map.of(taskId, 999L));
+        NativeMemoryUsageService.getInstance().refresh();
+
         NativeMemoryUsageTracker tracker = new NativeMemoryUsageTracker(() -> 0.5);
-        tracker.refresh();
-        Task task = mockTaskWithId(SearchTask.class, taskId);
+        Task task = createMockTask(SearchTask.class, taskId);
         Optional reason = tracker.checkAndMaybeGetCancellationReason(task);
         assertFalse(reason.isPresent());
     }
@@ -158,26 +144,27 @@ public void testEvaluateInertWhenBudgetZero() {
     public void testEvaluateNonCancellableTaskSkipped() {
         NativeMemoryUsageTracker.setNativeMemoryBudgetSupplier(() -> 1_000L);
         NativeMemoryUsageTracker.setSnapshotSupplier(() -> Map.of(1L, 999L));
+        NativeMemoryUsageService.getInstance().refresh();
+
         NativeMemoryUsageTracker tracker = new NativeMemoryUsageTracker(() -> 0.5);
-        tracker.refresh();
-        // Plain Task (not a CancellableTask) should be ignored.
-        Task task = new Task(1L, "type", "action", "description", TaskId.EMPTY_TASK_ID, java.util.Collections.emptyMap());
+        // Plain Task (not a CancellableTask) must be ignored by the evaluator.
+        Task task = new Task(1L, "type", "action", "description", TaskId.EMPTY_TASK_ID, Collections.emptyMap());
         Optional reason = tracker.checkAndMaybeGetCancellationReason(task);
         assertFalse(reason.isPresent());
     }
 
-    public void testEvaluateBelowThreshold() {
+    public void testEvaluateBelowThresholdNoCancellation() {
         long budget = 1_000L;
         double fraction = 0.5;
         long taskId = 7L;
-        // 400 < 500 (budget*fraction)
+        // 400 < 500 (budget * fraction)
         NativeMemoryUsageTracker.setNativeMemoryBudgetSupplier(() -> budget);
         NativeMemoryUsageTracker.setSnapshotSupplier(() -> Map.of(taskId, 400L));
+        NativeMemoryUsageService.getInstance().refresh();
+
         NativeMemoryUsageTracker tracker = new NativeMemoryUsageTracker(() -> fraction);
-        tracker.refresh();
-        Task task = mockTaskWithId(SearchTask.class, taskId);
-        Optional reason = tracker.checkAndMaybeGetCancellationReason(task);
-        assertFalse(reason.isPresent());
+        Task task = createMockTask(SearchTask.class, taskId);
+        assertFalse(tracker.checkAndMaybeGetCancellationReason(task).isPresent());
     }
 
     public void testEvaluateAtAndAboveThresholdReturnsReason() {
@@ -185,51 +172,54 @@ public void testEvaluateAtAndAboveThresholdReturnsReason() {
         double fraction = 0.5;
         long thresholdBytes = (long) (budget * fraction);
         long taskId = 7L;
-        long usage = thresholdBytes + 1L; // strictly above to keep score > 1
+        long usage = thresholdBytes + 1L;
         NativeMemoryUsageTracker.setNativeMemoryBudgetSupplier(() -> budget);
         NativeMemoryUsageTracker.setSnapshotSupplier(() -> Map.of(taskId, usage));
+        NativeMemoryUsageService.getInstance().refresh();
+
         NativeMemoryUsageTracker tracker = new NativeMemoryUsageTracker(() -> fraction);
-        tracker.refresh();
-        Task task = mockTaskWithId(SearchShardTask.class, taskId);
+        Task task = createMockTask(SearchShardTask.class, taskId);
         Optional reason = tracker.checkAndMaybeGetCancellationReason(task);
-        assertTrue(reason.isPresent());
+        assertTrue("usage above threshold must yield a cancellation reason", reason.isPresent());
         assertTrue("score must be at least 1", reason.get().getCancellationScore() >= 1);
         assertTrue(
-            "reason message should mention native memory usage",
+            "reason message should mention native memory",
             reason.get().getMessage().toLowerCase(java.util.Locale.ROOT).contains("native memory")
         );
     }
 
     public void testEvaluateScoreScalesWithUsage() {
-        // Score is floor(usage / threshold) per tracker's evaluator.
+        // Score is floor(usage / threshold) per the tracker's evaluator.
         long budget = 1_000L;
         double fraction = 0.1; // threshold = 100
         long taskId = 9L;
         NativeMemoryUsageTracker.setNativeMemoryBudgetSupplier(() -> budget);
         NativeMemoryUsageTracker.setSnapshotSupplier(() -> Map.of(taskId, 350L)); // 3x threshold
+        NativeMemoryUsageService.getInstance().refresh();
+
         NativeMemoryUsageTracker tracker = new NativeMemoryUsageTracker(() -> fraction);
-        tracker.refresh();
-        Task task = mockTaskWithId(SearchTask.class, taskId);
+        Task task = createMockTask(SearchTask.class, taskId);
         Optional reason = tracker.checkAndMaybeGetCancellationReason(task);
         assertTrue(reason.isPresent());
         assertEquals(3, reason.get().getCancellationScore());
     }
 
     public void testEvaluateFractionClampedToOne() {
-        // Fraction > 1.0 should be clamped to 1.0 — only tasks at or above the full budget cancel.
+        // Fraction > 1.0 must be clamped to 1.0 — only tasks at or above the full budget cancel.
         long budget = 1_000L;
         long taskId = 1L;
         NativeMemoryUsageTracker.setNativeMemoryBudgetSupplier(() -> budget);
         NativeMemoryUsageTracker.setSnapshotSupplier(() -> Map.of(taskId, 999L));
+        NativeMemoryUsageService.getInstance().refresh();
+
         NativeMemoryUsageTracker tracker = new NativeMemoryUsageTracker(() -> 5.0);
-        tracker.refresh();
-        Task task = mockTaskWithId(SearchTask.class, taskId);
-        // 999 < 1000 (clamped threshold) — must NOT cancel
+        Task task = createMockTask(SearchTask.class, taskId);
+        // 999 < 1000 (clamped threshold) — must NOT cancel.
         assertFalse(tracker.checkAndMaybeGetCancellationReason(task).isPresent());
 
         NativeMemoryUsageTracker.setSnapshotSupplier(() -> Map.of(taskId, 1_000L));
-        tracker.refresh();
-        // 1000 >= 1000 — cancel
+        NativeMemoryUsageService.getInstance().refresh();
+        // 1000 >= 1000 — must cancel.
         assertTrue(tracker.checkAndMaybeGetCancellationReason(task).isPresent());
     }
 
@@ -239,23 +229,22 @@ public void testStatsMaxAndAvgOverActiveTasks() {
         snapshot.put(2L, 200L);
         snapshot.put(3L, 300L);
         NativeMemoryUsageTracker.setSnapshotSupplier(() -> snapshot);
-        NativeMemoryUsageTracker tracker = new NativeMemoryUsageTracker(() -> 0.5);
-        tracker.refresh();
+        NativeMemoryUsageService.getInstance().refresh();
 
+        NativeMemoryUsageTracker tracker = new NativeMemoryUsageTracker(() -> 0.5);
         List tasks = List.of(
-            mockTaskWithId(SearchTask.class, 1L),
-            mockTaskWithId(SearchTask.class, 2L),
-            mockTaskWithId(SearchTask.class, 3L)
+            createMockTask(SearchTask.class, 1L),
+            createMockTask(SearchTask.class, 2L),
+            createMockTask(SearchTask.class, 3L)
         );
         NativeMemoryUsageTracker.Stats stats = (NativeMemoryUsageTracker.Stats) tracker.stats(tasks);
-        // Cancellation count is zero — no checkAndMaybeGetCancellationReason invocations counted yet.
+        // No checkAndMaybeGetCancellationReason invocations — cancellation count stays at zero.
         NativeMemoryUsageTracker.Stats expected = new NativeMemoryUsageTracker.Stats(0L, 300L, 200L);
         assertEquals(expected, stats);
     }
 
     public void testStatsEmptyTaskList() {
         NativeMemoryUsageTracker tracker = new NativeMemoryUsageTracker(() -> 0.5);
-        tracker.refresh();
         NativeMemoryUsageTracker.Stats stats = (NativeMemoryUsageTracker.Stats) tracker.stats(List.of());
         assertEquals(new NativeMemoryUsageTracker.Stats(0L, 0L, 0L), stats);
     }
@@ -284,39 +273,45 @@ public void testStatsEqualsAndHashCode() {
         assertEquals(a.hashCode(), b.hashCode());
     }
 
-    public void testIsNativeTrackingSupportedRequiresProvider() {
-        // isNativeTrackingSupported() returns false when no backend has installed a real
-        // supplier. Because the static supplier is a process-wide singleton, this assertion
-        // is best-effort: on Linux with prior installation it may be true, but on every
-        // platform it should agree with hasSnapshotProvider() AND a positive total memory
-        // reading. We verify the predicate is consistent with its inputs rather than
-        // hard-coding a value.
-        boolean supported = NativeMemoryUsageTracker.isNativeTrackingSupported();
-        if (supported) {
-            assertTrue(NativeMemoryUsageTracker.hasSnapshotProvider());
-            assertTrue(org.apache.lucene.util.Constants.LINUX);
-        }
-        // The negative direction always holds: non-Linux MUST report unsupported.
+    public void testIsNativeTrackingSupportedNonLinuxAlwaysFalse() {
+        // Negative direction always holds: non-Linux MUST report unsupported regardless of state.
         if (org.apache.lucene.util.Constants.LINUX == false) {
-            assertFalse(supported);
+            assertFalse(NativeMemoryUsageTracker.isNativeTrackingSupported());
         }
     }
 
+    public void testIsNativeTrackingSupportedRequiresProvider() {
+        // Without a snapshot provider the predicate must be false even on Linux.
+        // (resetForTesting() in @Before clears any previously installed supplier.)
+        assertFalse(NativeMemoryUsageTracker.isNativeTrackingSupported());
+    }
+
     public void testUpdateIsNoOp() {
         // update() is documented as a no-op; calling it must not throw or change snapshot state.
         NativeMemoryUsageTracker.setSnapshotSupplier(() -> Map.of(1L, 100L));
+        NativeMemoryUsageService.getInstance().refresh();
+
         NativeMemoryUsageTracker tracker = new NativeMemoryUsageTracker(() -> 0.5);
-        tracker.refresh();
-        Task task = mockTaskWithId(SearchTask.class, 1L);
+        Task task = createMockTask(SearchTask.class, 1L);
         tracker.update(task);
         assertEquals(100L, tracker.bytesForTask(task));
     }
 
+    public void testTrackerAcceptsInjectedService() {
+        // Package-private constructor lets a test isolate the service from the singleton.
+        NativeMemoryUsageService isolated = NativeMemoryUsageService.getInstance();
+        NativeMemoryUsageTracker tracker = new NativeMemoryUsageTracker(() -> 0.5, isolated);
+        assertNotNull(tracker);
+        assertEquals(TaskResourceUsageTrackerType.NATIVE_MEMORY_USAGE_TRACKER.getName(), tracker.name());
+    }
+
     /**
-     * Build a {@link CancellableTask} mock whose {@code getId()} returns the given id.
-     * The test-framework helper randomizes ids, which doesn't fit a snapshot-keyed test.
+     * Build a {@link CancellableTask} mock whose {@code getId()} returns the given id. The
+     * {@link org.opensearch.search.backpressure.SearchBackpressureTestHelpers#createMockTaskWithResourceStats}
+     * helper randomizes the id, which doesn't fit a snapshot-keyed test where evaluator
+     * lookups must hit a specific entry.
      */
-    private  T mockTaskWithId(Class type, long id) {
+    private  T createMockTask(Class type, long id) {
         T task = org.mockito.Mockito.mock(type);
         org.mockito.Mockito.when(task.getId()).thenReturn(id);
         org.mockito.Mockito.when(task.getTotalResourceStats())

From bb0c95f234f69387b1f9bf01f258f726a70d8fcc Mon Sep 17 00:00:00 2001
From: Pradeep L 
Date: Mon, 18 May 2026 16:20:28 +0530
Subject: [PATCH 07/10]  fixing rebase issues

Signed-off-by: Pradeep L 
---
 .../rust/src/lib.rs                           | 10 +-----
 .../rust/src/query_tracker.rs                 |  7 ++---
 .../be/datafusion/DataFusionPlugin.java       |  2 --
 .../nativelib/QueryRegistryLayout.java        |  2 +-
 .../NativeMemorySearchBackpressureIT.java     |  5 +--
 .../SearchBackpressureService.java            | 31 +++++--------------
 .../settings/NodeDuressSettings.java          |  9 ++++--
 .../settings/NodeDuressSettingsTests.java     |  4 ++-
 8 files changed, 24 insertions(+), 46 deletions(-)

diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/lib.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/lib.rs
index b1201c56c195e..066aa8214ac82 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/lib.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/lib.rs
@@ -13,14 +13,6 @@
 
 pub(crate) mod agg_mode;
 pub mod api;
-
-// Re-export the native logging bridge macros so this crate's modules can emit logs
-// that flow through `RustLoggerBridge` into the Java/Log4j sink. The standard
-// `log` crate macros (`log::info!`, `log::debug!`, …) are no-ops here because no
-// global `log` dispatcher is registered — using these macros instead is the same
-// pattern the `parquet-data-format` crate uses.
-pub use native_bridge_common::{log_debug, log_error, log_info};
-
 pub mod cache;
 pub mod cancellation;
 pub mod cross_rt_stream;
@@ -43,4 +35,4 @@ pub mod session_context;
 pub mod statistics_cache;
 pub mod udf;
 pub mod stats;
-pub mod task_monitors;
\ No newline at end of file
+pub mod task_monitors;
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/query_tracker.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/query_tracker.rs
index 900fbedf6f58f..45c66fb309916 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/query_tracker.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/query_tracker.rs
@@ -28,8 +28,6 @@ use tokio_util::sync::CancellationToken;
 use datafusion::common::DataFusionError;
 use datafusion::execution::memory_pool::{MemoryConsumer, MemoryPool, MemoryReservation};
 
-use crate::log_debug;
-
 // ---------------------------------------------------------------------------
 // Per-query memory pool
 // ---------------------------------------------------------------------------
@@ -353,10 +351,11 @@ impl Drop for QueryTrackingContext {
     fn drop(&mut self) {
         if let Some(tracker) = &self.tracker {
             tracker.mark_completed();
-            log_debug!(
-                "query completed ctx={}: wall={:.3}s, peak={}B",
+           debug!(
+               "Query completed ctx={}: wall={:.3}s, mem_current={}B, mem_peak={}B",
                 tracker.context_id,
                 tracker.wall_secs(),
+                tracker.memory_pool.current_bytes(),
                 tracker.memory_pool.peak_bytes(),
             );
             QUERY_REGISTRY.remove(&tracker.context_id);
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java
index bd1cb0da19707..8f13e5b2d3581 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java
@@ -13,7 +13,6 @@
 import org.opensearch.analytics.spi.AnalyticsSearchBackendPlugin;
 import org.opensearch.analytics.spi.QueryExecutionMetrics;
 import org.opensearch.be.datafusion.action.DataFusionStatsAction;
-import org.opensearch.be.datafusion.cache.CacheSettings;
 import org.opensearch.be.datafusion.nativelib.NativeBridge;
 import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
 import org.opensearch.cluster.node.DiscoveryNodes;
@@ -175,7 +174,6 @@ public Collection createComponents(
         // The OpenSearch task id is used as the DataFusion context_id at query launch
         // (see ShardScanInstructionHandler / DatafusionSearchExecEngine), so the map is
         // already keyed by Task#getId on the consumer side.
-        logger.info("installing native-memory snapshot supplier for search backpressure");
         NativeMemoryUsageTracker.setSnapshotSupplier(this::currentBytesByTaskId);
         NativeMemoryUsageTracker.setNativeMemoryBudgetSupplier(
             () -> DATAFUSION_MEMORY_POOL_LIMIT.get(clusterService.getSettings())
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/QueryRegistryLayout.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/QueryRegistryLayout.java
index 6cda9dc1cdb72..603a1ebccae93 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/QueryRegistryLayout.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/QueryRegistryLayout.java
@@ -1,4 +1,4 @@
-    /*
+/*
  * SPDX-License-Identifier: Apache-2.0
  *
  * The OpenSearch Contributors require contributions made to
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/backpressure/NativeMemorySearchBackpressureIT.java b/server/src/internalClusterTest/java/org/opensearch/search/backpressure/NativeMemorySearchBackpressureIT.java
index 1e6f3a08ea724..f04a19eaf1dd9 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/backpressure/NativeMemorySearchBackpressureIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/backpressure/NativeMemorySearchBackpressureIT.java
@@ -118,8 +118,9 @@ public final void setupNodeSettings() {
             .put(NodeDuressSettings.SETTING_HEAP_THRESHOLD.getKey(), 1.0) // disable heap duress
             .put(NodeDuressSettings.SETTING_NATIVE_MEMORY_THRESHOLD.getKey(), 0.0)
             .put(NodeDuressSettings.SETTING_NUM_SUCCESSIVE_BREACHES.getKey(), 1)
-            // node.native_memory.limit must be > 0 or the duress lambda short-circuits
-            // ("totalNative is zero"). 1 byte is enough — the probe reports a non-zero RssAnon.
+            // search_backpressure.node_duress.native_memory_limit must be > 0 or the duress
+            // lambda short-circuits ("totalNative is zero"). 1 byte is enough — the probe
+            // reports a non-zero RssAnon.
             .put(NodeDuressSettings.NODE_NATIVE_MEMORY_LIMIT_SETTING.getKey(), "1b")
             .build();
         assertAcked(client().admin().cluster().prepareUpdateSettings().setPersistentSettings(request).get());
diff --git a/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java b/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java
index dcf8abae1b042..a667cb456063e 100644
--- a/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java
+++ b/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java
@@ -136,12 +136,14 @@ public SearchBackpressureService(
                 put(ResourceType.NATIVE_MEMORY, new NodeDuressTracker(() -> {
                     double used = OsProbe.getInstance().getProcessNativeMemoryBytes();
                     double totalNative = settings.getNodeDuressSettings().getNodeNativeMemory();
-                    if (totalNative == 0) {
+                    if (totalNative <= 0) {
                         return false;
                     }
                     double usedFraction = used / totalNative;
                     if (usedFraction < 0.0d) {
+                        if (logger.isDebugEnabled()) {
                         logger.debug("native memory duress probe: signal unavailable (usedBytes={})", used);
+                        }
                         return false;
                     }
                     double threshold = settings.getNodeDuressSettings().getNativeMemoryThreshold();
@@ -157,7 +159,7 @@ public SearchBackpressureService(
                     return breached;
                 }, () -> settings.getNodeDuressSettings().getNumSuccessiveBreaches()));
             }
-        }, new TimeBasedExpiryTracker(System::nanoTime)),
+        }),
             getTrackers(
                 settings.getSearchTaskSettings()::getCpuTimeNanosThreshold,
                 settings.getSearchTaskSettings()::getHeapVarianceThreshold,
@@ -240,21 +242,6 @@ void doRun() {
         List searchShardTasks = getTaskByType(SearchShardTask.class);
 
         final boolean isNativeMemoryInDuress = nodeDuressTrackers.isNativeMemoryInDuress();
-        if (logger.isDebugEnabled()) {
-            if (isNativeMemoryInDuress) {
-                logger.debug(
-                    "native memory duress active, bypassing heap-dominance gate — searchTasks={}, searchShardTasks={}",
-                    searchTasks.size(),
-                    searchShardTasks.size()
-                );
-            } else {
-                logger.debug(
-                    "node in duress (non-native-memory) — searchTasks={}, searchShardTasks={}",
-                    searchTasks.size(),
-                    searchShardTasks.size()
-                );
-            }
-        }
 
         final Map, List> cancellableTasks;
         if (isNativeMemoryInDuress) {
@@ -316,11 +303,7 @@ void doRun() {
 
             // Stop cancelling tasks if there are no tokens in either of the two token buckets.
             if (rateLimitReached && ratioLimitReached) {
-                logger.warn(
-                    "cancellation limit reached for [{}], not cancelling task [{}]",
-                    taskType.getSimpleName(),
-                    taskCancellation.getTask().getId()
-                );
+                logger.debug("task cancellation limit reached");
                 searchBackpressureState.incrementLimitReachedCount();
                 break;
             }
@@ -441,7 +424,7 @@ public static TaskResourceUsageTrackers getTrackers(
         LongSupplier ElapsedTimeNanosSupplier,
         DoubleSupplier nativeMemoryPercentThresholdSupplier,
         ClusterSettings clusterSettings,
-        Setting heapWindowSizeSetting
+        Setting windowSizeSetting
     ) {
         TaskResourceUsageTrackers trackers = new TaskResourceUsageTrackers();
         trackers.addTracker(new CpuUsageTracker(cpuThresholdSupplier), TaskResourceUsageTrackerType.CPU_USAGE_TRACKER);
@@ -452,7 +435,7 @@ public static TaskResourceUsageTrackers getTrackers(
                     heapPercentThresholdSupplier,
                     heapMovingAverageWindowSize,
                     clusterSettings,
-                    heapWindowSizeSetting
+                    windowSizeSetting
                 ),
                 TaskResourceUsageTrackerType.HEAP_USAGE_TRACKER
             );
diff --git a/server/src/main/java/org/opensearch/search/backpressure/settings/NodeDuressSettings.java b/server/src/main/java/org/opensearch/search/backpressure/settings/NodeDuressSettings.java
index 3779fbc22d06d..b48a677ea2ede 100644
--- a/server/src/main/java/org/opensearch/search/backpressure/settings/NodeDuressSettings.java
+++ b/server/src/main/java/org/opensearch/search/backpressure/settings/NodeDuressSettings.java
@@ -89,12 +89,15 @@ private static class Defaults {
     );
 
     /**
-     * Absolute native-memory budget for this node, in bytes. When the value is {@link ByteSizeValue#ZERO}
-     * (default) the tracker treats the budget as unconfigured and reports {@code 0%}.
+     * Absolute native-memory budget for this node, in bytes, scoped to the search-backpressure
+     * duress probe. Independent from the node-level resource-tracker's
+     * {@code node.native_memory.limit} so the two features can be tuned separately. When this
+     * value is {@link ByteSizeValue#ZERO} (default), the duress probe treats the budget as
+     * unconfigured and stays inert.
      */
     private volatile ByteSizeValue nodeNativeMemory;
     public static final Setting NODE_NATIVE_MEMORY_LIMIT_SETTING = Setting.byteSizeSetting(
-        "node.native_memory.limit",
+        "search_backpressure.node_duress.native_memory_limit",
         ByteSizeValue.ZERO,
         Setting.Property.Dynamic,
         Setting.Property.NodeScope
diff --git a/server/src/test/java/org/opensearch/search/backpressure/settings/NodeDuressSettingsTests.java b/server/src/test/java/org/opensearch/search/backpressure/settings/NodeDuressSettingsTests.java
index 8850096547ca9..ffcc2b20763a1 100644
--- a/server/src/test/java/org/opensearch/search/backpressure/settings/NodeDuressSettingsTests.java
+++ b/server/src/test/java/org/opensearch/search/backpressure/settings/NodeDuressSettingsTests.java
@@ -87,7 +87,9 @@ public void testNodeNativeMemoryLimitIsDynamic() {
         );
         assertEquals(512L * 1024 * 1024, settings.getNodeNativeMemory());
 
-        clusterSettings.applySettings(Settings.builder().put(NodeDuressSettings.NODE_NATIVE_MEMORY_LIMIT_SETTING.getKey(), "1gb").build());
+        clusterSettings.applySettings(
+            Settings.builder().put(NodeDuressSettings.NODE_NATIVE_MEMORY_LIMIT_SETTING.getKey(), "1gb").build()
+        );
         assertEquals(1024L * 1024 * 1024, settings.getNodeNativeMemory());
     }
 

From 89737ba1a9bf1fbb53073d52eaf741e8e9e79d91 Mon Sep 17 00:00:00 2001
From: Pradeep L 
Date: Tue, 19 May 2026 02:33:12 +0530
Subject: [PATCH 08/10] fix: review comments

Signed-off-by: Pradeep L 
---
 .../analytics-backend-datafusion/rust/src/ffm.rs      |  1 -
 .../rust/src/query_tracker.rs                         |  5 +++--
 .../opensearch/be/datafusion/DataFusionPlugin.java    |  4 +---
 .../be/datafusion/nativelib/NativeBridge.java         | 11 +++--------
 .../be/datafusion/nativelib/QueryRegistryLayout.java  |  2 +-
 .../backpressure/SearchBackpressureService.java       |  3 +--
 .../settings/NodeDuressSettingsTests.java             |  4 +---
 7 files changed, 10 insertions(+), 20 deletions(-)

diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs
index d5b75de6b7aaa..b8fc8ce406b0e 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/ffm.rs
@@ -12,7 +12,6 @@ use std::slice;
 use std::str;
 use std::sync::Arc;
 
-use log::info;
 use native_bridge_common::ffm_safe;
 use parking_lot::RwLock;
 
diff --git a/sandbox/plugins/analytics-backend-datafusion/rust/src/query_tracker.rs b/sandbox/plugins/analytics-backend-datafusion/rust/src/query_tracker.rs
index 45c66fb309916..84d29376b6a1f 100644
--- a/sandbox/plugins/analytics-backend-datafusion/rust/src/query_tracker.rs
+++ b/sandbox/plugins/analytics-backend-datafusion/rust/src/query_tracker.rs
@@ -22,6 +22,7 @@ use std::sync::Arc;
 use std::time::Instant;
 
 use dashmap::DashMap;
+use log::debug;
 use once_cell::sync::Lazy;
 use tokio_util::sync::CancellationToken;
 
@@ -351,8 +352,8 @@ impl Drop for QueryTrackingContext {
     fn drop(&mut self) {
         if let Some(tracker) = &self.tracker {
             tracker.mark_completed();
-           debug!(
-               "Query completed ctx={}: wall={:.3}s, mem_current={}B, mem_peak={}B",
+          debug!(
+              "Query completed ctx={}: wall={:.3}s, mem_current={}B, mem_peak={}B",
                 tracker.context_id,
                 tracker.wall_secs(),
                 tracker.memory_pool.current_bytes(),
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java
index 8f13e5b2d3581..cf06ec8aff2b9 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/DataFusionPlugin.java
@@ -175,9 +175,7 @@ public Collection createComponents(
         // (see ShardScanInstructionHandler / DatafusionSearchExecEngine), so the map is
         // already keyed by Task#getId on the consumer side.
         NativeMemoryUsageTracker.setSnapshotSupplier(this::currentBytesByTaskId);
-        NativeMemoryUsageTracker.setNativeMemoryBudgetSupplier(
-            () -> DATAFUSION_MEMORY_POOL_LIMIT.get(clusterService.getSettings())
-        );
+        NativeMemoryUsageTracker.setNativeMemoryBudgetSupplier(() -> DATAFUSION_MEMORY_POOL_LIMIT.get(clusterService.getSettings()));
 
         this.substraitExtensions = loadSubstraitExtensions();
 
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java
index 631face32a91f..d261cb9514deb 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/NativeBridge.java
@@ -24,6 +24,7 @@
 import java.lang.foreign.ValueLayout;
 import java.lang.invoke.MethodHandle;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
@@ -758,17 +759,11 @@ public static Map getTopNQueriesByMemory(int n) {
             if (written == 0L) {
                 return Collections.emptyMap();
             }
-            // Defensive: Rust contract is `written <= cap_entries`. A higher value
-            // would mean the buffer was overrun, so refuse to decode past `n`.
             int rows = (int) Math.min(written, (long) n);
-            // LinkedHashMap preserves the heap-drain order Rust used so logs and
-            // tests see stable iteration; values are O(1) regardless.
-            Map out = new LinkedHashMap<>(rows);
+            Map out = new HashMap<>(rows);
             for (int i = 0; i < rows; i++) {
                 long ctxId = QueryRegistryLayout.readContextId(seg, i);
-                // putIfAbsent guards against context_id reuse, which is a Rust-side
-                // caller bug but cheap to defend against here.
-                out.putIfAbsent(ctxId, QueryRegistryLayout.readMetrics(seg, i));
+                out.put(ctxId, QueryRegistryLayout.readMetrics(seg, i));
             }
             return Collections.unmodifiableMap(out);
         }
diff --git a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/QueryRegistryLayout.java b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/QueryRegistryLayout.java
index 603a1ebccae93..568d57dc4851f 100644
--- a/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/QueryRegistryLayout.java
+++ b/sandbox/plugins/analytics-backend-datafusion/src/main/java/org/opensearch/be/datafusion/nativelib/QueryRegistryLayout.java
@@ -47,7 +47,7 @@ public final class QueryRegistryLayout {
 
     // Byte offsets within a single entry. Field order must match
     // ENTRY_LAYOUT and the Rust #[repr(C)] WireQueryMetric struct.
-    private static final long OFF_CONTEXT_ID    = 0L;
+    private static final long OFF_CONTEXT_ID = 0L;
     private static final long OFF_CURRENT_BYTES = 8L;
 
     static {
diff --git a/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java b/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java
index a667cb456063e..8f0ff8d7d1ff6 100644
--- a/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java
+++ b/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java
@@ -16,7 +16,6 @@
 import org.opensearch.common.lifecycle.AbstractLifecycleComponent;
 import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Setting;
-import org.opensearch.common.util.TimeBasedExpiryTracker;
 import org.opensearch.monitor.jvm.JvmStats;
 import org.opensearch.monitor.os.OsProbe;
 import org.opensearch.monitor.process.ProcessProbe;
@@ -142,7 +141,7 @@ public SearchBackpressureService(
                     double usedFraction = used / totalNative;
                     if (usedFraction < 0.0d) {
                         if (logger.isDebugEnabled()) {
-                        logger.debug("native memory duress probe: signal unavailable (usedBytes={})", used);
+                            logger.debug("native memory duress probe: signal unavailable (usedBytes={})", used);
                         }
                         return false;
                     }
diff --git a/server/src/test/java/org/opensearch/search/backpressure/settings/NodeDuressSettingsTests.java b/server/src/test/java/org/opensearch/search/backpressure/settings/NodeDuressSettingsTests.java
index ffcc2b20763a1..8850096547ca9 100644
--- a/server/src/test/java/org/opensearch/search/backpressure/settings/NodeDuressSettingsTests.java
+++ b/server/src/test/java/org/opensearch/search/backpressure/settings/NodeDuressSettingsTests.java
@@ -87,9 +87,7 @@ public void testNodeNativeMemoryLimitIsDynamic() {
         );
         assertEquals(512L * 1024 * 1024, settings.getNodeNativeMemory());
 
-        clusterSettings.applySettings(
-            Settings.builder().put(NodeDuressSettings.NODE_NATIVE_MEMORY_LIMIT_SETTING.getKey(), "1gb").build()
-        );
+        clusterSettings.applySettings(Settings.builder().put(NodeDuressSettings.NODE_NATIVE_MEMORY_LIMIT_SETTING.getKey(), "1gb").build());
         assertEquals(1024L * 1024 * 1024, settings.getNodeNativeMemory());
     }
 

From 281749de2c6558b9caa62e265ed3692fa706757e Mon Sep 17 00:00:00 2001
From: Pradeep L 
Date: Tue, 19 May 2026 16:07:18 +0530
Subject: [PATCH 09/10] remving index list from description

Signed-off-by: Pradeep L 
---
 .../analytics/exec/DefaultPlanExecutor.java   | 43 ++-----------------
 .../exec/task/AnalyticsQueryTask.java         | 14 +++---
 ...kloadGroupResourceUsageTrackerService.java | 11 ++---
 3 files changed, 14 insertions(+), 54 deletions(-)

diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/DefaultPlanExecutor.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/DefaultPlanExecutor.java
index ae7004f0d7e26..a37914bbe769b 100644
--- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/DefaultPlanExecutor.java
+++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/DefaultPlanExecutor.java
@@ -30,8 +30,6 @@
 import org.opensearch.analytics.planner.dag.PlanForker;
 import org.opensearch.analytics.planner.dag.QueryDAG;
 import org.opensearch.arrow.memory.ArrowAllocatorService;
-import org.opensearch.analytics.planner.dag.Stage;
-import org.opensearch.analytics.planner.rel.OpenSearchTableScan;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.Nullable;
 import org.opensearch.common.inject.Inject;
@@ -48,10 +46,8 @@
 import org.opensearch.transport.client.node.NodeClient;
 
 import java.util.ArrayList;
-import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 import java.util.concurrent.Executor;
 
 import static org.opensearch.action.search.TransportSearchAction.SEARCH_CANCEL_AFTER_TIME_INTERVAL_SETTING;
@@ -146,11 +142,10 @@ private void executeInternal(RelNode logicalFragment, ActionListener indices = collectIndices(dag);
         final AnalyticsQueryTask queryTask = (AnalyticsQueryTask) taskManager.register(
             "transport",
             "analytics_query",
-            new AnalyticsQueryTaskRequest(dag.queryId(), indices, null)
+            new AnalyticsQueryTaskRequest(dag.queryId(), null)
         );
         final QueryContext config = new QueryContext(dag, searchExecutor, queryTask, allocatorService);
 
@@ -196,13 +191,11 @@ protected void doExecute(Task task, ActionRequest request, ActionListener indices;
         private final TimeValue cancelAfterTimeInterval;
         private TaskId parentTaskId = TaskId.EMPTY_TASK_ID;
 
-        AnalyticsQueryTaskRequest(String queryId, List indices, @Nullable TimeValue cancelAfterTimeInterval) {
+        AnalyticsQueryTaskRequest(String queryId, @Nullable TimeValue cancelAfterTimeInterval) {
             this.queryId = queryId;
-            this.indices = List.copyOf(indices);
             this.cancelAfterTimeInterval = cancelAfterTimeInterval;
         }
 
@@ -218,37 +211,7 @@ public TaskId getParentTask() {
 
         @Override
         public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) {
-            return new AnalyticsQueryTask(id, type, action, queryId, indices, parentTaskId, headers, cancelAfterTimeInterval);
-        }
-    }
-
-    /**
-     * Walks the DAG and returns the distinct index names referenced by all
-     * {@link OpenSearchTableScan} leaves, preserving discovery order. Used to
-     * populate the coordinator task's description for the Tasks API and slow logs.
-     */
-    private static List collectIndices(QueryDAG dag) {
-        Set indices = new LinkedHashSet<>();
-        collectIndices(dag.rootStage(), indices);
-        return new ArrayList<>(indices);
-    }
-
-    private static void collectIndices(Stage stage, Set indices) {
-        if (stage.getFragment() != null) {
-            collectIndicesFromFragment(stage.getFragment(), indices);
-        }
-        for (Stage child : stage.getChildStages()) {
-            collectIndices(child, indices);
-        }
-    }
-
-    private static void collectIndicesFromFragment(RelNode node, Set indices) {
-        if (node instanceof OpenSearchTableScan scan) {
-            indices.add(scan.getTable().getQualifiedName().getLast());
-            return;
-        }
-        for (RelNode input : node.getInputs()) {
-            collectIndicesFromFragment(input, indices);
+            return new AnalyticsQueryTask(id, type, action, queryId, parentTaskId, headers, cancelAfterTimeInterval);
         }
     }
 
diff --git a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/task/AnalyticsQueryTask.java b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/task/AnalyticsQueryTask.java
index 7414d11f1871c..bb9ac23642f78 100644
--- a/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/task/AnalyticsQueryTask.java
+++ b/sandbox/plugins/analytics-engine/src/main/java/org/opensearch/analytics/exec/task/AnalyticsQueryTask.java
@@ -16,7 +16,6 @@
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.tasks.TaskId;
 
-import java.util.List;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Supplier;
@@ -33,7 +32,6 @@ public class AnalyticsQueryTask extends SearchTask {
     private static final Logger logger = LogManager.getLogger(AnalyticsQueryTask.class);
 
     private final String queryId;
-    private final List indices;
     private final TimeValue cancelAfterTimeInterval;
     private final AtomicReference onCancelCallback = new AtomicReference<>();
 
@@ -42,7 +40,6 @@ public AnalyticsQueryTask(
         String type,
         String action,
         String queryId,
-        List indices,
         TaskId parentTaskId,
         Map headers,
         @Nullable TimeValue cancelAfterTimeInterval
@@ -51,13 +48,12 @@ public AnalyticsQueryTask(
             id,
             type,
             action,
-            (Supplier) () -> "queryId[" + queryId + "] indices[" + String.join(",", indices) + "]",
+            (Supplier) () -> "queryId[" + queryId + "]",
             parentTaskId,
             headers,
             cancelAfterTimeInterval != null ? cancelAfterTimeInterval : TimeValue.MINUS_ONE
         );
         this.queryId = queryId;
-        this.indices = List.copyOf(indices);
         this.cancelAfterTimeInterval = cancelAfterTimeInterval;
     }
 
@@ -66,12 +62,12 @@ public boolean shouldCancelChildrenOnCancellation() {
         return true;
     }
 
-    public String getQueryId() {
-        return queryId;
+    public AnalyticsQueryTask(long id, String type, String action, String queryId, TaskId parentTaskId, Map headers) {
+        this(id, type, action, queryId, parentTaskId, headers, null);
     }
 
-    public List getIndices() {
-        return indices;
+    public String getQueryId() {
+        return queryId;
     }
 
     @Nullable
diff --git a/server/src/main/java/org/opensearch/wlm/tracker/WorkloadGroupResourceUsageTrackerService.java b/server/src/main/java/org/opensearch/wlm/tracker/WorkloadGroupResourceUsageTrackerService.java
index b189e8442eeb2..5f6bcbdb01949 100644
--- a/server/src/main/java/org/opensearch/wlm/tracker/WorkloadGroupResourceUsageTrackerService.java
+++ b/server/src/main/java/org/opensearch/wlm/tracker/WorkloadGroupResourceUsageTrackerService.java
@@ -24,11 +24,12 @@
  * This class tracks resource usage per WorkloadGroup
  */
 public class WorkloadGroupResourceUsageTrackerService {
-    // Explicit set (CPU + MEMORY) rather than {@code EnumSet.allOf(ResourceType.class)}. The
-    // NATIVE_MEMORY entry was added to {@link ResourceType} only to flow duress signals
-    // through NodeDuressTrackers for search backpressure; WLM does not account off-heap
-    // memory per workload group and iterating it here would run the no-op calculator,
-    // produce zero usage, and clutter stats output.
+    // CPU + MEMORY only. NATIVE_MEMORY exists in {@link ResourceType} to flow duress
+    // signals through NodeDuressTrackers for search backpressure, but WLM has no
+    // per-workload-group native-memory accounting yet — adding it here would just run
+    // the no-op calculator and report zero usage.
+    // TODO: when WLM gains per-group native-memory accounting (real usage calculator
+    // and per-task hook), add ResourceType.NATIVE_MEMORY to this set.
     public static final EnumSet TRACKED_RESOURCES = EnumSet.of(ResourceType.CPU, ResourceType.MEMORY);
     private final TaskResourceTrackingService taskResourceTrackingService;
 

From e048863d7feedf8fc7d5f4ce9261703fe5450b63 Mon Sep 17 00:00:00 2001
From: Bukhtawar Khan 
Date: Tue, 19 May 2026 22:43:43 +0530
Subject: [PATCH 10/10] Fix native memory duress probe and simplify
 configuration
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

1. Duress probe now uses NativeMemoryUsageService pool usage (sum of
   per-task snapshot) instead of OsProbe.getProcessNativeMemoryBytes().
   Process RSS overcounts — it includes Netty direct buffers, thread
   stacks, mmap, not just DataFusion's pool.

2. Remove NODE_NATIVE_MEMORY_LIMIT_SETTING from NodeDuressSettings —
   redundant with NativeMemoryUsageService.getBudgetBytes() installed
   by the backend plugin. Two budget settings that can diverge is a
   misconfiguration trap.

3. Replace platform gate (isNativeTrackingSupported: Linux/macOS only)
   with hasSnapshotProvider() — feature is active when a backend
   installs a supplier, not based on OS.

Signed-off-by: Bukhtawar Khan 
---
 .../common/settings/ClusterSettings.java      |  1 -
 .../SearchBackpressureService.java            | 17 ++++++-------
 .../settings/NodeDuressSettings.java          | 24 -------------------
 3 files changed, 9 insertions(+), 33 deletions(-)

diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
index 35825cd10157c..37eb9cac44d6a 100644
--- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
+++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
@@ -725,7 +725,6 @@ public void apply(Settings value, Settings current, Settings previous) {
                 NodeDuressSettings.SETTING_NUM_SUCCESSIVE_BREACHES,
                 NodeDuressSettings.SETTING_CPU_THRESHOLD,
                 NodeDuressSettings.SETTING_HEAP_THRESHOLD,
-                NodeDuressSettings.NODE_NATIVE_MEMORY_LIMIT_SETTING,
                 NodeDuressSettings.SETTING_NATIVE_MEMORY_THRESHOLD,
                 SearchTaskSettings.SETTING_CANCELLATION_RATIO,
                 SearchTaskSettings.SETTING_CANCELLATION_RATE,
diff --git a/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java b/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java
index 8f0ff8d7d1ff6..f5da8c4c2a9ee 100644
--- a/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java
+++ b/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java
@@ -61,7 +61,6 @@
 import java.util.stream.Collectors;
 
 import static org.opensearch.search.backpressure.trackers.HeapUsageTracker.isHeapTrackingSupported;
-import static org.opensearch.search.backpressure.trackers.NativeMemoryUsageTracker.isNativeTrackingSupported;
 
 /**
  * SearchBackpressureService is responsible for monitoring and cancelling in-flight search tasks if they are
@@ -90,7 +89,8 @@ public class SearchBackpressureService extends AbstractLifecycleComponent implem
         TaskResourceUsageTrackerType.ELAPSED_TIME_TRACKER,
         (nodeDuressTrackers) -> true,
         TaskResourceUsageTrackerType.NATIVE_MEMORY_USAGE_TRACKER,
-        (nodeDuressTrackers) -> isNativeTrackingSupported() && nodeDuressTrackers.isResourceInDuress(ResourceType.NATIVE_MEMORY)
+        (nodeDuressTrackers) -> NativeMemoryUsageService.getInstance().hasSnapshotProvider()
+            && nodeDuressTrackers.isResourceInDuress(ResourceType.NATIVE_MEMORY)
     );
 
     private volatile Scheduler.Cancellable scheduledFuture;
@@ -133,18 +133,19 @@ public SearchBackpressureService(
                     )
                 );
                 put(ResourceType.NATIVE_MEMORY, new NodeDuressTracker(() -> {
-                    double used = OsProbe.getInstance().getProcessNativeMemoryBytes();
-                    double totalNative = settings.getNodeDuressSettings().getNodeNativeMemory();
-                    if (totalNative <= 0) {
+                    NativeMemoryUsageService svc = NativeMemoryUsageService.getInstance();
+                    long budget = svc.getBudgetBytes();
+                    if (budget <= 0L) {
                         return false;
                     }
-                    double usedFraction = used / totalNative;
-                    if (usedFraction < 0.0d) {
+                    double used = OsProbe.getInstance().getProcessNativeMemoryBytes();
+                    if (used < 0.0d) {
                         if (logger.isDebugEnabled()) {
                             logger.debug("native memory duress probe: signal unavailable (usedBytes={})", used);
                         }
                         return false;
                     }
+                    double usedFraction = used / budget;
                     double threshold = settings.getNodeDuressSettings().getNativeMemoryThreshold();
                     boolean breached = usedFraction >= threshold;
                     if (logger.isDebugEnabled()) {
@@ -445,7 +446,7 @@ public static TaskResourceUsageTrackers getTrackers(
             new ElapsedTimeTracker(ElapsedTimeNanosSupplier, System::nanoTime),
             TaskResourceUsageTrackerType.ELAPSED_TIME_TRACKER
         );
-        if (isNativeTrackingSupported()) {
+        if (NativeMemoryUsageService.getInstance().hasSnapshotProvider()) {
             trackers.addTracker(
                 new NativeMemoryUsageTracker(nativeMemoryPercentThresholdSupplier),
                 TaskResourceUsageTrackerType.NATIVE_MEMORY_USAGE_TRACKER
diff --git a/server/src/main/java/org/opensearch/search/backpressure/settings/NodeDuressSettings.java b/server/src/main/java/org/opensearch/search/backpressure/settings/NodeDuressSettings.java
index b48a677ea2ede..a2c77ac167732 100644
--- a/server/src/main/java/org/opensearch/search/backpressure/settings/NodeDuressSettings.java
+++ b/server/src/main/java/org/opensearch/search/backpressure/settings/NodeDuressSettings.java
@@ -11,7 +11,6 @@
 import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Setting;
 import org.opensearch.common.settings.Settings;
-import org.opensearch.core.common.unit.ByteSizeValue;
 
 /**
  * Defines the settings for a node to be considered in duress.
@@ -88,20 +87,6 @@ private static class Defaults {
         Setting.Property.NodeScope
     );
 
-    /**
-     * Absolute native-memory budget for this node, in bytes, scoped to the search-backpressure
-     * duress probe. Independent from the node-level resource-tracker's
-     * {@code node.native_memory.limit} so the two features can be tuned separately. When this
-     * value is {@link ByteSizeValue#ZERO} (default), the duress probe treats the budget as
-     * unconfigured and stays inert.
-     */
-    private volatile ByteSizeValue nodeNativeMemory;
-    public static final Setting NODE_NATIVE_MEMORY_LIMIT_SETTING = Setting.byteSizeSetting(
-        "search_backpressure.node_duress.native_memory_limit",
-        ByteSizeValue.ZERO,
-        Setting.Property.Dynamic,
-        Setting.Property.NodeScope
-    );
 
     public NodeDuressSettings(Settings settings, ClusterSettings clusterSettings) {
         numSuccessiveBreaches = SETTING_NUM_SUCCESSIVE_BREACHES.get(settings);
@@ -116,8 +101,6 @@ public NodeDuressSettings(Settings settings, ClusterSettings clusterSettings) {
         nativeMemoryThreshold = SETTING_NATIVE_MEMORY_THRESHOLD.get(settings);
         clusterSettings.addSettingsUpdateConsumer(SETTING_NATIVE_MEMORY_THRESHOLD, this::setNativeMemoryThreshold);
 
-        nodeNativeMemory = NODE_NATIVE_MEMORY_LIMIT_SETTING.get(settings);
-        clusterSettings.addSettingsUpdateConsumer(NODE_NATIVE_MEMORY_LIMIT_SETTING, this::setNodeNativeMemory);
     }
 
     public int getNumSuccessiveBreaches() {
@@ -152,12 +135,5 @@ private void setNativeMemoryThreshold(double nativeMemoryThreshold) {
         this.nativeMemoryThreshold = nativeMemoryThreshold;
     }
 
-    public long getNodeNativeMemory() {
-        return nodeNativeMemory.getBytes();
-    }
-
-    public void setNodeNativeMemory(ByteSizeValue nativeMemoryLimitBytes) {
-        this.nodeNativeMemory = nativeMemoryLimitBytes;
-    }
 
 }