From 2a9ae39d92720df6e8590ece57ecd61d3561550c Mon Sep 17 00:00:00 2001 From: Juan Gomez Date: Fri, 27 Feb 2026 15:04:47 -0600 Subject: [PATCH 1/5] Adding ibv_query_rt_values_ex verb Adding a unit test, also as an example of use --- src/ibverbs/device_context.rs | 113 +++++++++++++++++++++++++++++++++- 1 file changed, 110 insertions(+), 3 deletions(-) diff --git a/src/ibverbs/device_context.rs b/src/ibverbs/device_context.rs index f1c8802..f637899 100644 --- a/src/ibverbs/device_context.rs +++ b/src/ibverbs/device_context.rs @@ -7,13 +7,15 @@ use std::io; use std::mem::MaybeUninit; use std::ptr::{self, NonNull}; use std::sync::Arc; +use std::time::Duration; +use bitmask_enum::bitmask; use rdma_mummy_sys::{ ibv_alloc_pd, ibv_close_device, ibv_context, ibv_device_attr_ex, ibv_get_device_guid, ibv_get_device_name, ibv_gid_entry, ibv_mtu, ibv_port_attr, ibv_port_state, ibv_query_device_ex, ibv_query_gid, ibv_query_gid_ex, - ibv_query_gid_table, ibv_query_gid_type, ibv_query_port, IBV_GID_TYPE_IB, IBV_GID_TYPE_ROCE_V1, - IBV_GID_TYPE_ROCE_V2, IBV_GID_TYPE_SYSFS_IB_ROCE_V1, IBV_GID_TYPE_SYSFS_ROCE_V2, IBV_LINK_LAYER_ETHERNET, - IBV_LINK_LAYER_INFINIBAND, IBV_LINK_LAYER_UNSPECIFIED, + ibv_query_gid_table, ibv_query_gid_type, ibv_query_port, ibv_query_rt_values_ex, ibv_values_ex, ibv_values_mask, + IBV_GID_TYPE_IB, IBV_GID_TYPE_ROCE_V1, IBV_GID_TYPE_ROCE_V2, IBV_GID_TYPE_SYSFS_IB_ROCE_V1, + IBV_GID_TYPE_SYSFS_ROCE_V2, IBV_LINK_LAYER_ETHERNET, IBV_LINK_LAYER_INFINIBAND, IBV_LINK_LAYER_UNSPECIFIED, }; use serde::{Deserialize, Serialize}; @@ -22,6 +24,22 @@ use super::completion::{CompletionChannel, CompletionQueueBuilder, CreateComplet use super::device::{DeviceInfo, TransportType}; use super::protection_domain::ProtectionDomain; +/// Error returned by [`DeviceContext::query_rt_values_ex`] for querying real-time values. +#[derive(Debug, thiserror::Error)] +#[error("failed to query RT values")] +#[non_exhaustive] +pub struct QueryRtValuesError(#[from] pub QueryRtValuesErrorKind); + +/// The enum type for [`QueryRtValuesError`]. +#[derive(Debug, thiserror::Error)] +#[error(transparent)] +#[non_exhaustive] +pub enum QueryRtValuesErrorKind { + Ibverbs(#[from] io::Error), + #[error("operation not supported by driver")] + NotSupported, +} + /// Error returned by [`DeviceContext::alloc_pd`] for allocating a new RDMA PD. #[derive(Debug, thiserror::Error)] #[error("failed to alloc protection domain")] @@ -106,6 +124,35 @@ pub enum QueryGidErrorKind { Ibverbs(#[from] io::Error), } +/// Bitmask of values to request (or that were returned) by [`DeviceContext::query_rt_values_ex`]. +/// +/// Set the desired bits before calling [`DeviceContext::query_rt_values_ex`]; on success the +/// returned [`RtValues::comp_mask`] indicates which fields were actually populated by the driver. +#[bitmask(u32)] +#[bitmask_config(vec_debug)] +pub enum ValuesMask { + /// Query / indicates the raw hardware clock value ([`RtValues::raw_clock`]). + RawClock = ibv_values_mask::IBV_VALUES_MASK_RAW_CLOCK.0 as _, +} + +/// Real-time values queried from an RDMA device via [`DeviceContext::query_rt_values_ex`]. +pub struct RtValues { + inner: ibv_values_ex, +} + +impl RtValues { + /// Returns the raw hardware clock as a [`Duration`] since an arbitrary epoch (device boot or + /// reset). Only meaningful when [`ValuesMask::RawClock`] is set in [`RtValues::comp_mask`]. + pub fn raw_clock(&self) -> Duration { + Duration::new(self.inner.raw_clock.tv_sec as u64, self.inner.raw_clock.tv_nsec as u32) + } + + /// Returns the `comp_mask` indicating which fields were actually populated by the driver. + pub fn comp_mask(&self) -> ValuesMask { + ValuesMask::from(self.inner.comp_mask) + } +} + /// A Global Unique Indentifier (GUID) for the RDMA device. Usually assigned to the device by its /// vendor during the manufacturing, may contain part of the MAC address on the ethernet device. #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] @@ -682,6 +729,42 @@ impl DeviceContext { Ok(entries) } + /// Query real-time values from the RDMA device. + /// + /// Set bits in `mask` to request which values to retrieve. Currently the only defined bit is + /// [`ValuesMask::RawClock`], which retrieves the device's free-running hardware clock — useful + /// for correlating CQ completion timestamps with wall-clock time. + /// + /// Returns [`QueryRtValuesErrorKind::NotSupported`] if the driver does not implement this + /// operation. + /// + /// # Example + /// + /// ```no_run + /// use sideway::ibverbs::device::DeviceList; + /// use sideway::ibverbs::device_context::ValuesMask; + /// + /// let device_list = DeviceList::new().unwrap(); + /// let device = device_list.get(0).unwrap(); + /// let context = device.open().unwrap(); + /// + /// let rt = context.query_rt_values_ex(ValuesMask::RawClock).unwrap(); + /// println!("HW clock: {:?}", rt.raw_clock()); + /// ``` + pub fn query_rt_values_ex(&self, mask: ValuesMask) -> Result { + let mut values = ibv_values_ex { + comp_mask: mask.bits(), + raw_clock: libc::timespec { tv_sec: 0, tv_nsec: 0 }, + }; + unsafe { + match ibv_query_rt_values_ex(self.context.as_ptr(), &mut values) { + 0 => Ok(RtValues { inner: values }), + ret if ret == libc::EOPNOTSUPP => Err(QueryRtValuesErrorKind::NotSupported.into()), + ret => Err(QueryRtValuesErrorKind::Ibverbs(io::Error::from_raw_os_error(ret)).into()), + } + } + } + /// # Safety /// /// Return the handle of device context. @@ -725,6 +808,30 @@ mod tests { use super::*; use crate::ibverbs::device::{self, DeviceInfo}; + #[test] + fn test_query_rt_values_ex() -> Result<(), Box> { + let device_list = device::DeviceList::new()?; + for device in &device_list { + let ctx = device.open().unwrap(); + match ctx.query_rt_values_ex(ValuesMask::RawClock) { + Ok(values) => { + // comp_mask must have RawClock set when the driver supports it + assert!(values.comp_mask().contains(ValuesMask::RawClock)); + // A running device should have a non-zero clock + assert!(values.raw_clock().as_nanos() > 0); + }, + Err(e) => { + // NotSupported is acceptable on some drivers / simulators + assert!( + matches!(e.0, QueryRtValuesErrorKind::NotSupported), + "unexpected error: {e}" + ); + }, + } + } + Ok(()) + } + #[test] fn test_mtu_conversion() { assert_eq!(Mtu::from(ibv_mtu::IBV_MTU_256), Mtu::Mtu256); From 943ef5c23b1df4e542f2c5347c78c72dab117c81 Mon Sep 17 00:00:00 2001 From: Juan Gomez Date: Fri, 27 Feb 2026 16:22:21 -0500 Subject: [PATCH 2/5] Update src/ibverbs/device_context.rs Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- src/ibverbs/device_context.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/ibverbs/device_context.rs b/src/ibverbs/device_context.rs index f637899..d96d6a4 100644 --- a/src/ibverbs/device_context.rs +++ b/src/ibverbs/device_context.rs @@ -752,13 +752,13 @@ impl DeviceContext { /// println!("HW clock: {:?}", rt.raw_clock()); /// ``` pub fn query_rt_values_ex(&self, mask: ValuesMask) -> Result { - let mut values = ibv_values_ex { - comp_mask: mask.bits(), - raw_clock: libc::timespec { tv_sec: 0, tv_nsec: 0 }, - }; + let mut values = std::mem::MaybeUninit::::uninit(); unsafe { - match ibv_query_rt_values_ex(self.context.as_ptr(), &mut values) { - 0 => Ok(RtValues { inner: values }), + (*values.as_mut_ptr()).comp_mask = mask.bits(); + match ibv_query_rt_values_ex(self.context.as_ptr(), values.as_mut_ptr()) { + 0 => Ok(RtValues { + inner: values.assume_init(), + }), ret if ret == libc::EOPNOTSUPP => Err(QueryRtValuesErrorKind::NotSupported.into()), ret => Err(QueryRtValuesErrorKind::Ibverbs(io::Error::from_raw_os_error(ret)).into()), } From deb24c2eabb1ea8ac05f01127036e71033f0f17f Mon Sep 17 00:00:00 2001 From: Juan Gomez Date: Sat, 28 Feb 2026 15:48:37 -0600 Subject: [PATCH 3/5] Renaming Rt to RealTime --- src/ibverbs/device_context.rs | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/ibverbs/device_context.rs b/src/ibverbs/device_context.rs index d96d6a4..793a9f1 100644 --- a/src/ibverbs/device_context.rs +++ b/src/ibverbs/device_context.rs @@ -28,13 +28,13 @@ use super::protection_domain::ProtectionDomain; #[derive(Debug, thiserror::Error)] #[error("failed to query RT values")] #[non_exhaustive] -pub struct QueryRtValuesError(#[from] pub QueryRtValuesErrorKind); +pub struct QueryRealTimeValuesError(#[from] pub QueryRealTimeValuesErrorKind); -/// The enum type for [`QueryRtValuesError`]. +/// The enum type for [`QueryRealTimeValuesError`]. #[derive(Debug, thiserror::Error)] #[error(transparent)] #[non_exhaustive] -pub enum QueryRtValuesErrorKind { +pub enum QueryRealTimeValuesErrorKind { Ibverbs(#[from] io::Error), #[error("operation not supported by driver")] NotSupported, @@ -127,22 +127,22 @@ pub enum QueryGidErrorKind { /// Bitmask of values to request (or that were returned) by [`DeviceContext::query_rt_values_ex`]. /// /// Set the desired bits before calling [`DeviceContext::query_rt_values_ex`]; on success the -/// returned [`RtValues::comp_mask`] indicates which fields were actually populated by the driver. +/// returned [`RealTimeValues::comp_mask`] indicates which fields were actually populated by the driver. #[bitmask(u32)] #[bitmask_config(vec_debug)] pub enum ValuesMask { - /// Query / indicates the raw hardware clock value ([`RtValues::raw_clock`]). + /// Query / indicates the raw hardware clock value ([`RealTimeValues::raw_clock`]). RawClock = ibv_values_mask::IBV_VALUES_MASK_RAW_CLOCK.0 as _, } /// Real-time values queried from an RDMA device via [`DeviceContext::query_rt_values_ex`]. -pub struct RtValues { +pub struct RealTimeValues { inner: ibv_values_ex, } -impl RtValues { +impl RealTimeValues { /// Returns the raw hardware clock as a [`Duration`] since an arbitrary epoch (device boot or - /// reset). Only meaningful when [`ValuesMask::RawClock`] is set in [`RtValues::comp_mask`]. + /// reset). Only meaningful when [`ValuesMask::RawClock`] is set in [`RealTimeValues::comp_mask`]. pub fn raw_clock(&self) -> Duration { Duration::new(self.inner.raw_clock.tv_sec as u64, self.inner.raw_clock.tv_nsec as u32) } @@ -735,7 +735,7 @@ impl DeviceContext { /// [`ValuesMask::RawClock`], which retrieves the device's free-running hardware clock — useful /// for correlating CQ completion timestamps with wall-clock time. /// - /// Returns [`QueryRtValuesErrorKind::NotSupported`] if the driver does not implement this + /// Returns [`QueryRealTimeValuesErrorKind::NotSupported`] if the driver does not implement this /// operation. /// /// # Example @@ -751,16 +751,16 @@ impl DeviceContext { /// let rt = context.query_rt_values_ex(ValuesMask::RawClock).unwrap(); /// println!("HW clock: {:?}", rt.raw_clock()); /// ``` - pub fn query_rt_values_ex(&self, mask: ValuesMask) -> Result { + pub fn query_rt_values_ex(&self, mask: ValuesMask) -> Result { let mut values = std::mem::MaybeUninit::::uninit(); unsafe { (*values.as_mut_ptr()).comp_mask = mask.bits(); match ibv_query_rt_values_ex(self.context.as_ptr(), values.as_mut_ptr()) { - 0 => Ok(RtValues { + 0 => Ok(RealTimeValues { inner: values.assume_init(), }), - ret if ret == libc::EOPNOTSUPP => Err(QueryRtValuesErrorKind::NotSupported.into()), - ret => Err(QueryRtValuesErrorKind::Ibverbs(io::Error::from_raw_os_error(ret)).into()), + ret if ret == libc::EOPNOTSUPP => Err(QueryRealTimeValuesErrorKind::NotSupported.into()), + ret => Err(QueryRealTimeValuesErrorKind::Ibverbs(io::Error::from_raw_os_error(ret)).into()), } } } @@ -823,7 +823,7 @@ mod tests { Err(e) => { // NotSupported is acceptable on some drivers / simulators assert!( - matches!(e.0, QueryRtValuesErrorKind::NotSupported), + matches!(e.0, QueryRealTimeValuesErrorKind::NotSupported), "unexpected error: {e}" ); }, From 7e040c9d24a363298dc57824ce5374f99be8b24f Mon Sep 17 00:00:00 2001 From: Juan Gomez Date: Sat, 28 Feb 2026 15:51:19 -0600 Subject: [PATCH 4/5] return Option from RealTimeValues::raw_clock --- src/ibverbs/device_context.rs | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/src/ibverbs/device_context.rs b/src/ibverbs/device_context.rs index 793a9f1..70732b2 100644 --- a/src/ibverbs/device_context.rs +++ b/src/ibverbs/device_context.rs @@ -142,9 +142,17 @@ pub struct RealTimeValues { impl RealTimeValues { /// Returns the raw hardware clock as a [`Duration`] since an arbitrary epoch (device boot or - /// reset). Only meaningful when [`ValuesMask::RawClock`] is set in [`RealTimeValues::comp_mask`]. - pub fn raw_clock(&self) -> Duration { - Duration::new(self.inner.raw_clock.tv_sec as u64, self.inner.raw_clock.tv_nsec as u32) + /// reset), or [`None`] if [`ValuesMask::RawClock`] was not set in [`RealTimeValues::comp_mask`] + /// (i.e. the driver did not populate this field). + pub fn raw_clock(&self) -> Option { + if self.comp_mask().contains(ValuesMask::RawClock) { + Some(Duration::new( + self.inner.raw_clock.tv_sec as u64, + self.inner.raw_clock.tv_nsec as u32, + )) + } else { + None + } } /// Returns the `comp_mask` indicating which fields were actually populated by the driver. @@ -818,7 +826,13 @@ mod tests { // comp_mask must have RawClock set when the driver supports it assert!(values.comp_mask().contains(ValuesMask::RawClock)); // A running device should have a non-zero clock - assert!(values.raw_clock().as_nanos() > 0); + assert!( + values + .raw_clock() + .expect("RawClock bit set but raw_clock() returned None") + .as_nanos() + > 0 + ); }, Err(e) => { // NotSupported is acceptable on some drivers / simulators From f81c0515e9704180c043d48925d506fc76ced69c Mon Sep 17 00:00:00 2001 From: Juan Gomez Date: Mon, 2 Mar 2026 13:45:18 -0600 Subject: [PATCH 5/5] feat: Adding ibv_query_rt_values_ex verb Moving from timestamps to clock ticks. The hardware reports raw tick counts from the last reboot, not timespecs in secs or nsecs as the C data type suggests. So the interface changed to clearly represent these semantics. --- src/ibverbs/device_context.rs | 49 +++++++++++++++++++++++++---------- 1 file changed, 35 insertions(+), 14 deletions(-) diff --git a/src/ibverbs/device_context.rs b/src/ibverbs/device_context.rs index 70732b2..d29a2cf 100644 --- a/src/ibverbs/device_context.rs +++ b/src/ibverbs/device_context.rs @@ -7,7 +7,6 @@ use std::io; use std::mem::MaybeUninit; use std::ptr::{self, NonNull}; use std::sync::Arc; -use std::time::Duration; use bitmask_enum::bitmask; use rdma_mummy_sys::{ @@ -135,21 +134,43 @@ pub enum ValuesMask { RawClock = ibv_values_mask::IBV_VALUES_MASK_RAW_CLOCK.0 as _, } +/// Raw hardware clock counter returned by [`DeviceContext::query_rt_values_ex`]. +/// +/// The two fields are the high and low halves of a free-running hardware tick counter in +/// device-specific units. They are **not** wall-clock seconds and nanoseconds despite the +/// underlying C `timespec` field names. To convert to real time, combine the parts and divide +/// by the device clock frequency (`hca_core_clock` from `ibv_query_device_ex`). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RawClock { + /// High part of the hardware counter (maps to `timespec.tv_sec` in the C struct). + pub counter_hi: u64, + /// Low part of the hardware counter (maps to `timespec.tv_nsec` in the C struct). + pub counter_lo: u64, +} + +impl RawClock { + /// Combine both halves into a single 128-bit tick value. + pub fn to_ticks(&self) -> u128 { + ((self.counter_hi as u128) << 64) | (self.counter_lo as u128) + } +} + /// Real-time values queried from an RDMA device via [`DeviceContext::query_rt_values_ex`]. pub struct RealTimeValues { inner: ibv_values_ex, } impl RealTimeValues { - /// Returns the raw hardware clock as a [`Duration`] since an arbitrary epoch (device boot or - /// reset), or [`None`] if [`ValuesMask::RawClock`] was not set in [`RealTimeValues::comp_mask`] - /// (i.e. the driver did not populate this field). - pub fn raw_clock(&self) -> Option { + /// Returns the raw hardware clock counter, or [`None`] if [`ValuesMask::RawClock`] was not + /// set in [`RealTimeValues::comp_mask`] (i.e. the driver did not populate this field). + /// + /// See [`RawClock`] for details on how to interpret the returned value. + pub fn raw_clock(&self) -> Option { if self.comp_mask().contains(ValuesMask::RawClock) { - Some(Duration::new( - self.inner.raw_clock.tv_sec as u64, - self.inner.raw_clock.tv_nsec as u32, - )) + Some(RawClock { + counter_hi: self.inner.raw_clock.tv_sec as u64, + counter_lo: self.inner.raw_clock.tv_nsec as u64, + }) } else { None } @@ -826,12 +847,12 @@ mod tests { // comp_mask must have RawClock set when the driver supports it assert!(values.comp_mask().contains(ValuesMask::RawClock)); // A running device should have a non-zero clock + let clock = values + .raw_clock() + .expect("RawClock bit set but raw_clock() returned None"); assert!( - values - .raw_clock() - .expect("RawClock bit set but raw_clock() returned None") - .as_nanos() - > 0 + clock.counter_hi > 0 || clock.counter_lo > 0, + "raw clock counter should be non-zero" ); }, Err(e) => {