feat(observability): propagate W3C trace context across nico-api's network boundaries

Cargo.toml

@@ -52,6 +52,7 @@ sqlx-macros-core = { version = "0.9" }
 
 opentelemetry = { version = "0.31.0", features = ["logs"] }
 opentelemetry_sdk = { version = "0.31.0", features = ["logs", "rt-tokio"] }
+opentelemetry-http = "0.31.0"
 opentelemetry-otlp = { version = "0.31.1", features = ["metrics"] }
 opentelemetry-prometheus = "0.31.0"
 opentelemetry-semantic-conventions = "0.31.0"
@@ -195,6 +196,10 @@ ratatui = "0.30.0"
 rcgen = "0.14.7"
 regex = "1.11.2"
 reqwest = { default-features = false, version = "0.13.3" }
+reqwest-middleware = "0.5"
+reqwest-tracing = { version = "0.7", default-features = false, features = [
+  "opentelemetry_0_31",
+] }
 resolv-conf = "0.7.0"
 ringbuf = "0.5.0"
 rsa = "0.9.9"

crates/api-core/Cargo.toml

@@ -76,6 +76,7 @@ nras = { path = "../nras" }
 spancounter = { path = "../spancounter" }
 state-controller = { path = "../state-controller" }
 sqlx-query-tracing = { path = "../sqlx-query-tracing" }
+trace-propagation = { path = "../trace-propagation" }
 
 # External dependencies. PLEASE KEEP ALPHABETIZED ORDER.
 arc-swap = { workspace = true }
@@ -132,6 +133,8 @@ reqwest = { workspace = true, default-features = false, features = [
   "rustls",
   "stream",
 ] }
+reqwest-middleware = { workspace = true }
+reqwest-tracing = { workspace = true }
 rsa = { workspace = true }
 rumqttc = { workspace = true }
 rustls = { workspace = true }

crates/api-core/src/handlers/redfish.rs

@@ -393,7 +393,7 @@ pub(crate) async fn create_client(
         rpc::forge::BmcMetaDataGetResponse,
         http::Uri,
         HeaderMap,
-        reqwest::Client,
+        reqwest_middleware::ClientWithMiddleware,
     ),
     CarbideError,
 > {
@@ -450,15 +450,20 @@ pub(crate) async fn create_client(
             .connect_timeout(std::time::Duration::from_secs(5)) // Limit connections to 5 seconds
             .timeout(std::time::Duration::from_secs(60)); // Limit the overall request to 60 seconds
 
-        match builder.build() {
+        let client = match builder.build() {
             Ok(client) => client,
             Err(err) => {
                 tracing::error!(%err, "build_http_client");
                 return Err(CarbideError::internal(format!(
                     "Http building failed: {err}"
                 )));
             }
-        }
+        };
+        // The `reqwest-tracing` middleware injects the current span's W3C trace context into every
+        // outgoing request (#2438).
+        reqwest_middleware::ClientBuilder::new(client)
+            .with(reqwest_tracing::TracingMiddleware::default())
+            .build()
     };
     Ok((metadata, new_uri, headers, http_client))
 }
@@ -558,15 +563,15 @@ impl TestBehavior {
     }
 }
 
-// Subset of the data we care about from reqwest::Error, so that we can mock it (we can't build our
-// own reqwest::Error as its constructors are all private.)
+// Subset of the data we care about from the HTTP error, so that we can mock it (we can't build our
+// own reqwest error as its constructors are all private.)
 pub struct RequestErrorInfo {
     pub status_code: Option<http::status::StatusCode>,
     pub description: String,
 }
 
-impl From<reqwest::Error> for RequestErrorInfo {
-    fn from(e: reqwest::Error) -> Self {
+impl From<reqwest_middleware::Error> for RequestErrorInfo {
+    fn from(e: reqwest_middleware::Error) -> Self {
         Self {
             status_code: e.status(),
             description: e.to_string(),

crates/api-core/src/logging/api_logs.rs

@@ -163,6 +163,13 @@ where
                 tenant.organization_id = tracing::field::Empty,
             );
 
+            // Continue any distributed trace the caller started: make the inbound W3C trace
+            // context (`traceparent`/`tracestate`) the parent of this request span. The shared
+            // tower layer serves both the gRPC and REST listeners, so this single extractor
+            // covers both ingress paths. No-op when there is no valid inbound context, leaving
+            // the request span a fresh root (issue #2438).
+            trace_propagation::set_span_parent_from_headers(&request_span, request.headers());
+
             // Try to extract the gRPC service and method from the URI
             let mut grpc_method: Option<String> = None;
             let mut grpc_service: Option<String> = None;