feat: allow disabling of APM Tracing

src/common/headers.cpp

@@ -44,13 +44,6 @@ bool add_header(ngx_pool_t &pool, ngx_list_t &headers, std::string_view key,
 
   const auto key_size = key.size();
 
-  // This trick tells ngx_http_header_module to reflect the header value
-  // in the actual response. Otherwise the header will be ignored and client
-  // will never see it. To date the value must be just non zero.
-  // Source:
-  // <https://web.archive.org/web/20240409072840/https://www.nginx.com/resources/wiki/start/topics/examples/headers_management/>
-  h->hash = 1;
-
   // HTTP proxy module expects the header to has a lowercased key value
   // Instead of allocating twice the same key, `h->key` and `h->lowcase_key`
   // use the same data.
@@ -61,8 +54,47 @@ bool add_header(ngx_pool_t &pool, ngx_list_t &headers, std::string_view key,
   }
   h->lowcase_key = h->key.data;
 
+  // In request headers, the hash should be calculated from the lowercase key.
+  // See ngx_http_parse_header_line in ngx_http_parse.c
+  // Response headers OTOH use either 1 or 0, with 0 meaning "skip this header".
+  h->hash = ngx_hash_key(h->lowcase_key, key.size());
+
   h->value = nginx::to_ngx_str(&pool, value);
   return true;
 }
 
+bool remove_header(ngx_list_t &headers, std::string_view key) {
+  auto key_lc = std::unique_ptr<u_char[]>{new u_char[key.size()]};
+  std::transform(key.begin(), key.end(), key_lc.get(),
+                 datadog::nginx::to_lower);
+  ngx_uint_t key_hash = ngx_hash_key(key_lc.get(), key.size());
+
+  ngx_list_part_t *part = &headers.part;
+  ngx_table_elt_t *h = static_cast<ngx_table_elt_t *>(part->elts);
+  for (std::size_t i = 0;; i++) {
+    if (i >= part->nelts) {
+      if (part->next == nullptr) {
+        break;
+      }
+
+      part = part->next;
+      h = static_cast<ngx_table_elt_t *>(part->elts);
+      i = 0;
+    }
+
+    if (h[i].hash != key_hash || key.size() != h[i].key.len ||
+        memcmp(key_lc.get(), h[i].lowcase_key, key.size()) != 0) {
+      continue;
+    }
+
+    part->nelts--;
+    if (i < part->nelts) {
+      memmove(&h[i], &h[i + 1], (part->nelts - i) * sizeof(*h));
+    }
+    return true;
+  }
+
+  return false;
+}
+
 }  // namespace datadog::common

src/common/headers.h

@@ -50,4 +50,20 @@ ngx_table_elt_t *search_header(ngx_list_t &headers, std::string_view key);
 bool add_header(ngx_pool_t &pool, ngx_list_t &headers, std::string_view key,
                 std::string_view value);
 
+/// Deletes a request header with the specified key from a NGINX-request header
+/// list.
+///
+/// @param headers
+///     A reference to an NGINX-style list (`ngx_list_t`) containing
+///     `ngx_table_elt_t` elements, typically representing HTTP headers.
+///
+/// @param key
+///     A string view representing the name of the header to delete.
+///     The comparison is case-insensitive.
+///
+/// @return
+///     `true` if a header with the given key was found and deleted;
+///     `false` if no header with the given key exists in the list.
+bool remove_header(ngx_list_t &headers, std::string_view key);
+
 }  // namespace datadog::common

src/datadog_conf.h

@@ -57,6 +57,10 @@ struct sampling_rule_t {
 };
 
 struct datadog_main_conf_t {
+  // DD_APM_TRACING_ENABLED
+  // Whether we discard almost all traces not setting  _dd.p.ts
+  ngx_flag_t apm_tracing_enabled{NGX_CONF_UNSET};
+
   std::unordered_map<std::string, ngx_http_complex_value_t *> tags;
   // `are_propagation_styles_locked` is whether the tracer's propagation styles
   // have been set, either by an explicit `datadog_propagation_styles`

src/datadog_context.cpp

@@ -8,6 +8,7 @@
 #include "datadog/span.h"
 #include "datadog_handler.h"
 #include "dd.h"
+#include "ngx_header_writer.h"
 #include "ngx_http_datadog_module.h"
 #ifdef WITH_WAF
 #include "security/context.h"
@@ -16,13 +17,27 @@
 
 namespace datadog {
 namespace nginx {
+namespace {
+
+#ifdef WITH_WAF
+bool is_apm_tracing_enabled(ngx_http_request_t *request) {
+  auto main_conf = static_cast<datadog_main_conf_t *>(
+      ngx_http_get_module_main_conf(request, ngx_http_datadog_module));
+  if (main_conf == nullptr) return false;
+
+  return main_conf->apm_tracing_enabled;
+}
+#endif
+
+}  // namespace
 
 DatadogContext::DatadogContext(ngx_http_request_t *request,
                                ngx_http_core_loc_conf_t *core_loc_conf,
                                datadog_loc_conf_t *loc_conf)
 #ifdef WITH_WAF
     : sec_ctx_{security::Context::maybe_create(
-          *loc_conf, security::Library::max_saved_output_data())}
+          security::Library::max_saved_output_data(),
+          is_apm_tracing_enabled(request))}
 #endif
 {
   if (loc_conf->enable_tracing) {
@@ -324,5 +339,27 @@ void destroy_datadog_context(ngx_http_request_t *request) noexcept {
   ngx_http_set_ctx(request, nullptr, ngx_http_datadog_module);
 }
 
+ngx_int_t DatadogContext::on_precontent_phase(ngx_http_request_t *request) {
+  // inject headers in the precontent phase into the request headers
+  // These headers will be copied by ngx_http_proxy_create_request on the
+  // content phase into the outgoing request headers (probably)
+  RequestTracing &trace = single_trace();
+  dd::Span &span = trace.active_span();
+  span.set_tag("span.kind", "client");
+
+#ifdef WITH_WAF
+  if (auto sec_ctx = get_security_context()) {
+    if (sec_ctx->has_matches()) {
+      span.set_source(datadog::tracing::Source::appsec);
+    }
+  }
+#endif
+
+  NgxHeaderWriter writer(request);
+  span.inject(writer);
+
+  return NGX_DECLINED;
+}
+
 }  // namespace nginx
 }  // namespace datadog

src/datadog_context.h

@@ -51,6 +51,8 @@ class DatadogContext {
 
   void on_log_request(ngx_http_request_t* request);
 
+  ngx_int_t on_precontent_phase(ngx_http_request_t* request);
+
   ngx_str_t lookup_span_variable_value(ngx_http_request_t* request,
                                        std::string_view key);
 

src/datadog_handler.cpp

@@ -175,5 +175,22 @@ ngx_int_t on_output_body_filter(ngx_http_request_t *request,
   }
 }
 
+ngx_int_t on_precontent_phase(ngx_http_request_t *request) noexcept {
+  auto *context = get_datadog_context(request);
+  if (!context) {
+    return NGX_DECLINED;
+  }
+
+  try {
+    return context->on_precontent_phase(request);
+  } catch (const std::exception &e) {
+    telemetry::log::error(e.what(), CURRENT_FRAME(request));
+    ngx_log_error(NGX_LOG_ERR, request->connection->log, 0,
+                  "Datadog instrumentation failed for request %p: %s", request,
+                  e.what());
+    return NGX_ERROR;
+  }
+}
+
 }  // namespace nginx
 }  // namespace datadog

src/datadog_handler.h

@@ -30,5 +30,7 @@ ngx_int_t request_body_filter(ngx_http_request_t *r,
 ngx_int_t on_output_body_filter(ngx_http_request_t *r,
                                 ngx_chain_t *chain) noexcept;
 
+ngx_int_t on_precontent_phase(ngx_http_request_t *request) noexcept;
+
 }  // namespace nginx
 }  // namespace datadog

src/ngx_header_writer.h

@@ -32,6 +32,10 @@ class NgxHeaderWriter : public datadog::tracing::DictWriter {
       common::add_header(*pool_, request_->headers_in.headers, key, value);
     }
   }
+
+  void erase(std::string_view key) override {
+    common::remove_header(request_->headers_in.headers, key);
+  }
 };
 
 }  // namespace nginx

src/ngx_http_datadog_module.cpp

@@ -3,9 +3,7 @@
 #include <cassert>
 #include <cstdlib>
 #include <exception>
-#include <iterator>
 #include <memory>
-#include <new>
 #include <string_view>
 #include <utility>
 
@@ -313,6 +311,11 @@ static ngx_int_t datadog_module_init(ngx_conf_t *cf) noexcept {
   }
 #endif
 
+  if (set_handler(cf->log, core_main_config, NGX_HTTP_PRECONTENT_PHASE,
+                  on_precontent_phase) != NGX_OK) {
+    return NGX_ERROR;
+  }
+
   // Add default span tags.
   const auto tags = TracingLibrary::default_tags();
   if (!tags.empty()) {

src/request_tracing.cpp

@@ -1,7 +1,6 @@
 #include "request_tracing.h"
 
 #include <datadog/dict_writer.h>
-#include <datadog/injection_options.h>
 #include <datadog/span.h>
 #include <datadog/span_config.h>
 #include <datadog/trace_segment.h>
@@ -19,7 +18,6 @@
 #include "dd.h"
 #include "global_tracer.h"
 #include "ngx_header_reader.h"
-#include "ngx_header_writer.h"
 #include "ngx_http_datadog_module.h"
 #include "string_util.h"
 #include "tracing_library.h"
@@ -240,12 +238,6 @@ RequestTracing::RequestTracing(ngx_http_request_t *request,
   // We care about sampling rules for the request span only, because it's the
   // only span that could be the root span.
   set_sample_rate_tag(request_, loc_conf_, *request_span_);
-
-  // Inject the active span
-  NgxHeaderWriter writer(request_);
-  auto &span = active_span();
-  span.set_tag("span.kind", "client");
-  span.inject(writer);
 }
 
 void RequestTracing::on_change_block(ngx_http_core_loc_conf_t *core_loc_conf,
@@ -275,12 +267,6 @@ void RequestTracing::on_change_block(ngx_http_core_loc_conf_t *core_loc_conf,
   // We care about sampling rules for the request span only, because it's the
   // only span that could be the root span.
   set_sample_rate_tag(request_, loc_conf_, *request_span_);
-
-  // Inject the active span
-  NgxHeaderWriter writer(request_);
-  auto &span = active_span();
-  span.set_tag("span.kind", "client");
-  span.inject(writer);
 }
 
 dd::Span &RequestTracing::active_span() {

src/security/context.cpp

@@ -197,8 +197,11 @@ auto catch_exceptions(std::string_view name, const ngx_http_request_t &req,
 
 namespace datadog::nginx::security {
 
-Context::Context(std::shared_ptr<OwnedDdwafHandle> handle)
-    : stage_{new std::atomic<stage>{}}, waf_handle_{std::move(handle)} {
+Context::Context(std::shared_ptr<OwnedDdwafHandle> handle,
+                 bool apm_tracing_enabled)
+    : stage_{new std::atomic<stage>{}},
+      waf_handle_{std::move(handle)},
+      apm_tracing_enabled_{apm_tracing_enabled} {
   if (!waf_handle_) {
     return;
   }
@@ -210,13 +213,14 @@ Context::Context(std::shared_ptr<OwnedDdwafHandle> handle)
 }
 
 std::unique_ptr<Context> Context::maybe_create(
-    datadog_loc_conf_t &loc_conf,
-    std::optional<std::size_t> max_saved_output_data) {
+    std::optional<std::size_t> max_saved_output_data,
+    bool apm_tracing_enabled) {
   std::shared_ptr<OwnedDdwafHandle> handle = Library::get_handle();
   if (!handle) {
     return {};
   }
-  auto res = std::unique_ptr<Context>{new Context{std::move(handle)}};
+  auto res = std::unique_ptr<Context>{
+      new Context{std::move(handle), apm_tracing_enabled}};
   if (max_saved_output_data) {
     res->max_saved_output_data_ = *max_saved_output_data;
   }
@@ -1786,6 +1790,10 @@ void Context::report_matches(ngx_http_request_t &request, dd::Span &span) {
 
   report_match(request, span.trace_segment(), span, results_);
   results_.clear();
+
+  if (!apm_tracing_enabled_) {
+    span.set_source(tracing::Source::appsec);
+  }
 }
 
 void Context::report_client_ip(dd::Span &span) const {

src/security/context.h

@@ -52,13 +52,14 @@ struct OwnedDdwafContext
 };
 
 class Context {
-  Context(std::shared_ptr<OwnedDdwafHandle> waf_handle);
+  Context(std::shared_ptr<OwnedDdwafHandle> waf_handle,
+          bool apm_tracing_enabled);
 
  public:
   // returns a new context or an empty unique_ptr if the waf is not active
   static std::unique_ptr<Context> maybe_create(
-      datadog_loc_conf_t &loc_conf,
-      std::optional<std::size_t> max_saved_output_data);
+      std::optional<std::size_t> max_saved_output_data,
+      bool apm_tracing_enabled);
 
   ngx_int_t request_body_filter(ngx_http_request_t &request, ngx_chain_t *chain,
                                 dd::Span &span) noexcept;
@@ -87,6 +88,8 @@ class Context {
   std::optional<BlockSpecification> run_waf_end(ngx_http_request_t &request,
                                                 dd::Span &span);
 
+  bool has_matches() const noexcept;
+
  private:
   bool do_on_request_start(ngx_http_request_t &request, dd::Span &span);
   ngx_int_t do_request_body_filter(ngx_http_request_t &request,
@@ -96,7 +99,6 @@ class Context {
                                   ngx_chain_t *chain, dd::Span &span);
   void do_on_main_log_request(ngx_http_request_t &request, dd::Span &span);
 
-  bool has_matches() const noexcept;
   void report_matches(ngx_http_request_t &request, dd::Span &span);
   void report_client_ip(dd::Span &span) const;
 
@@ -198,6 +200,8 @@ class Context {
   static inline constexpr std::size_t kDefaultMaxSavedOutputData = 256 * 1024;
   std::size_t max_saved_output_data_{kDefaultMaxSavedOutputData};
 
+  bool apm_tracing_enabled_;
+
   struct FilterCtx {
     ngx_chain_t *out;  // the buffered request or response body
     ngx_chain_t **out_latest{&out};

src/tracing/directives.h

@@ -51,6 +51,14 @@ constexpr datadog::nginx::directive tracing_directives[] = {
         offsetof(datadog_loc_conf_t, enable_tracing),
         nullptr,
     },
+    {
+        "datadog_apm_tracing_enabled",
+        NGX_HTTP_MAIN_CONF | NGX_CONF_TAKE1,
+        ngx_conf_set_flag_slot,
+        NGX_HTTP_MAIN_CONF_OFFSET,
+        offsetof(datadog_main_conf_t, apm_tracing_enabled),
+        nullptr,
+    },
     {
         "datadog_trace_locations",
         anywhere | NGX_CONF_TAKE1,

src/tracing_library.cpp

@@ -55,6 +55,10 @@ dd::Expected<dd::Tracer> TracingLibrary::make_tracer(
   config.integration_version = NGINX_VERSION;
   config.service = "nginx";
 
+  if (nginx_conf.apm_tracing_enabled != NGX_CONF_UNSET) {
+    config.tracing_enabled = {nginx_conf.apm_tracing_enabled == 1};
+  }
+
   if (!nginx_conf.propagation_styles.empty()) {
     config.injection_styles = config.extraction_styles =
         nginx_conf.propagation_styles;