From dd04bd91c34bf28e58d62c64a3bbdfe921e39cc8 Mon Sep 17 00:00:00 2001 From: Zach Montoya Date: Tue, 27 Jan 2026 14:53:33 -0800 Subject: [PATCH 01/29] Implement initial OTLP traces + trace stats with the .NET weblog example. Notably, this creates a new scenario APM_TRACING_OTLP to enable the environment variables needed to configure the SDK to export traces as OTLP. --- .github/workflows/run-end-to-end.yml | 5 + tests/otel/test_tracing_otlp.py | 132 ++++++++++++++++++ utils/_context/_scenarios/__init__.py | 16 +++ utils/_context/_scenarios/endtoend.py | 17 ++- utils/interfaces/_open_telemetry.py | 49 ++++++- utils/proxy/_deserializer.py | 2 + .../scripts/ci_orchestrators/workflow_data.py | 1 + 7 files changed, 220 insertions(+), 2 deletions(-) create mode 100644 tests/otel/test_tracing_otlp.py diff --git a/.github/workflows/run-end-to-end.yml b/.github/workflows/run-end-to-end.yml index 3e6ef05f7b4..6c3785efd2f 100644 --- a/.github/workflows/run-end-to-end.yml +++ b/.github/workflows/run-end-to-end.yml @@ -235,6 +235,11 @@ jobs: env: DD_API_KEY: ${{ secrets.DD_API_KEY }} DD_APP_KEY: ${{ secrets.DD_APPLICATION_KEY }} + - name: Run APM_TRACING_OTLP scenario + if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"APM_TRACING_OTLP"') + run: ./run.sh APM_TRACING_OTLP + env: + DD_API_KEY: ${{ secrets.DD_API_KEY }} - name: Run APM_TRACING_EFFICIENT_PAYLOAD scenario if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"APM_TRACING_EFFICIENT_PAYLOAD"') run: ./run.sh APM_TRACING_EFFICIENT_PAYLOAD diff --git a/tests/otel/test_tracing_otlp.py b/tests/otel/test_tracing_otlp.py new file mode 100644 index 00000000000..9c4721729e0 --- /dev/null +++ b/tests/otel/test_tracing_otlp.py @@ -0,0 +1,132 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the the Apache License Version 2.0. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2024 Datadog, Inc. + +import json +from utils import weblog, interfaces, scenarios, features, incomplete_test_app +from utils._logger import logger +from typing import Any, Iterator + + +# Assert that the histogram has only one recorded data point matching the overall duration +def assert_single_histogram_data_point(duration: int, bucket_counts: list[int], explicit_bounds: list[float]): + for i in range(len(explicit_bounds)): + is_first_index = i == 0 + is_last_index = i == len(explicit_bounds) - 1 + + if is_first_index and is_last_index: + assert bucket_counts[i] == 1 + assert duration <= explicit_bounds[i] + break + + if int(bucket_counts[i]) == 1: + lower_bound = float('-inf') if is_first_index else explicit_bounds[i-1] + upper_bound = float('inf') if is_last_index else explicit_bounds[i] + + if is_last_index: + assert duration > lower_bound and duration < upper_bound + else: + assert duration > lower_bound and duration <= upper_bound + +def get_keyvalue_generator(attributes: list[dict]) -> Iterator[tuple[str, Any]]: + for keyValue in attributes: + if keyValue["value"].get("string_value"): + yield keyValue["key"], keyValue["value"]["string_value"] + elif keyValue["value"].get("stringValue"): + yield keyValue["key"], keyValue["value"]["stringValue"] + elif keyValue["value"].get("bool_value"): + yield keyValue["key"], keyValue["value"]["bool_value"] + elif keyValue["value"].get("boolValue"): + yield keyValue["key"], keyValue["value"]["boolValue"] + elif keyValue["value"].get("int_value"): + yield keyValue["key"], keyValue["value"]["int_value"] + elif keyValue["value"].get("intValue"): + yield keyValue["key"], keyValue["value"]["intValue"] + elif keyValue["value"].get("double_value"): + yield keyValue["key"], keyValue["value"]["double_value"] + elif keyValue["value"].get("doubleValue"): + yield keyValue["key"], keyValue["value"]["doubleValue"] + elif keyValue["value"].get("array_value"): + yield keyValue["key"], keyValue["value"]["array_value"] + elif keyValue["value"].get("arrayValue"): + yield keyValue["key"], keyValue["value"]["arrayValue"] + elif keyValue["value"].get("kvlist_value"): + yield keyValue["key"], keyValue["value"]["kvlist_value"] + elif keyValue["value"].get("kvlistValue"): + yield keyValue["key"], keyValue["value"]["kvlistValue"] + elif keyValue["value"].get("bytes_value"): + yield keyValue["key"], keyValue["value"]["bytes_value"] + elif keyValue["value"].get("bytesValue"): + yield keyValue["key"], keyValue["value"]["bytesValue"] + else: + raise ValueError(f"Unknown attribute value: {keyValue["value"]}") + + +# @scenarios.apm_tracing_e2e_otel +@features.otel_api +@scenarios.apm_tracing_otlp +class Test_Otel_Tracing_OTLP: + def setup_tracing(self): + self.req = weblog.get("/") + + # Note: Both camelcase and snake_case are allowed by the ProtoJSON Format (https://protobuf.dev/programming-guides/json/) + def test_tracing(self): + data = list(interfaces.open_telemetry.get_otel_spans(self.req)) + # _logger.debug(data) + assert len(data) == 1 + resource_span, span = data[0] + + # Assert resource attributes (we only expect string values) + attributes = {keyValue["key"]: keyValue["value"].get("string_value") or keyValue["value"].get("stringValue") for keyValue in resource_span.get("resource").get("attributes")} + assert attributes.get("service.name") == "weblog" + assert attributes.get("service.version") == "1.0.0" + assert attributes.get("deployment.environment.name") == "system-tests" or attributes.get("deployment.environment") == "system-tests" + # assert attributes.get("telemetry.sdk.name") == "datadog" + assert "telemetry.sdk.language" in attributes + assert "telemetry.sdk.version" in attributes + assert "git.commit.sha" in attributes + assert "git.repository_url" in attributes + assert "runtime-id" in attributes + + # Assert spans + assert span.get("name") == "GET /" + assert span.get("kind") == "SPAN_KIND_SERVER" + assert span.get("start_time_unix_nano") or span.get("startTimeUnixNano") + assert span.get("end_time_unix_nano") or span.get("endTimeUnixNano") + assert span.get("attributes") is not None + + # Assert HTTP tags + # Convert attributes list to a dictionary, but for now only handle KeyValue objects with stringValue + # attributes = {keyValue["key"]: keyValue["value"]["string_value"] or keyValue["value"]["stringValue"] for keyValue in span.get("attributes")} + span_attributes = dict(get_keyvalue_generator(span.get("attributes"))) + assert span_attributes.get("http.method") == "GET" + assert span_attributes.get("http.status_code") == "200" # We may want to convert this to int later + assert span_attributes.get("http.url") == "http://localhost:7777/" + + # Assert trace stats + for metric, data_point in interfaces.open_telemetry.get_trace_stats("GET /"): + # If we run another export, we'll have the same aggregation keys with zero counts. Skip them. + if not data_point.get("sum"): + continue + + assert metric.get("name") == "request.latencies" + assert metric.get("description") == "Summary of request latencies" + assert metric.get("unit") == "ns" + assert metric.get("histogram").get("aggregationTemporality") == "AGGREGATION_TEMPORALITY_DELTA" + + assert int(data_point.get("count")) == 1 + assert sum(map(int, data_point.get("bucketCounts"))) == int(data_point.get("count")) + assert len(data_point.get("bucketCounts")) == len(data_point.get("explicitBounds")) + assert data_point.get("explicitBounds") == sorted(data_point.get("explicitBounds")) + assert_single_histogram_data_point(data_point.get("sum"), list(map(int, data_point.get("bucketCounts"))), data_point.get("explicitBounds")) + + attributes = dict(get_keyvalue_generator(data_point.get("attributes"))) + assert "Name" in attributes + assert attributes.get("Resource") == "GET /" + assert attributes.get("Type") == "web" + assert attributes.get("StatusCode") == "200" + assert attributes.get("TopLevel") == "True" + assert attributes.get("Error") == "False" + assert len(attributes) == 6 + + diff --git a/utils/_context/_scenarios/__init__.py b/utils/_context/_scenarios/__init__.py index ca1e06a7e9b..bb537381006 100644 --- a/utils/_context/_scenarios/__init__.py +++ b/utils/_context/_scenarios/__init__.py @@ -568,6 +568,22 @@ class _Scenarios: require_api_key=True, doc="", ) + apm_tracing_otlp = EndToEndScenario( + "APM_TRACING_OTLP", + weblog_env={ + "DD_TRACE_OTEL_ENABLED": "true", + "DD_TRACE_STATS_COMPUTATION_ENABLED": "true", + "OTEL_TRACES_EXPORTER": "otlp", + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": f"http://proxy:{ProxyPorts.open_telemetry_weblog}/v1/traces", + "OTEL_EXPORTER_OTLP_TRACES_HEADERS": "dd-protocol=otlp,dd-otlp-path=agent", + "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT": f"http://proxy:{ProxyPorts.open_telemetry_weblog}/v1/metrics", + "OTEL_EXPORTER_OTLP_METRICS_HEADERS": "dd-otlp-path=agent", + }, + backend_interface_timeout=5, + require_api_key=True, + include_opentelemetry=True, + doc="", + ) apm_tracing_efficient_payload = EndToEndScenario( "APM_TRACING_EFFICIENT_PAYLOAD", diff --git a/utils/_context/_scenarios/endtoend.py b/utils/_context/_scenarios/endtoend.py index 8a4e41a6692..45456e462a7 100644 --- a/utils/_context/_scenarios/endtoend.py +++ b/utils/_context/_scenarios/endtoend.py @@ -244,6 +244,7 @@ def __init__( runtime_metrics_enabled: bool = False, backend_interface_timeout: int = 0, include_buddies: bool = False, + include_opentelemetry: bool = False, require_api_key: bool = False, other_weblog_containers: tuple[type[TestedContainer], ...] = (), ) -> None: @@ -323,6 +324,7 @@ def __init__( self.agent_interface_timeout = agent_interface_timeout self.backend_interface_timeout = backend_interface_timeout self._library_interface_timeout = library_interface_timeout + self.include_opentelemetry = include_opentelemetry def configure(self, config: pytest.Config): if self._require_api_key and "DD_API_KEY" not in os.environ and not self.replay: @@ -330,6 +332,7 @@ def configure(self, config: pytest.Config): self.weblog_infra.configure(config) self._set_containers_dependancies() + self.weblog_container.environment["DD_API_KEY"] = os.environ.get("DD_API_KEY") super().configure(config) interfaces.agent.configure(self.host_log_folder, replay=self.replay) @@ -339,6 +342,9 @@ def configure(self, config: pytest.Config): interfaces.library_stdout.configure(self.host_log_folder, replay=self.replay) interfaces.agent_stdout.configure(self.host_log_folder, replay=self.replay) + if self.include_opentelemetry: + interfaces.open_telemetry.configure(self.host_log_folder, replay=self.replay) + for container in self.buddies: container.interface.configure(self.host_log_folder, replay=self.replay) @@ -412,7 +418,7 @@ def _get_weblog_system_info(self): def _start_interfaces_watchdog(self): super().start_interfaces_watchdog( - [interfaces.library, interfaces.agent] + [container.interface for container in self.buddies] + [interfaces.library, interfaces.agent] + [container.interface for container in self.buddies] + [interfaces.open_telemetry] if self.include_opentelemetry else [] ) def _set_weblog_domain(self): @@ -472,6 +478,10 @@ def _wait_and_stop_containers(self, *, force_interface_timout_to_zero: bool): interfaces.backend.load_data_from_logs() + if self.include_opentelemetry: + interfaces.open_telemetry.load_data_from_logs() + interfaces.open_telemetry.check_deserialization_errors() + else: self._wait_interface( interfaces.library, 0 if force_interface_timout_to_zero else self.library_interface_timeout @@ -496,6 +506,11 @@ def _wait_and_stop_containers(self, *, force_interface_timout_to_zero: bool): interfaces.backend, 0 if force_interface_timout_to_zero else self.backend_interface_timeout ) + if self.include_opentelemetry: + self._wait_interface( + interfaces.open_telemetry, 0 if force_interface_timout_to_zero else self.backend_interface_timeout + ) + def _wait_interface(self, interface: ProxyBasedInterfaceValidator, timeout: int): logger.terminal.write_sep("-", f"Wait for {interface} ({timeout}s)") logger.terminal.flush() diff --git a/utils/interfaces/_open_telemetry.py b/utils/interfaces/_open_telemetry.py index 54c9381631c..f4acfb19db6 100644 --- a/utils/interfaces/_open_telemetry.py +++ b/utils/interfaces/_open_telemetry.py @@ -35,6 +35,53 @@ def get_otel_trace_id(self, request: HttpResponse): for span in scope_span.get("spans"): for attribute in span.get("attributes", []): attr_key = attribute.get("key") - attr_val = attribute.get("value").get("stringValue") + attr_val = attribute.get("value").get("string_value") or attribute.get("value").get("stringValue") if attr_key == "http.request.headers.user-agent" and rid in attr_val: yield span.get("traceId") + elif attr_key == "http.useragent" and rid in attr_val: + yield span.get("traceId") + + def get_otel_spans(self, request: HttpResponse): + paths = ["/api/v0.2/traces", "/v1/traces"] + rid = request.get_rid() + + if rid: + logger.debug(f"Try to find traces related to request {rid}") + + for data in self.get_data(path_filters=paths): + resource_spans = data.get("request").get("content").get("resource_spans") or data.get("request").get("content").get("resourceSpans") + for resource_span in resource_spans: + scope_spans = resource_span.get("scope_spans") or resource_span.get("scopeSpans") + for scope_span in scope_spans: + for span in scope_span.get("spans"): + for attribute in span.get("attributes", []): + attr_key = attribute.get("key") + attr_val = attribute.get("value").get("string_value") or attribute.get("value").get("stringValue") + if attr_key == "http.request.headers.user-agent" and rid in attr_val: + yield resource_span, span + break # Skip to next span + elif attr_key == "http.useragent" and rid in attr_val: + yield resource_span, span + break # Skip to next span + + def get_trace_stats(self, resource: str): + paths = ["/api/v0.2/stats", "/v1/metrics"] + + for data in self.get_data(path_filters=paths): + resource_metrics = data.get("request").get("content").get("resource_metrics") or data.get("request").get("content").get("resourceMetrics") + if not resource_metrics: + continue + + for resource_metric in resource_metrics: + scope_metrics = resource_metric.get("scope_metrics") or resource_metric.get("scopeMetrics") + for scope_metric in scope_metrics: + if scope_metric.get("scope").get("name") == "datadog.trace.metrics": + for metric in scope_metric.get("metrics"): + if metric.get("name") == "request.latencies": + data_points = metric.get("histogram").get("data_points") or metric.get("histogram").get("dataPoints") + for data_point in data_points: + for attribute in data_point.get("attributes", []): + attr_key = attribute.get("key") + attr_val = attribute.get("value").get("string_value") or attribute.get("value").get("stringValue") + if attr_key == "Resource" and attr_val == resource: + yield metric, data_point diff --git a/utils/proxy/_deserializer.py b/utils/proxy/_deserializer.py index 40b44f0303e..087c0e61895 100644 --- a/utils/proxy/_deserializer.py +++ b/utils/proxy/_deserializer.py @@ -191,6 +191,8 @@ def json_load(): return MessageToDict(ExportMetricsServiceResponse.FromString(content)) if path == "/v1/logs": return MessageToDict(ExportLogsServiceResponse.FromString(content)) + if path == "/api/v0.2/stats": + return MessageToDict(ExportMetricsServiceRequest.FromString(content)) if path == "/api/v0.2/traces": result = MessageToDict(TracePayload.FromString(content)) _deserialized_nested_json_from_trace_payloads(result, interface) diff --git a/utils/scripts/ci_orchestrators/workflow_data.py b/utils/scripts/ci_orchestrators/workflow_data.py index 008b42f44b2..7011c8aeed3 100644 --- a/utils/scripts/ci_orchestrators/workflow_data.py +++ b/utils/scripts/ci_orchestrators/workflow_data.py @@ -588,6 +588,7 @@ def _is_supported(library: str, weblog: str, scenario: str, _ci_environment: str "endtoend": [ "AGENT_NOT_SUPPORTING_SPAN_EVENTS", "APM_TRACING_E2E_OTEL", + "APM_TRACING_OTLP", "APM_TRACING_E2E_SINGLE_SPAN", "APPSEC_API_SECURITY", "APPSEC_API_SECURITY_NO_RESPONSE_BODY", From ebbbb5adefe3faa8d2d48d6a87fda741fe86e8eb Mon Sep 17 00:00:00 2001 From: Zach Montoya Date: Fri, 20 Feb 2026 14:17:42 -0800 Subject: [PATCH 02/29] Skip assertions for the Trace Metrics, as we don't have a spec for this yet --- tests/otel/test_tracing_otlp.py | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/tests/otel/test_tracing_otlp.py b/tests/otel/test_tracing_otlp.py index 9c4721729e0..4007f892089 100644 --- a/tests/otel/test_tracing_otlp.py +++ b/tests/otel/test_tracing_otlp.py @@ -102,31 +102,3 @@ def test_tracing(self): assert span_attributes.get("http.method") == "GET" assert span_attributes.get("http.status_code") == "200" # We may want to convert this to int later assert span_attributes.get("http.url") == "http://localhost:7777/" - - # Assert trace stats - for metric, data_point in interfaces.open_telemetry.get_trace_stats("GET /"): - # If we run another export, we'll have the same aggregation keys with zero counts. Skip them. - if not data_point.get("sum"): - continue - - assert metric.get("name") == "request.latencies" - assert metric.get("description") == "Summary of request latencies" - assert metric.get("unit") == "ns" - assert metric.get("histogram").get("aggregationTemporality") == "AGGREGATION_TEMPORALITY_DELTA" - - assert int(data_point.get("count")) == 1 - assert sum(map(int, data_point.get("bucketCounts"))) == int(data_point.get("count")) - assert len(data_point.get("bucketCounts")) == len(data_point.get("explicitBounds")) - assert data_point.get("explicitBounds") == sorted(data_point.get("explicitBounds")) - assert_single_histogram_data_point(data_point.get("sum"), list(map(int, data_point.get("bucketCounts"))), data_point.get("explicitBounds")) - - attributes = dict(get_keyvalue_generator(data_point.get("attributes"))) - assert "Name" in attributes - assert attributes.get("Resource") == "GET /" - assert attributes.get("Type") == "web" - assert attributes.get("StatusCode") == "200" - assert attributes.get("TopLevel") == "True" - assert attributes.get("Error") == "False" - assert len(attributes) == 6 - - From 7fee5a2fc26c9aaa7a0e1308aeaca37162a05460 Mon Sep 17 00:00:00 2001 From: Zach Montoya Date: Fri, 20 Feb 2026 14:25:35 -0800 Subject: [PATCH 03/29] Run formatter --- tests/otel/test_tracing_otlp.py | 94 +++++++++++++++------------ utils/_context/_scenarios/endtoend.py | 6 +- utils/interfaces/_open_telemetry.py | 39 +++++++---- 3 files changed, 81 insertions(+), 58 deletions(-) diff --git a/tests/otel/test_tracing_otlp.py b/tests/otel/test_tracing_otlp.py index 4007f892089..746ed4c201e 100644 --- a/tests/otel/test_tracing_otlp.py +++ b/tests/otel/test_tracing_otlp.py @@ -2,10 +2,9 @@ # This product includes software developed at Datadog (https://www.datadoghq.com/). # Copyright 2024 Datadog, Inc. -import json -from utils import weblog, interfaces, scenarios, features, incomplete_test_app -from utils._logger import logger -from typing import Any, Iterator +from utils import weblog, interfaces, scenarios, features +from typing import Any +from collections.abc import Iterator # Assert that the histogram has only one recorded data point matching the overall duration @@ -20,46 +19,49 @@ def assert_single_histogram_data_point(duration: int, bucket_counts: list[int], break if int(bucket_counts[i]) == 1: - lower_bound = float('-inf') if is_first_index else explicit_bounds[i-1] - upper_bound = float('inf') if is_last_index else explicit_bounds[i] + lower_bound = float("-inf") if is_first_index else explicit_bounds[i - 1] + upper_bound = float("inf") if is_last_index else explicit_bounds[i] if is_last_index: - assert duration > lower_bound and duration < upper_bound + assert duration > lower_bound + assert duration < upper_bound else: - assert duration > lower_bound and duration <= upper_bound + assert duration > lower_bound + assert duration <= upper_bound + def get_keyvalue_generator(attributes: list[dict]) -> Iterator[tuple[str, Any]]: - for keyValue in attributes: - if keyValue["value"].get("string_value"): - yield keyValue["key"], keyValue["value"]["string_value"] - elif keyValue["value"].get("stringValue"): - yield keyValue["key"], keyValue["value"]["stringValue"] - elif keyValue["value"].get("bool_value"): - yield keyValue["key"], keyValue["value"]["bool_value"] - elif keyValue["value"].get("boolValue"): - yield keyValue["key"], keyValue["value"]["boolValue"] - elif keyValue["value"].get("int_value"): - yield keyValue["key"], keyValue["value"]["int_value"] - elif keyValue["value"].get("intValue"): - yield keyValue["key"], keyValue["value"]["intValue"] - elif keyValue["value"].get("double_value"): - yield keyValue["key"], keyValue["value"]["double_value"] - elif keyValue["value"].get("doubleValue"): - yield keyValue["key"], keyValue["value"]["doubleValue"] - elif keyValue["value"].get("array_value"): - yield keyValue["key"], keyValue["value"]["array_value"] - elif keyValue["value"].get("arrayValue"): - yield keyValue["key"], keyValue["value"]["arrayValue"] - elif keyValue["value"].get("kvlist_value"): - yield keyValue["key"], keyValue["value"]["kvlist_value"] - elif keyValue["value"].get("kvlistValue"): - yield keyValue["key"], keyValue["value"]["kvlistValue"] - elif keyValue["value"].get("bytes_value"): - yield keyValue["key"], keyValue["value"]["bytes_value"] - elif keyValue["value"].get("bytesValue"): - yield keyValue["key"], keyValue["value"]["bytesValue"] + for key_value in attributes: + if key_value["value"].get("string_value"): + yield key_value["key"], key_value["value"]["string_value"] + elif key_value["value"].get("stringValue"): + yield key_value["key"], key_value["value"]["stringValue"] + elif key_value["value"].get("bool_value"): + yield key_value["key"], key_value["value"]["bool_value"] + elif key_value["value"].get("boolValue"): + yield key_value["key"], key_value["value"]["boolValue"] + elif key_value["value"].get("int_value"): + yield key_value["key"], key_value["value"]["int_value"] + elif key_value["value"].get("intValue"): + yield key_value["key"], key_value["value"]["intValue"] + elif key_value["value"].get("double_value"): + yield key_value["key"], key_value["value"]["double_value"] + elif key_value["value"].get("doubleValue"): + yield key_value["key"], key_value["value"]["doubleValue"] + elif key_value["value"].get("array_value"): + yield key_value["key"], key_value["value"]["array_value"] + elif key_value["value"].get("arrayValue"): + yield key_value["key"], key_value["value"]["arrayValue"] + elif key_value["value"].get("kvlist_value"): + yield key_value["key"], key_value["value"]["kvlist_value"] + elif key_value["value"].get("kvlistValue"): + yield key_value["key"], key_value["value"]["kvlistValue"] + elif key_value["value"].get("bytes_value"): + yield key_value["key"], key_value["value"]["bytes_value"] + elif key_value["value"].get("bytesValue"): + yield key_value["key"], key_value["value"]["bytesValue"] else: - raise ValueError(f"Unknown attribute value: {keyValue["value"]}") + raise ValueError(f"Unknown attribute value: {key_value['value']}") # @scenarios.apm_tracing_e2e_otel @@ -77,10 +79,16 @@ def test_tracing(self): resource_span, span = data[0] # Assert resource attributes (we only expect string values) - attributes = {keyValue["key"]: keyValue["value"].get("string_value") or keyValue["value"].get("stringValue") for keyValue in resource_span.get("resource").get("attributes")} + attributes = { + key_value["key"]: key_value["value"].get("string_value") or key_value["value"].get("stringValue") + for key_value in resource_span.get("resource").get("attributes") + } assert attributes.get("service.name") == "weblog" assert attributes.get("service.version") == "1.0.0" - assert attributes.get("deployment.environment.name") == "system-tests" or attributes.get("deployment.environment") == "system-tests" + assert ( + attributes.get("deployment.environment.name") == "system-tests" + or attributes.get("deployment.environment") == "system-tests" + ) # assert attributes.get("telemetry.sdk.name") == "datadog" assert "telemetry.sdk.language" in attributes assert "telemetry.sdk.version" in attributes @@ -96,9 +104,9 @@ def test_tracing(self): assert span.get("attributes") is not None # Assert HTTP tags - # Convert attributes list to a dictionary, but for now only handle KeyValue objects with stringValue - # attributes = {keyValue["key"]: keyValue["value"]["string_value"] or keyValue["value"]["stringValue"] for keyValue in span.get("attributes")} + # Convert attributes list to a dictionary, but for now only handle key_value objects with stringValue + # attributes = {key_value["key"]: key_value["value"]["string_value"] or key_value["value"]["stringValue"] for key_value in span.get("attributes")} span_attributes = dict(get_keyvalue_generator(span.get("attributes"))) assert span_attributes.get("http.method") == "GET" - assert span_attributes.get("http.status_code") == "200" # We may want to convert this to int later + assert span_attributes.get("http.status_code") == "200" # We may want to convert this to int later assert span_attributes.get("http.url") == "http://localhost:7777/" diff --git a/utils/_context/_scenarios/endtoend.py b/utils/_context/_scenarios/endtoend.py index 45456e462a7..c5a8be07738 100644 --- a/utils/_context/_scenarios/endtoend.py +++ b/utils/_context/_scenarios/endtoend.py @@ -418,7 +418,11 @@ def _get_weblog_system_info(self): def _start_interfaces_watchdog(self): super().start_interfaces_watchdog( - [interfaces.library, interfaces.agent] + [container.interface for container in self.buddies] + [interfaces.open_telemetry] if self.include_opentelemetry else [] + [interfaces.library, interfaces.agent] + + [container.interface for container in self.buddies] + + [interfaces.open_telemetry] + if self.include_opentelemetry + else [] ) def _set_weblog_domain(self): diff --git a/utils/interfaces/_open_telemetry.py b/utils/interfaces/_open_telemetry.py index f4acfb19db6..4abd7e75446 100644 --- a/utils/interfaces/_open_telemetry.py +++ b/utils/interfaces/_open_telemetry.py @@ -35,10 +35,12 @@ def get_otel_trace_id(self, request: HttpResponse): for span in scope_span.get("spans"): for attribute in span.get("attributes", []): attr_key = attribute.get("key") - attr_val = attribute.get("value").get("string_value") or attribute.get("value").get("stringValue") - if attr_key == "http.request.headers.user-agent" and rid in attr_val: - yield span.get("traceId") - elif attr_key == "http.useragent" and rid in attr_val: + attr_val = attribute.get("value").get("string_value") or attribute.get("value").get( + "stringValue" + ) + if (attr_key == "http.request.headers.user-agent" and rid in attr_val) or ( + attr_key == "http.useragent" and rid in attr_val + ): yield span.get("traceId") def get_otel_spans(self, request: HttpResponse): @@ -49,26 +51,31 @@ def get_otel_spans(self, request: HttpResponse): logger.debug(f"Try to find traces related to request {rid}") for data in self.get_data(path_filters=paths): - resource_spans = data.get("request").get("content").get("resource_spans") or data.get("request").get("content").get("resourceSpans") + resource_spans = data.get("request").get("content").get("resource_spans") or data.get("request").get( + "content" + ).get("resourceSpans") for resource_span in resource_spans: scope_spans = resource_span.get("scope_spans") or resource_span.get("scopeSpans") for scope_span in scope_spans: for span in scope_span.get("spans"): for attribute in span.get("attributes", []): attr_key = attribute.get("key") - attr_val = attribute.get("value").get("string_value") or attribute.get("value").get("stringValue") - if attr_key == "http.request.headers.user-agent" and rid in attr_val: - yield resource_span, span - break # Skip to next span - elif attr_key == "http.useragent" and rid in attr_val: + attr_val = attribute.get("value").get("string_value") or attribute.get("value").get( + "stringValue" + ) + if (attr_key == "http.request.headers.user-agent" and rid in attr_val) or ( + attr_key == "http.useragent" and rid in attr_val + ): yield resource_span, span - break # Skip to next span + break # Skip to next span def get_trace_stats(self, resource: str): paths = ["/api/v0.2/stats", "/v1/metrics"] for data in self.get_data(path_filters=paths): - resource_metrics = data.get("request").get("content").get("resource_metrics") or data.get("request").get("content").get("resourceMetrics") + resource_metrics = data.get("request").get("content").get("resource_metrics") or data.get("request").get( + "content" + ).get("resourceMetrics") if not resource_metrics: continue @@ -78,10 +85,14 @@ def get_trace_stats(self, resource: str): if scope_metric.get("scope").get("name") == "datadog.trace.metrics": for metric in scope_metric.get("metrics"): if metric.get("name") == "request.latencies": - data_points = metric.get("histogram").get("data_points") or metric.get("histogram").get("dataPoints") + data_points = metric.get("histogram").get("data_points") or metric.get("histogram").get( + "dataPoints" + ) for data_point in data_points: for attribute in data_point.get("attributes", []): attr_key = attribute.get("key") - attr_val = attribute.get("value").get("string_value") or attribute.get("value").get("stringValue") + attr_val = attribute.get("value").get("string_value") or attribute.get( + "value" + ).get("stringValue") if attr_key == "Resource" and attr_val == resource: yield metric, data_point From 5f8b2a8f60c06c6e4cbe1758acaca1e0da21ba32 Mon Sep 17 00:00:00 2001 From: Zach Montoya Date: Fri, 20 Feb 2026 14:41:51 -0800 Subject: [PATCH 04/29] Remove "/api/v0.2/stats" subpath for the OpenTelemetry interface so we only use the "/v1/metrics" subpath --- utils/interfaces/_open_telemetry.py | 2 +- utils/proxy/_deserializer.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/utils/interfaces/_open_telemetry.py b/utils/interfaces/_open_telemetry.py index 4abd7e75446..20049211ff8 100644 --- a/utils/interfaces/_open_telemetry.py +++ b/utils/interfaces/_open_telemetry.py @@ -70,7 +70,7 @@ def get_otel_spans(self, request: HttpResponse): break # Skip to next span def get_trace_stats(self, resource: str): - paths = ["/api/v0.2/stats", "/v1/metrics"] + paths = ["/v1/metrics"] for data in self.get_data(path_filters=paths): resource_metrics = data.get("request").get("content").get("resource_metrics") or data.get("request").get( diff --git a/utils/proxy/_deserializer.py b/utils/proxy/_deserializer.py index 087c0e61895..40b44f0303e 100644 --- a/utils/proxy/_deserializer.py +++ b/utils/proxy/_deserializer.py @@ -191,8 +191,6 @@ def json_load(): return MessageToDict(ExportMetricsServiceResponse.FromString(content)) if path == "/v1/logs": return MessageToDict(ExportLogsServiceResponse.FromString(content)) - if path == "/api/v0.2/stats": - return MessageToDict(ExportMetricsServiceRequest.FromString(content)) if path == "/api/v0.2/traces": result = MessageToDict(TracePayload.FromString(content)) _deserialized_nested_json_from_trace_payloads(result, interface) From 221b0318deedd997f192e0ba092914df52dc9300 Mon Sep 17 00:00:00 2001 From: Zach Montoya Date: Fri, 20 Feb 2026 14:45:04 -0800 Subject: [PATCH 05/29] Add manifest entries for languages --- manifests/dotnet.yml | 1 + manifests/golang.yml | 1 + manifests/java.yml | 1 + manifests/nodejs.yml | 1 + manifests/php.yml | 1 + manifests/python.yml | 1 + manifests/ruby.yml | 1 + manifests/rust.yml | 1 + 8 files changed, 8 insertions(+) diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml index 58e72b9f79d..d0926d52bd1 100644 --- a/manifests/dotnet.yml +++ b/manifests/dotnet.yml @@ -734,6 +734,7 @@ manifest: tests/k8s_lib_injection/test_k8s_lib_injection_appsec.py::TestK8sLibInjectionAppsecClusterEnabled: v3.36.0 tests/k8s_lib_injection/test_k8s_lib_injection_appsec.py::TestK8sLibInjectionAppsecDisabledByDefault: v3.36.0 tests/otel/test_context_propagation.py::Test_Otel_Context_Propagation_Default_Propagator_Api: v3.9.0 + tests/otel/test_tracing_otlp.py::Test_Otel_Tracing_OTLP: missing_feature tests/parametric/test_128_bit_traceids.py::Test_128_Bit_Traceids: v2.42.0 tests/parametric/test_config_consistency.py::Test_Config_Dogstatsd: missing_feature (does not support hostname) tests/parametric/test_config_consistency.py::Test_Config_RateLimit: v3.4.1 diff --git a/manifests/golang.yml b/manifests/golang.yml index bb1ca16a981..ba55e32b560 100644 --- a/manifests/golang.yml +++ b/manifests/golang.yml @@ -921,6 +921,7 @@ manifest: - weblog_declaration: "*": incomplete_test_app (endpoint not implemented) net-http: v1.70.1 + tests/otel/test_tracing_otlp.py::Test_Otel_Tracing_OTLP: missing_feature tests/parametric/test_128_bit_traceids.py::Test_128_Bit_Traceids: v1.50.0 tests/parametric/test_config_consistency.py::Test_Config_Dogstatsd: v1.72.0-dev tests/parametric/test_config_consistency.py::Test_Config_RateLimit: v1.67.0 diff --git a/manifests/java.yml b/manifests/java.yml index e56b7917251..c15a4fdfd11 100644 --- a/manifests/java.yml +++ b/manifests/java.yml @@ -3250,6 +3250,7 @@ manifest: spring-boot-openliberty: v1.58.2+06122213c8 # Modified by easy win activation script uds-spring-boot: v1.58.2+06122213c8 # Modified by easy win activation script spring-boot-payara: v1.58.2+06122213c8 # Modified by easy win activation script + tests/otel/test_tracing_otlp.py::Test_Otel_Tracing_OTLP: missing_feature tests/parametric/test_128_bit_traceids.py::Test_128_Bit_Traceids: v1.12.0 tests/parametric/test_128_bit_traceids.py::Test_128_Bit_Traceids::test_datadog_128_bit_generation_enabled_by_default: - declaration: missing_feature (Implemented in 1.24.0) diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml index a43e75b517f..6552447bfc6 100644 --- a/manifests/nodejs.yml +++ b/manifests/nodejs.yml @@ -1723,6 +1723,7 @@ manifest: fastify: *ref_5_26_0 tests/otel/test_context_propagation.py::Test_Otel_Context_Propagation_Default_Propagator_Api::test_propagation_extract: incomplete_test_app (Node.js extract endpoint doesn't seem to be working.) tests/otel/test_context_propagation.py::Test_Otel_Context_Propagation_Default_Propagator_Api::test_propagation_inject: incomplete_test_app (Node.js inject endpoint doesn't seem to be working.) + tests/otel/test_tracing_otlp.py::Test_Otel_Tracing_OTLP: missing_feature tests/parametric/test_128_bit_traceids.py::Test_128_Bit_Traceids: *ref_3_0_0 tests/parametric/test_128_bit_traceids.py::Test_128_Bit_Traceids::test_datadog_128_bit_generation_enabled_by_default: - declaration: missing_feature (Implemented in 4.19.0 & 3.40.0) diff --git a/manifests/php.yml b/manifests/php.yml index d5794bb0498..d7497574d8b 100644 --- a/manifests/php.yml +++ b/manifests/php.yml @@ -539,6 +539,7 @@ manifest: tests/k8s_lib_injection/test_k8s_lib_injection_profiling.py::TestK8sLibInjectioProfilingClusterOverride: v1.9.0 tests/k8s_lib_injection/test_k8s_lib_injection_profiling.py::TestK8sLibInjectioProfilingDisabledByDefault: v1.9.0 tests/otel/test_context_propagation.py::Test_Otel_Context_Propagation_Default_Propagator_Api: incomplete_test_app (endpoint not implemented) + tests/otel/test_tracing_otlp.py::Test_Otel_Tracing_OTLP: missing_feature tests/parametric/test_128_bit_traceids.py::Test_128_Bit_Traceids: v0.84.0 tests/parametric/test_config_consistency.py::Test_Config_Dogstatsd: v1.9.0 tests/parametric/test_config_consistency.py::Test_Config_RateLimit: v1.5.0 diff --git a/manifests/python.yml b/manifests/python.yml index ddcdff86a70..4546dc08f00 100644 --- a/manifests/python.yml +++ b/manifests/python.yml @@ -1403,6 +1403,7 @@ manifest: flask-poc: v2.19.0 uds-flask: v4.3.1 # Modified by easy win activation script uwsgi-poc: v4.3.1 # Modified by easy win activation script + tests/otel/test_tracing_otlp.py::Test_Otel_Tracing_OTLP: missing_feature tests/parametric/test_128_bit_traceids.py::Test_128_Bit_Traceids: v2.6.0 tests/parametric/test_config_consistency.py::Test_Config_Dogstatsd: v3.3.0.dev tests/parametric/test_config_consistency.py::Test_Config_RateLimit: v2.15.0 diff --git a/manifests/ruby.yml b/manifests/ruby.yml index 81ae7be2154..a003e51624e 100644 --- a/manifests/ruby.yml +++ b/manifests/ruby.yml @@ -1136,6 +1136,7 @@ manifest: "*": incomplete_test_app (endpoint not implemented) rails72: v2.0.0 tests/otel/test_context_propagation.py::Test_Otel_Context_Propagation_Default_Propagator_Api::test_propagation_extract: incomplete_test_app (Ruby extract seems to fail even though it should be supported) + tests/otel/test_tracing_otlp.py::Test_Otel_Tracing_OTLP: missing_feature tests/parametric/test_128_bit_traceids.py::Test_128_Bit_Traceids: v1.17.0 tests/parametric/test_128_bit_traceids.py::Test_128_Bit_Traceids::test_datadog_128_bit_propagation_tid_chars: # Modified by easy win activation script - declaration: missing_feature (not implemented) diff --git a/manifests/rust.yml b/manifests/rust.yml index 9908665b134..6145325c5e8 100644 --- a/manifests/rust.yml +++ b/manifests/rust.yml @@ -10,6 +10,7 @@ manifest: tests/docker_ssi/test_docker_ssi_appsec.py::TestDockerSSIAppsecFeatures::test_telemetry_source_ssi: missing_feature tests/ffe/test_dynamic_evaluation.py: missing_feature tests/ffe/test_exposures.py: missing_feature + tests/otel/test_tracing_otlp.py::Test_Otel_Tracing_OTLP: missing_feature tests/parametric/test_128_bit_traceids.py::Test_128_Bit_Traceids: v0.0.1 tests/parametric/test_128_bit_traceids.py::Test_128_Bit_Traceids::test_b3multi_128_bit_generation_disabled: missing_feature (propagation style not supported) tests/parametric/test_128_bit_traceids.py::Test_128_Bit_Traceids::test_b3multi_128_bit_generation_enabled: missing_feature (propagation style not supported) From cacbab5d2d9fbf80183186b48468c9a021593e24 Mon Sep 17 00:00:00 2001 From: Zach Montoya Date: Thu, 5 Mar 2026 14:10:50 -0800 Subject: [PATCH 06/29] Remove DD_TRACE_OTEL_ENABLED=true from the APM_TRACING_OTLP weblog env, since users do not need this feature for OTLP export to work --- utils/_context/_scenarios/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/utils/_context/_scenarios/__init__.py b/utils/_context/_scenarios/__init__.py index bb537381006..50ff4a5fcd4 100644 --- a/utils/_context/_scenarios/__init__.py +++ b/utils/_context/_scenarios/__init__.py @@ -571,7 +571,6 @@ class _Scenarios: apm_tracing_otlp = EndToEndScenario( "APM_TRACING_OTLP", weblog_env={ - "DD_TRACE_OTEL_ENABLED": "true", "DD_TRACE_STATS_COMPUTATION_ENABLED": "true", "OTEL_TRACES_EXPORTER": "otlp", "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": f"http://proxy:{ProxyPorts.open_telemetry_weblog}/v1/traces", From 14b1995be35068bf065d332c99bcdcb611fc6b9e Mon Sep 17 00:00:00 2001 From: Zach Montoya Date: Thu, 5 Mar 2026 14:13:07 -0800 Subject: [PATCH 07/29] Remove all helper methods and setup related to OTLP trace stats, since we're not immediately implementing that --- tests/otel/test_tracing_otlp.py | 23 ---------------------- utils/_context/_scenarios/__init__.py | 3 --- utils/interfaces/_open_telemetry.py | 28 --------------------------- 3 files changed, 54 deletions(-) diff --git a/tests/otel/test_tracing_otlp.py b/tests/otel/test_tracing_otlp.py index 746ed4c201e..fac734d707f 100644 --- a/tests/otel/test_tracing_otlp.py +++ b/tests/otel/test_tracing_otlp.py @@ -7,29 +7,6 @@ from collections.abc import Iterator -# Assert that the histogram has only one recorded data point matching the overall duration -def assert_single_histogram_data_point(duration: int, bucket_counts: list[int], explicit_bounds: list[float]): - for i in range(len(explicit_bounds)): - is_first_index = i == 0 - is_last_index = i == len(explicit_bounds) - 1 - - if is_first_index and is_last_index: - assert bucket_counts[i] == 1 - assert duration <= explicit_bounds[i] - break - - if int(bucket_counts[i]) == 1: - lower_bound = float("-inf") if is_first_index else explicit_bounds[i - 1] - upper_bound = float("inf") if is_last_index else explicit_bounds[i] - - if is_last_index: - assert duration > lower_bound - assert duration < upper_bound - else: - assert duration > lower_bound - assert duration <= upper_bound - - def get_keyvalue_generator(attributes: list[dict]) -> Iterator[tuple[str, Any]]: for key_value in attributes: if key_value["value"].get("string_value"): diff --git a/utils/_context/_scenarios/__init__.py b/utils/_context/_scenarios/__init__.py index 50ff4a5fcd4..52da2baad53 100644 --- a/utils/_context/_scenarios/__init__.py +++ b/utils/_context/_scenarios/__init__.py @@ -571,12 +571,9 @@ class _Scenarios: apm_tracing_otlp = EndToEndScenario( "APM_TRACING_OTLP", weblog_env={ - "DD_TRACE_STATS_COMPUTATION_ENABLED": "true", "OTEL_TRACES_EXPORTER": "otlp", "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": f"http://proxy:{ProxyPorts.open_telemetry_weblog}/v1/traces", "OTEL_EXPORTER_OTLP_TRACES_HEADERS": "dd-protocol=otlp,dd-otlp-path=agent", - "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT": f"http://proxy:{ProxyPorts.open_telemetry_weblog}/v1/metrics", - "OTEL_EXPORTER_OTLP_METRICS_HEADERS": "dd-otlp-path=agent", }, backend_interface_timeout=5, require_api_key=True, diff --git a/utils/interfaces/_open_telemetry.py b/utils/interfaces/_open_telemetry.py index 20049211ff8..03f7fca030a 100644 --- a/utils/interfaces/_open_telemetry.py +++ b/utils/interfaces/_open_telemetry.py @@ -68,31 +68,3 @@ def get_otel_spans(self, request: HttpResponse): ): yield resource_span, span break # Skip to next span - - def get_trace_stats(self, resource: str): - paths = ["/v1/metrics"] - - for data in self.get_data(path_filters=paths): - resource_metrics = data.get("request").get("content").get("resource_metrics") or data.get("request").get( - "content" - ).get("resourceMetrics") - if not resource_metrics: - continue - - for resource_metric in resource_metrics: - scope_metrics = resource_metric.get("scope_metrics") or resource_metric.get("scopeMetrics") - for scope_metric in scope_metrics: - if scope_metric.get("scope").get("name") == "datadog.trace.metrics": - for metric in scope_metric.get("metrics"): - if metric.get("name") == "request.latencies": - data_points = metric.get("histogram").get("data_points") or metric.get("histogram").get( - "dataPoints" - ) - for data_point in data_points: - for attribute in data_point.get("attributes", []): - attr_key = attribute.get("key") - attr_val = attribute.get("value").get("string_value") or attribute.get( - "value" - ).get("stringValue") - if attr_key == "Resource" and attr_val == resource: - yield metric, data_point From 57ed6bc04c4f7f297d3d5916a8778ea8422bad3c Mon Sep 17 00:00:00 2001 From: Zach Montoya Date: Thu, 5 Mar 2026 18:25:00 -0800 Subject: [PATCH 08/29] Add more comprehensive testing, especially around the http/json encoding specifically: - At runtime determine if the request is JSON - If JSON, look up proto field names by their camelCase representation. Otherwise, look up field names by their snake_case representation - If JSON, assert that the 'traceId' and 'spanId' fields are case-insensitive hexadecimal strings, rather than base64-encoded strings - If JSON, assert that enums (e.g. span.kind and span.status.code) are encoded using an integer, not a string representation of the enum value name - Regardless of protocol, get the time before and after the test HTTP request is issued, and assert that the span's reported 'start_time_unix_nano' and 'end_time_unix_nano' fall in this range - Regardless of protocol, make the 'http.method' and 'http.status_code' span attribute assertions more flexible by also testing against their stable OpenTelemetry HTTP equivalents of 'http.request.method' and 'http.response.status_code', respectively --- tests/otel/test_tracing_otlp.py | 112 +++++++++++++++++++++++----- utils/interfaces/_open_telemetry.py | 7 +- 2 files changed, 95 insertions(+), 24 deletions(-) diff --git a/tests/otel/test_tracing_otlp.py b/tests/otel/test_tracing_otlp.py index fac734d707f..d833ce00e7e 100644 --- a/tests/otel/test_tracing_otlp.py +++ b/tests/otel/test_tracing_otlp.py @@ -2,11 +2,46 @@ # This product includes software developed at Datadog (https://www.datadoghq.com/). # Copyright 2024 Datadog, Inc. +import time +import re +from enum import Enum from utils import weblog, interfaces, scenarios, features from typing import Any from collections.abc import Iterator +def _snake_to_camel(snake_key: str) -> str: + parts = snake_key.split("_") + return parts[0].lower() + "".join(p.capitalize() for p in parts[1:]) + + +def get_otlp_key(d: dict[str, Any] | None, snake_case_key: str, *, is_json: bool, default: Any = None) -> Any: # noqa: ANN401 + """Look up a field by its snake_case name when is_json is false, or its camelCase equivalent when is_json is true. + Fields must be camelCase for JSON Protobuf encoding. See https://opentelemetry.io/docs/specs/otlp/#json-protobuf-encoding + """ + if d is None: + return default + key = _snake_to_camel(snake_case_key) if is_json else snake_case_key + return d.get(key, default) + + +# See https://github.com/open-telemetry/opentelemetry-proto/blob/v1.9.0/opentelemetry/proto/trace/v1/trace.proto#L153 +class SpanKind(Enum): + UNSPECIFIED = 0 + INTERNAL = 1 + SERVER = 2 + CLIENT = 3 + PRODUCER = 4 + CONSUMER = 5 + + +# See https://github.com/open-telemetry/opentelemetry-proto/blob/v1.9.0/opentelemetry/proto/trace/v1/trace.proto#L316 +class StatusCode(Enum): + STATUS_CODE_UNSET = 0 + STATUS_CODE_OK = 1 + STATUS_CODE_ERROR = 2 + + def get_keyvalue_generator(attributes: list[dict]) -> Iterator[tuple[str, Any]]: for key_value in attributes: if key_value["value"].get("string_value"): @@ -45,45 +80,82 @@ def get_keyvalue_generator(attributes: list[dict]) -> Iterator[tuple[str, Any]]: @features.otel_api @scenarios.apm_tracing_otlp class Test_Otel_Tracing_OTLP: - def setup_tracing(self): + def setup_single_server_trace(self): + self.start_time_ns = time.time_ns() self.req = weblog.get("/") + self.end_time_ns = time.time_ns() - # Note: Both camelcase and snake_case are allowed by the ProtoJSON Format (https://protobuf.dev/programming-guides/json/) - def test_tracing(self): + def test_single_server_trace(self): data = list(interfaces.open_telemetry.get_otel_spans(self.req)) - # _logger.debug(data) + + # Assert that there is only one OTLP request containing the desired server span assert len(data) == 1 - resource_span, span = data[0] + request, content, span = data[0] + + # Determine if JSON Protobuf Encoding was used for the OTLP request (rather than Binary Protobuf) + # We need to assert that we match the OTLP specification, which has some odd encoding rules when using JSON: https://opentelemetry.io/docs/specs/otlp/#json-protobuf-encoding + request_headers = {key.lower(): value for key, value in request.get("headers")} + is_json = request_headers.get("content-type") == "application/json" + + # Assert that there is only one resource span (i.e. SDK) in the OTLP request + resource_spans = get_otlp_key(content, "resource_spans", is_json=is_json) + expected_key = _snake_to_camel("resource_spans") if is_json else "resource_spans" + assert resource_spans is not None, f"missing '{expected_key}' on content: {content}" + assert len(resource_spans) == 1, f"expected 1 resource span, got {len(resource_spans)}" + resource_span = resource_spans[0] - # Assert resource attributes (we only expect string values) attributes = { - key_value["key"]: key_value["value"].get("string_value") or key_value["value"].get("stringValue") + key_value["key"]: get_otlp_key(key_value["value"], "string_value", is_json=is_json) for key_value in resource_span.get("resource").get("attributes") } + + # Assert that the resource attributes contain the service-level attributes that were configured for weblog assert attributes.get("service.name") == "weblog" assert attributes.get("service.version") == "1.0.0" assert ( attributes.get("deployment.environment.name") == "system-tests" or attributes.get("deployment.environment") == "system-tests" ) + + # Assert that the resource attributes contain the tracer-level attributes we expect # assert attributes.get("telemetry.sdk.name") == "datadog" assert "telemetry.sdk.language" in attributes assert "telemetry.sdk.version" in attributes - assert "git.commit.sha" in attributes - assert "git.repository_url" in attributes assert "runtime-id" in attributes + # assert "git.commit.sha" in attributes + # assert "git.repository_url" in attributes - # Assert spans - assert span.get("name") == "GET /" - assert span.get("kind") == "SPAN_KIND_SERVER" - assert span.get("start_time_unix_nano") or span.get("startTimeUnixNano") - assert span.get("end_time_unix_nano") or span.get("endTimeUnixNano") - assert span.get("attributes") is not None + # Assert that the `traceId` and `spanId` JSON fields are valid case-insensitive hexadecimal strings, not base64-encoded strings as defined in the standard Protobuf JSON Mapping. + # See https://opentelemetry.io/docs/specs/otlp/#json-protobuf-encoding + if is_json: + assert re.match(r"^[0-9a-fA-F]{32}$", span.get("traceId")), ( + f"traceId is not a valid case-insensitive hexadecimal string, got {span.get('traceId')}" + ) + assert re.match(r"^[0-9a-fA-F]{16}$", span.get("spanId")), ( + f"spanId is not a valid case-insensitive hexadecimal string, got {span.get('spanId')}" + ) + + # Assert that the span fields match the expected values + span_start_time_ns = int(get_otlp_key(span, "start_time_unix_nano", is_json=is_json)) + span_end_time_ns = int(get_otlp_key(span, "end_time_unix_nano", is_json=is_json)) + assert span_start_time_ns >= self.start_time_ns + assert span_end_time_ns >= span_start_time_ns + assert span_end_time_ns <= self.end_time_ns + + assert get_otlp_key(span, "name", is_json=is_json) + assert get_otlp_key(span, "kind", is_json=is_json) == SpanKind.SERVER.value + assert get_otlp_key(span, "attributes", is_json=is_json) is not None + assert ( + get_otlp_key(span, "status", is_json=is_json) is None + or get_otlp_key(span, "status", is_json=is_json).get("code") == StatusCode.STATUS_CODE_UNSET.value + ) # Assert HTTP tags # Convert attributes list to a dictionary, but for now only handle key_value objects with stringValue - # attributes = {key_value["key"]: key_value["value"]["string_value"] or key_value["value"]["stringValue"] for key_value in span.get("attributes")} - span_attributes = dict(get_keyvalue_generator(span.get("attributes"))) - assert span_attributes.get("http.method") == "GET" - assert span_attributes.get("http.status_code") == "200" # We may want to convert this to int later - assert span_attributes.get("http.url") == "http://localhost:7777/" + span_attributes = dict(get_keyvalue_generator(get_otlp_key(span, "attributes", is_json=is_json))) + method = span_attributes.get("http.method") or span_attributes.get("http.request.method") + status_code = span_attributes.get("http.status_code") or span_attributes.get("http.response.status_code") + assert method == "GET", f"HTTP method is not GET, got {method}" + assert status_code is not None + assert int(status_code) == 200, f"HTTP status code is not 200, got {int(status_code)}" + # assert span_attributes.get("http.url") == "http://localhost:7777/" diff --git a/utils/interfaces/_open_telemetry.py b/utils/interfaces/_open_telemetry.py index 03f7fca030a..23836c50a99 100644 --- a/utils/interfaces/_open_telemetry.py +++ b/utils/interfaces/_open_telemetry.py @@ -51,9 +51,8 @@ def get_otel_spans(self, request: HttpResponse): logger.debug(f"Try to find traces related to request {rid}") for data in self.get_data(path_filters=paths): - resource_spans = data.get("request").get("content").get("resource_spans") or data.get("request").get( - "content" - ).get("resourceSpans") + content = data.get("request").get("content") + resource_spans = content.get("resource_spans") or content.get("resourceSpans") for resource_span in resource_spans: scope_spans = resource_span.get("scope_spans") or resource_span.get("scopeSpans") for scope_span in scope_spans: @@ -66,5 +65,5 @@ def get_otel_spans(self, request: HttpResponse): if (attr_key == "http.request.headers.user-agent" and rid in attr_val) or ( attr_key == "http.useragent" and rid in attr_val ): - yield resource_span, span + yield data.get("request"), content, span break # Skip to next span From 6a1df0763c23d61ede7c70d6663145092c66b4b9 Mon Sep 17 00:00:00 2001 From: Zach Montoya Date: Thu, 5 Mar 2026 18:30:20 -0800 Subject: [PATCH 09/29] Small refactoring of code and comments --- tests/otel/test_tracing_otlp.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/tests/otel/test_tracing_otlp.py b/tests/otel/test_tracing_otlp.py index d833ce00e7e..b881a9ee093 100644 --- a/tests/otel/test_tracing_otlp.py +++ b/tests/otel/test_tracing_otlp.py @@ -109,24 +109,21 @@ def test_single_server_trace(self): for key_value in resource_span.get("resource").get("attributes") } - # Assert that the resource attributes contain the service-level attributes that were configured for weblog + # Assert that the resource attributes contain the service-level attributes and tracer-level attributes we expect + # TODO: Assert the following attributes: runtime-id, git.commit.sha, git.repository_url assert attributes.get("service.name") == "weblog" assert attributes.get("service.version") == "1.0.0" assert ( attributes.get("deployment.environment.name") == "system-tests" or attributes.get("deployment.environment") == "system-tests" ) - - # Assert that the resource attributes contain the tracer-level attributes we expect - # assert attributes.get("telemetry.sdk.name") == "datadog" + assert attributes.get("telemetry.sdk.name") == "datadog" assert "telemetry.sdk.language" in attributes assert "telemetry.sdk.version" in attributes - assert "runtime-id" in attributes - # assert "git.commit.sha" in attributes - # assert "git.repository_url" in attributes # Assert that the `traceId` and `spanId` JSON fields are valid case-insensitive hexadecimal strings, not base64-encoded strings as defined in the standard Protobuf JSON Mapping. # See https://opentelemetry.io/docs/specs/otlp/#json-protobuf-encoding + # TODO: Assert against trace_id and span_id fields in the protobuf encoding as well if is_json: assert re.match(r"^[0-9a-fA-F]{32}$", span.get("traceId")), ( f"traceId is not a valid case-insensitive hexadecimal string, got {span.get('traceId')}" @@ -158,4 +155,3 @@ def test_single_server_trace(self): assert method == "GET", f"HTTP method is not GET, got {method}" assert status_code is not None assert int(status_code) == 200, f"HTTP status code is not 200, got {int(status_code)}" - # assert span_attributes.get("http.url") == "http://localhost:7777/" From 746148b7ed59967bc0e6f0a104ab52c03091990b Mon Sep 17 00:00:00 2001 From: Zach Montoya Date: Tue, 10 Mar 2026 14:39:09 -0700 Subject: [PATCH 10/29] Add test case asserting that an incoming trace context's sampling decision of 0 is respected for OTLP traces by default --- tests/otel/test_tracing_otlp.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/otel/test_tracing_otlp.py b/tests/otel/test_tracing_otlp.py index b881a9ee093..2ec2498bfa0 100644 --- a/tests/otel/test_tracing_otlp.py +++ b/tests/otel/test_tracing_otlp.py @@ -86,6 +86,7 @@ def setup_single_server_trace(self): self.end_time_ns = time.time_ns() def test_single_server_trace(self): + """Validates the required elements of the OTLP payload for a single trace""" data = list(interfaces.open_telemetry.get_otel_spans(self.req)) # Assert that there is only one OTLP request containing the desired server span @@ -155,3 +156,13 @@ def test_single_server_trace(self): assert method == "GET", f"HTTP method is not GET, got {method}" assert status_code is not None assert int(status_code) == 200, f"HTTP status code is not 200, got {int(status_code)}" + + def setup_unsampled_trace(self): + self.req = weblog.get("/", headers={"traceparent": "00-11111111111111110000000000000001-0000000000000001-00"}) + + def test_unsampled_trace(self): + """Validates that the spans from a non-sampled trace are not exported.""" + data = list(interfaces.open_telemetry.get_otel_spans(self.req)) + + # Assert that the span from this test case was not exported + assert len(data) == 0, f"Expected no weblog spans in the OTLP trace payload, got {data}" From 919b5efde5b7498e2477caab3065ae49b58ace07 Mon Sep 17 00:00:00 2001 From: Zach Montoya Date: Tue, 10 Mar 2026 14:45:54 -0700 Subject: [PATCH 11/29] Move enums to dd_constants.py --- tests/otel/test_tracing_otlp.py | 19 +------------------ utils/dd_constants.py | 8 ++++++++ 2 files changed, 9 insertions(+), 18 deletions(-) diff --git a/tests/otel/test_tracing_otlp.py b/tests/otel/test_tracing_otlp.py index 2ec2498bfa0..562949d3e37 100644 --- a/tests/otel/test_tracing_otlp.py +++ b/tests/otel/test_tracing_otlp.py @@ -4,8 +4,8 @@ import time import re -from enum import Enum from utils import weblog, interfaces, scenarios, features +from utils.dd_constants import SpanKind, StatusCode from typing import Any from collections.abc import Iterator @@ -25,23 +25,6 @@ def get_otlp_key(d: dict[str, Any] | None, snake_case_key: str, *, is_json: bool return d.get(key, default) -# See https://github.com/open-telemetry/opentelemetry-proto/blob/v1.9.0/opentelemetry/proto/trace/v1/trace.proto#L153 -class SpanKind(Enum): - UNSPECIFIED = 0 - INTERNAL = 1 - SERVER = 2 - CLIENT = 3 - PRODUCER = 4 - CONSUMER = 5 - - -# See https://github.com/open-telemetry/opentelemetry-proto/blob/v1.9.0/opentelemetry/proto/trace/v1/trace.proto#L316 -class StatusCode(Enum): - STATUS_CODE_UNSET = 0 - STATUS_CODE_OK = 1 - STATUS_CODE_ERROR = 2 - - def get_keyvalue_generator(attributes: list[dict]) -> Iterator[tuple[str, Any]]: for key_value in attributes: if key_value["value"].get("string_value"): diff --git a/utils/dd_constants.py b/utils/dd_constants.py index b2bf0e3b539..507704bee78 100644 --- a/utils/dd_constants.py +++ b/utils/dd_constants.py @@ -108,6 +108,7 @@ class SamplingMechanism(IntEnum): AI_GUARD = 13 +# See https://github.com/open-telemetry/opentelemetry-proto/blob/v1.9.0/opentelemetry/proto/trace/v1/trace.proto#L153 class SpanKind(IntEnum): UNSPECIFIED = 0 INTERNAL = 1 @@ -115,3 +116,10 @@ class SpanKind(IntEnum): CLIENT = 3 PRODUCER = 4 CONSUMER = 5 + + +# See https://github.com/open-telemetry/opentelemetry-proto/blob/v1.9.0/opentelemetry/proto/trace/v1/trace.proto#L316 +class StatusCode(IntEnum): + STATUS_CODE_UNSET = 0 + STATUS_CODE_OK = 1 + STATUS_CODE_ERROR = 2 From 0d6ff15b73520a924fae9dddc0c0d0058c89ab07 Mon Sep 17 00:00:00 2001 From: Mikayla Toffler Date: Fri, 13 Mar 2026 11:07:33 -0400 Subject: [PATCH 12/29] Support protobuf format in MessageToDict and modify helpers accoridngly --- tests/otel/test_tracing_otlp.py | 80 +++++++++++++++-------------- utils/interfaces/_open_telemetry.py | 16 ++++-- utils/proxy/_deserializer.py | 12 ++--- 3 files changed, 59 insertions(+), 49 deletions(-) diff --git a/tests/otel/test_tracing_otlp.py b/tests/otel/test_tracing_otlp.py index b881a9ee093..93cfe86d330 100644 --- a/tests/otel/test_tracing_otlp.py +++ b/tests/otel/test_tracing_otlp.py @@ -17,7 +17,10 @@ def _snake_to_camel(snake_key: str) -> str: def get_otlp_key(d: dict[str, Any] | None, snake_case_key: str, *, is_json: bool, default: Any = None) -> Any: # noqa: ANN401 """Look up a field by its snake_case name when is_json is false, or its camelCase equivalent when is_json is true. - Fields must be camelCase for JSON Protobuf encoding. See https://opentelemetry.io/docs/specs/otlp/#json-protobuf-encoding + + Binary Protobuf is deserialised with preserving_proto_field_name=True (snake_case keys, integer enums). + JSON Protobuf encoding uses camelCase keys per the OTLP spec. + See https://opentelemetry.io/docs/specs/otlp/#json-protobuf-encoding """ if d is None: return default @@ -44,36 +47,39 @@ class StatusCode(Enum): def get_keyvalue_generator(attributes: list[dict]) -> Iterator[tuple[str, Any]]: for key_value in attributes: - if key_value["value"].get("string_value"): - yield key_value["key"], key_value["value"]["string_value"] - elif key_value["value"].get("stringValue"): - yield key_value["key"], key_value["value"]["stringValue"] - elif key_value["value"].get("bool_value"): - yield key_value["key"], key_value["value"]["bool_value"] - elif key_value["value"].get("boolValue"): - yield key_value["key"], key_value["value"]["boolValue"] - elif key_value["value"].get("int_value"): - yield key_value["key"], key_value["value"]["int_value"] - elif key_value["value"].get("intValue"): - yield key_value["key"], key_value["value"]["intValue"] - elif key_value["value"].get("double_value"): - yield key_value["key"], key_value["value"]["double_value"] - elif key_value["value"].get("doubleValue"): - yield key_value["key"], key_value["value"]["doubleValue"] - elif key_value["value"].get("array_value"): - yield key_value["key"], key_value["value"]["array_value"] - elif key_value["value"].get("arrayValue"): - yield key_value["key"], key_value["value"]["arrayValue"] - elif key_value["value"].get("kvlist_value"): - yield key_value["key"], key_value["value"]["kvlist_value"] - elif key_value["value"].get("kvlistValue"): - yield key_value["key"], key_value["value"]["kvlistValue"] - elif key_value["value"].get("bytes_value"): - yield key_value["key"], key_value["value"]["bytes_value"] - elif key_value["value"].get("bytesValue"): - yield key_value["key"], key_value["value"]["bytesValue"] + v = key_value["value"] + # Use `is not None` rather than truthiness so zero/false/empty values are not skipped. + # Binary protobuf uses snake_case keys; JSON encoding uses camelCase. Handle both. + if v.get("string_value") is not None: + yield key_value["key"], v["string_value"] + elif v.get("stringValue") is not None: + yield key_value["key"], v["stringValue"] + elif v.get("bool_value") is not None: + yield key_value["key"], v["bool_value"] + elif v.get("boolValue") is not None: + yield key_value["key"], v["boolValue"] + elif v.get("int_value") is not None: + yield key_value["key"], v["int_value"] + elif v.get("intValue") is not None: + yield key_value["key"], v["intValue"] + elif v.get("double_value") is not None: + yield key_value["key"], v["double_value"] + elif v.get("doubleValue") is not None: + yield key_value["key"], v["doubleValue"] + elif v.get("array_value") is not None: + yield key_value["key"], v["array_value"] + elif v.get("arrayValue") is not None: + yield key_value["key"], v["arrayValue"] + elif v.get("kvlist_value") is not None: + yield key_value["key"], v["kvlist_value"] + elif v.get("kvlistValue") is not None: + yield key_value["key"], v["kvlistValue"] + elif v.get("bytes_value") is not None: + yield key_value["key"], v["bytes_value"] + elif v.get("bytesValue") is not None: + yield key_value["key"], v["bytesValue"] else: - raise ValueError(f"Unknown attribute value: {key_value['value']}") + raise ValueError(f"Unknown attribute value: {v}") # @scenarios.apm_tracing_e2e_otel @@ -99,8 +105,7 @@ def test_single_server_trace(self): # Assert that there is only one resource span (i.e. SDK) in the OTLP request resource_spans = get_otlp_key(content, "resource_spans", is_json=is_json) - expected_key = _snake_to_camel("resource_spans") if is_json else "resource_spans" - assert resource_spans is not None, f"missing '{expected_key}' on content: {content}" + assert resource_spans is not None, f"missing '{_snake_to_camel('resource_spans')}' on content: {content}" assert len(resource_spans) == 1, f"expected 1 resource span, got {len(resource_spans)}" resource_span = resource_spans[0] @@ -117,9 +122,9 @@ def test_single_server_trace(self): attributes.get("deployment.environment.name") == "system-tests" or attributes.get("deployment.environment") == "system-tests" ) - assert attributes.get("telemetry.sdk.name") == "datadog" + # assert attributes.get("telemetry.sdk.name") == "datadog" assert "telemetry.sdk.language" in attributes - assert "telemetry.sdk.version" in attributes + # assert "telemetry.sdk.version" in attributes # Assert that the `traceId` and `spanId` JSON fields are valid case-insensitive hexadecimal strings, not base64-encoded strings as defined in the standard Protobuf JSON Mapping. # See https://opentelemetry.io/docs/specs/otlp/#json-protobuf-encoding @@ -142,10 +147,9 @@ def test_single_server_trace(self): assert get_otlp_key(span, "name", is_json=is_json) assert get_otlp_key(span, "kind", is_json=is_json) == SpanKind.SERVER.value assert get_otlp_key(span, "attributes", is_json=is_json) is not None - assert ( - get_otlp_key(span, "status", is_json=is_json) is None - or get_otlp_key(span, "status", is_json=is_json).get("code") == StatusCode.STATUS_CODE_UNSET.value - ) + status = get_otlp_key(span, "status", is_json=is_json) + # An absent or empty status dict both mean STATUS_CODE_UNSET (protobuf default = 0). + assert not status or status.get("code", StatusCode.STATUS_CODE_UNSET.value) == StatusCode.STATUS_CODE_UNSET.value # Assert HTTP tags # Convert attributes list to a dictionary, but for now only handle key_value objects with stringValue diff --git a/utils/interfaces/_open_telemetry.py b/utils/interfaces/_open_telemetry.py index 23836c50a99..a93ba39fdcd 100644 --- a/utils/interfaces/_open_telemetry.py +++ b/utils/interfaces/_open_telemetry.py @@ -23,6 +23,7 @@ def ingest_file(self, src_path: str): return super().ingest_file(src_path) def get_otel_trace_id(self, request: HttpResponse): + # Paths filter intercepted OTLP export requests (weblog → proxy), not weblog or backend URLs. paths = ["/api/v0.2/traces", "/v1/traces"] rid = request.get_rid() @@ -30,18 +31,23 @@ def get_otel_trace_id(self, request: HttpResponse): logger.debug(f"Try to find traces related to request {rid}") for data in self.get_data(path_filters=paths): - for resource_span in data.get("request").get("content").get("resourceSpans"): - for scope_span in resource_span.get("scopeSpans"): - for span in scope_span.get("spans"): + content = data.get("request").get("content") + # Binary protobuf (deserialised with preserving_proto_field_name=True) uses snake_case; + # JSON encoding uses camelCase. Handle both. + resource_spans = content.get("resource_spans") or content.get("resourceSpans") or [] + for resource_span in resource_spans: + scope_spans = resource_span.get("scope_spans") or resource_span.get("scopeSpans") or [] + for scope_span in scope_spans: + for span in scope_span.get("spans", []): for attribute in span.get("attributes", []): attr_key = attribute.get("key") - attr_val = attribute.get("value").get("string_value") or attribute.get("value").get( + attr_val = attribute.get("value", {}).get("string_value") or attribute.get("value", {}).get( "stringValue" ) if (attr_key == "http.request.headers.user-agent" and rid in attr_val) or ( attr_key == "http.useragent" and rid in attr_val ): - yield span.get("traceId") + yield span.get("trace_id") or span.get("traceId") def get_otel_spans(self, request: HttpResponse): paths = ["/api/v0.2/traces", "/v1/traces"] diff --git a/utils/proxy/_deserializer.py b/utils/proxy/_deserializer.py index 5f6b964fbb0..fe76394e314 100644 --- a/utils/proxy/_deserializer.py +++ b/utils/proxy/_deserializer.py @@ -178,17 +178,17 @@ def json_load(): assert isinstance(content, bytes) dd_protocol = get_header_value("dd-protocol", message["headers"]) if dd_protocol == "otlp" and "traces" in path: - return MessageToDict(ExportTraceServiceRequest.FromString(content)) + return MessageToDict(ExportTraceServiceRequest.FromString(content), preserving_proto_field_name=True, use_integers_for_enums=True) if dd_protocol == "otlp" and "metrics" in path: - return MessageToDict(ExportMetricsServiceRequest.FromString(content)) + return MessageToDict(ExportMetricsServiceRequest.FromString(content), preserving_proto_field_name=True, use_integers_for_enums=True) if dd_protocol == "otlp" and "logs" in path: - return MessageToDict(ExportLogsServiceRequest.FromString(content)) + return MessageToDict(ExportLogsServiceRequest.FromString(content), preserving_proto_field_name=True, use_integers_for_enums=True) if path == "/v1/traces": - return MessageToDict(ExportTraceServiceResponse.FromString(content)) + return MessageToDict(ExportTraceServiceResponse.FromString(content), preserving_proto_field_name=True, use_integers_for_enums=True) if path == "/v1/metrics": - return MessageToDict(ExportMetricsServiceResponse.FromString(content)) + return MessageToDict(ExportMetricsServiceResponse.FromString(content), preserving_proto_field_name=True, use_integers_for_enums=True) if path == "/v1/logs": - return MessageToDict(ExportLogsServiceResponse.FromString(content)) + return MessageToDict(ExportLogsServiceResponse.FromString(content), preserving_proto_field_name=True, use_integers_for_enums=True) if path == "/api/v0.2/traces": result = MessageToDict(TracePayload.FromString(content)) _deserialized_nested_json_from_trace_payloads(result, interface) From ca4dc74b8dc33d49e88f49beb1e09b3da1c0f0ec Mon Sep 17 00:00:00 2001 From: Zach Montoya Date: Mon, 16 Mar 2026 13:39:24 -0700 Subject: [PATCH 13/29] Remove snake_case proto fields PART ONE. Since JSON must be expressed in lowerCamelCase (according to the OpenTelemetry spec), we can consolidate our parsing and assertions on that style of field names --- tests/otel/test_tracing_otlp.py | 58 +++++++---------------------- utils/interfaces/_open_telemetry.py | 18 +++------ utils/proxy/_deserializer.py | 12 +++--- 3 files changed, 26 insertions(+), 62 deletions(-) diff --git a/tests/otel/test_tracing_otlp.py b/tests/otel/test_tracing_otlp.py index 93cfe86d330..53740c7d50f 100644 --- a/tests/otel/test_tracing_otlp.py +++ b/tests/otel/test_tracing_otlp.py @@ -10,24 +10,6 @@ from collections.abc import Iterator -def _snake_to_camel(snake_key: str) -> str: - parts = snake_key.split("_") - return parts[0].lower() + "".join(p.capitalize() for p in parts[1:]) - - -def get_otlp_key(d: dict[str, Any] | None, snake_case_key: str, *, is_json: bool, default: Any = None) -> Any: # noqa: ANN401 - """Look up a field by its snake_case name when is_json is false, or its camelCase equivalent when is_json is true. - - Binary Protobuf is deserialised with preserving_proto_field_name=True (snake_case keys, integer enums). - JSON Protobuf encoding uses camelCase keys per the OTLP spec. - See https://opentelemetry.io/docs/specs/otlp/#json-protobuf-encoding - """ - if d is None: - return default - key = _snake_to_camel(snake_case_key) if is_json else snake_case_key - return d.get(key, default) - - # See https://github.com/open-telemetry/opentelemetry-proto/blob/v1.9.0/opentelemetry/proto/trace/v1/trace.proto#L153 class SpanKind(Enum): UNSPECIFIED = 0 @@ -50,32 +32,18 @@ def get_keyvalue_generator(attributes: list[dict]) -> Iterator[tuple[str, Any]]: v = key_value["value"] # Use `is not None` rather than truthiness so zero/false/empty values are not skipped. # Binary protobuf uses snake_case keys; JSON encoding uses camelCase. Handle both. - if v.get("string_value") is not None: - yield key_value["key"], v["string_value"] - elif v.get("stringValue") is not None: + if v.get("stringValue") is not None: yield key_value["key"], v["stringValue"] - elif v.get("bool_value") is not None: - yield key_value["key"], v["bool_value"] elif v.get("boolValue") is not None: yield key_value["key"], v["boolValue"] - elif v.get("int_value") is not None: - yield key_value["key"], v["int_value"] elif v.get("intValue") is not None: yield key_value["key"], v["intValue"] - elif v.get("double_value") is not None: - yield key_value["key"], v["double_value"] elif v.get("doubleValue") is not None: yield key_value["key"], v["doubleValue"] - elif v.get("array_value") is not None: - yield key_value["key"], v["array_value"] elif v.get("arrayValue") is not None: yield key_value["key"], v["arrayValue"] - elif v.get("kvlist_value") is not None: - yield key_value["key"], v["kvlist_value"] elif v.get("kvlistValue") is not None: yield key_value["key"], v["kvlistValue"] - elif v.get("bytes_value") is not None: - yield key_value["key"], v["bytes_value"] elif v.get("bytesValue") is not None: yield key_value["key"], v["bytesValue"] else: @@ -104,13 +72,13 @@ def test_single_server_trace(self): is_json = request_headers.get("content-type") == "application/json" # Assert that there is only one resource span (i.e. SDK) in the OTLP request - resource_spans = get_otlp_key(content, "resource_spans", is_json=is_json) - assert resource_spans is not None, f"missing '{_snake_to_camel('resource_spans')}' on content: {content}" + resource_spans = content["resourceSpans"] + assert resource_spans is not None, f"missing 'resourceSpans' on content: {content}" assert len(resource_spans) == 1, f"expected 1 resource span, got {len(resource_spans)}" resource_span = resource_spans[0] attributes = { - key_value["key"]: get_otlp_key(key_value["value"], "string_value", is_json=is_json) + key_value["key"]: key_value["value"]["stringValue"] for key_value in resource_span.get("resource").get("attributes") } @@ -138,22 +106,24 @@ def test_single_server_trace(self): ) # Assert that the span fields match the expected values - span_start_time_ns = int(get_otlp_key(span, "start_time_unix_nano", is_json=is_json)) - span_end_time_ns = int(get_otlp_key(span, "end_time_unix_nano", is_json=is_json)) + span_start_time_ns = int(span["startTimeUnixNano"]) + span_end_time_ns = int(span["endTimeUnixNano"]) assert span_start_time_ns >= self.start_time_ns assert span_end_time_ns >= span_start_time_ns assert span_end_time_ns <= self.end_time_ns - assert get_otlp_key(span, "name", is_json=is_json) - assert get_otlp_key(span, "kind", is_json=is_json) == SpanKind.SERVER.value - assert get_otlp_key(span, "attributes", is_json=is_json) is not None - status = get_otlp_key(span, "status", is_json=is_json) + assert span["name"] + assert span["kind"] == SpanKind.SERVER.value + assert span["attributes"] is not None + status = span["status"] # An absent or empty status dict both mean STATUS_CODE_UNSET (protobuf default = 0). - assert not status or status.get("code", StatusCode.STATUS_CODE_UNSET.value) == StatusCode.STATUS_CODE_UNSET.value + assert ( + not status or status.get("code", StatusCode.STATUS_CODE_UNSET.value) == StatusCode.STATUS_CODE_UNSET.value + ) # Assert HTTP tags # Convert attributes list to a dictionary, but for now only handle key_value objects with stringValue - span_attributes = dict(get_keyvalue_generator(get_otlp_key(span, "attributes", is_json=is_json))) + span_attributes = dict(get_keyvalue_generator(span["attributes"])) method = span_attributes.get("http.method") or span_attributes.get("http.request.method") status_code = span_attributes.get("http.status_code") or span_attributes.get("http.response.status_code") assert method == "GET", f"HTTP method is not GET, got {method}" diff --git a/utils/interfaces/_open_telemetry.py b/utils/interfaces/_open_telemetry.py index a93ba39fdcd..c7809ec356d 100644 --- a/utils/interfaces/_open_telemetry.py +++ b/utils/interfaces/_open_telemetry.py @@ -32,18 +32,14 @@ def get_otel_trace_id(self, request: HttpResponse): for data in self.get_data(path_filters=paths): content = data.get("request").get("content") - # Binary protobuf (deserialised with preserving_proto_field_name=True) uses snake_case; - # JSON encoding uses camelCase. Handle both. - resource_spans = content.get("resource_spans") or content.get("resourceSpans") or [] + resource_spans = content.get("resourceSpans") or [] for resource_span in resource_spans: - scope_spans = resource_span.get("scope_spans") or resource_span.get("scopeSpans") or [] + scope_spans = resource_span.get("scopeSpans") or [] for scope_span in scope_spans: for span in scope_span.get("spans", []): for attribute in span.get("attributes", []): attr_key = attribute.get("key") - attr_val = attribute.get("value", {}).get("string_value") or attribute.get("value", {}).get( - "stringValue" - ) + attr_val = attribute.get("value", {}).get("stringValue") if (attr_key == "http.request.headers.user-agent" and rid in attr_val) or ( attr_key == "http.useragent" and rid in attr_val ): @@ -58,16 +54,14 @@ def get_otel_spans(self, request: HttpResponse): for data in self.get_data(path_filters=paths): content = data.get("request").get("content") - resource_spans = content.get("resource_spans") or content.get("resourceSpans") + resource_spans = content.get("resourceSpans") for resource_span in resource_spans: - scope_spans = resource_span.get("scope_spans") or resource_span.get("scopeSpans") + scope_spans = resource_span.get("scopeSpans") for scope_span in scope_spans: for span in scope_span.get("spans"): for attribute in span.get("attributes", []): attr_key = attribute.get("key") - attr_val = attribute.get("value").get("string_value") or attribute.get("value").get( - "stringValue" - ) + attr_val = attribute.get("value").get("stringValue") if (attr_key == "http.request.headers.user-agent" and rid in attr_val) or ( attr_key == "http.useragent" and rid in attr_val ): diff --git a/utils/proxy/_deserializer.py b/utils/proxy/_deserializer.py index fe76394e314..79c36072092 100644 --- a/utils/proxy/_deserializer.py +++ b/utils/proxy/_deserializer.py @@ -178,17 +178,17 @@ def json_load(): assert isinstance(content, bytes) dd_protocol = get_header_value("dd-protocol", message["headers"]) if dd_protocol == "otlp" and "traces" in path: - return MessageToDict(ExportTraceServiceRequest.FromString(content), preserving_proto_field_name=True, use_integers_for_enums=True) + return MessageToDict(ExportTraceServiceRequest.FromString(content), preserving_proto_field_name=False, use_integers_for_enums=True) if dd_protocol == "otlp" and "metrics" in path: - return MessageToDict(ExportMetricsServiceRequest.FromString(content), preserving_proto_field_name=True, use_integers_for_enums=True) + return MessageToDict(ExportMetricsServiceRequest.FromString(content), preserving_proto_field_name=False, use_integers_for_enums=True) if dd_protocol == "otlp" and "logs" in path: - return MessageToDict(ExportLogsServiceRequest.FromString(content), preserving_proto_field_name=True, use_integers_for_enums=True) + return MessageToDict(ExportLogsServiceRequest.FromString(content), preserving_proto_field_name=False, use_integers_for_enums=True) if path == "/v1/traces": - return MessageToDict(ExportTraceServiceResponse.FromString(content), preserving_proto_field_name=True, use_integers_for_enums=True) + return MessageToDict(ExportTraceServiceResponse.FromString(content), preserving_proto_field_name=False, use_integers_for_enums=True) if path == "/v1/metrics": - return MessageToDict(ExportMetricsServiceResponse.FromString(content), preserving_proto_field_name=True, use_integers_for_enums=True) + return MessageToDict(ExportMetricsServiceResponse.FromString(content), preserving_proto_field_name=False, use_integers_for_enums=True) if path == "/v1/logs": - return MessageToDict(ExportLogsServiceResponse.FromString(content), preserving_proto_field_name=True, use_integers_for_enums=True) + return MessageToDict(ExportLogsServiceResponse.FromString(content), preserving_proto_field_name=False, use_integers_for_enums=True) if path == "/api/v0.2/traces": result = MessageToDict(TracePayload.FromString(content)) _deserialized_nested_json_from_trace_payloads(result, interface) From bfcd7a9851f0daf42fed27e5f6bfb5a7d2faf691 Mon Sep 17 00:00:00 2001 From: Zach Montoya Date: Mon, 16 Mar 2026 16:01:09 -0700 Subject: [PATCH 14/29] Move flattening of OTLP attribute dictionaries from test_tracing_otlp.py to the proxy, in utils/proxy/traces/otlp_v1.py --- tests/otel/test_tracing_otlp.py | 36 ++-------------- utils/interfaces/_open_telemetry.py | 29 ++++++------- utils/proxy/_deserializer.py | 8 +++- utils/proxy/traces/otlp_v1.py | 67 +++++++++++++++++++++++++++++ 4 files changed, 89 insertions(+), 51 deletions(-) create mode 100644 utils/proxy/traces/otlp_v1.py diff --git a/tests/otel/test_tracing_otlp.py b/tests/otel/test_tracing_otlp.py index 53740c7d50f..e0ec89aec3a 100644 --- a/tests/otel/test_tracing_otlp.py +++ b/tests/otel/test_tracing_otlp.py @@ -6,8 +6,6 @@ import re from enum import Enum from utils import weblog, interfaces, scenarios, features -from typing import Any -from collections.abc import Iterator # See https://github.com/open-telemetry/opentelemetry-proto/blob/v1.9.0/opentelemetry/proto/trace/v1/trace.proto#L153 @@ -27,29 +25,6 @@ class StatusCode(Enum): STATUS_CODE_ERROR = 2 -def get_keyvalue_generator(attributes: list[dict]) -> Iterator[tuple[str, Any]]: - for key_value in attributes: - v = key_value["value"] - # Use `is not None` rather than truthiness so zero/false/empty values are not skipped. - # Binary protobuf uses snake_case keys; JSON encoding uses camelCase. Handle both. - if v.get("stringValue") is not None: - yield key_value["key"], v["stringValue"] - elif v.get("boolValue") is not None: - yield key_value["key"], v["boolValue"] - elif v.get("intValue") is not None: - yield key_value["key"], v["intValue"] - elif v.get("doubleValue") is not None: - yield key_value["key"], v["doubleValue"] - elif v.get("arrayValue") is not None: - yield key_value["key"], v["arrayValue"] - elif v.get("kvlistValue") is not None: - yield key_value["key"], v["kvlistValue"] - elif v.get("bytesValue") is not None: - yield key_value["key"], v["bytesValue"] - else: - raise ValueError(f"Unknown attribute value: {v}") - - # @scenarios.apm_tracing_e2e_otel @features.otel_api @scenarios.apm_tracing_otlp @@ -57,7 +32,6 @@ class Test_Otel_Tracing_OTLP: def setup_single_server_trace(self): self.start_time_ns = time.time_ns() self.req = weblog.get("/") - self.end_time_ns = time.time_ns() def test_single_server_trace(self): data = list(interfaces.open_telemetry.get_otel_spans(self.req)) @@ -77,10 +51,7 @@ def test_single_server_trace(self): assert len(resource_spans) == 1, f"expected 1 resource span, got {len(resource_spans)}" resource_span = resource_spans[0] - attributes = { - key_value["key"]: key_value["value"]["stringValue"] - for key_value in resource_span.get("resource").get("attributes") - } + attributes = resource_span.get("resource", {}).get("attributes", {}) # Assert that the resource attributes contain the service-level attributes and tracer-level attributes we expect # TODO: Assert the following attributes: runtime-id, git.commit.sha, git.repository_url @@ -110,12 +81,11 @@ def test_single_server_trace(self): span_end_time_ns = int(span["endTimeUnixNano"]) assert span_start_time_ns >= self.start_time_ns assert span_end_time_ns >= span_start_time_ns - assert span_end_time_ns <= self.end_time_ns assert span["name"] assert span["kind"] == SpanKind.SERVER.value assert span["attributes"] is not None - status = span["status"] + status = span.get("status", {}) # An absent or empty status dict both mean STATUS_CODE_UNSET (protobuf default = 0). assert ( not status or status.get("code", StatusCode.STATUS_CODE_UNSET.value) == StatusCode.STATUS_CODE_UNSET.value @@ -123,7 +93,7 @@ def test_single_server_trace(self): # Assert HTTP tags # Convert attributes list to a dictionary, but for now only handle key_value objects with stringValue - span_attributes = dict(get_keyvalue_generator(span["attributes"])) + span_attributes = span["attributes"] method = span_attributes.get("http.method") or span_attributes.get("http.request.method") status_code = span_attributes.get("http.status_code") or span_attributes.get("http.response.status_code") assert method == "GET", f"HTTP method is not GET, got {method}" diff --git a/utils/interfaces/_open_telemetry.py b/utils/interfaces/_open_telemetry.py index c7809ec356d..aa893b9a510 100644 --- a/utils/interfaces/_open_telemetry.py +++ b/utils/interfaces/_open_telemetry.py @@ -37,13 +37,11 @@ def get_otel_trace_id(self, request: HttpResponse): scope_spans = resource_span.get("scopeSpans") or [] for scope_span in scope_spans: for span in scope_span.get("spans", []): - for attribute in span.get("attributes", []): - attr_key = attribute.get("key") - attr_val = attribute.get("value", {}).get("stringValue") - if (attr_key == "http.request.headers.user-agent" and rid in attr_val) or ( - attr_key == "http.useragent" and rid in attr_val - ): - yield span.get("trace_id") or span.get("traceId") + attributes = span.get("attributes", {}) + request_headers_user_agent_value = attributes.get("http.request.headers.user-agent", "") + user_agent_value = attributes.get("http.useragent", "") + if rid in request_headers_user_agent_value or rid in user_agent_value: + yield span.get("trace_id") or span.get("traceId") def get_otel_spans(self, request: HttpResponse): paths = ["/api/v0.2/traces", "/v1/traces"] @@ -54,16 +52,15 @@ def get_otel_spans(self, request: HttpResponse): for data in self.get_data(path_filters=paths): content = data.get("request").get("content") - resource_spans = content.get("resourceSpans") + logger.debug(f"[get_otel_spans] content: {content}") + resource_spans = content.get("resourceSpans") or [] for resource_span in resource_spans: scope_spans = resource_span.get("scopeSpans") for scope_span in scope_spans: for span in scope_span.get("spans"): - for attribute in span.get("attributes", []): - attr_key = attribute.get("key") - attr_val = attribute.get("value").get("stringValue") - if (attr_key == "http.request.headers.user-agent" and rid in attr_val) or ( - attr_key == "http.useragent" and rid in attr_val - ): - yield data.get("request"), content, span - break # Skip to next span + attributes = span.get("attributes", {}) + request_headers_user_agent_value = attributes.get("http.request.headers.user-agent", "") + user_agent_value = attributes.get("http.useragent", "") + if rid in request_headers_user_agent_value or rid in user_agent_value: + yield data.get("request"), content, span + break # Skip to next span diff --git a/utils/proxy/_deserializer.py b/utils/proxy/_deserializer.py index 79c36072092..b87d3fd489d 100644 --- a/utils/proxy/_deserializer.py +++ b/utils/proxy/_deserializer.py @@ -27,6 +27,7 @@ ) from ._decoders.protobuf_schemas import MetricPayload, TracePayload, SketchPayload, BackendResponsePayload from .traces.trace_v1 import deserialize_v1_trace, _uncompress_agent_v1_trace, decode_appsec_s_value +from .traces.otlp_v1 import deserialize_otlp_v1_trace from .utils import logger @@ -126,6 +127,9 @@ def json_load(): return None if content_type and any(mime_type in content_type for mime_type in ("application/json", "text/json")): + # For OTLP traces, flatten some attributes to simplify the payload for testing purposes + if path == "/v1/traces": + return deserialize_otlp_v1_trace(json_load()) return json_load() if path == "/v0.7/config": # Kyle, please add content-type header :) @@ -178,13 +182,13 @@ def json_load(): assert isinstance(content, bytes) dd_protocol = get_header_value("dd-protocol", message["headers"]) if dd_protocol == "otlp" and "traces" in path: - return MessageToDict(ExportTraceServiceRequest.FromString(content), preserving_proto_field_name=False, use_integers_for_enums=True) + return deserialize_otlp_v1_trace(MessageToDict(ExportTraceServiceRequest.FromString(content), preserving_proto_field_name=False, use_integers_for_enums=True)) if dd_protocol == "otlp" and "metrics" in path: return MessageToDict(ExportMetricsServiceRequest.FromString(content), preserving_proto_field_name=False, use_integers_for_enums=True) if dd_protocol == "otlp" and "logs" in path: return MessageToDict(ExportLogsServiceRequest.FromString(content), preserving_proto_field_name=False, use_integers_for_enums=True) if path == "/v1/traces": - return MessageToDict(ExportTraceServiceResponse.FromString(content), preserving_proto_field_name=False, use_integers_for_enums=True) + return deserialize_otlp_v1_trace(MessageToDict(ExportTraceServiceResponse.FromString(content), preserving_proto_field_name=False, use_integers_for_enums=True)) if path == "/v1/metrics": return MessageToDict(ExportMetricsServiceResponse.FromString(content), preserving_proto_field_name=False, use_integers_for_enums=True) if path == "/v1/logs": diff --git a/utils/proxy/traces/otlp_v1.py b/utils/proxy/traces/otlp_v1.py new file mode 100644 index 00000000000..d88fcc28950 --- /dev/null +++ b/utils/proxy/traces/otlp_v1.py @@ -0,0 +1,67 @@ +from collections.abc import Iterator +from typing import Any + + +def _flatten_otlp_attributes(attributes: list[dict]) -> Iterator[tuple[str, Any]]: + for key_value in attributes: + v = key_value["value"] + # Use `is not None` rather than truthiness so zero/false/empty values are not skipped. + if v.get("stringValue") is not None: + yield key_value["key"], v["stringValue"] + elif v.get("boolValue") is not None: + yield key_value["key"], v["boolValue"] + elif v.get("intValue") is not None: + yield key_value["key"], v["intValue"] + elif v.get("doubleValue") is not None: + yield key_value["key"], v["doubleValue"] + elif v.get("arrayValue") is not None: + yield key_value["key"], v["arrayValue"] + elif v.get("kvlistValue") is not None: + yield key_value["key"], v["kvlistValue"] + elif v.get("bytesValue") is not None: + yield key_value["key"], v["bytesValue"] + else: + raise ValueError(f"Unknown attribute value: {v}") + + +def deserialize_otlp_v1_trace(content: dict) -> dict: + # Iterate the OTLP payload to flatten any attributes dictionary + # Attributes are represented in the following way: + # - {"key": "value": { "stringValue": }} + # - {"key": "value": { "boolValue": }} + # - etc. + # + # We'll remap them to simple key-value pairs {"key": , "key2": , etc.} + for resource_span in content.get("resourceSpans", []): + + resource = resource_span.get("resource", {}) + if resource: + remapped_attributes = dict(_flatten_otlp_attributes(resource.get("attributes", []))) + resource["attributes"] = remapped_attributes + + for scope_span in resource_span.get("scopeSpans", []): + scope = scope_span.get("scope", {}) + scope_attributes = scope.get("attributes", []) + if scope and scope_attributes: + remapped_attributes = dict(_flatten_otlp_attributes(scope_attributes)) + scope["attributes"] = remapped_attributes + + for span in scope_span.get("spans", []): + span_attributes = span.get("attributes", []) + if span_attributes: + remapped_attributes = dict(_flatten_otlp_attributes(span_attributes)) + span["attributes"] = remapped_attributes + + for event in span.get("events", []): + event_attributes = event.get("attributes", []) + if event_attributes: + remapped_attributes = dict(_flatten_otlp_attributes(event_attributes)) + event["attributes"] = remapped_attributes + + for link in span.get("links", []): + link_attributes = link.get("attributes", []) + if link_attributes: + remapped_attributes = dict(_flatten_otlp_attributes(link_attributes)) + link["attributes"] = remapped_attributes + + return content From d0ce4dde286e35d5fe3b32d3aedff552dc9d69d1 Mon Sep 17 00:00:00 2001 From: Zach Montoya Date: Mon, 16 Mar 2026 16:02:27 -0700 Subject: [PATCH 15/29] Run the formatter --- utils/proxy/_deserializer.py | 40 +++++++++++++++++++++++++++++------ utils/proxy/traces/otlp_v1.py | 1 - 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/utils/proxy/_deserializer.py b/utils/proxy/_deserializer.py index b87d3fd489d..bf13871981d 100644 --- a/utils/proxy/_deserializer.py +++ b/utils/proxy/_deserializer.py @@ -182,17 +182,45 @@ def json_load(): assert isinstance(content, bytes) dd_protocol = get_header_value("dd-protocol", message["headers"]) if dd_protocol == "otlp" and "traces" in path: - return deserialize_otlp_v1_trace(MessageToDict(ExportTraceServiceRequest.FromString(content), preserving_proto_field_name=False, use_integers_for_enums=True)) + return deserialize_otlp_v1_trace( + MessageToDict( + ExportTraceServiceRequest.FromString(content), + preserving_proto_field_name=False, + use_integers_for_enums=True, + ) + ) if dd_protocol == "otlp" and "metrics" in path: - return MessageToDict(ExportMetricsServiceRequest.FromString(content), preserving_proto_field_name=False, use_integers_for_enums=True) + return MessageToDict( + ExportMetricsServiceRequest.FromString(content), + preserving_proto_field_name=False, + use_integers_for_enums=True, + ) if dd_protocol == "otlp" and "logs" in path: - return MessageToDict(ExportLogsServiceRequest.FromString(content), preserving_proto_field_name=False, use_integers_for_enums=True) + return MessageToDict( + ExportLogsServiceRequest.FromString(content), + preserving_proto_field_name=False, + use_integers_for_enums=True, + ) if path == "/v1/traces": - return deserialize_otlp_v1_trace(MessageToDict(ExportTraceServiceResponse.FromString(content), preserving_proto_field_name=False, use_integers_for_enums=True)) + return deserialize_otlp_v1_trace( + MessageToDict( + ExportTraceServiceResponse.FromString(content), + preserving_proto_field_name=False, + use_integers_for_enums=True, + ) + ) if path == "/v1/metrics": - return MessageToDict(ExportMetricsServiceResponse.FromString(content), preserving_proto_field_name=False, use_integers_for_enums=True) + return MessageToDict( + ExportMetricsServiceResponse.FromString(content), + preserving_proto_field_name=False, + use_integers_for_enums=True, + ) if path == "/v1/logs": - return MessageToDict(ExportLogsServiceResponse.FromString(content), preserving_proto_field_name=False, use_integers_for_enums=True) + return MessageToDict( + ExportLogsServiceResponse.FromString(content), + preserving_proto_field_name=False, + use_integers_for_enums=True, + ) if path == "/api/v0.2/traces": result = MessageToDict(TracePayload.FromString(content)) _deserialized_nested_json_from_trace_payloads(result, interface) diff --git a/utils/proxy/traces/otlp_v1.py b/utils/proxy/traces/otlp_v1.py index d88fcc28950..b8c18e66a04 100644 --- a/utils/proxy/traces/otlp_v1.py +++ b/utils/proxy/traces/otlp_v1.py @@ -33,7 +33,6 @@ def deserialize_otlp_v1_trace(content: dict) -> dict: # # We'll remap them to simple key-value pairs {"key": , "key2": , etc.} for resource_span in content.get("resourceSpans", []): - resource = resource_span.get("resource", {}) if resource: remapped_attributes = dict(_flatten_otlp_attributes(resource.get("attributes", []))) From 43271c40e7b5c20c23e5deadb28ecebf20e4f927 Mon Sep 17 00:00:00 2001 From: Zach Montoya Date: Tue, 17 Mar 2026 08:41:33 -0700 Subject: [PATCH 16/29] Fix imports after merge --- tests/otel/test_tracing_otlp.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/otel/test_tracing_otlp.py b/tests/otel/test_tracing_otlp.py index b0e1d3aa7f4..211dcd6d8fa 100644 --- a/tests/otel/test_tracing_otlp.py +++ b/tests/otel/test_tracing_otlp.py @@ -5,6 +5,7 @@ import time import re from utils import weblog, interfaces, scenarios, features +from utils.dd_constants import SpanKind, StatusCode # @scenarios.apm_tracing_e2e_otel From 3c77f0ec7a805e3de99cd3e0ff6380b82f415b18 Mon Sep 17 00:00:00 2001 From: Zach Montoya Date: Fri, 20 Mar 2026 09:54:53 -0700 Subject: [PATCH 17/29] Remove unnecessary transformation for OTLP traces response --- utils/proxy/_deserializer.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/utils/proxy/_deserializer.py b/utils/proxy/_deserializer.py index 9b6ce054f2b..78bf2945736 100644 --- a/utils/proxy/_deserializer.py +++ b/utils/proxy/_deserializer.py @@ -202,12 +202,10 @@ def json_load(): use_integers_for_enums=True, ) if path == "/v1/traces": - return deserialize_otlp_v1_trace( - MessageToDict( - ExportTraceServiceResponse.FromString(content), - preserving_proto_field_name=False, - use_integers_for_enums=True, - ) + return MessageToDict( + ExportTraceServiceResponse.FromString(content), + preserving_proto_field_name=False, + use_integers_for_enums=True, ) if path == "/v1/metrics": return MessageToDict( From 8d1af4773fa7246db162960e943a3cb35e614d98 Mon Sep 17 00:00:00 2001 From: Zach Montoya Date: Fri, 20 Mar 2026 10:22:42 -0700 Subject: [PATCH 18/29] Fix possible bug in calculating which interfaces to start. This would explain why other scenarios are facing issues with the "Library not ready" messages --- utils/_context/_scenarios/endtoend.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/utils/_context/_scenarios/endtoend.py b/utils/_context/_scenarios/endtoend.py index 92fe0266b93..b35dc53295e 100644 --- a/utils/_context/_scenarios/endtoend.py +++ b/utils/_context/_scenarios/endtoend.py @@ -365,12 +365,11 @@ def _get_weblog_system_info(self): logger.stdout("") def _start_interfaces_watchdog(self): + open_telemetry_interfaces = [interfaces.open_telemetry] if self.include_opentelemetry else [] super().start_interfaces_watchdog( [interfaces.library, interfaces.agent] + [container.interface for container in self.buddies] - + [interfaces.open_telemetry] - if self.include_opentelemetry - else [] + + open_telemetry_interfaces ) def _set_weblog_domain(self): From 2c037be3387eaeb91558f525992145d86c893798 Mon Sep 17 00:00:00 2001 From: Zach Montoya Date: Fri, 20 Mar 2026 12:37:50 -0700 Subject: [PATCH 19/29] Fix Python linting issues --- utils/_context/_scenarios/endtoend.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/utils/_context/_scenarios/endtoend.py b/utils/_context/_scenarios/endtoend.py index b35dc53295e..bf9520884f7 100644 --- a/utils/_context/_scenarios/endtoend.py +++ b/utils/_context/_scenarios/endtoend.py @@ -365,7 +365,9 @@ def _get_weblog_system_info(self): logger.stdout("") def _start_interfaces_watchdog(self): - open_telemetry_interfaces = [interfaces.open_telemetry] if self.include_opentelemetry else [] + open_telemetry_interfaces: list[ProxyBasedInterfaceValidator] = ( + [interfaces.open_telemetry] if self.include_opentelemetry else [] + ) super().start_interfaces_watchdog( [interfaces.library, interfaces.agent] + [container.interface for container in self.buddies] From 5f73ad4c0b824806a60def6824c154c4ac722112 Mon Sep 17 00:00:00 2001 From: Zach Montoya Date: Fri, 20 Mar 2026 14:28:24 -0700 Subject: [PATCH 20/29] Skip NGINX for new OTLP tracing tests --- manifests/cpp_nginx.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/manifests/cpp_nginx.yml b/manifests/cpp_nginx.yml index 5e7df1d66f7..1a268b1c640 100644 --- a/manifests/cpp_nginx.yml +++ b/manifests/cpp_nginx.yml @@ -289,6 +289,7 @@ manifest: tests/integrations/test_service_overrides.py::Test_SqlServiceNameSource: irrelevant (Only implemented for Java) tests/integrations/test_sql.py::Test_Sql: missing_feature (Endpoint is not implemented on weblog) tests/otel/test_context_propagation.py::Test_Otel_Context_Propagation_Default_Propagator_Api: irrelevant (library does not implement OpenTelemetry) + tests/otel/test_tracing_otlp.py::Test_Otel_Tracing_OTLP: irrelevant (library does not implement OpenTelemetry) tests/otel_tracing_e2e/test_e2e.py::Test_OTelLogE2E: irrelevant tests/otel_tracing_e2e/test_e2e.py::Test_OTelMetricE2E: irrelevant tests/otel_tracing_e2e/test_e2e.py::Test_OTelTracingE2E: irrelevant From b1fa3b828332c787036f932239cc5bf9a48b3bf7 Mon Sep 17 00:00:00 2001 From: Zach Montoya Date: Tue, 24 Mar 2026 13:43:36 -0700 Subject: [PATCH 21/29] Fix the assertion for the span start time by issuing a 'date -u +%s%N' call to the weblog container --- tests/otel/test_tracing_otlp.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/otel/test_tracing_otlp.py b/tests/otel/test_tracing_otlp.py index 211dcd6d8fa..c2c99956b1e 100644 --- a/tests/otel/test_tracing_otlp.py +++ b/tests/otel/test_tracing_otlp.py @@ -2,9 +2,10 @@ # This product includes software developed at Datadog (https://www.datadoghq.com/). # Copyright 2024 Datadog, Inc. -import time import re -from utils import weblog, interfaces, scenarios, features + +from utils import context, features, interfaces, scenarios, weblog +from utils._context._scenarios.endtoend import EndToEndScenario from utils.dd_constants import SpanKind, StatusCode @@ -13,7 +14,14 @@ @scenarios.apm_tracing_otlp class Test_Otel_Tracing_OTLP: def setup_single_server_trace(self): - self.start_time_ns = time.time_ns() + # Get the start time of the weblog container in nanoseconds + assert isinstance(context.scenario, EndToEndScenario) + exit_code, output = context.scenario.weblog_container.execute_command("date -u +%s%N") + assert exit_code == 0, f"self.start_time_ns: date -u +%s%N in weblog container failed: {output!r}" + stripped = output.strip() + assert stripped, f"empty output from date -u +%s%N in weblog container: {output!r}" + + self.start_time_ns = int(stripped) self.req = weblog.get("/") def test_single_server_trace(self): From 2287e1bb2d8dba3dbb533ecc22357fdf8c3e38ca Mon Sep 17 00:00:00 2001 From: Zach Montoya Date: Tue, 24 Mar 2026 13:44:18 -0700 Subject: [PATCH 22/29] Update dotnet manifest with latest development version that implements the tracing OTLP export feature --- manifests/dotnet.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml index a9d363ef4e2..5fb7e964501 100644 --- a/manifests/dotnet.yml +++ b/manifests/dotnet.yml @@ -756,7 +756,7 @@ manifest: tests/k8s_lib_injection/test_k8s_lib_injection_appsec.py::TestK8sLibInjectionAppsecClusterEnabled: v3.36.0 tests/k8s_lib_injection/test_k8s_lib_injection_appsec.py::TestK8sLibInjectionAppsecDisabledByDefault: v3.36.0 tests/otel/test_context_propagation.py::Test_Otel_Context_Propagation_Default_Propagator_Api: v3.9.0 - tests/otel/test_tracing_otlp.py::Test_Otel_Tracing_OTLP: missing_feature + tests/otel/test_tracing_otlp.py::Test_Otel_Tracing_OTLP: v3.41.0 tests/otel_tracing_e2e/test_e2e.py::Test_OTelLogE2E: irrelevant tests/otel_tracing_e2e/test_e2e.py::Test_OTelMetricE2E: irrelevant tests/otel_tracing_e2e/test_e2e.py::Test_OTelTracingE2E: irrelevant From 4d386dca7ff846b656a37b0aecf9e74c4433484e Mon Sep 17 00:00:00 2001 From: Zach Montoya Date: Thu, 26 Mar 2026 08:37:40 -0700 Subject: [PATCH 23/29] Remove usage of DD_API_KEY from new APM_TRACING_OTLP scenario --- .github/workflows/run-end-to-end.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/run-end-to-end.yml b/.github/workflows/run-end-to-end.yml index e6ac962b475..b2dafc25f6f 100644 --- a/.github/workflows/run-end-to-end.yml +++ b/.github/workflows/run-end-to-end.yml @@ -238,8 +238,6 @@ jobs: - name: Run APM_TRACING_OTLP scenario if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"APM_TRACING_OTLP"') run: ./run.sh APM_TRACING_OTLP - env: - DD_API_KEY: ${{ secrets.DD_API_KEY }} - name: Run APM_TRACING_EFFICIENT_PAYLOAD scenario if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"APM_TRACING_EFFICIENT_PAYLOAD"') run: ./run.sh APM_TRACING_EFFICIENT_PAYLOAD From 84d68be38477dcdf41250a6479ca76560711b7ac Mon Sep 17 00:00:00 2001 From: Zach Montoya Date: Thu, 26 Mar 2026 09:32:18 -0700 Subject: [PATCH 24/29] Add several new assertions: - Assert that resource attribute telemetry.sdk.name=datadog - Assert that span attribute span.type=web - Assert that span attribute operation.name is present --- tests/otel/test_tracing_otlp.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/otel/test_tracing_otlp.py b/tests/otel/test_tracing_otlp.py index c2c99956b1e..3a6d9d4d7a5 100644 --- a/tests/otel/test_tracing_otlp.py +++ b/tests/otel/test_tracing_otlp.py @@ -53,7 +53,7 @@ def test_single_server_trace(self): attributes.get("deployment.environment.name") == "system-tests" or attributes.get("deployment.environment") == "system-tests" ) - # assert attributes.get("telemetry.sdk.name") == "datadog" + assert attributes.get("telemetry.sdk.name") == "datadog" assert "telemetry.sdk.language" in attributes # assert "telemetry.sdk.version" in attributes @@ -76,16 +76,20 @@ def test_single_server_trace(self): assert span["name"] assert span["kind"] == SpanKind.SERVER.value - assert span["attributes"] is not None status = span.get("status", {}) # An absent or empty status dict both mean STATUS_CODE_UNSET (protobuf default = 0). assert ( not status or status.get("code", StatusCode.STATUS_CODE_UNSET.value) == StatusCode.STATUS_CODE_UNSET.value ) + # Assert core span attributes + assert span["attributes"] is not None + span_attributes = span["attributes"] + assert span_attributes["span.type"] == "web" + assert span_attributes["operation.name"] is not None + # Assert HTTP tags # Convert attributes list to a dictionary, but for now only handle key_value objects with stringValue - span_attributes = span["attributes"] method = span_attributes.get("http.method") or span_attributes.get("http.request.method") status_code = span_attributes.get("http.status_code") or span_attributes.get("http.response.status_code") assert method == "GET", f"HTTP method is not GET, got {method}" From 470194c2430bf2f8d2c1684c2e68ded16c3ee2b2 Mon Sep 17 00:00:00 2001 From: Zach Montoya Date: Thu, 26 Mar 2026 09:38:07 -0700 Subject: [PATCH 25/29] Skip dotnet for the updated Test_Otel_Tracing_OTLP tests because we don't implement the latest required span attributes --- manifests/dotnet.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml index a1c1377946c..721fae3871e 100644 --- a/manifests/dotnet.yml +++ b/manifests/dotnet.yml @@ -756,7 +756,7 @@ manifest: tests/k8s_lib_injection/test_k8s_lib_injection_appsec.py::TestK8sLibInjectionAppsecClusterEnabled: v3.36.0 tests/k8s_lib_injection/test_k8s_lib_injection_appsec.py::TestK8sLibInjectionAppsecDisabledByDefault: v3.36.0 tests/otel/test_context_propagation.py::Test_Otel_Context_Propagation_Default_Propagator_Api: v3.9.0 - tests/otel/test_tracing_otlp.py::Test_Otel_Tracing_OTLP: v3.41.0 + tests/otel/test_tracing_otlp.py::Test_Otel_Tracing_OTLP: missing_feature tests/otel_tracing_e2e/test_e2e.py::Test_OTelLogE2E: irrelevant tests/otel_tracing_e2e/test_e2e.py::Test_OTelMetricE2E: irrelevant tests/otel_tracing_e2e/test_e2e.py::Test_OTelTracingE2E: irrelevant From 00b44371e0be8ef57af809c063940e90c3720ab5 Mon Sep 17 00:00:00 2001 From: Zach Montoya Date: Thu, 26 Mar 2026 18:27:42 -0700 Subject: [PATCH 26/29] Update the core span attribute assertions: - span.attributes["resource.name"] must match the span.Name field - span.attributes["service.name"] must not be present or it must match the reported "weblog" service --- tests/otel/test_tracing_otlp.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/otel/test_tracing_otlp.py b/tests/otel/test_tracing_otlp.py index 3a6d9d4d7a5..a807a4d6605 100644 --- a/tests/otel/test_tracing_otlp.py +++ b/tests/otel/test_tracing_otlp.py @@ -85,6 +85,8 @@ def test_single_server_trace(self): # Assert core span attributes assert span["attributes"] is not None span_attributes = span["attributes"] + assert span_attributes.get("service.name") == "weblog" or span_attributes.get("service.name") == None + assert span_attributes["resource.name"] == span["name"] assert span_attributes["span.type"] == "web" assert span_attributes["operation.name"] is not None From 3c52d474ead139c8141e36726d673d7296fceac4 Mon Sep 17 00:00:00 2001 From: Zach Montoya Date: Thu, 26 Mar 2026 18:29:51 -0700 Subject: [PATCH 27/29] Fix linting issue --- tests/otel/test_tracing_otlp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/otel/test_tracing_otlp.py b/tests/otel/test_tracing_otlp.py index a807a4d6605..e2082fd2da8 100644 --- a/tests/otel/test_tracing_otlp.py +++ b/tests/otel/test_tracing_otlp.py @@ -85,7 +85,7 @@ def test_single_server_trace(self): # Assert core span attributes assert span["attributes"] is not None span_attributes = span["attributes"] - assert span_attributes.get("service.name") == "weblog" or span_attributes.get("service.name") == None + assert span_attributes.get("service.name") == "weblog" or span_attributes.get("service.name") is None assert span_attributes["resource.name"] == span["name"] assert span_attributes["span.type"] == "web" assert span_attributes["operation.name"] is not None From 89a04803c086cbab4337c1a874b2f1727efbcfe2 Mon Sep 17 00:00:00 2001 From: Zach Montoya Date: Fri, 27 Mar 2026 09:05:46 -0700 Subject: [PATCH 28/29] Remove the API Key requirement from the scenario --- utils/_context/_scenarios/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/utils/_context/_scenarios/__init__.py b/utils/_context/_scenarios/__init__.py index 882e80540da..2b8abf6223d 100644 --- a/utils/_context/_scenarios/__init__.py +++ b/utils/_context/_scenarios/__init__.py @@ -587,7 +587,6 @@ class _Scenarios: "OTEL_EXPORTER_OTLP_TRACES_HEADERS": "dd-protocol=otlp,dd-otlp-path=agent", }, backend_interface_timeout=5, - require_api_key=True, include_opentelemetry=True, doc="", ) From 783437d830600f8047a67a37b0d9d60b028a3e9b Mon Sep 17 00:00:00 2001 From: Zach Montoya Date: Mon, 30 Mar 2026 14:26:28 -0700 Subject: [PATCH 29/29] Remove unnecessary use of DD_API_KEY in weblog container --- utils/_context/_scenarios/endtoend.py | 1 - 1 file changed, 1 deletion(-) diff --git a/utils/_context/_scenarios/endtoend.py b/utils/_context/_scenarios/endtoend.py index bf9520884f7..3b276ebf344 100644 --- a/utils/_context/_scenarios/endtoend.py +++ b/utils/_context/_scenarios/endtoend.py @@ -292,7 +292,6 @@ def configure(self, config: pytest.Config): self.weblog_infra.configure(config) self._set_containers_dependancies() - self.weblog_container.environment["DD_API_KEY"] = os.environ.get("DD_API_KEY") super().configure(config) interfaces.agent.configure(self.host_log_folder, replay=self.replay)