diff --git a/.github/workflows/run-end-to-end.yml b/.github/workflows/run-end-to-end.yml index 7970e5569e0..96172d0da10 100644 --- a/.github/workflows/run-end-to-end.yml +++ b/.github/workflows/run-end-to-end.yml @@ -235,6 +235,9 @@ jobs: env: DD_API_KEY: ${{ secrets.DD_API_KEY }} DD_APP_KEY: ${{ secrets.DD_APPLICATION_KEY }} + - name: Run APM_TRACING_OTLP scenario + if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"APM_TRACING_OTLP"') + run: ./run.sh APM_TRACING_OTLP - name: Run APM_TRACING_EFFICIENT_PAYLOAD scenario if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"APM_TRACING_EFFICIENT_PAYLOAD"') run: ./run.sh APM_TRACING_EFFICIENT_PAYLOAD diff --git a/manifests/cpp_nginx.yml b/manifests/cpp_nginx.yml index 536dde96fc2..53ac1cd67aa 100644 --- a/manifests/cpp_nginx.yml +++ b/manifests/cpp_nginx.yml @@ -291,6 +291,7 @@ manifest: tests/integrations/test_service_overrides.py::Test_SqlServiceNameSource: irrelevant (Only implemented for Java) tests/integrations/test_sql.py::Test_Sql: missing_feature (Endpoint is not implemented on weblog) tests/otel/test_context_propagation.py::Test_Otel_Context_Propagation_Default_Propagator_Api: irrelevant (library does not implement OpenTelemetry) + tests/otel/test_tracing_otlp.py::Test_Otel_Tracing_OTLP: irrelevant (library does not implement OpenTelemetry) tests/otel_tracing_e2e/test_e2e.py::Test_OTelLogE2E: irrelevant tests/otel_tracing_e2e/test_e2e.py::Test_OTelMetricE2E: irrelevant tests/otel_tracing_e2e/test_e2e.py::Test_OTelTracingE2E: irrelevant diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml index e49fb25082f..a1ad8143c32 100644 --- a/manifests/dotnet.yml +++ b/manifests/dotnet.yml @@ -756,6 +756,7 @@ manifest: tests/k8s_lib_injection/test_k8s_lib_injection_appsec.py::TestK8sLibInjectionAppsecClusterEnabled: v3.36.0 tests/k8s_lib_injection/test_k8s_lib_injection_appsec.py::TestK8sLibInjectionAppsecDisabledByDefault: v3.36.0 tests/otel/test_context_propagation.py::Test_Otel_Context_Propagation_Default_Propagator_Api: v3.9.0 + tests/otel/test_tracing_otlp.py::Test_Otel_Tracing_OTLP: missing_feature tests/otel_tracing_e2e/test_e2e.py::Test_OTelLogE2E: irrelevant tests/otel_tracing_e2e/test_e2e.py::Test_OTelMetricE2E: irrelevant tests/otel_tracing_e2e/test_e2e.py::Test_OTelTracingE2E: irrelevant diff --git a/manifests/golang.yml b/manifests/golang.yml index 6fb9dc15273..6e26f717217 100644 --- a/manifests/golang.yml +++ b/manifests/golang.yml @@ -989,6 +989,7 @@ manifest: - weblog_declaration: "*": incomplete_test_app (endpoint not implemented) net-http: v1.70.1 + tests/otel/test_tracing_otlp.py::Test_Otel_Tracing_OTLP: missing_feature tests/otel_tracing_e2e/test_e2e.py::Test_OTelLogE2E: irrelevant tests/otel_tracing_e2e/test_e2e.py::Test_OTelMetricE2E: irrelevant tests/otel_tracing_e2e/test_e2e.py::Test_OTelTracingE2E: irrelevant diff --git a/manifests/java.yml b/manifests/java.yml index f502d1b3e06..226bef0a08e 100644 --- a/manifests/java.yml +++ b/manifests/java.yml @@ -3474,6 +3474,7 @@ manifest: spring-boot-openliberty: v1.58.2+06122213c8 # Modified by easy win activation script uds-spring-boot: v1.58.2+06122213c8 # Modified by easy win activation script spring-boot-payara: v1.58.2+06122213c8 # Modified by easy win activation script + tests/otel/test_tracing_otlp.py::Test_Otel_Tracing_OTLP: missing_feature tests/otel_tracing_e2e/test_e2e.py::Test_OTelLogE2E: irrelevant tests/otel_tracing_e2e/test_e2e.py::Test_OTelMetricE2E: irrelevant tests/otel_tracing_e2e/test_e2e.py::Test_OTelTracingE2E: irrelevant diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml index 13a553b27e5..b3860eff279 100644 --- a/manifests/nodejs.yml +++ b/manifests/nodejs.yml @@ -1860,6 +1860,7 @@ manifest: fastify: *ref_5_26_0 tests/otel/test_context_propagation.py::Test_Otel_Context_Propagation_Default_Propagator_Api::test_propagation_extract: incomplete_test_app (Node.js extract endpoint doesn't seem to be working.) tests/otel/test_context_propagation.py::Test_Otel_Context_Propagation_Default_Propagator_Api::test_propagation_inject: incomplete_test_app (Node.js inject endpoint doesn't seem to be working.) + tests/otel/test_tracing_otlp.py::Test_Otel_Tracing_OTLP: missing_feature tests/otel_tracing_e2e/test_e2e.py::Test_OTelLogE2E: irrelevant tests/otel_tracing_e2e/test_e2e.py::Test_OTelMetricE2E: irrelevant tests/otel_tracing_e2e/test_e2e.py::Test_OTelTracingE2E: irrelevant diff --git a/manifests/php.yml b/manifests/php.yml index 7d80cea05af..69d6acdf4cf 100644 --- a/manifests/php.yml +++ b/manifests/php.yml @@ -620,6 +620,7 @@ manifest: tests/k8s_lib_injection/test_k8s_lib_injection_profiling.py::TestK8sLibInjectioProfilingClusterOverride: v1.9.0 tests/k8s_lib_injection/test_k8s_lib_injection_profiling.py::TestK8sLibInjectioProfilingDisabledByDefault: v1.9.0 tests/otel/test_context_propagation.py::Test_Otel_Context_Propagation_Default_Propagator_Api: incomplete_test_app (endpoint not implemented) + tests/otel/test_tracing_otlp.py::Test_Otel_Tracing_OTLP: missing_feature tests/otel_tracing_e2e/test_e2e.py::Test_OTelLogE2E: irrelevant tests/otel_tracing_e2e/test_e2e.py::Test_OTelMetricE2E: irrelevant tests/otel_tracing_e2e/test_e2e.py::Test_OTelTracingE2E: irrelevant diff --git a/manifests/python.yml b/manifests/python.yml index e0e054a183b..b4488027845 100644 --- a/manifests/python.yml +++ b/manifests/python.yml @@ -1507,6 +1507,7 @@ manifest: flask-poc: v2.19.0 uds-flask: v4.3.1 # Modified by easy win activation script uwsgi-poc: v4.3.1 # Modified by easy win activation script + tests/otel/test_tracing_otlp.py::Test_Otel_Tracing_OTLP: missing_feature tests/otel_tracing_e2e/test_e2e.py::Test_OTelLogE2E: irrelevant tests/otel_tracing_e2e/test_e2e.py::Test_OTelMetricE2E: irrelevant tests/otel_tracing_e2e/test_e2e.py::Test_OTelTracingE2E: irrelevant diff --git a/manifests/ruby.yml b/manifests/ruby.yml index 5ce54d75df1..83491653b1d 100644 --- a/manifests/ruby.yml +++ b/manifests/ruby.yml @@ -1241,6 +1241,7 @@ manifest: "*": incomplete_test_app (endpoint not implemented) rails72: v2.0.0 tests/otel/test_context_propagation.py::Test_Otel_Context_Propagation_Default_Propagator_Api::test_propagation_extract: incomplete_test_app (Ruby extract seems to fail even though it should be supported) + tests/otel/test_tracing_otlp.py::Test_Otel_Tracing_OTLP: missing_feature tests/otel_tracing_e2e/test_e2e.py::Test_OTelLogE2E: irrelevant tests/otel_tracing_e2e/test_e2e.py::Test_OTelMetricE2E: irrelevant tests/otel_tracing_e2e/test_e2e.py::Test_OTelTracingE2E: irrelevant diff --git a/manifests/rust.yml b/manifests/rust.yml index 1712885a96f..4c0cd0c8a41 100644 --- a/manifests/rust.yml +++ b/manifests/rust.yml @@ -47,6 +47,7 @@ manifest: tests/integrations/test_mongo.py::Test_Mongo: missing_feature (Endpoint is not implemented on weblog) tests/integrations/test_service_overrides.py::Test_SqlServiceNameSource: irrelevant (Only implemented for Java) tests/integrations/test_sql.py::Test_Sql: missing_feature (Endpoint is not implemented on weblog) + tests/otel/test_tracing_otlp.py::Test_Otel_Tracing_OTLP: missing_feature tests/otel_tracing_e2e/test_e2e.py::Test_OTelLogE2E: irrelevant tests/otel_tracing_e2e/test_e2e.py::Test_OTelMetricE2E: irrelevant tests/otel_tracing_e2e/test_e2e.py::Test_OTelTracingE2E: irrelevant diff --git a/tests/otel/test_tracing_otlp.py b/tests/otel/test_tracing_otlp.py new file mode 100644 index 00000000000..e2082fd2da8 --- /dev/null +++ b/tests/otel/test_tracing_otlp.py @@ -0,0 +1,109 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the the Apache License Version 2.0. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2024 Datadog, Inc. + +import re + +from utils import context, features, interfaces, scenarios, weblog +from utils._context._scenarios.endtoend import EndToEndScenario +from utils.dd_constants import SpanKind, StatusCode + + +# @scenarios.apm_tracing_e2e_otel +@features.otel_api +@scenarios.apm_tracing_otlp +class Test_Otel_Tracing_OTLP: + def setup_single_server_trace(self): + # Get the start time of the weblog container in nanoseconds + assert isinstance(context.scenario, EndToEndScenario) + exit_code, output = context.scenario.weblog_container.execute_command("date -u +%s%N") + assert exit_code == 0, f"self.start_time_ns: date -u +%s%N in weblog container failed: {output!r}" + stripped = output.strip() + assert stripped, f"empty output from date -u +%s%N in weblog container: {output!r}" + + self.start_time_ns = int(stripped) + self.req = weblog.get("/") + + def test_single_server_trace(self): + """Validates the required elements of the OTLP payload for a single trace""" + data = list(interfaces.open_telemetry.get_otel_spans(self.req)) + + # Assert that there is only one OTLP request containing the desired server span + assert len(data) == 1 + request, content, span = data[0] + + # Determine if JSON Protobuf Encoding was used for the OTLP request (rather than Binary Protobuf) + # We need to assert that we match the OTLP specification, which has some odd encoding rules when using JSON: https://opentelemetry.io/docs/specs/otlp/#json-protobuf-encoding + request_headers = {key.lower(): value for key, value in request.get("headers")} + is_json = request_headers.get("content-type") == "application/json" + + # Assert that there is only one resource span (i.e. SDK) in the OTLP request + resource_spans = content["resourceSpans"] + assert resource_spans is not None, f"missing 'resourceSpans' on content: {content}" + assert len(resource_spans) == 1, f"expected 1 resource span, got {len(resource_spans)}" + resource_span = resource_spans[0] + + attributes = resource_span.get("resource", {}).get("attributes", {}) + + # Assert that the resource attributes contain the service-level attributes and tracer-level attributes we expect + # TODO: Assert the following attributes: runtime-id, git.commit.sha, git.repository_url + assert attributes.get("service.name") == "weblog" + assert attributes.get("service.version") == "1.0.0" + assert ( + attributes.get("deployment.environment.name") == "system-tests" + or attributes.get("deployment.environment") == "system-tests" + ) + assert attributes.get("telemetry.sdk.name") == "datadog" + assert "telemetry.sdk.language" in attributes + # assert "telemetry.sdk.version" in attributes + + # Assert that the `traceId` and `spanId` JSON fields are valid case-insensitive hexadecimal strings, not base64-encoded strings as defined in the standard Protobuf JSON Mapping. + # See https://opentelemetry.io/docs/specs/otlp/#json-protobuf-encoding + # TODO: Assert against trace_id and span_id fields in the protobuf encoding as well + if is_json: + assert re.match(r"^[0-9a-fA-F]{32}$", span.get("traceId")), ( + f"traceId is not a valid case-insensitive hexadecimal string, got {span.get('traceId')}" + ) + assert re.match(r"^[0-9a-fA-F]{16}$", span.get("spanId")), ( + f"spanId is not a valid case-insensitive hexadecimal string, got {span.get('spanId')}" + ) + + # Assert that the span fields match the expected values + span_start_time_ns = int(span["startTimeUnixNano"]) + span_end_time_ns = int(span["endTimeUnixNano"]) + assert span_start_time_ns >= self.start_time_ns + assert span_end_time_ns >= span_start_time_ns + + assert span["name"] + assert span["kind"] == SpanKind.SERVER.value + status = span.get("status", {}) + # An absent or empty status dict both mean STATUS_CODE_UNSET (protobuf default = 0). + assert ( + not status or status.get("code", StatusCode.STATUS_CODE_UNSET.value) == StatusCode.STATUS_CODE_UNSET.value + ) + + # Assert core span attributes + assert span["attributes"] is not None + span_attributes = span["attributes"] + assert span_attributes.get("service.name") == "weblog" or span_attributes.get("service.name") is None + assert span_attributes["resource.name"] == span["name"] + assert span_attributes["span.type"] == "web" + assert span_attributes["operation.name"] is not None + + # Assert HTTP tags + # Convert attributes list to a dictionary, but for now only handle key_value objects with stringValue + method = span_attributes.get("http.method") or span_attributes.get("http.request.method") + status_code = span_attributes.get("http.status_code") or span_attributes.get("http.response.status_code") + assert method == "GET", f"HTTP method is not GET, got {method}" + assert status_code is not None + assert int(status_code) == 200, f"HTTP status code is not 200, got {int(status_code)}" + + def setup_unsampled_trace(self): + self.req = weblog.get("/", headers={"traceparent": "00-11111111111111110000000000000001-0000000000000001-00"}) + + def test_unsampled_trace(self): + """Validates that the spans from a non-sampled trace are not exported.""" + data = list(interfaces.open_telemetry.get_otel_spans(self.req)) + + # Assert that the span from this test case was not exported + assert len(data) == 0, f"Expected no weblog spans in the OTLP trace payload, got {data}" diff --git a/utils/_context/_scenarios/__init__.py b/utils/_context/_scenarios/__init__.py index fd98887a8ec..2b8abf6223d 100644 --- a/utils/_context/_scenarios/__init__.py +++ b/utils/_context/_scenarios/__init__.py @@ -579,6 +579,17 @@ class _Scenarios: require_api_key=True, doc="", ) + apm_tracing_otlp = EndToEndScenario( + "APM_TRACING_OTLP", + weblog_env={ + "OTEL_TRACES_EXPORTER": "otlp", + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": f"http://proxy:{ProxyPorts.open_telemetry_weblog}/v1/traces", + "OTEL_EXPORTER_OTLP_TRACES_HEADERS": "dd-protocol=otlp,dd-otlp-path=agent", + }, + backend_interface_timeout=5, + include_opentelemetry=True, + doc="", + ) apm_tracing_efficient_payload = EndToEndScenario( "APM_TRACING_EFFICIENT_PAYLOAD", diff --git a/utils/_context/_scenarios/endtoend.py b/utils/_context/_scenarios/endtoend.py index d8f5efc34f3..3b276ebf344 100644 --- a/utils/_context/_scenarios/endtoend.py +++ b/utils/_context/_scenarios/endtoend.py @@ -204,6 +204,7 @@ def __init__( runtime_metrics_enabled: bool = False, backend_interface_timeout: int = 0, include_buddies: bool = False, + include_opentelemetry: bool = False, require_api_key: bool = False, other_weblog_containers: tuple[type[TestedContainer], ...] = (), ) -> None: @@ -283,6 +284,7 @@ def __init__( self.agent_interface_timeout = agent_interface_timeout self.backend_interface_timeout = backend_interface_timeout self._library_interface_timeout = library_interface_timeout + self.include_opentelemetry = include_opentelemetry def configure(self, config: pytest.Config): if self._require_api_key and "DD_API_KEY" not in os.environ and not self.replay: @@ -299,6 +301,9 @@ def configure(self, config: pytest.Config): interfaces.library_stdout.configure(self.host_log_folder, replay=self.replay) interfaces.agent_stdout.configure(self.host_log_folder, replay=self.replay) + if self.include_opentelemetry: + interfaces.open_telemetry.configure(self.host_log_folder, replay=self.replay) + for container in self.buddies: container.interface.configure(self.host_log_folder, replay=self.replay) @@ -359,8 +364,13 @@ def _get_weblog_system_info(self): logger.stdout("") def _start_interfaces_watchdog(self): + open_telemetry_interfaces: list[ProxyBasedInterfaceValidator] = ( + [interfaces.open_telemetry] if self.include_opentelemetry else [] + ) super().start_interfaces_watchdog( - [interfaces.library, interfaces.agent] + [container.interface for container in self.buddies] + [interfaces.library, interfaces.agent] + + [container.interface for container in self.buddies] + + open_telemetry_interfaces ) def _set_weblog_domain(self): @@ -420,6 +430,10 @@ def _wait_and_stop_containers(self, *, force_interface_timout_to_zero: bool): interfaces.backend.load_data_from_logs() + if self.include_opentelemetry: + interfaces.open_telemetry.load_data_from_logs() + interfaces.open_telemetry.check_deserialization_errors() + else: self._wait_interface( interfaces.library, 0 if force_interface_timout_to_zero else self.library_interface_timeout @@ -444,6 +458,11 @@ def _wait_and_stop_containers(self, *, force_interface_timout_to_zero: bool): interfaces.backend, 0 if force_interface_timout_to_zero else self.backend_interface_timeout ) + if self.include_opentelemetry: + self._wait_interface( + interfaces.open_telemetry, 0 if force_interface_timout_to_zero else self.backend_interface_timeout + ) + def _wait_interface(self, interface: ProxyBasedInterfaceValidator, timeout: int): logger.terminal.write_sep("-", f"Wait for {interface} ({timeout}s)") logger.terminal.flush() diff --git a/utils/dd_constants.py b/utils/dd_constants.py index b2bf0e3b539..507704bee78 100644 --- a/utils/dd_constants.py +++ b/utils/dd_constants.py @@ -108,6 +108,7 @@ class SamplingMechanism(IntEnum): AI_GUARD = 13 +# See https://github.com/open-telemetry/opentelemetry-proto/blob/v1.9.0/opentelemetry/proto/trace/v1/trace.proto#L153 class SpanKind(IntEnum): UNSPECIFIED = 0 INTERNAL = 1 @@ -115,3 +116,10 @@ class SpanKind(IntEnum): CLIENT = 3 PRODUCER = 4 CONSUMER = 5 + + +# See https://github.com/open-telemetry/opentelemetry-proto/blob/v1.9.0/opentelemetry/proto/trace/v1/trace.proto#L316 +class StatusCode(IntEnum): + STATUS_CODE_UNSET = 0 + STATUS_CODE_OK = 1 + STATUS_CODE_ERROR = 2 diff --git a/utils/interfaces/_open_telemetry.py b/utils/interfaces/_open_telemetry.py index 54c9381631c..aa893b9a510 100644 --- a/utils/interfaces/_open_telemetry.py +++ b/utils/interfaces/_open_telemetry.py @@ -23,6 +23,7 @@ def ingest_file(self, src_path: str): return super().ingest_file(src_path) def get_otel_trace_id(self, request: HttpResponse): + # Paths filter intercepted OTLP export requests (weblog → proxy), not weblog or backend URLs. paths = ["/api/v0.2/traces", "/v1/traces"] rid = request.get_rid() @@ -30,11 +31,36 @@ def get_otel_trace_id(self, request: HttpResponse): logger.debug(f"Try to find traces related to request {rid}") for data in self.get_data(path_filters=paths): - for resource_span in data.get("request").get("content").get("resourceSpans"): - for scope_span in resource_span.get("scopeSpans"): + content = data.get("request").get("content") + resource_spans = content.get("resourceSpans") or [] + for resource_span in resource_spans: + scope_spans = resource_span.get("scopeSpans") or [] + for scope_span in scope_spans: + for span in scope_span.get("spans", []): + attributes = span.get("attributes", {}) + request_headers_user_agent_value = attributes.get("http.request.headers.user-agent", "") + user_agent_value = attributes.get("http.useragent", "") + if rid in request_headers_user_agent_value or rid in user_agent_value: + yield span.get("trace_id") or span.get("traceId") + + def get_otel_spans(self, request: HttpResponse): + paths = ["/api/v0.2/traces", "/v1/traces"] + rid = request.get_rid() + + if rid: + logger.debug(f"Try to find traces related to request {rid}") + + for data in self.get_data(path_filters=paths): + content = data.get("request").get("content") + logger.debug(f"[get_otel_spans] content: {content}") + resource_spans = content.get("resourceSpans") or [] + for resource_span in resource_spans: + scope_spans = resource_span.get("scopeSpans") + for scope_span in scope_spans: for span in scope_span.get("spans"): - for attribute in span.get("attributes", []): - attr_key = attribute.get("key") - attr_val = attribute.get("value").get("stringValue") - if attr_key == "http.request.headers.user-agent" and rid in attr_val: - yield span.get("traceId") + attributes = span.get("attributes", {}) + request_headers_user_agent_value = attributes.get("http.request.headers.user-agent", "") + user_agent_value = attributes.get("http.useragent", "") + if rid in request_headers_user_agent_value or rid in user_agent_value: + yield data.get("request"), content, span + break # Skip to next span diff --git a/utils/proxy/_deserializer.py b/utils/proxy/_deserializer.py index 0da8c8390cb..2e2b4c6d2f0 100644 --- a/utils/proxy/_deserializer.py +++ b/utils/proxy/_deserializer.py @@ -28,6 +28,7 @@ from ._decoders.protobuf_schemas import MetricPayload, TracePayload, SketchPayload, BackendResponsePayload from .trace_bytes_decoding import decode_trace_bytes_ascii, unpack_trace_bytes_msgpack from .traces.trace_v1 import deserialize_v1_trace, _uncompress_agent_v1_trace, decode_appsec_s_value +from .traces.otlp_v1 import deserialize_otlp_v1_trace from .utils import logger @@ -127,6 +128,9 @@ def json_load(): return None if content_type and any(mime_type in content_type for mime_type in ("application/json", "text/json")): + # For OTLP traces, flatten some attributes to simplify the payload for testing purposes + if path == "/v1/traces": + return deserialize_otlp_v1_trace(json_load()) return json_load() if path == "/v0.7/config": # Kyle, please add content-type header :) @@ -179,17 +183,43 @@ def json_load(): assert isinstance(content, bytes) dd_protocol = get_header_value("dd-protocol", message["headers"]) if dd_protocol == "otlp" and "traces" in path: - return MessageToDict(ExportTraceServiceRequest.FromString(content)) + return deserialize_otlp_v1_trace( + MessageToDict( + ExportTraceServiceRequest.FromString(content), + preserving_proto_field_name=False, + use_integers_for_enums=True, + ) + ) if dd_protocol == "otlp" and "metrics" in path: - return MessageToDict(ExportMetricsServiceRequest.FromString(content)) + return MessageToDict( + ExportMetricsServiceRequest.FromString(content), + preserving_proto_field_name=False, + use_integers_for_enums=True, + ) if dd_protocol == "otlp" and "logs" in path: - return MessageToDict(ExportLogsServiceRequest.FromString(content)) + return MessageToDict( + ExportLogsServiceRequest.FromString(content), + preserving_proto_field_name=False, + use_integers_for_enums=True, + ) if path == "/v1/traces": - return MessageToDict(ExportTraceServiceResponse.FromString(content)) + return MessageToDict( + ExportTraceServiceResponse.FromString(content), + preserving_proto_field_name=False, + use_integers_for_enums=True, + ) if path == "/v1/metrics": - return MessageToDict(ExportMetricsServiceResponse.FromString(content)) + return MessageToDict( + ExportMetricsServiceResponse.FromString(content), + preserving_proto_field_name=False, + use_integers_for_enums=True, + ) if path == "/v1/logs": - return MessageToDict(ExportLogsServiceResponse.FromString(content)) + return MessageToDict( + ExportLogsServiceResponse.FromString(content), + preserving_proto_field_name=False, + use_integers_for_enums=True, + ) if path == "/api/v0.2/traces": result = MessageToDict(TracePayload.FromString(content)) _deserialized_nested_json_from_trace_payloads(result, interface) diff --git a/utils/proxy/traces/otlp_v1.py b/utils/proxy/traces/otlp_v1.py new file mode 100644 index 00000000000..b8c18e66a04 --- /dev/null +++ b/utils/proxy/traces/otlp_v1.py @@ -0,0 +1,66 @@ +from collections.abc import Iterator +from typing import Any + + +def _flatten_otlp_attributes(attributes: list[dict]) -> Iterator[tuple[str, Any]]: + for key_value in attributes: + v = key_value["value"] + # Use `is not None` rather than truthiness so zero/false/empty values are not skipped. + if v.get("stringValue") is not None: + yield key_value["key"], v["stringValue"] + elif v.get("boolValue") is not None: + yield key_value["key"], v["boolValue"] + elif v.get("intValue") is not None: + yield key_value["key"], v["intValue"] + elif v.get("doubleValue") is not None: + yield key_value["key"], v["doubleValue"] + elif v.get("arrayValue") is not None: + yield key_value["key"], v["arrayValue"] + elif v.get("kvlistValue") is not None: + yield key_value["key"], v["kvlistValue"] + elif v.get("bytesValue") is not None: + yield key_value["key"], v["bytesValue"] + else: + raise ValueError(f"Unknown attribute value: {v}") + + +def deserialize_otlp_v1_trace(content: dict) -> dict: + # Iterate the OTLP payload to flatten any attributes dictionary + # Attributes are represented in the following way: + # - {"key": "value": { "stringValue": }} + # - {"key": "value": { "boolValue": }} + # - etc. + # + # We'll remap them to simple key-value pairs {"key": , "key2": , etc.} + for resource_span in content.get("resourceSpans", []): + resource = resource_span.get("resource", {}) + if resource: + remapped_attributes = dict(_flatten_otlp_attributes(resource.get("attributes", []))) + resource["attributes"] = remapped_attributes + + for scope_span in resource_span.get("scopeSpans", []): + scope = scope_span.get("scope", {}) + scope_attributes = scope.get("attributes", []) + if scope and scope_attributes: + remapped_attributes = dict(_flatten_otlp_attributes(scope_attributes)) + scope["attributes"] = remapped_attributes + + for span in scope_span.get("spans", []): + span_attributes = span.get("attributes", []) + if span_attributes: + remapped_attributes = dict(_flatten_otlp_attributes(span_attributes)) + span["attributes"] = remapped_attributes + + for event in span.get("events", []): + event_attributes = event.get("attributes", []) + if event_attributes: + remapped_attributes = dict(_flatten_otlp_attributes(event_attributes)) + event["attributes"] = remapped_attributes + + for link in span.get("links", []): + link_attributes = link.get("attributes", []) + if link_attributes: + remapped_attributes = dict(_flatten_otlp_attributes(link_attributes)) + link["attributes"] = remapped_attributes + + return content diff --git a/utils/scripts/ci_orchestrators/workflow_data.py b/utils/scripts/ci_orchestrators/workflow_data.py index 75afc379a55..64c38ed3864 100644 --- a/utils/scripts/ci_orchestrators/workflow_data.py +++ b/utils/scripts/ci_orchestrators/workflow_data.py @@ -592,6 +592,7 @@ def _is_supported(library: str, weblog: str, scenario: str, _ci_environment: str "endtoend": [ "AGENT_NOT_SUPPORTING_SPAN_EVENTS", "APM_TRACING_E2E_OTEL", + "APM_TRACING_OTLP", "APM_TRACING_E2E_SINGLE_SPAN", "APPSEC_API_SECURITY", "APPSEC_API_SECURITY_NO_RESPONSE_BODY",