From 2267c39a71d4f868e552bbd667e7837e1e2ca18f Mon Sep 17 00:00:00 2001 From: Florentin Labelle Date: Mon, 23 Mar 2026 13:47:49 +0100 Subject: [PATCH 1/6] appsec: add smoke tests for apm standalone --- .github/workflows/run-end-to-end.yml | 6 + .../agent-interface-validation-methods.md | 4 +- manifests/agent.yml | 5 +- manifests/cpp_nginx.yml | 1 + manifests/dotnet.yml | 1 + manifests/golang.yml | 1 + manifests/java.yml | 1 + manifests/nodejs.yml | 1 + manifests/php.yml | 1 + manifests/python.yml | 1 + manifests/ruby.yml | 1 + tests/appsec/test_agent_level_smoke_tests.py | 152 ++++++++++++++++++ utils/_context/_scenarios/__init__.py | 34 ++++ utils/_features.py | 12 +- utils/interfaces/_agent.py | 123 +++++++++++++- 15 files changed, 336 insertions(+), 8 deletions(-) create mode 100644 tests/appsec/test_agent_level_smoke_tests.py diff --git a/.github/workflows/run-end-to-end.yml b/.github/workflows/run-end-to-end.yml index 453ab89cc90..f0c052ccc44 100644 --- a/.github/workflows/run-end-to-end.yml +++ b/.github/workflows/run-end-to-end.yml @@ -174,6 +174,12 @@ jobs: - name: Run APPSEC_STANDALONE scenario if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"APPSEC_STANDALONE"') run: ./run.sh APPSEC_STANDALONE + - name: Run APPSEC_APM_STANDALONE scenario + if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"APPSEC_APM_STANDALONE"') + run: ./run.sh APPSEC_APM_STANDALONE + - name: Run APPSEC_STANDALONE_APM_STANDALONE scenario + if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"APPSEC_STANDALONE_APM_STANDALONE"') + run: ./run.sh APPSEC_STANDALONE_APM_STANDALONE - name: Run APPSEC_STANDALONE_API_SECURITY scenario if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"APPSEC_STANDALONE_API_SECURITY"') run: ./run.sh APPSEC_STANDALONE_API_SECURITY diff --git a/docs/edit/agent-interface-validation-methods.md b/docs/edit/agent-interface-validation-methods.md index 561d5dcdd23..899db6c7af5 100644 --- a/docs/edit/agent-interface-validation-methods.md +++ b/docs/edit/agent-interface-validation-methods.md @@ -89,7 +89,7 @@ def test_agent_trace_forwarding(self): def test_appsec_agent_forwarding(self): r = weblog.get("/", headers={"X-Attack": "' OR 1=1--"}) - def appsec_validator(data, payload, chunk, span, appsec_data): + def appsec_validator(data, span, appsec_data): return "triggers" in appsec_data interfaces.agent.validate_appsec(r, appsec_validator) @@ -186,4 +186,4 @@ This means: - [Interface Initialization](../../utils/interfaces/__init__.py) - [Library Interface Validation Methods](./library-interface-validation-methods.md) - [End-to-End Testing Guide](../execute/README.md) -- [Adding New Tests](./add-new-test.md) \ No newline at end of file +- [Adding New Tests](./add-new-test.md) diff --git a/manifests/agent.yml b/manifests/agent.yml index 3b1bc10f016..819ff57e0fa 100644 --- a/manifests/agent.yml +++ b/manifests/agent.yml @@ -3,7 +3,10 @@ manifest: tests/apm_tracing_e2e/test_single_span.py::Test_SingleSpan: - declaration: missing_feature (Single Spans is not available in agents pre 7.40) - component_version: '<7.40.0' + component_version: "<7.40.0" + tests/appsec/test_agent_level_smoke_tests.py::Test_AppSecAPMStandalone: + - component_version: "<7.77.0-0" + declaration: irrelevant (APM Standalone option was added in 7.77.0) tests/integrations/test_open_telemetry.py::Test_MsSql::test_error_exception_event: - component_version: "<7.75.0-0" declaration: irrelevant (new behavior implement in 7.75.0) diff --git a/manifests/cpp_nginx.yml b/manifests/cpp_nginx.yml index 536dde96fc2..823b342a4e8 100644 --- a/manifests/cpp_nginx.yml +++ b/manifests/cpp_nginx.yml @@ -50,6 +50,7 @@ manifest: tests/appsec/iast/test_security_controls.py: irrelevant (ASM is not implemented in C++) tests/appsec/iast/test_vulnerability_schema.py: irrelevant (ASM is not implemented in C++) tests/appsec/rasp/: irrelevant (ASM is not implemented in C++) + tests/appsec/test_agent_level_smoke_tests.py::Test_AppSecAPMStandalone: missing_feature tests/appsec/test_alpha.py: irrelevant (ASM is not implemented in C++) tests/appsec/test_asm_standalone.py::Test_APISecurityStandalone: '>=1.12.0' # Modified by easy win activation script tests/appsec/test_asm_standalone.py::Test_APISecurityStandalone::test_appsec_propagation_does_not_force_schema_collection: missing_feature # Created by easy win activation script diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml index e49fb25082f..b1ab87d7578 100644 --- a/manifests/dotnet.yml +++ b/manifests/dotnet.yml @@ -295,6 +295,7 @@ manifest: tests/appsec/rasp/test_ssrf.py::Test_Ssrf_Telemetry_V2: v3.26.3 tests/appsec/rasp/test_ssrf.py::Test_Ssrf_UrlQuery: v2.51.0 tests/appsec/rasp/test_ssrf.py::Test_Ssrf_Waf_Version: v3.4.1 + tests/appsec/test_agent_level_smoke_tests.py::Test_AppSecAPMStandalone: missing_feature tests/appsec/test_alpha.py::Test_Basic: v1.28.6 tests/appsec/test_asm_standalone.py::Test_APISecurityStandalone: v3.17.0 tests/appsec/test_asm_standalone.py::Test_AppSecStandalone_UpstreamPropagation_V2: v3.12.0 diff --git a/manifests/golang.yml b/manifests/golang.yml index 6fb9dc15273..39ec4a67bfd 100644 --- a/manifests/golang.yml +++ b/manifests/golang.yml @@ -294,6 +294,7 @@ manifest: tests/appsec/rasp/test_ssrf.py::Test_Ssrf_Telemetry_V2: v2.0.0 tests/appsec/rasp/test_ssrf.py::Test_Ssrf_UrlQuery: v1.65.1 tests/appsec/rasp/test_ssrf.py::Test_Ssrf_Waf_Version: v2.0.0 + tests/appsec/test_agent_level_smoke_tests.py::Test_AppSecAPMStandalone: missing_feature tests/appsec/test_alpha.py::Test_Basic: - weblog_declaration: "*": v1.34.0 diff --git a/manifests/java.yml b/manifests/java.yml index f502d1b3e06..871c70e0114 100644 --- a/manifests/java.yml +++ b/manifests/java.yml @@ -1493,6 +1493,7 @@ manifest: - weblog_declaration: "*": v1.40.0 spring-boot-3-native: irrelevant (GraalVM. Tracing support only) + tests/appsec/test_agent_level_smoke_tests.py::Test_AppSecAPMStandalone: missing_feature tests/appsec/test_alpha.py::Test_Basic: - weblog_declaration: "*": v0.87.0 diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml index 13a553b27e5..9ed69677045 100644 --- a/manifests/nodejs.yml +++ b/manifests/nodejs.yml @@ -909,6 +909,7 @@ manifest: fastify: *ref_5_66_0 nextjs: missing_feature tests/appsec/rasp/test_ssrf.py::Test_Ssrf_Waf_Version: *ref_5_25_0 + tests/appsec/test_agent_level_smoke_tests.py::Test_AppSecAPMStandalone: missing_feature tests/appsec/test_alpha.py::Test_Basic: v2.0.0 tests/appsec/test_asm_standalone.py::BaseSCAStandaloneTelemetry::test_app_dependencies_loaded: - weblog_declaration: diff --git a/manifests/php.yml b/manifests/php.yml index 7d80cea05af..db7bba10d5d 100644 --- a/manifests/php.yml +++ b/manifests/php.yml @@ -284,6 +284,7 @@ manifest: tests/appsec/rasp/test_ssrf.py::Test_Ssrf_Telemetry_V2: missing_feature tests/appsec/rasp/test_ssrf.py::Test_Ssrf_UrlQuery: v1.7.0 tests/appsec/rasp/test_ssrf.py::Test_Ssrf_Waf_Version: v1.7.0 + tests/appsec/test_agent_level_smoke_tests.py::Test_AppSecAPMStandalone: missing_feature tests/appsec/test_asm_standalone.py::Test_APISecurityStandalone: v1.11.0 tests/appsec/test_asm_standalone.py::Test_AppSecStandalone_NotEnabled: v1.6.2 tests/appsec/test_asm_standalone.py::Test_AppSecStandalone_UpstreamPropagation_V2: v1.8.0 diff --git a/manifests/python.yml b/manifests/python.yml index e0e054a183b..5fd9d2095dc 100644 --- a/manifests/python.yml +++ b/manifests/python.yml @@ -476,6 +476,7 @@ manifest: "*": v2.10.0 *django: v4.3.0-dev0 (with httpx support) tests/appsec/rasp/test_ssrf.py::Test_Ssrf_Waf_Version: v2.15.0 + tests/appsec/test_agent_level_smoke_tests.py::Test_AppSecAPMStandalone: v4.6.0 tests/appsec/test_alpha.py::Test_Basic: - weblog_declaration: "*": v1.1.0-rc2 diff --git a/manifests/ruby.yml b/manifests/ruby.yml index a3550bcdfec..5c181023fc7 100644 --- a/manifests/ruby.yml +++ b/manifests/ruby.yml @@ -378,6 +378,7 @@ manifest: tests/appsec/rasp/test_ssrf.py::Test_Ssrf_Telemetry_V2: missing_feature tests/appsec/rasp/test_ssrf.py::Test_Ssrf_UrlQuery: v2.14.0 tests/appsec/rasp/test_ssrf.py::Test_Ssrf_Waf_Version: missing_feature # requires Telemetry V2 + tests/appsec/test_agent_level_smoke_tests.py::Test_AppSecAPMStandalone: missing_feature tests/appsec/test_asm_standalone.py::Test_APISecurityStandalone: v2.18.0 tests/appsec/test_asm_standalone.py::Test_AppSecStandalone_NotEnabled: v2.13.0 tests/appsec/test_asm_standalone.py::Test_AppSecStandalone_UpstreamPropagation_V2: v2.13.0 diff --git a/tests/appsec/test_agent_level_smoke_tests.py b/tests/appsec/test_agent_level_smoke_tests.py new file mode 100644 index 00000000000..cc0543a23a5 --- /dev/null +++ b/tests/appsec/test_agent_level_smoke_tests.py @@ -0,0 +1,152 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the the Apache License Version 2.0. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2021 Datadog, Inc. + +"""AppSec smoke tests for the appsec_apm_standalone scenario.""" + +from utils import features, interfaces, remote_config as rc, weblog, scenarios + +SMOKE_RC_RULE_ID = "smoke-rc-0001" +SMOKE_RC_RULE_FILE: tuple[str, dict[str, object]] = ( + "datadog/2/ASM_DD/rules/config", + { + "version": "2.2", + "metadata": {"rules_version": "2.71.8182"}, + "rules": [ + { + "id": SMOKE_RC_RULE_ID, + "name": "Smoke RC rule", + "tags": { + "type": "attack_tool", + "category": "attack_attempt", + "confidence": "1", + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.headers.no_cookies", + "key_path": ["x-smoke-test"], + } + ], + "regex": "^rc-smoke$", + }, + "operator": "match_regex", + } + ], + "transformers": [], + } + ], + }, +) + + +class AgentLevelSmokeTests: + def setup_lfi_smoke(self) -> None: + self.r = weblog.get("/rasp/lfi", params={"file": "../etc/passwd"}) + + def test_lfi_smoke(self) -> None: + assert self.r.status_code == 200 + + interfaces.agent.assert_rasp_attack( + self.r, + "rasp-930-100", + { + "resource": {"address": "server.io.fs.file", "value": "../etc/passwd"}, + "params": {"address": "server.request.query", "value": "../etc/passwd"}, + }, + ) + + def setup_api_security_smoke(self) -> None: + self.r = weblog.get("/waf") + + def test_api_security_smoke(self) -> None: + assert any( + any(key.startswith("_dd.appsec.s.") for key in span.meta) for _, span in interfaces.agent.get_spans(self.r) + ) + + def setup_telemetry_smoke(self) -> None: + weblog.get("/") + self.r = weblog.get("/waf", headers={"User-Agent": "Arachni/v1"}) + + def test_telemetry_smoke(self) -> None: + telemetry_data = list(interfaces.agent.get_telemetry_data(flatten_message_batches=False)) + + assert telemetry_data, "Agent should forward telemetry data from the library" + + found_metrics = False + found_waf_metric = False + + for data in telemetry_data: + content = data.get("request", {}).get("content", {}) + request_type = content.get("request_type") + + if request_type == "generate-metrics": + found_metrics = True + + payload = content.get("payload", {}) + series = payload.get("series", []) + + for metric_series in series: + metric_name = metric_series.get("metric", "") + if metric_name.startswith("waf."): + found_waf_metric = True + break + + if found_metrics and found_waf_metric: + break + + assert found_metrics + assert found_waf_metric + + def setup_remote_config_smoke(self) -> None: + self.config_state = rc.tracer_rc_state.reset().set_config(*SMOKE_RC_RULE_FILE).apply().state + self.r = weblog.get("/waf", headers={"X-Smoke-Test": "rc-smoke"}) + + def test_remote_config_smoke(self) -> None: + assert self.config_state == rc.ApplyState.ACKNOWLEDGED, ( + f"Remote config should be acknowledged, got {self.config_state}" + ) + + assert any( + span.meta.get("appsec.event") == "true" + and any( + trigger.get("rule", {}).get("id") == SMOKE_RC_RULE_ID for trigger in appsec_data.get("triggers", []) + ) + for _, span, appsec_data in interfaces.agent.get_appsec_data(self.r) + ), "Agent should forward AppSec events for the RC-updated rule" + + def setup_attack_detection_smoke(self) -> None: + rc.tracer_rc_state.reset().apply() + self.r = weblog.get("/waf", headers={"User-Agent": "Arachni/v1"}) + + def test_attack_detection_smoke(self) -> None: + found_attack = False + has_waf_version = False + has_appsec_data = False + + for _, span, appsec_data in interfaces.agent.get_appsec_data(self.r): + span_meta = span.get("meta", {}) or span.get("attributes", {}) + if span_meta.get("appsec.event") == "true": + found_attack = True + + if "_dd.appsec.waf.version" in span_meta: + has_waf_version = True + + if appsec_data is not None: + has_appsec_data = True + + if has_waf_version and has_appsec_data: + break + + assert found_attack, "Agent should forward detected attacks in span metadata" + assert has_waf_version, "Agent spans should include WAF version metadata" + assert has_appsec_data, "Agent spans should include AppSec payload (JSON or metastruct)" + + +@features.appsec_apm_standalone +@scenarios.appsec_apm_standalone +@scenarios.appsec_standalone_apm_standalone +class Test_AppSecAPMStandalone(AgentLevelSmokeTests): + pass diff --git a/utils/_context/_scenarios/__init__.py b/utils/_context/_scenarios/__init__.py index fd98887a8ec..ea67e0a06f3 100644 --- a/utils/_context/_scenarios/__init__.py +++ b/utils/_context/_scenarios/__init__.py @@ -428,6 +428,40 @@ class _Scenarios: scenario_groups=[scenario_groups.appsec], ) + appsec_apm_standalone = EndToEndScenario( + "APPSEC_APM_STANDALONE", + rc_api_enabled=True, + weblog_env={ + "DD_APM_TRACING_ENABLED": "true", + "DD_TELEMETRY_METRICS_ENABLED": "true", + "DD_TELEMETRY_METRICS_INTERVAL_SECONDS": "2.0", + "DD_API_SECURITY_REQUEST_SAMPLE_RATE": "1.0", + "DD_API_SECURITY_SAMPLE_DELAY": "0.0", + }, + agent_env={ + "DD_INFRASTRUCTURE_MODE": "none", + }, + doc="Appsec with APM Standalone (infra opt out)", + scenario_groups=[scenario_groups.appsec], + ) + + appsec_standalone_apm_standalone = EndToEndScenario( + "APPSEC_STANDALONE_APM_STANDALONE", + rc_api_enabled=True, + weblog_env={ + "DD_APM_TRACING_ENABLED": "false", + "DD_TELEMETRY_METRICS_ENABLED": "true", + "DD_TELEMETRY_METRICS_INTERVAL_SECONDS": "2.0", + "DD_API_SECURITY_REQUEST_SAMPLE_RATE": "1.0", + "DD_API_SECURITY_SAMPLE_DELAY": "0.0", + }, + agent_env={ + "DD_INFRASTRUCTURE_MODE": "none", + }, + doc="Appsec standalone mode (APM opt out) with APM Standalone (infra opt out)", + scenario_groups=[scenario_groups.appsec], + ) + # Combined scenario for API Security in standalone mode appsec_standalone_api_security = EndToEndScenario( "APPSEC_STANDALONE_API_SECURITY", diff --git a/utils/_features.py b/utils/_features.py index efe41c9da59..b08ff24c5a7 100644 --- a/utils/_features.py +++ b/utils/_features.py @@ -2790,14 +2790,22 @@ def service_override_source(test_object): """ return _mark_test_object(test_object, feature_id=545, owner=_Owner.language_platform) + @staticmethod + def appsec_apm_standalone(test_object): + """Ensure that AppSec works correctly with the infra product disabled + + https://feature-parity.us1.prod.dog/#/?feature=546 + """ + return _mark_test_object(test_object, feature_id=546, owner=_Owner.asm) + @staticmethod def base_service(test_object): """_dd.base_service meta tag is set on spans whose service name differs from the global service. Preserves the originating service context when integrations override the service name. - https://feature-parity.us1.prod.dog/#/?feature=546 + https://feature-parity.us1.prod.dog/#/?feature=549 """ - return _mark_test_object(test_object, feature_id=546, owner=_Owner.idm) + return _mark_test_object(test_object, feature_id=549, owner=_Owner.idm) features = _Features() diff --git a/utils/interfaces/_agent.py b/utils/interfaces/_agent.py index e638c9cb242..55c8e33f303 100644 --- a/utils/interfaces/_agent.py +++ b/utils/interfaces/_agent.py @@ -4,15 +4,21 @@ """Validate data flow between agent and backend""" -from collections.abc import Callable, Generator, Iterable +import base64 +import binascii import copy +import json import threading +from collections.abc import Callable, Generator, Iterable +from typing import Any + +import msgpack -from utils.dd_types import DataDogAgentTrace, DataDogAgentSpan, AgentTraceFormat from utils._logger import logger +from utils._weblog import HttpResponse +from utils.dd_types import AgentTraceFormat, DataDogAgentSpan, DataDogAgentTrace from utils.interfaces._core import ProxyBasedInterfaceValidator from utils.interfaces._misc_validators import HeadersPresenceValidator -from utils._weblog import HttpResponse class AgentInterfaceValidator(ProxyBasedInterfaceValidator): @@ -26,6 +32,117 @@ def ingest_file(self, src_path: str): self.ready.set() return super().ingest_file(src_path) + @staticmethod + def _extract_appsec_data(span: DataDogAgentSpan) -> dict[str, Any] | None: + appsec_json = span.meta.get("_dd.appsec.json") + if isinstance(appsec_json, dict): + return appsec_json + if isinstance(appsec_json, str): + try: + decoded = json.loads(appsec_json) + except json.JSONDecodeError: + decoded = None + if isinstance(decoded, dict): + return decoded + + meta_struct = span.get("meta_struct") or span.get("metaStruct") or None + if not isinstance(meta_struct, dict): + return None + + appsec_payload = meta_struct.get("appsec") + if isinstance(appsec_payload, dict): + return appsec_payload + + if isinstance(appsec_payload, bytes): + decoded_payload = msgpack.loads(appsec_payload, raw=False, strict_map_key=False, unicode_errors="replace") + return decoded_payload if isinstance(decoded_payload, dict) else None + + if isinstance(appsec_payload, str): + try: + payload = base64.b64decode(appsec_payload) + except (ValueError, binascii.Error): + payload = appsec_payload.encode("utf-8") + decoded_payload = msgpack.loads(payload, raw=False, strict_map_key=False, unicode_errors="replace") + return decoded_payload if isinstance(decoded_payload, dict) else None + + return None + + def get_appsec_data( + self, request: HttpResponse + ) -> Generator[tuple[dict[str, Any], DataDogAgentSpan, dict[str, Any]], Any, None]: + for data, span in self.get_spans(request): + appsec_data = self._extract_appsec_data(span) + if appsec_data is None: + continue + + yield data, span, appsec_data + + def assert_rasp_attack( + self, + request: HttpResponse, + rule: str, + parameters: dict | None = None, + ) -> None: + def validator(_: dict, appsec_data: dict[str, Any]) -> bool: + triggers = appsec_data.get("triggers") + assert isinstance(triggers, list), "'triggers' is not a list" + assert triggers, "no appsec triggers found" + + for trigger in triggers: + obtained_rule_id = trigger.get("rule", {}).get("id") + if obtained_rule_id != rule: + continue + + if parameters is None: + return True + + for match in trigger.get("rule_matches", []): + for obtained_parameters in match.get("parameters", []): + if not isinstance(obtained_parameters, dict): + continue + + ok = True + for name, fields in parameters.items(): + if name not in obtained_parameters: + ok = False + break + + obtained_param = obtained_parameters[name] + if not isinstance(obtained_param, dict): + ok = False + break + + address = fields.get("address") + if obtained_param.get("address") != address: + ok = False + break + + if "value" in fields and obtained_param.get("value") != fields["value"]: + ok = False + break + + if "key_path" in fields and obtained_param.get("key_path") != fields["key_path"]: + ok = False + break + + if ok: + return True + + return False + + for data, _, appsec_data in self.get_appsec_data(request=request): + if validator(data, appsec_data): + return + + raise AssertionError("No AppSec payload found for the request") + + def validate_appsec(self, request: HttpResponse, validator: Callable): + for data, span, appsec_data in self.get_appsec_data(request=request): + if validator(data, span, appsec_data): + return + + raise ValueError("No data validate this test") + def get_profiling_data(self): yield from self.get_data(path_filters="/api/v2/profile") From b630a1afb9a4538f9147665e3eb4aab8c0506354 Mon Sep 17 00:00:00 2001 From: Florentin Labelle Date: Thu, 26 Mar 2026 15:29:22 +0100 Subject: [PATCH 2/6] comment to fill things --- manifests/cpp_nginx.yml | 2 +- manifests/dotnet.yml | 2 +- manifests/golang.yml | 2 +- manifests/java.yml | 2 +- manifests/nodejs.yml | 2 +- manifests/php.yml | 2 +- manifests/ruby.yml | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/manifests/cpp_nginx.yml b/manifests/cpp_nginx.yml index 823b342a4e8..f00845b0321 100644 --- a/manifests/cpp_nginx.yml +++ b/manifests/cpp_nginx.yml @@ -50,7 +50,7 @@ manifest: tests/appsec/iast/test_security_controls.py: irrelevant (ASM is not implemented in C++) tests/appsec/iast/test_vulnerability_schema.py: irrelevant (ASM is not implemented in C++) tests/appsec/rasp/: irrelevant (ASM is not implemented in C++) - tests/appsec/test_agent_level_smoke_tests.py::Test_AppSecAPMStandalone: missing_feature + # tests/appsec/test_agent_level_smoke_tests.py::Test_AppSecAPMStandalone: missing_feature tests/appsec/test_alpha.py: irrelevant (ASM is not implemented in C++) tests/appsec/test_asm_standalone.py::Test_APISecurityStandalone: '>=1.12.0' # Modified by easy win activation script tests/appsec/test_asm_standalone.py::Test_APISecurityStandalone::test_appsec_propagation_does_not_force_schema_collection: missing_feature # Created by easy win activation script diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml index b1ab87d7578..866d8820ed9 100644 --- a/manifests/dotnet.yml +++ b/manifests/dotnet.yml @@ -295,7 +295,7 @@ manifest: tests/appsec/rasp/test_ssrf.py::Test_Ssrf_Telemetry_V2: v3.26.3 tests/appsec/rasp/test_ssrf.py::Test_Ssrf_UrlQuery: v2.51.0 tests/appsec/rasp/test_ssrf.py::Test_Ssrf_Waf_Version: v3.4.1 - tests/appsec/test_agent_level_smoke_tests.py::Test_AppSecAPMStandalone: missing_feature + # tests/appsec/test_agent_level_smoke_tests.py::Test_AppSecAPMStandalone: missing_feature tests/appsec/test_alpha.py::Test_Basic: v1.28.6 tests/appsec/test_asm_standalone.py::Test_APISecurityStandalone: v3.17.0 tests/appsec/test_asm_standalone.py::Test_AppSecStandalone_UpstreamPropagation_V2: v3.12.0 diff --git a/manifests/golang.yml b/manifests/golang.yml index 39ec4a67bfd..8196ae361d5 100644 --- a/manifests/golang.yml +++ b/manifests/golang.yml @@ -294,7 +294,7 @@ manifest: tests/appsec/rasp/test_ssrf.py::Test_Ssrf_Telemetry_V2: v2.0.0 tests/appsec/rasp/test_ssrf.py::Test_Ssrf_UrlQuery: v1.65.1 tests/appsec/rasp/test_ssrf.py::Test_Ssrf_Waf_Version: v2.0.0 - tests/appsec/test_agent_level_smoke_tests.py::Test_AppSecAPMStandalone: missing_feature + # tests/appsec/test_agent_level_smoke_tests.py::Test_AppSecAPMStandalone: missing_feature tests/appsec/test_alpha.py::Test_Basic: - weblog_declaration: "*": v1.34.0 diff --git a/manifests/java.yml b/manifests/java.yml index 871c70e0114..2c13e29120b 100644 --- a/manifests/java.yml +++ b/manifests/java.yml @@ -1493,7 +1493,7 @@ manifest: - weblog_declaration: "*": v1.40.0 spring-boot-3-native: irrelevant (GraalVM. Tracing support only) - tests/appsec/test_agent_level_smoke_tests.py::Test_AppSecAPMStandalone: missing_feature + # tests/appsec/test_agent_level_smoke_tests.py::Test_AppSecAPMStandalone: missing_feature tests/appsec/test_alpha.py::Test_Basic: - weblog_declaration: "*": v0.87.0 diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml index 9ed69677045..8fd63637ee0 100644 --- a/manifests/nodejs.yml +++ b/manifests/nodejs.yml @@ -909,7 +909,7 @@ manifest: fastify: *ref_5_66_0 nextjs: missing_feature tests/appsec/rasp/test_ssrf.py::Test_Ssrf_Waf_Version: *ref_5_25_0 - tests/appsec/test_agent_level_smoke_tests.py::Test_AppSecAPMStandalone: missing_feature + # tests/appsec/test_agent_level_smoke_tests.py::Test_AppSecAPMStandalone: missing_feature tests/appsec/test_alpha.py::Test_Basic: v2.0.0 tests/appsec/test_asm_standalone.py::BaseSCAStandaloneTelemetry::test_app_dependencies_loaded: - weblog_declaration: diff --git a/manifests/php.yml b/manifests/php.yml index db7bba10d5d..4fcaf41fdcb 100644 --- a/manifests/php.yml +++ b/manifests/php.yml @@ -284,7 +284,7 @@ manifest: tests/appsec/rasp/test_ssrf.py::Test_Ssrf_Telemetry_V2: missing_feature tests/appsec/rasp/test_ssrf.py::Test_Ssrf_UrlQuery: v1.7.0 tests/appsec/rasp/test_ssrf.py::Test_Ssrf_Waf_Version: v1.7.0 - tests/appsec/test_agent_level_smoke_tests.py::Test_AppSecAPMStandalone: missing_feature + # tests/appsec/test_agent_level_smoke_tests.py::Test_AppSecAPMStandalone: missing_feature tests/appsec/test_asm_standalone.py::Test_APISecurityStandalone: v1.11.0 tests/appsec/test_asm_standalone.py::Test_AppSecStandalone_NotEnabled: v1.6.2 tests/appsec/test_asm_standalone.py::Test_AppSecStandalone_UpstreamPropagation_V2: v1.8.0 diff --git a/manifests/ruby.yml b/manifests/ruby.yml index 5c181023fc7..901b5aeca83 100644 --- a/manifests/ruby.yml +++ b/manifests/ruby.yml @@ -378,7 +378,7 @@ manifest: tests/appsec/rasp/test_ssrf.py::Test_Ssrf_Telemetry_V2: missing_feature tests/appsec/rasp/test_ssrf.py::Test_Ssrf_UrlQuery: v2.14.0 tests/appsec/rasp/test_ssrf.py::Test_Ssrf_Waf_Version: missing_feature # requires Telemetry V2 - tests/appsec/test_agent_level_smoke_tests.py::Test_AppSecAPMStandalone: missing_feature + # tests/appsec/test_agent_level_smoke_tests.py::Test_AppSecAPMStandalone: missing_feature tests/appsec/test_asm_standalone.py::Test_APISecurityStandalone: v2.18.0 tests/appsec/test_asm_standalone.py::Test_AppSecStandalone_NotEnabled: v2.13.0 tests/appsec/test_asm_standalone.py::Test_AppSecStandalone_UpstreamPropagation_V2: v2.13.0 From 9c677ee1fef58a09262396bc198d8f50f934de4a Mon Sep 17 00:00:00 2001 From: Florentin Labelle Date: Thu, 26 Mar 2026 18:00:18 +0100 Subject: [PATCH 3/6] enable tests for the different tracers --- manifests/cpp_nginx.yml | 2 +- manifests/dotnet.yml | 2 +- manifests/golang.yml | 6 +++++- manifests/java.yml | 5 ++++- manifests/php.yml | 2 +- manifests/ruby.yml | 3 ++- tests/appsec/test_agent_level_smoke_tests.py | 7 ------- 7 files changed, 14 insertions(+), 13 deletions(-) diff --git a/manifests/cpp_nginx.yml b/manifests/cpp_nginx.yml index f00845b0321..823b342a4e8 100644 --- a/manifests/cpp_nginx.yml +++ b/manifests/cpp_nginx.yml @@ -50,7 +50,7 @@ manifest: tests/appsec/iast/test_security_controls.py: irrelevant (ASM is not implemented in C++) tests/appsec/iast/test_vulnerability_schema.py: irrelevant (ASM is not implemented in C++) tests/appsec/rasp/: irrelevant (ASM is not implemented in C++) - # tests/appsec/test_agent_level_smoke_tests.py::Test_AppSecAPMStandalone: missing_feature + tests/appsec/test_agent_level_smoke_tests.py::Test_AppSecAPMStandalone: missing_feature tests/appsec/test_alpha.py: irrelevant (ASM is not implemented in C++) tests/appsec/test_asm_standalone.py::Test_APISecurityStandalone: '>=1.12.0' # Modified by easy win activation script tests/appsec/test_asm_standalone.py::Test_APISecurityStandalone::test_appsec_propagation_does_not_force_schema_collection: missing_feature # Created by easy win activation script diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml index 866d8820ed9..1b42c8f28e6 100644 --- a/manifests/dotnet.yml +++ b/manifests/dotnet.yml @@ -295,7 +295,7 @@ manifest: tests/appsec/rasp/test_ssrf.py::Test_Ssrf_Telemetry_V2: v3.26.3 tests/appsec/rasp/test_ssrf.py::Test_Ssrf_UrlQuery: v2.51.0 tests/appsec/rasp/test_ssrf.py::Test_Ssrf_Waf_Version: v3.4.1 - # tests/appsec/test_agent_level_smoke_tests.py::Test_AppSecAPMStandalone: missing_feature + tests/appsec/test_agent_level_smoke_tests.py::Test_AppSecAPMStandalone: v3.41.0 tests/appsec/test_alpha.py::Test_Basic: v1.28.6 tests/appsec/test_asm_standalone.py::Test_APISecurityStandalone: v3.17.0 tests/appsec/test_asm_standalone.py::Test_AppSecStandalone_UpstreamPropagation_V2: v3.12.0 diff --git a/manifests/golang.yml b/manifests/golang.yml index 8196ae361d5..2df658988d4 100644 --- a/manifests/golang.yml +++ b/manifests/golang.yml @@ -294,7 +294,11 @@ manifest: tests/appsec/rasp/test_ssrf.py::Test_Ssrf_Telemetry_V2: v2.0.0 tests/appsec/rasp/test_ssrf.py::Test_Ssrf_UrlQuery: v1.65.1 tests/appsec/rasp/test_ssrf.py::Test_Ssrf_Waf_Version: v2.0.0 - # tests/appsec/test_agent_level_smoke_tests.py::Test_AppSecAPMStandalone: missing_feature + tests/appsec/test_agent_level_smoke_tests.py::Test_AppSecAPMStandalone: v2.7.0 + tests/appsec/test_agent_level_smoke_tests.py::Test_AppSecAPMStandalone::test_lfi_smoke: + - weblog_declaration: + "*": irrelevant (LFi detection requires orchestrion) + net-http-orchestrion: v2.7.0 tests/appsec/test_alpha.py::Test_Basic: - weblog_declaration: "*": v1.34.0 diff --git a/manifests/java.yml b/manifests/java.yml index 2c13e29120b..750aa7a456a 100644 --- a/manifests/java.yml +++ b/manifests/java.yml @@ -1493,7 +1493,10 @@ manifest: - weblog_declaration: "*": v1.40.0 spring-boot-3-native: irrelevant (GraalVM. Tracing support only) - # tests/appsec/test_agent_level_smoke_tests.py::Test_AppSecAPMStandalone: missing_feature + tests/appsec/test_agent_level_smoke_tests.py::Test_AppSecAPMStandalone: + - weblog_declaration: + "*": v1.60.3 + spring-boot-3-native: irrelevant (GraalVM. Tracing support only) tests/appsec/test_alpha.py::Test_Basic: - weblog_declaration: "*": v0.87.0 diff --git a/manifests/php.yml b/manifests/php.yml index 4fcaf41fdcb..48dbf5cbd43 100644 --- a/manifests/php.yml +++ b/manifests/php.yml @@ -284,7 +284,7 @@ manifest: tests/appsec/rasp/test_ssrf.py::Test_Ssrf_Telemetry_V2: missing_feature tests/appsec/rasp/test_ssrf.py::Test_Ssrf_UrlQuery: v1.7.0 tests/appsec/rasp/test_ssrf.py::Test_Ssrf_Waf_Version: v1.7.0 - # tests/appsec/test_agent_level_smoke_tests.py::Test_AppSecAPMStandalone: missing_feature + tests/appsec/test_agent_level_smoke_tests.py::Test_AppSecAPMStandalone: v1.18.0 tests/appsec/test_asm_standalone.py::Test_APISecurityStandalone: v1.11.0 tests/appsec/test_asm_standalone.py::Test_AppSecStandalone_NotEnabled: v1.6.2 tests/appsec/test_asm_standalone.py::Test_AppSecStandalone_UpstreamPropagation_V2: v1.8.0 diff --git a/manifests/ruby.yml b/manifests/ruby.yml index 901b5aeca83..6aaa64a1aa2 100644 --- a/manifests/ruby.yml +++ b/manifests/ruby.yml @@ -378,7 +378,8 @@ manifest: tests/appsec/rasp/test_ssrf.py::Test_Ssrf_Telemetry_V2: missing_feature tests/appsec/rasp/test_ssrf.py::Test_Ssrf_UrlQuery: v2.14.0 tests/appsec/rasp/test_ssrf.py::Test_Ssrf_Waf_Version: missing_feature # requires Telemetry V2 - # tests/appsec/test_agent_level_smoke_tests.py::Test_AppSecAPMStandalone: missing_feature + tests/appsec/test_agent_level_smoke_tests.py::Test_AppSecAPMStandalone: v2.31.0 + tests/appsec/test_agent_level_smoke_tests.py::Test_AppSecAPMStandalone::test_lfi_smoke: missing_feature tests/appsec/test_asm_standalone.py::Test_APISecurityStandalone: v2.18.0 tests/appsec/test_asm_standalone.py::Test_AppSecStandalone_NotEnabled: v2.13.0 tests/appsec/test_asm_standalone.py::Test_AppSecStandalone_UpstreamPropagation_V2: v2.13.0 diff --git a/tests/appsec/test_agent_level_smoke_tests.py b/tests/appsec/test_agent_level_smoke_tests.py index cc0543a23a5..5b3a5454525 100644 --- a/tests/appsec/test_agent_level_smoke_tests.py +++ b/tests/appsec/test_agent_level_smoke_tests.py @@ -123,7 +123,6 @@ def setup_attack_detection_smoke(self) -> None: def test_attack_detection_smoke(self) -> None: found_attack = False - has_waf_version = False has_appsec_data = False for _, span, appsec_data in interfaces.agent.get_appsec_data(self.r): @@ -131,17 +130,11 @@ def test_attack_detection_smoke(self) -> None: if span_meta.get("appsec.event") == "true": found_attack = True - if "_dd.appsec.waf.version" in span_meta: - has_waf_version = True - if appsec_data is not None: has_appsec_data = True - - if has_waf_version and has_appsec_data: break assert found_attack, "Agent should forward detected attacks in span metadata" - assert has_waf_version, "Agent spans should include WAF version metadata" assert has_appsec_data, "Agent spans should include AppSec payload (JSON or metastruct)" From b4b99d657a374b170beb908a3277311844fdd858 Mon Sep 17 00:00:00 2001 From: Florentin Labelle Date: Fri, 27 Mar 2026 10:39:30 +0100 Subject: [PATCH 4/6] fix format --- manifests/java.yml | 4 ++-- manifests/nodejs.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/manifests/java.yml b/manifests/java.yml index 750aa7a456a..14bbd0e754e 100644 --- a/manifests/java.yml +++ b/manifests/java.yml @@ -1494,8 +1494,8 @@ manifest: "*": v1.40.0 spring-boot-3-native: irrelevant (GraalVM. Tracing support only) tests/appsec/test_agent_level_smoke_tests.py::Test_AppSecAPMStandalone: - - weblog_declaration: - "*": v1.60.3 + - weblog_declaration: + "*": v1.60.3 spring-boot-3-native: irrelevant (GraalVM. Tracing support only) tests/appsec/test_alpha.py::Test_Basic: - weblog_declaration: diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml index 8fd63637ee0..a9834551f88 100644 --- a/manifests/nodejs.yml +++ b/manifests/nodejs.yml @@ -909,7 +909,7 @@ manifest: fastify: *ref_5_66_0 nextjs: missing_feature tests/appsec/rasp/test_ssrf.py::Test_Ssrf_Waf_Version: *ref_5_25_0 - # tests/appsec/test_agent_level_smoke_tests.py::Test_AppSecAPMStandalone: missing_feature + tests/appsec/test_agent_level_smoke_tests.py::Test_AppSecAPMStandalone: v5.92.0 tests/appsec/test_alpha.py::Test_Basic: v2.0.0 tests/appsec/test_asm_standalone.py::BaseSCAStandaloneTelemetry::test_app_dependencies_loaded: - weblog_declaration: From 13031a0833ca69f756f91e532a4609d9795e2203 Mon Sep 17 00:00:00 2001 From: Florentin Labelle Date: Mon, 30 Mar 2026 11:16:25 +0200 Subject: [PATCH 5/6] v1 trace payload support + small refactor --- tests/appsec/test_agent_level_smoke_tests.py | 31 +++-- utils/interfaces/_agent.py | 123 +++---------------- 2 files changed, 40 insertions(+), 114 deletions(-) diff --git a/tests/appsec/test_agent_level_smoke_tests.py b/tests/appsec/test_agent_level_smoke_tests.py index 5b3a5454525..3751688376f 100644 --- a/tests/appsec/test_agent_level_smoke_tests.py +++ b/tests/appsec/test_agent_level_smoke_tests.py @@ -49,14 +49,29 @@ def setup_lfi_smoke(self) -> None: def test_lfi_smoke(self) -> None: assert self.r.status_code == 200 - interfaces.agent.assert_rasp_attack( - self.r, - "rasp-930-100", - { - "resource": {"address": "server.io.fs.file", "value": "../etc/passwd"}, - "params": {"address": "server.request.query", "value": "../etc/passwd"}, - }, - ) + expected_rule = "rasp-930-100" + expected_params = { + "resource": {"address": "server.io.fs.file", "value": "../etc/passwd"}, + "params": {"address": "server.request.query", "value": "../etc/passwd"}, + } + + for _, _, appsec_data in interfaces.agent.get_appsec_data(self.r): + for trigger in appsec_data.get("triggers", []): + if trigger.get("rule", {}).get("id") != expected_rule: + continue + for match in trigger.get("rule_matches", []): + for params in match.get("parameters", []): + if not isinstance(params, dict): + continue + if all( + isinstance(params.get(name), dict) + and params[name].get("address") == fields.get("address") + and ("value" not in fields or params[name].get("value") == fields["value"]) + for name, fields in expected_params.items() + ): + return + + raise AssertionError(f"No RASP attack found for rule {expected_rule}") def setup_api_security_smoke(self) -> None: self.r = weblog.get("/waf") diff --git a/utils/interfaces/_agent.py b/utils/interfaces/_agent.py index 55c8e33f303..5bad2ffb1b5 100644 --- a/utils/interfaces/_agent.py +++ b/utils/interfaces/_agent.py @@ -5,9 +5,7 @@ """Validate data flow between agent and backend""" import base64 -import binascii import copy -import json import threading from collections.abc import Callable, Generator, Iterable from typing import Any @@ -32,116 +30,29 @@ def ingest_file(self, src_path: str): self.ready.set() return super().ingest_file(src_path) - @staticmethod - def _extract_appsec_data(span: DataDogAgentSpan) -> dict[str, Any] | None: - appsec_json = span.meta.get("_dd.appsec.json") - if isinstance(appsec_json, dict): - return appsec_json - if isinstance(appsec_json, str): - try: - decoded = json.loads(appsec_json) - except json.JSONDecodeError: - decoded = None - if isinstance(decoded, dict): - return decoded - - meta_struct = span.get("meta_struct") or span.get("metaStruct") or None - if not isinstance(meta_struct, dict): - return None - - appsec_payload = meta_struct.get("appsec") - if isinstance(appsec_payload, dict): - return appsec_payload - - if isinstance(appsec_payload, bytes): - decoded_payload = msgpack.loads(appsec_payload, raw=False, strict_map_key=False, unicode_errors="replace") - return decoded_payload if isinstance(decoded_payload, dict) else None - - if isinstance(appsec_payload, str): - try: - payload = base64.b64decode(appsec_payload) - except (ValueError, binascii.Error): - payload = appsec_payload.encode("utf-8") - decoded_payload = msgpack.loads(payload, raw=False, strict_map_key=False, unicode_errors="replace") - return decoded_payload if isinstance(decoded_payload, dict) else None - - return None - def get_appsec_data( self, request: HttpResponse ) -> Generator[tuple[dict[str, Any], DataDogAgentSpan, dict[str, Any]], Any, None]: for data, span in self.get_spans(request): - appsec_data = self._extract_appsec_data(span) - if appsec_data is None: - continue + json_payload = span.meta.get("_dd.appsec.json") + if json_payload is not None: + yield data, span, json_payload - yield data, span, appsec_data + legacy_metastruct = span.get("metaStruct", {}).get("appsec") + v1_metastruct = span.meta.get("appsec") + payload = legacy_metastruct or v1_metastruct - def assert_rasp_attack( - self, - request: HttpResponse, - rule: str, - parameters: dict | None = None, - ) -> None: - def validator(_: dict, appsec_data: dict[str, Any]) -> bool: - triggers = appsec_data.get("triggers") - assert isinstance(triggers, list), "'triggers' is not a list" - assert triggers, "no appsec triggers found" - - for trigger in triggers: - obtained_rule_id = trigger.get("rule", {}).get("id") - if obtained_rule_id != rule: - continue - - if parameters is None: - return True - - for match in trigger.get("rule_matches", []): - for obtained_parameters in match.get("parameters", []): - if not isinstance(obtained_parameters, dict): - continue - - ok = True - for name, fields in parameters.items(): - if name not in obtained_parameters: - ok = False - break - - obtained_param = obtained_parameters[name] - if not isinstance(obtained_param, dict): - ok = False - break - - address = fields.get("address") - if obtained_param.get("address") != address: - ok = False - break - - if "value" in fields and obtained_param.get("value") != fields["value"]: - ok = False - break - - if "key_path" in fields and obtained_param.get("key_path") != fields["key_path"]: - ok = False - break - - if ok: - return True - - return False - - for data, _, appsec_data in self.get_appsec_data(request=request): - if validator(data, appsec_data): - return - - raise AssertionError("No AppSec payload found for the request") - - def validate_appsec(self, request: HttpResponse, validator: Callable): - for data, span, appsec_data in self.get_appsec_data(request=request): - if validator(data, span, appsec_data): - return - - raise ValueError("No data validate this test") + try: + b64_decoded = base64.b64decode(payload) + msgpack_decoded = msgpack.loads(b64_decoded, raw=False, strict_map_key=False, unicode_errors="replace") + yield data, span, msgpack_decoded + except Exception: + logger.warning( + "Failed to decode appsec payload for request %s on span %s", + request.get_rid(), + span.get_span_name(), + exc_info=True, + ) def get_profiling_data(self): yield from self.get_data(path_filters="/api/v2/profile") From 4042a1951a03044e4fde5bc7f068f4881d9cc11a Mon Sep 17 00:00:00 2001 From: Florentin Labelle Date: Mon, 30 Mar 2026 11:22:12 +0200 Subject: [PATCH 6/6] linter --- utils/interfaces/_agent.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/utils/interfaces/_agent.py b/utils/interfaces/_agent.py index 5bad2ffb1b5..8230116dd22 100644 --- a/utils/interfaces/_agent.py +++ b/utils/interfaces/_agent.py @@ -42,17 +42,20 @@ def get_appsec_data( v1_metastruct = span.meta.get("appsec") payload = legacy_metastruct or v1_metastruct - try: - b64_decoded = base64.b64decode(payload) - msgpack_decoded = msgpack.loads(b64_decoded, raw=False, strict_map_key=False, unicode_errors="replace") - yield data, span, msgpack_decoded - except Exception: - logger.warning( - "Failed to decode appsec payload for request %s on span %s", - request.get_rid(), - span.get_span_name(), - exc_info=True, - ) + if isinstance(payload, str): + try: + b64_decoded = base64.b64decode(payload) + msgpack_decoded = msgpack.loads( + b64_decoded, raw=False, strict_map_key=False, unicode_errors="replace" + ) + yield data, span, msgpack_decoded + except Exception: + logger.warning( + "Failed to decode appsec payload for request %s on span %s", + request.get_rid(), + span.get_span_name(), + exc_info=True, + ) def get_profiling_data(self): yield from self.get_data(path_filters="/api/v2/profile")