diff --git a/.github/workflows/run-end-to-end.yml b/.github/workflows/run-end-to-end.yml index 453ab89cc90..f0c052ccc44 100644 --- a/.github/workflows/run-end-to-end.yml +++ b/.github/workflows/run-end-to-end.yml @@ -174,6 +174,12 @@ jobs: - name: Run APPSEC_STANDALONE scenario if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"APPSEC_STANDALONE"') run: ./run.sh APPSEC_STANDALONE + - name: Run APPSEC_APM_STANDALONE scenario + if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"APPSEC_APM_STANDALONE"') + run: ./run.sh APPSEC_APM_STANDALONE + - name: Run APPSEC_STANDALONE_APM_STANDALONE scenario + if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"APPSEC_STANDALONE_APM_STANDALONE"') + run: ./run.sh APPSEC_STANDALONE_APM_STANDALONE - name: Run APPSEC_STANDALONE_API_SECURITY scenario if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"APPSEC_STANDALONE_API_SECURITY"') run: ./run.sh APPSEC_STANDALONE_API_SECURITY diff --git a/docs/edit/agent-interface-validation-methods.md b/docs/edit/agent-interface-validation-methods.md index 561d5dcdd23..899db6c7af5 100644 --- a/docs/edit/agent-interface-validation-methods.md +++ b/docs/edit/agent-interface-validation-methods.md @@ -89,7 +89,7 @@ def test_agent_trace_forwarding(self): def test_appsec_agent_forwarding(self): r = weblog.get("/", headers={"X-Attack": "' OR 1=1--"}) - def appsec_validator(data, payload, chunk, span, appsec_data): + def appsec_validator(data, span, appsec_data): return "triggers" in appsec_data interfaces.agent.validate_appsec(r, appsec_validator) @@ -186,4 +186,4 @@ This means: - [Interface Initialization](../../utils/interfaces/__init__.py) - [Library Interface Validation Methods](./library-interface-validation-methods.md) - [End-to-End Testing Guide](../execute/README.md) -- [Adding New Tests](./add-new-test.md) \ No newline at end of file +- [Adding New Tests](./add-new-test.md) diff --git a/manifests/agent.yml b/manifests/agent.yml index 3b1bc10f016..819ff57e0fa 100644 --- a/manifests/agent.yml +++ b/manifests/agent.yml @@ -3,7 +3,10 @@ manifest: tests/apm_tracing_e2e/test_single_span.py::Test_SingleSpan: - declaration: missing_feature (Single Spans is not available in agents pre 7.40) - component_version: '<7.40.0' + component_version: "<7.40.0" + tests/appsec/test_agent_level_smoke_tests.py::Test_AppSecAPMStandalone: + - component_version: "<7.77.0-0" + declaration: irrelevant (APM Standalone option was added in 7.77.0) tests/integrations/test_open_telemetry.py::Test_MsSql::test_error_exception_event: - component_version: "<7.75.0-0" declaration: irrelevant (new behavior implement in 7.75.0) diff --git a/manifests/cpp_nginx.yml b/manifests/cpp_nginx.yml index 536dde96fc2..823b342a4e8 100644 --- a/manifests/cpp_nginx.yml +++ b/manifests/cpp_nginx.yml @@ -50,6 +50,7 @@ manifest: tests/appsec/iast/test_security_controls.py: irrelevant (ASM is not implemented in C++) tests/appsec/iast/test_vulnerability_schema.py: irrelevant (ASM is not implemented in C++) tests/appsec/rasp/: irrelevant (ASM is not implemented in C++) + tests/appsec/test_agent_level_smoke_tests.py::Test_AppSecAPMStandalone: missing_feature tests/appsec/test_alpha.py: irrelevant (ASM is not implemented in C++) tests/appsec/test_asm_standalone.py::Test_APISecurityStandalone: '>=1.12.0' # Modified by easy win activation script tests/appsec/test_asm_standalone.py::Test_APISecurityStandalone::test_appsec_propagation_does_not_force_schema_collection: missing_feature # Created by easy win activation script diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml index e49fb25082f..1b42c8f28e6 100644 --- a/manifests/dotnet.yml +++ b/manifests/dotnet.yml @@ -295,6 +295,7 @@ manifest: tests/appsec/rasp/test_ssrf.py::Test_Ssrf_Telemetry_V2: v3.26.3 tests/appsec/rasp/test_ssrf.py::Test_Ssrf_UrlQuery: v2.51.0 tests/appsec/rasp/test_ssrf.py::Test_Ssrf_Waf_Version: v3.4.1 + tests/appsec/test_agent_level_smoke_tests.py::Test_AppSecAPMStandalone: v3.41.0 tests/appsec/test_alpha.py::Test_Basic: v1.28.6 tests/appsec/test_asm_standalone.py::Test_APISecurityStandalone: v3.17.0 tests/appsec/test_asm_standalone.py::Test_AppSecStandalone_UpstreamPropagation_V2: v3.12.0 diff --git a/manifests/golang.yml b/manifests/golang.yml index 6fb9dc15273..2df658988d4 100644 --- a/manifests/golang.yml +++ b/manifests/golang.yml @@ -294,6 +294,11 @@ manifest: tests/appsec/rasp/test_ssrf.py::Test_Ssrf_Telemetry_V2: v2.0.0 tests/appsec/rasp/test_ssrf.py::Test_Ssrf_UrlQuery: v1.65.1 tests/appsec/rasp/test_ssrf.py::Test_Ssrf_Waf_Version: v2.0.0 + tests/appsec/test_agent_level_smoke_tests.py::Test_AppSecAPMStandalone: v2.7.0 + tests/appsec/test_agent_level_smoke_tests.py::Test_AppSecAPMStandalone::test_lfi_smoke: + - weblog_declaration: + "*": irrelevant (LFi detection requires orchestrion) + net-http-orchestrion: v2.7.0 tests/appsec/test_alpha.py::Test_Basic: - weblog_declaration: "*": v1.34.0 diff --git a/manifests/java.yml b/manifests/java.yml index f502d1b3e06..14bbd0e754e 100644 --- a/manifests/java.yml +++ b/manifests/java.yml @@ -1493,6 +1493,10 @@ manifest: - weblog_declaration: "*": v1.40.0 spring-boot-3-native: irrelevant (GraalVM. Tracing support only) + tests/appsec/test_agent_level_smoke_tests.py::Test_AppSecAPMStandalone: + - weblog_declaration: + "*": v1.60.3 + spring-boot-3-native: irrelevant (GraalVM. Tracing support only) tests/appsec/test_alpha.py::Test_Basic: - weblog_declaration: "*": v0.87.0 diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml index 13a553b27e5..a9834551f88 100644 --- a/manifests/nodejs.yml +++ b/manifests/nodejs.yml @@ -909,6 +909,7 @@ manifest: fastify: *ref_5_66_0 nextjs: missing_feature tests/appsec/rasp/test_ssrf.py::Test_Ssrf_Waf_Version: *ref_5_25_0 + tests/appsec/test_agent_level_smoke_tests.py::Test_AppSecAPMStandalone: v5.92.0 tests/appsec/test_alpha.py::Test_Basic: v2.0.0 tests/appsec/test_asm_standalone.py::BaseSCAStandaloneTelemetry::test_app_dependencies_loaded: - weblog_declaration: diff --git a/manifests/php.yml b/manifests/php.yml index 7d80cea05af..48dbf5cbd43 100644 --- a/manifests/php.yml +++ b/manifests/php.yml @@ -284,6 +284,7 @@ manifest: tests/appsec/rasp/test_ssrf.py::Test_Ssrf_Telemetry_V2: missing_feature tests/appsec/rasp/test_ssrf.py::Test_Ssrf_UrlQuery: v1.7.0 tests/appsec/rasp/test_ssrf.py::Test_Ssrf_Waf_Version: v1.7.0 + tests/appsec/test_agent_level_smoke_tests.py::Test_AppSecAPMStandalone: v1.18.0 tests/appsec/test_asm_standalone.py::Test_APISecurityStandalone: v1.11.0 tests/appsec/test_asm_standalone.py::Test_AppSecStandalone_NotEnabled: v1.6.2 tests/appsec/test_asm_standalone.py::Test_AppSecStandalone_UpstreamPropagation_V2: v1.8.0 diff --git a/manifests/python.yml b/manifests/python.yml index e0e054a183b..5fd9d2095dc 100644 --- a/manifests/python.yml +++ b/manifests/python.yml @@ -476,6 +476,7 @@ manifest: "*": v2.10.0 *django: v4.3.0-dev0 (with httpx support) tests/appsec/rasp/test_ssrf.py::Test_Ssrf_Waf_Version: v2.15.0 + tests/appsec/test_agent_level_smoke_tests.py::Test_AppSecAPMStandalone: v4.6.0 tests/appsec/test_alpha.py::Test_Basic: - weblog_declaration: "*": v1.1.0-rc2 diff --git a/manifests/ruby.yml b/manifests/ruby.yml index a3550bcdfec..6aaa64a1aa2 100644 --- a/manifests/ruby.yml +++ b/manifests/ruby.yml @@ -378,6 +378,8 @@ manifest: tests/appsec/rasp/test_ssrf.py::Test_Ssrf_Telemetry_V2: missing_feature tests/appsec/rasp/test_ssrf.py::Test_Ssrf_UrlQuery: v2.14.0 tests/appsec/rasp/test_ssrf.py::Test_Ssrf_Waf_Version: missing_feature # requires Telemetry V2 + tests/appsec/test_agent_level_smoke_tests.py::Test_AppSecAPMStandalone: v2.31.0 + tests/appsec/test_agent_level_smoke_tests.py::Test_AppSecAPMStandalone::test_lfi_smoke: missing_feature tests/appsec/test_asm_standalone.py::Test_APISecurityStandalone: v2.18.0 tests/appsec/test_asm_standalone.py::Test_AppSecStandalone_NotEnabled: v2.13.0 tests/appsec/test_asm_standalone.py::Test_AppSecStandalone_UpstreamPropagation_V2: v2.13.0 diff --git a/tests/appsec/test_agent_level_smoke_tests.py b/tests/appsec/test_agent_level_smoke_tests.py new file mode 100644 index 00000000000..3751688376f --- /dev/null +++ b/tests/appsec/test_agent_level_smoke_tests.py @@ -0,0 +1,160 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the the Apache License Version 2.0. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2021 Datadog, Inc. + +"""AppSec smoke tests for the appsec_apm_standalone scenario.""" + +from utils import features, interfaces, remote_config as rc, weblog, scenarios + +SMOKE_RC_RULE_ID = "smoke-rc-0001" +SMOKE_RC_RULE_FILE: tuple[str, dict[str, object]] = ( + "datadog/2/ASM_DD/rules/config", + { + "version": "2.2", + "metadata": {"rules_version": "2.71.8182"}, + "rules": [ + { + "id": SMOKE_RC_RULE_ID, + "name": "Smoke RC rule", + "tags": { + "type": "attack_tool", + "category": "attack_attempt", + "confidence": "1", + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.headers.no_cookies", + "key_path": ["x-smoke-test"], + } + ], + "regex": "^rc-smoke$", + }, + "operator": "match_regex", + } + ], + "transformers": [], + } + ], + }, +) + + +class AgentLevelSmokeTests: + def setup_lfi_smoke(self) -> None: + self.r = weblog.get("/rasp/lfi", params={"file": "../etc/passwd"}) + + def test_lfi_smoke(self) -> None: + assert self.r.status_code == 200 + + expected_rule = "rasp-930-100" + expected_params = { + "resource": {"address": "server.io.fs.file", "value": "../etc/passwd"}, + "params": {"address": "server.request.query", "value": "../etc/passwd"}, + } + + for _, _, appsec_data in interfaces.agent.get_appsec_data(self.r): + for trigger in appsec_data.get("triggers", []): + if trigger.get("rule", {}).get("id") != expected_rule: + continue + for match in trigger.get("rule_matches", []): + for params in match.get("parameters", []): + if not isinstance(params, dict): + continue + if all( + isinstance(params.get(name), dict) + and params[name].get("address") == fields.get("address") + and ("value" not in fields or params[name].get("value") == fields["value"]) + for name, fields in expected_params.items() + ): + return + + raise AssertionError(f"No RASP attack found for rule {expected_rule}") + + def setup_api_security_smoke(self) -> None: + self.r = weblog.get("/waf") + + def test_api_security_smoke(self) -> None: + assert any( + any(key.startswith("_dd.appsec.s.") for key in span.meta) for _, span in interfaces.agent.get_spans(self.r) + ) + + def setup_telemetry_smoke(self) -> None: + weblog.get("/") + self.r = weblog.get("/waf", headers={"User-Agent": "Arachni/v1"}) + + def test_telemetry_smoke(self) -> None: + telemetry_data = list(interfaces.agent.get_telemetry_data(flatten_message_batches=False)) + + assert telemetry_data, "Agent should forward telemetry data from the library" + + found_metrics = False + found_waf_metric = False + + for data in telemetry_data: + content = data.get("request", {}).get("content", {}) + request_type = content.get("request_type") + + if request_type == "generate-metrics": + found_metrics = True + + payload = content.get("payload", {}) + series = payload.get("series", []) + + for metric_series in series: + metric_name = metric_series.get("metric", "") + if metric_name.startswith("waf."): + found_waf_metric = True + break + + if found_metrics and found_waf_metric: + break + + assert found_metrics + assert found_waf_metric + + def setup_remote_config_smoke(self) -> None: + self.config_state = rc.tracer_rc_state.reset().set_config(*SMOKE_RC_RULE_FILE).apply().state + self.r = weblog.get("/waf", headers={"X-Smoke-Test": "rc-smoke"}) + + def test_remote_config_smoke(self) -> None: + assert self.config_state == rc.ApplyState.ACKNOWLEDGED, ( + f"Remote config should be acknowledged, got {self.config_state}" + ) + + assert any( + span.meta.get("appsec.event") == "true" + and any( + trigger.get("rule", {}).get("id") == SMOKE_RC_RULE_ID for trigger in appsec_data.get("triggers", []) + ) + for _, span, appsec_data in interfaces.agent.get_appsec_data(self.r) + ), "Agent should forward AppSec events for the RC-updated rule" + + def setup_attack_detection_smoke(self) -> None: + rc.tracer_rc_state.reset().apply() + self.r = weblog.get("/waf", headers={"User-Agent": "Arachni/v1"}) + + def test_attack_detection_smoke(self) -> None: + found_attack = False + has_appsec_data = False + + for _, span, appsec_data in interfaces.agent.get_appsec_data(self.r): + span_meta = span.get("meta", {}) or span.get("attributes", {}) + if span_meta.get("appsec.event") == "true": + found_attack = True + + if appsec_data is not None: + has_appsec_data = True + break + + assert found_attack, "Agent should forward detected attacks in span metadata" + assert has_appsec_data, "Agent spans should include AppSec payload (JSON or metastruct)" + + +@features.appsec_apm_standalone +@scenarios.appsec_apm_standalone +@scenarios.appsec_standalone_apm_standalone +class Test_AppSecAPMStandalone(AgentLevelSmokeTests): + pass diff --git a/utils/_context/_scenarios/__init__.py b/utils/_context/_scenarios/__init__.py index fd98887a8ec..ea67e0a06f3 100644 --- a/utils/_context/_scenarios/__init__.py +++ b/utils/_context/_scenarios/__init__.py @@ -428,6 +428,40 @@ class _Scenarios: scenario_groups=[scenario_groups.appsec], ) + appsec_apm_standalone = EndToEndScenario( + "APPSEC_APM_STANDALONE", + rc_api_enabled=True, + weblog_env={ + "DD_APM_TRACING_ENABLED": "true", + "DD_TELEMETRY_METRICS_ENABLED": "true", + "DD_TELEMETRY_METRICS_INTERVAL_SECONDS": "2.0", + "DD_API_SECURITY_REQUEST_SAMPLE_RATE": "1.0", + "DD_API_SECURITY_SAMPLE_DELAY": "0.0", + }, + agent_env={ + "DD_INFRASTRUCTURE_MODE": "none", + }, + doc="Appsec with APM Standalone (infra opt out)", + scenario_groups=[scenario_groups.appsec], + ) + + appsec_standalone_apm_standalone = EndToEndScenario( + "APPSEC_STANDALONE_APM_STANDALONE", + rc_api_enabled=True, + weblog_env={ + "DD_APM_TRACING_ENABLED": "false", + "DD_TELEMETRY_METRICS_ENABLED": "true", + "DD_TELEMETRY_METRICS_INTERVAL_SECONDS": "2.0", + "DD_API_SECURITY_REQUEST_SAMPLE_RATE": "1.0", + "DD_API_SECURITY_SAMPLE_DELAY": "0.0", + }, + agent_env={ + "DD_INFRASTRUCTURE_MODE": "none", + }, + doc="Appsec standalone mode (APM opt out) with APM Standalone (infra opt out)", + scenario_groups=[scenario_groups.appsec], + ) + # Combined scenario for API Security in standalone mode appsec_standalone_api_security = EndToEndScenario( "APPSEC_STANDALONE_API_SECURITY", diff --git a/utils/_features.py b/utils/_features.py index efe41c9da59..b08ff24c5a7 100644 --- a/utils/_features.py +++ b/utils/_features.py @@ -2790,14 +2790,22 @@ def service_override_source(test_object): """ return _mark_test_object(test_object, feature_id=545, owner=_Owner.language_platform) + @staticmethod + def appsec_apm_standalone(test_object): + """Ensure that AppSec works correctly with the infra product disabled + + https://feature-parity.us1.prod.dog/#/?feature=546 + """ + return _mark_test_object(test_object, feature_id=546, owner=_Owner.asm) + @staticmethod def base_service(test_object): """_dd.base_service meta tag is set on spans whose service name differs from the global service. Preserves the originating service context when integrations override the service name. - https://feature-parity.us1.prod.dog/#/?feature=546 + https://feature-parity.us1.prod.dog/#/?feature=549 """ - return _mark_test_object(test_object, feature_id=546, owner=_Owner.idm) + return _mark_test_object(test_object, feature_id=549, owner=_Owner.idm) features = _Features() diff --git a/utils/interfaces/_agent.py b/utils/interfaces/_agent.py index e638c9cb242..8230116dd22 100644 --- a/utils/interfaces/_agent.py +++ b/utils/interfaces/_agent.py @@ -4,15 +4,19 @@ """Validate data flow between agent and backend""" -from collections.abc import Callable, Generator, Iterable +import base64 import copy import threading +from collections.abc import Callable, Generator, Iterable +from typing import Any + +import msgpack -from utils.dd_types import DataDogAgentTrace, DataDogAgentSpan, AgentTraceFormat from utils._logger import logger +from utils._weblog import HttpResponse +from utils.dd_types import AgentTraceFormat, DataDogAgentSpan, DataDogAgentTrace from utils.interfaces._core import ProxyBasedInterfaceValidator from utils.interfaces._misc_validators import HeadersPresenceValidator -from utils._weblog import HttpResponse class AgentInterfaceValidator(ProxyBasedInterfaceValidator): @@ -26,6 +30,33 @@ def ingest_file(self, src_path: str): self.ready.set() return super().ingest_file(src_path) + def get_appsec_data( + self, request: HttpResponse + ) -> Generator[tuple[dict[str, Any], DataDogAgentSpan, dict[str, Any]], Any, None]: + for data, span in self.get_spans(request): + json_payload = span.meta.get("_dd.appsec.json") + if json_payload is not None: + yield data, span, json_payload + + legacy_metastruct = span.get("metaStruct", {}).get("appsec") + v1_metastruct = span.meta.get("appsec") + payload = legacy_metastruct or v1_metastruct + + if isinstance(payload, str): + try: + b64_decoded = base64.b64decode(payload) + msgpack_decoded = msgpack.loads( + b64_decoded, raw=False, strict_map_key=False, unicode_errors="replace" + ) + yield data, span, msgpack_decoded + except Exception: + logger.warning( + "Failed to decode appsec payload for request %s on span %s", + request.get_rid(), + span.get_span_name(), + exc_info=True, + ) + def get_profiling_data(self): yield from self.get_data(path_filters="/api/v2/profile")