Merge branch 'main' into jordan.gonzalez/gitlab/secrets-from-kv

Author: duncanista

.github/workflows/system_tests.yml

@@ -7,6 +7,8 @@ on:
   pull_request:
     branches:
       - "**"
+  schedule:
+    - cron:  '00 03 * * *'
 
 jobs:
   build:
@@ -27,12 +29,15 @@ jobs:
     needs:
       - build
     uses: DataDog/system-tests/.github/workflows/system-tests.yml@main
-    secrets: inherit
+    secrets:
+      DD_API_KEY: ${{ secrets.DD_API_KEY }}
+      TEST_OPTIMIZATION_API_KEY: ${{ secrets.TEST_OPTIMIZATION_API_KEY }}
     permissions:
       contents: read
       packages: write
     with:
       library: python_lambda
       binaries_artifact: binaries
-      scenarios_groups: appsec
+      scenarios_groups: tracer_release
       skip_empty_scenarios: true
+      push_to_test_optimization: true

Dockerfile

@@ -25,17 +25,34 @@ ENV PATH=/root/.cargo/bin:$PATH
 
 # Install datadog_lambda and dependencies from local
 COPY . .
+
 RUN pip install --no-cache-dir . -t ./python/lib/$runtime/site-packages
 
 # Remove botocore (40MB) to reduce package size. aws-xray-sdk
 # installs it, while it's already provided by the Lambda Runtime.
 RUN rm -rf ./python/lib/$runtime/site-packages/botocore*
 RUN rm -rf ./python/lib/$runtime/site-packages/setuptools
 RUN rm -rf ./python/lib/$runtime/site-packages/jsonschema/tests
-RUN rm -f ./python/lib/$runtime/site-packages/ddtrace/appsec/_iast/_ast/iastpatch*.so
-RUN rm -rf ./python/lib/$runtime/site-packages/ddtrace/appsec/_iast/_taint_tracking/_vendor
-RUN rm -f ./python/lib/$runtime/site-packages/ddtrace/appsec/_iast/_taint_tracking/*.so
-RUN rm -f ./python/lib/$runtime/site-packages/ddtrace/appsec/_iast/_stacktrace*.so
+RUN rm -rf ./python/lib/$runtime/site-packages/ddtrace/appsec/_iast
+# CI Visibility paths/integrations
+RUN rm -rf \
+    ./python/lib/$runtime/site-packages/ddtrace/contrib/coverage/ \
+    ./python/lib/$runtime/site-packages/ddtrace/contrib/pytest/ \
+    ./python/lib/$runtime/site-packages/ddtrace/contrib/pytest_bdd/ \
+    ./python/lib/$runtime/site-packages/ddtrace/contrib/pytest_benchmark/ \
+    ./python/lib/$runtime/site-packages/ddtrace/contrib/selenium/ \
+    ./python/lib/$runtime/site-packages/ddtrace/contrib/unittest/ \
+    ./python/lib/$runtime/site-packages/ddtrace/ext/ci_visibility \
+    ./python/lib/$runtime/site-packages/ddtrace/ext/test_visibility \
+    ./python/lib/$runtime/site-packages/ddtrace/internal/ci_visibility \
+    ./python/lib/$runtime/site-packages/ddtrace/internal/coverage \
+    ./python/lib/$runtime/site-packages/ddtrace/internal/test_visibility \
+    ./python/lib/$runtime/site-packages/ddtrace/testing/
+
+# Dogshell
+RUN rm -rf ./python/lib/$runtime/site-packages/datadog/dogshell
+RUN rm -rf ./python/lib/$runtime/site-packages/bin/dog*
+
 # remove *.dist-info directories except any entry_points.txt files and METADATA files required for Appsec Software Composition Analysis
 RUN find ./python/lib/$runtime/site-packages/*.dist-info \
         -type f \
@@ -49,7 +66,8 @@ RUN rm -rf \
         ./python/lib/$runtime/site-packages/urllib3* \
         ./python/lib/$runtime/site-packages/certifi* \
         ./python/lib/$runtime/site-packages/idna* \
-        ./python/lib/$runtime/site-packages/charset_normalizer*
+        ./python/lib/$runtime/site-packages/charset_normalizer* \
+        ./python/lib/$runtime/site-packages/*__mypyc*.so  # from charset_normalizer
 
 # Precompile all .pyc files and remove .py files. This speeds up load time.
 # Compile with optimization level 2 (-OO) and PYTHONNODEBUGRANGES=1 to redtce
@@ -58,10 +76,18 @@ RUN rm -rf \
 # https://docs.python.org/3.11/using/cmdline.html#cmdoption-O
 # https://docs.python.org/3/using/cmdline.html#envvar-PYTHONNODEBUGRANGES
 RUN PYTHONNODEBUGRANGES=1 python -OO -m compileall -b ./python/lib/$runtime/site-packages
-# remove all .py files except ddtrace/contrib/*/patch.py which are necessary
-# for ddtrace.patch to discover instrumationation packages.
-RUN find ./python/lib/$runtime/site-packages -name \*.py | grep -v ddtrace/contrib | xargs rm -rf
-RUN find ./python/lib/$runtime/site-packages/ddtrace/contrib -name \*.py | grep -v patch.py | xargs rm -rf
+# remove all .py files
+# DEV: ddtrace>=4.7.0rc3 checks for .pyc files in addition to .py files for instrumentation
+# discovery (DataDog/dd-trace-py#17196), so we can safely remove all .py files.
+# For older versions, we need to keep patch.py files for instrumentation discovery.
+RUN pip install --quiet packaging && \
+    DDTRACE_VERSION=$(grep "^Version:" ./python/lib/$runtime/site-packages/ddtrace-*.dist-info/METADATA | awk '{print $2}') && \
+    if python -c "from packaging.version import Version; exit(0 if Version('$DDTRACE_VERSION') >= Version('4.7.0rc3') else 1)"; then \
+        find ./python/lib/$runtime/site-packages -name \*.py | xargs rm -rf; \
+    else \
+        find ./python/lib/$runtime/site-packages -name \*.py | grep -v ddtrace/contrib | xargs rm -rf && \
+        find ./python/lib/$runtime/site-packages/ddtrace/contrib -name \*.py | grep -v patch.py | xargs rm -rf; \
+    fi
 RUN find ./python/lib/$runtime/site-packages -name __pycache__ -type d -exec rm -r {} \+
 
 # When building ddtrace from branch, remove extra source files.  These are
@@ -72,6 +98,7 @@ RUN find ./python/lib/$runtime/site-packages/ddtrace -name \*.cc -delete
 RUN find ./python/lib/$runtime/site-packages/ddtrace -name \*.h -delete
 RUN find ./python/lib/$runtime/site-packages/ddtrace -name \*.hpp -delete
 RUN find ./python/lib/$runtime/site-packages/ddtrace -name \*.pyx -delete
+RUN find ./python/lib/$runtime/site-packages/ddtrace -name \*.pyi -delete
 
 # Strip debug symbols and symbols that are not needed for relocation
 # processing  using strip --strip-unneeded for all .so files. This is to

datadog_lambda/config.py

@@ -141,6 +141,12 @@ def _reset(self):
         "enabled" if config.fips_mode_enabled else "not enabled",
     )
 
+# disable css to prevent double counting in lambda
+os.environ["DD_TRACE_STATS_COMPUTATION_ENABLED"] = "false"
+
+# unset css aliases to ensure it is disabled
+if "DD_TRACE_COMPUTE_STATS" in os.environ:
+    del os.environ["DD_TRACE_COMPUTE_STATS"]
 
 if (
     "DD_INSTRUMENTATION_TELEMETRY_ENABLED" not in os.environ

datadog_lambda/durable.py

@@ -0,0 +1,63 @@
+# Unless explicitly stated otherwise all files in this repository are licensed
+# under the Apache License Version 2.0.
+# This product includes software developed at Datadog (https://www.datadoghq.com/).
+# Copyright 2019 Datadog, Inc.
+import logging
+import re
+
+logger = logging.getLogger(__name__)
+
+
+def _parse_durable_execution_arn(arn):
+    """
+    Parses a DurableExecutionArn to extract execution name and ID.
+    ARN format:
+        arn:aws:lambda:{region}:{account}:function:{func}:{version}/durable-execution/{name}/{id}
+    Returns (execution_name, execution_id) or None if parsing fails.
+    """
+    match = re.search(r"/durable-execution/([^/]+)/([^/]+)$", arn)
+    if not match:
+        return None
+    execution_name, execution_id = match.group(1), match.group(2)
+    if not execution_name or not execution_id:
+        return None
+    return execution_name, execution_id
+
+
+def extract_durable_function_tags(event):
+    """
+    Extracts durable function tags from the Lambda event payload.
+    Returns a dict with durable function tags, or an empty dict if the event
+    is not a durable function invocation.
+    """
+    if not isinstance(event, dict):
+        return {}
+
+    durable_execution_arn = event.get("DurableExecutionArn")
+    if not isinstance(durable_execution_arn, str):
+        return {}
+
+    parsed = _parse_durable_execution_arn(durable_execution_arn)
+    if not parsed:
+        logger.error("Failed to parse DurableExecutionArn: %s", durable_execution_arn)
+        return {}
+
+    execution_name, execution_id = parsed
+    return {
+        "aws_lambda.durable_function.execution_name": execution_name,
+        "aws_lambda.durable_function.execution_id": execution_id,
+    }
+
+
+VALID_DURABLE_STATUSES = {"SUCCEEDED", "FAILED", "STOPPED", "TIMED_OUT"}
+
+
+def extract_durable_execution_status(response, event):
+    if not isinstance(event, dict) or "DurableExecutionArn" not in event:
+        return None
+    if not isinstance(response, dict):
+        return None
+    status = response.get("Status")
+    if status not in VALID_DURABLE_STATUSES:
+        return None
+    return status

datadog_lambda/tracing.py

@@ -1449,6 +1449,7 @@ def create_function_execution_span(
     trace_context_source,
     merge_xray_traces,
     trigger_tags,
+    durable_function_tags=None,
     parent_span=None,
     span_pointers=None,
 ):
@@ -1459,6 +1460,7 @@ def create_function_execution_span(
         function_arn = ":".join(tk[0:7]) if len(tk) > 7 else function_arn
         function_version = tk[7] if len(tk) > 7 else "$LATEST"
         tags = {
+            "span.kind": "server",
             "cold_start": str(is_cold_start).lower(),
             "function_arn": function_arn,
             "function_version": function_version,
@@ -1477,6 +1479,8 @@ def create_function_execution_span(
     if trace_context_source == TraceContextSource.XRAY and merge_xray_traces:
         tags["_dd.parent_source"] = trace_context_source
     tags.update(trigger_tags)
+    if durable_function_tags:
+        tags.update(durable_function_tags)
     tracer.set_tags(_dd_origin)
     # Determine service name based on config and env var
     if config.service:

datadog_lambda/version.py

@@ -1 +1 @@
-__version__ = "8.123.0.dev0"
+__version__ = "8.124.0.dev0"

datadog_lambda/wrapper.py

@@ -42,6 +42,10 @@
     tracer,
     propagator,
 )
+from datadog_lambda.durable import (
+    extract_durable_function_tags,
+    extract_durable_execution_status,
+)
 from datadog_lambda.trigger import (
     extract_trigger_tags,
     extract_http_status_code_tag,
@@ -152,7 +156,7 @@ def __init__(self, func):
             if config.trace_extractor:
                 extractor_parts = config.trace_extractor.rsplit(".", 1)
                 if len(extractor_parts) == 2:
-                    (mod_name, extractor_name) = extractor_parts
+                    mod_name, extractor_name = extractor_parts
                     modified_extractor_name = modify_module_name(mod_name)
                     extractor_module = import_module(modified_extractor_name)
                     self.trace_extractor = getattr(extractor_module, extractor_name)
@@ -243,6 +247,7 @@ def _before(self, event, context):
                 submit_invocations_metric(context)
 
             self.trigger_tags = extract_trigger_tags(event, context)
+            self.durable_function_tags = extract_durable_function_tags(event)
             # Extract Datadog trace context and source from incoming requests
             dd_context, trace_context_source, event_source = extract_dd_trace_context(
                 event,
@@ -280,6 +285,7 @@ def _before(self, event, context):
                     trace_context_source=trace_context_source,
                     merge_xray_traces=config.merge_xray_traces,
                     trigger_tags=self.trigger_tags,
+                    durable_function_tags=self.durable_function_tags,
                     parent_span=self.inferred_span,
                     span_pointers=calculate_span_pointers(event_source, event),
                 )
@@ -289,7 +295,7 @@ def _before(self, event, context):
             else:
                 set_correlation_ids()
             if config.profiling_enabled and profiler and is_new_sandbox():
-                self.prof.start(stop_on_exit=False, profile_children=True)
+                self.prof.start()
             logger.debug("datadog_lambda_wrapper _before() done")
         except Exception as e:
             logger.error(format_err_with_traceback(e))
@@ -302,6 +308,16 @@ def _after(self, event, context):
 
             status_code = extract_http_status_code_tag(self.trigger_tags, self.response)
 
+            # Skip creating cold start spans in managed instances mode
+            # In managed instances, the tracer library handles cold start independently
+            should_trace_cold_start = (
+                config.cold_start_tracing
+                and is_new_sandbox()
+                and not is_managed_instances_mode()
+            )
+            if should_trace_cold_start:
+                trace_ctx = tracer.current_trace_context()
+
             if self.span:
                 if config.appsec_enabled and not self.blocking_response:
                     asm_start_response(
@@ -327,6 +343,13 @@ def _after(self, event, context):
                 if status_code:
                     self.span.set_tag("http.status_code", status_code)
 
+                durable_status = extract_durable_execution_status(self.response, event)
+                if durable_status:
+                    self.span.set_tag(
+                        "aws_lambda.durable_function.execution_status",
+                        durable_status,
+                    )
+
                 self.span.finish()
 
             if status_code:
@@ -339,15 +362,6 @@ def _after(self, event, context):
                 create_dd_dummy_metadata_subsegment(
                     self.trigger_tags, XraySubsegment.LAMBDA_FUNCTION_TAGS_KEY
                 )
-            # Skip creating cold start spans in managed instances mode
-            # In managed instances, the tracer library handles cold start independently
-            should_trace_cold_start = (
-                config.cold_start_tracing
-                and is_new_sandbox()
-                and not is_managed_instances_mode()
-            )
-            if should_trace_cold_start:
-                trace_ctx = tracer.current_trace_context()
 
             if self.inferred_span:
                 if status_code:

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "datadog_lambda"
-version = "8.123.0.dev0"
+version = "8.124.0.dev0"
 description = "The Datadog AWS Lambda Library"
 authors = ["Datadog, Inc. <dev@datadoghq.com>"]
 license = "Apache-2.0"
@@ -31,7 +31,7 @@ datadog = ">=0.51.0,<1.0.0"
 wrapt = "^1.11.2"
 ddtrace = [
     {version = ">=3.19.1,<4", python = ">=3.8,<3.10"},
-    {version = ">=4.1.1,<5", python = ">=3.10"}
+    {version = ">=4.1.1,<5,!=4.6.*", python = ">=3.10"}
 ]
 ujson = ">=5.9.0"
 botocore = { version = "^1.34.0", optional = true }