chore(build, crashtracker): build w/ debug symbols

.github/workflows/build_python_3.yml

@@ -70,14 +70,22 @@ jobs:
       # `platform.mac_ver()` reports incorrect MacOS version at 11.0
       # See: https://stackoverflow.com/a/65402241
       CIBW_ENVIRONMENT_MACOS: CMAKE_BUILD_PARALLEL_LEVEL=24 SYSTEM_VERSION_COMPAT=0 CMAKE_ARGS="-DNATIVE_TESTING=OFF"
+      # cibuildwheel repair will copy anything's under /output directory from the
+      # build container to the host machine. This is a bit hacky way, but seems
+      # to be the only way getting debug symbols out from the container while
+      # we don't mess up with RECORD file.
       CIBW_REPAIR_WHEEL_COMMAND_LINUX: |
+        mkdir -p /output/debugwheelhouse &&
+        python scripts/extract_debug_symbols.py {wheel} --output-dir /output/debugwheelhouse &&
         python scripts/zip_filter.py {wheel} \*.c \*.cpp \*.cc \*.h \*.hpp \*.pyx \*.md &&
         mkdir ./tempwheelhouse &&
         unzip -l {wheel} | grep '\.so' &&
         auditwheel repair -w ./tempwheelhouse {wheel} &&
         mv ./tempwheelhouse/*.whl {dest_dir} &&
         rm -rf ./tempwheelhouse
       CIBW_REPAIR_WHEEL_COMMAND_MACOS: |
+        mkdir -p ./debugwheelhouse &&
+        python scripts/extract_debug_symbols.py {wheel} --output-dir ./debugwheelhouse &&
         python scripts/zip_filter.py {wheel} \*.c \*.cpp \*.cc \*.h \*.hpp \*.pyx \*.md &&
         MACOSX_DEPLOYMENT_TARGET=12.7 delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel}
       CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: python scripts/zip_filter.py "{wheel}" "*.c" "*.cpp" "*.cc" "*.h" "*.hpp" "*.pyx" "*.md" && mv "{wheel}" "{dest_dir}"
@@ -126,3 +134,11 @@ jobs:
         with:
           name: wheels-${{ env.ARTIFACT_NAME }}
           path: ./wheelhouse/*.whl
+
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        if: runner.os != 'Windows'
+        with:
+          name: debug-symbols-${{ env.ARTIFACT_NAME }}
+          path: |
+            ./debugwheelhouse/*.zip
+            ./wheelhouse/debugwheelhouse/*.zip

.gitignore

@@ -195,3 +195,6 @@ tests/appsec/iast/fixtures/taint_sinks/not_exists.txt
 # .env file
 .env
 .envrc
+
+*.debug
+*.dSYM/

.gitlab/download-wheels-from-gh-actions.sh

@@ -67,7 +67,7 @@ fi
 
 echo "Github workflow finished. Downloading wheels"
 # download all wheels
-gh run download $RUN_ID --repo DataDog/dd-trace-py
+gh run download $RUN_ID --repo DataDog/dd-trace-py --pattern "wheels-*" --pattern "source-dist*"
 
 cd ..
 

ddtrace/appsec/_iast/_taint_tracking/native.cpp

@@ -42,11 +42,12 @@ static PyMethodDef AspectsMethods[] = {
     { nullptr, nullptr, 0, nullptr }
 };
 
-static struct PyModuleDef aspects = { PyModuleDef_HEAD_INIT,
-                                      .m_name = PY_MODULE_NAME_ASPECTS,
-                                      .m_doc = "Taint tracking Aspects",
-                                      .m_size = -1,
-                                      .m_methods = AspectsMethods };
+// Mark the module as used to prevent it from being stripped.
+static struct PyModuleDef aspects __attribute__((used)) = { PyModuleDef_HEAD_INIT,
+                                                            .m_name = PY_MODULE_NAME_ASPECTS,
+                                                            .m_doc = "Taint tracking Aspects",
+                                                            .m_size = -1,
+                                                            .m_methods = AspectsMethods };
 
 static PyMethodDef OpsMethods[] = {
     { "new_pyobject_id", (PyCFunction)api_new_pyobject_id, METH_FASTCALL, "new pyobject id" },
@@ -55,11 +56,12 @@ static PyMethodDef OpsMethods[] = {
     { nullptr, nullptr, 0, nullptr }
 };
 
-static struct PyModuleDef ops = { PyModuleDef_HEAD_INIT,
-                                  .m_name = PY_MODULE_NAME_ASPECTS,
-                                  .m_doc = "Taint tracking operations",
-                                  .m_size = -1,
-                                  .m_methods = OpsMethods };
+// Mark the module as used to prevent it from being stripped.
+static struct PyModuleDef ops __attribute__((used)) = { PyModuleDef_HEAD_INIT,
+                                                        .m_name = PY_MODULE_NAME_ASPECTS,
+                                                        .m_doc = "Taint tracking operations",
+                                                        .m_size = -1,
+                                                        .m_methods = OpsMethods };
 
 /**
  * This function initializes the native module.

ddtrace/internal/datadog/profiling/cmake/AnalysisFunc.cmake

@@ -26,9 +26,6 @@ function(add_ddup_config target)
                                                  "$<$<CONFIG:RelWithDebInfo>:-Os;-ggdb3>" -fno-semantic-interposition)
     endif()
 
-    # Common link options
-    target_link_options(${target} PRIVATE "$<$<CONFIG:RelWithDebInfo>:>")
-
     if(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
         # macOS-specific linker options
         target_link_options(${target} PRIVATE "$<$<CONFIG:Release>:-Wl,-dead_strip>")
@@ -46,11 +43,19 @@ function(add_ddup_config target)
             -Wl,--exclude-libs,ALL)
     endif()
 
-    # If we can IPO, then do so
+    # If we can IPO, then do so.
     check_ipo_supported(RESULT result)
 
     if(result)
-        set_property(TARGET ${target} PROPERTY INTERPROCEDURAL_OPTIMIZATION TRUE)
+        if(CMAKE_CXX_COMPILER_ID MATCHES "AppleClang")
+            # When using AppleClang, explicitly use thin LTO to match Rust's thin LTO strategy. And set the object path
+            # for debug symbols.
+            target_compile_options(${target} PRIVATE -flto=thin)
+            target_link_options(${target} PRIVATE -flto=thin)
+            target_link_options(${target} PRIVATE -Wl,-object_path_lto,${CMAKE_CURRENT_BINARY_DIR}/${target}_lto.o)
+        else()
+            set_property(TARGET ${target} PROPERTY INTERPROCEDURAL_OPTIMIZATION TRUE)
+        endif()
     endif()
 
     # Propagate sanitizers
@@ -85,4 +90,5 @@ function(add_ddup_config target)
     # The main targets, ddup, crashtracker, stack_v2, and dd_wrapper are built as dynamic libraries, so PIC is required.
     # And setting this is also fine for tests as they're loading those dynamic libraries.
     set_target_properties(${target} PROPERTIES POSITION_INDEPENDENT_CODE ON)
+
 endfunction()

docs/debug_symbols.rst

@@ -0,0 +1,107 @@
+Debugging Native Extensions with Debug Symbols
+==============================================
+
+dd-trace-py is built with debug symbols by default, and packaged separately from the main wheel files to reduce the size of the primary distribution packages.
+
+Debug Symbol Files
+------------------
+
+The project generates debug symbols during the build process:
+
+- **Linux**: ``.debug`` files (using ``objcopy --only-keep-debug``)
+- **macOS**: ``.dSYM`` bundles (using ``dsymutil``)
+
+These debug symbols are extracted from the main wheels and packaged into separate `.zip` files with the naming convention:
+
+::
+
+    {original-wheel-name}-debug-symbols.zip
+
+For example:
+
+- ``ddtrace-1.20.0-cp39-cp39-linux_x86_64.whl`` → ``ddtrace-1.20.0-cp39-cp39-linux_x86_64-debug-symbols.zip``
+- ``ddtrace-1.20.0-cp39-cp39-macosx_10_9_x86_64.whl`` → ``ddtrace-1.20.0-cp39-cp39-macosx_10_9_x86_64-debug-symbols.zip``
+
+Build Process
+-------------
+
+The debug symbols are handled automatically during the CI build process:
+
+1. Wheels are built with debug symbols included
+2. Debug symbols are extracted using the ``scripts/extract_debug_symbols.py`` script
+3. Debug symbols are removed from the main wheel to reduce size
+4. Separate debug symbol packages are created and uploaded as artifacts
+
+Usage
+-----
+
+To use debug symbols for debugging or crash analysis:
+
+1. Download the appropriate debug symbol package for your platform and Python version
+2. Extract the debug symbol files to the same directory as the corresponding `.so` files.
+   Typically, the site-packages directory where ddtrace is installed.
+3. Your debugger or crash analysis tool should automatically find the debug symbols
+4. To view assembly with code side by side, you also need the source code, and
+   set substitute paths in your debugger to the source code directory. For example,
+   for ``_stack_v2.cpython-313-x86_64-linux-gnu.so`` is compiled from
+   echion as specified in ``ddtrace/internal/datadog/profiling/stack_v2/CMakeLists.txt``.
+   So you first need to check out the echion repository and checkout the commit hash.
+   Then, set substitute paths in gdb to the echion source code directory.
+   Typically, if you run ``dias /m <symbol>`` in gdb, it will tell you the full
+   file path of the source code as the following:
+
+   .. code-block:: bash
+
+       (gdb) disas /m Frame::read
+       Dump of assembler code for function _ZN5Frame4readEP19_PyInterpreterFramePS1_:
+       269     /project/build/cmake.linux-x86_64-cpython-313/ddtrace.internal.datadog.profiling.stack_v2._stack_v2/_deps/echion-src/echion/frame.cc: No such file or directory.
+          0x000000000000ece4 <+0>:     push   %r12
+          0x000000000000ece6 <+2>:     mov    %rdi,%r8
+          0x000000000000ece9 <+5>:     push   %rbp
+          0x000000000000ecea <+6>:     mov    %rsi,%rbp
+          0x000000000000eced <+9>:     push   %rbx
+          0x000000000000ecee <+10>:    sub    $0x60,%rsp
+
+       270     in /project/build/cmake.linux-x86_64-cpython-313/ddtrace.internal.datadog.profiling.stack_v2._stack_v2/_deps/echion-src/echion/frame.cc
+       271     in /project/build/cmake.linux-x86_64-cpython-313/ddtrace.internal.datadog.profiling.stack_v2._stack_v2/_deps/echion-src/echion/frame.cc
+
+   Then you can set substitute paths in gdb to the echion source code directory
+
+   .. code-block:: bash
+
+       (gdb) set substitute-path /project/build/cmake.linux-x86_64-cpython-313/ddtrace.internal.datadog.profiling.stack_v2._stack_v2/_deps/echion-src/echion /path/to/echion/source/code
+
+   Run ``dias /m Frame::read`` again to see the assembly with code side by side.
+
+   .. code-block:: bash
+
+       (gdb) disas /m Frame::read
+       Dump of assembler code for function _ZN5Frame4readEP19_PyInterpreterFramePS1_:
+       warning: Source file is more recent than executable.
+       269     {
+          0x000000000000ece4 <+0>:     push   %r12
+          0x000000000000ece6 <+2>:     mov    %rdi,%r8
+          0x000000000000ece9 <+5>:     push   %rbp
+          0x000000000000ecea <+6>:     mov    %rsi,%rbp
+          0x000000000000eced <+9>:     push   %rbx
+          0x000000000000ecee <+10>:    sub    $0x60,%rsp
+
+       270     #if PY_VERSION_HEX >= 0x030b0000
+       271         _PyInterpreterFrame iframe;
+
+       272     #if PY_VERSION_HEX >= 0x030d0000
+       273         // From Python versions 3.13, f_executable can have objects other than
+       274         // code objects for an internal frame. We need to skip some frames if
+       275         // its f_executable is not code as suggested here:
+       276         // https://github.com/python/cpython/issues/100987#issuecomment-1485556487
+       277         PyObject f_executable;
+
+       278
+       279         for (; frame_addr; frame_addr = frame_addr->previous)
+          0x000000000000ecf7 <+19>:    test   %r8,%r8
+          0x000000000000ecfa <+22>:    je     0xed91 <_ZN5Frame4readEP19_PyInterpreterFramePS1_+173>
+          0x000000000000ed88 <+164>:   mov    0x8(%rbx),%r8
+          0x000000000000ed8c <+168>:   jmp    0xecf7 <_ZN5Frame4readEP19_PyInterpreterFramePS1_+19>
+
+      On lldb, you can find the source code full path by running ``image lookup -n Frame::read --verbose``,
+      and set the source code path using ``settings set target.source-map <expected-path> <actual-path>``.

docs/index.rst

@@ -283,6 +283,7 @@ Indices and tables
     basic_usage
     advanced_usage
     build_system
+    debug_symbols
     benchmarks
     contributing
     troubleshooting

docs/spelling_wordlist.txt

@@ -95,6 +95,7 @@ dramatiq
 Dramatiq
 dsn
 dunder
+echion
 eg
 elasticsearch
 elasticsearch1
@@ -116,6 +117,7 @@ flamegraph
 fnmatch
 formatter
 freezegun
+gdb
 genai
 generativeai
 gevent
@@ -340,4 +342,4 @@ wsgi
 xfail
 yaaredis
 openai-agents
-validators
+validators