diff --git a/.github/.trivyignore b/.github/.trivyignore index 1f9f11bd30..e9de2222cc 100644 --- a/.github/.trivyignore +++ b/.github/.trivyignore @@ -1,9 +1,16 @@ +# ============================= +# Accepted Risk Vulnerabilities +# ============================= + +# Accepting risk due to Python 3.8 support. +CVE-2025-50181 # Requires misconfiguration of urllib3, which agent does not do without intervention +CVE-2025-66418 # Malicious servers could cause high resource consumption +CVE-2025-66471 # Malicious servers could cause high resource consumption +CVE-2026-21441 # Improper Handling of Highly Compressed Data (Data Amplification) + # ======================= # Ignored Vulnerabilities # ======================= -# Accepting risk due to Python 3.8 support. -CVE-2025-50181 - # Not relevant, only affects Pyodide CVE-2025-50182 diff --git a/.github/containers/Dockerfile b/.github/containers/Dockerfile index 3f370a4a45..7538ea11f2 100644 --- a/.github/containers/Dockerfile +++ b/.github/containers/Dockerfile @@ -74,18 +74,6 @@ RUN if [ "${TARGETPLATFORM}" = "linux/amd64" ]; then export ARCH="x86_64"; else rm -rf /tmp/addlicense && \ chmod +x /usr/local/bin/addlicense -# Build librdkafka from source -ARG LIBRDKAFKA_VERSION=2.1.1 -RUN cd /tmp && \ - wget https://github.com/confluentinc/librdkafka/archive/refs/tags/v${LIBRDKAFKA_VERSION}.zip -O ./librdkafka.zip && \ - unzip ./librdkafka.zip && \ - rm ./librdkafka.zip && \ - cd ./librdkafka-${LIBRDKAFKA_VERSION} && \ - ./configure && \ - make all install && \ - cd /tmp && \ - rm -rf ./librdkafka-${LIBRDKAFKA_VERSION} - # Setup ODBC config RUN if [ "${TARGETPLATFORM}" = "linux/amd64" ]; then export ARCH="x86_64"; else export ARCH="aarch64"; fi && \ sed -i "s|Driver=psqlodbca.so|Driver=/usr/lib/${ARCH}-linux-gnu/odbc/psqlodbca.so|g" /etc/odbcinst.ini && \ @@ -109,13 +97,11 @@ ENV PATH="${HOME}/.local/bin:${PATH}" ENV UV_PYTHON_PREFERENCE="only-managed" ENV UV_LINK_MODE="copy" -# Install PyPy versions and rename shims -RUN uv python install -f pp3.11 pp3.10 -RUN mv "${HOME}/.local/bin/python3.11" "${HOME}/.local/bin/pypy3.11" && \ - mv "${HOME}/.local/bin/python3.10" "${HOME}/.local/bin/pypy3.10" - -# Install CPython versions -RUN uv python install -f cp3.14 cp3.14t cp3.13 cp3.12 cp3.11 cp3.10 cp3.9 cp3.8 +# Install CPython and PyPy versions +RUN uv python install -f \ + cp3.14 cp3.13 cp3.12 cp3.11 cp3.10 cp3.9 cp3.8 \ + pp3.11 pp3.10 \ + cp3.14t # Set default Python version to CPython 3.13 RUN uv python install -f --default cp3.13 @@ -130,7 +116,7 @@ EOF ENV UV_PYTHON_DOWNLOADS=never # Install tools with uv in isolated environments -RUN uv tool install tox==4.23.2 --with tox-uv && \ +RUN uv tool install tox --with tox-uv && \ uv tool install ruff && \ uv tool install pre-commit --with pre-commit-uv && \ uv tool install asv --with virtualenv diff --git a/.github/scripts/install_azure_functions_worker.sh b/.github/scripts/install_azure_functions_worker.sh index eea074abfd..6a6c68be95 100755 --- a/.github/scripts/install_azure_functions_worker.sh +++ b/.github/scripts/install_azure_functions_worker.sh @@ -34,7 +34,7 @@ ${PIP} install pip-tools build invoke # Install proto build dependencies $( cd ${BUILD_DIR}/workers/ && ${PIPCOMPILE} -o ${BUILD_DIR}/requirements.txt ) -${PIP} install -r ${BUILD_DIR}/requirements.txt +${PIP} install 'setuptools<82' -r ${BUILD_DIR}/requirements.txt # Build proto files into pb2 files (invoke handles fixing include paths for the protos) cd ${BUILD_DIR}/workers/tests && ${INVOKE} -c test_setup build-protos diff --git a/.github/workflows/addlicense.yml b/.github/workflows/addlicense.yml index 8d66691ff7..f357a8b093 100644 --- a/.github/workflows/addlicense.yml +++ b/.github/workflows/addlicense.yml @@ -39,7 +39,7 @@ jobs: permissions: contents: read steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # 5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # 6.0.2 - name: Fetch git tags run: | diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 513e467f29..9aa5825a79 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -25,24 +25,51 @@ concurrency: cancel-in-progress: true jobs: - # Benchmarks + # =========================== + # Aggregate Benchmark Results + # =========================== + benchmarks: + runs-on: ubuntu-24.04 + if: always() # Always run, even on cancellation or failure + needs: + - benchmark + + steps: + - name: Status + run: | + if [[ "${{ contains(needs.*.result, 'cancelled') }}" == "true" ]]; then + echo "Workflow cancelled." + exit 1 + elif [[ "${{ contains(needs.*.result, 'failure') }}" == "true" ]]; then + echo "One or more jobs failed." + exit 1 + else + echo "All jobs completed successfully." + exit 0 + fi + + # ================================== + # Benchmarks Run Directly On Runners + # ================================== benchmark: runs-on: ubuntu-24.04 timeout-minutes: 30 strategy: + fail-fast: false matrix: python: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] env: ASV_FACTOR: "1.1" BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # 5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # 6.0.2 with: fetch-depth: 0 - - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # 6.0.0 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # 6.2.0 with: python-version: "${{ matrix.python }}" @@ -62,9 +89,10 @@ jobs: - name: Run Benchmark run: | + echo "Running continuous benchmarking between base commit ${BASE_SHA} and head commit ${HEAD_SHA}" asv continuous \ --show-stderr \ --split \ --factor "${ASV_FACTOR}" \ --python=${{ matrix.python }} \ - "${BASE_SHA}" "${GITHUB_SHA}" + "${BASE_SHA}" "${HEAD_SHA}" diff --git a/.github/workflows/build-ci-image.yml b/.github/workflows/build-ci-image.yml index ab183f48a2..d85b0d89fc 100644 --- a/.github/workflows/build-ci-image.yml +++ b/.github/workflows/build-ci-image.yml @@ -43,14 +43,14 @@ jobs: name: Docker Build ${{ matrix.platform }} steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # 5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # 6.0.2 with: persist-credentials: false fetch-depth: 0 - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # 3.11.1 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # 3.12.0 # Lowercase image name and append -ci - name: Generate Image Name @@ -60,7 +60,7 @@ jobs: - name: Generate Docker Metadata (Tags and Labels) id: meta - uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # 5.8.0 + uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # 5.10.0 with: images: ghcr.io/${{ steps.image-name.outputs.IMAGE_NAME }} flavor: | @@ -75,7 +75,7 @@ jobs: - name: Login to GitHub Container Registry if: github.event_name != 'pull_request' - uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # 3.6.0 + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # 3.7.0 with: registry: ghcr.io username: ${{ github.repository_owner }} @@ -83,7 +83,7 @@ jobs: - name: Build and Push Image by Digest id: build - uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # 6.18.0 + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # 6.19.2 with: context: .github/containers platforms: ${{ matrix.platform }} @@ -97,7 +97,7 @@ jobs: touch "${{ runner.temp }}/digests/${digest#sha256:}" - name: Upload Digest - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # 5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # 6.0.0 with: name: digests-${{ matrix.cache_tag }} path: ${{ runner.temp }}/digests/* @@ -114,7 +114,7 @@ jobs: steps: - name: Download Digests - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # 6.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # 7.0.0 with: path: ${{ runner.temp }}/digests pattern: digests-* @@ -122,14 +122,14 @@ jobs: - name: Login to GitHub Container Registry if: github.event_name != 'pull_request' - uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # 3.6.0 + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # 3.7.0 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Set up Docker Buildx - uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # 3.11.1 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # 3.12.0 # Lowercase image name and append -ci - name: Generate Image Name @@ -139,7 +139,7 @@ jobs: - name: Generate Docker Metadata (Tags and Labels) id: meta - uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # 5.8.0 + uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # 5.10.0 with: images: ghcr.io/${{ steps.image-name.outputs.IMAGE_NAME }} flavor: | diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 8b469eaacb..9f3e4f0a4f 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -69,14 +69,14 @@ jobs: runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # 5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # 6.0.2 with: persist-credentials: false fetch-depth: 0 - name: Setup QEMU if: runner.os == 'Linux' - uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # 3.6.0 + uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # 3.7.0 with: platforms: arm64 @@ -97,7 +97,7 @@ jobs: CIBW_TEST_SKIP: "*-win_arm64" - name: Upload Artifacts - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # 5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # 6.0.0 with: name: ${{ github.job }}-${{ matrix.wheel }} path: ./wheelhouse/*.whl @@ -109,12 +109,12 @@ jobs: runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # 5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # 6.0.2 with: persist-credentials: false fetch-depth: 0 - - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # 6.0.0 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # 6.2.0 with: python-version: "3.13" @@ -134,7 +134,7 @@ jobs: openssl md5 -binary "dist/${tarball}" | xxd -p | tr -d '\n' > "dist/${md5_file}" - name: Upload Artifacts - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # 5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # 6.0.0 with: name: ${{ github.job }}-sdist path: | @@ -166,7 +166,7 @@ jobs: environment: ${{ matrix.pypi-instance }} steps: - - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # 6.0.0 + - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # 7.0.0 with: path: ./dist/ merge-multiple: true @@ -196,7 +196,7 @@ jobs: repository-url: https://test.pypi.org/legacy/ - name: Attest - uses: actions/attest-build-provenance@977bb373ede98d70efdf65b84cb5f73e068dcc2a # 3.0.0 + uses: actions/attest-build-provenance@96278af6caaf10aea03fd8d33a09a777ca52d62f # 3.2.0 id: attest with: subject-path: | diff --git a/.github/workflows/mega-linter.yml b/.github/workflows/mega-linter.yml index 8f74866d43..e8efe75ada 100644 --- a/.github/workflows/mega-linter.yml +++ b/.github/workflows/mega-linter.yml @@ -45,7 +45,7 @@ jobs: steps: # Git Checkout - name: Checkout Code - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # 5.0.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # 6.0.2 with: token: ${{ secrets.PAT || secrets.GITHUB_TOKEN }} fetch-depth: 0 # Required for pushing commits to PRs @@ -53,7 +53,7 @@ jobs: # MegaLinter - name: MegaLinter id: ml - uses: oxsecurity/megalinter/flavors/python@62c799d895af9bcbca5eacfebca29d527f125a57 # 9.1.0 + uses: oxsecurity/megalinter/flavors/python@42bb470545e359597e7f12156947c436e4e3fb9a # 9.3.0 env: # All available variables are described in documentation # https://megalinter.io/latest/configuration/ @@ -68,7 +68,7 @@ jobs: # Upload MegaLinter artifacts - name: Archive production artifacts if: success() || failure() - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # 5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # 6.0.0 with: name: MegaLinter reports include-hidden-files: "true" @@ -109,7 +109,7 @@ jobs: run: sudo chown -Rc $UID .git/ - name: Commit and push applied linter fixes - uses: stefanzweifel/git-auto-commit-action@28e16e81777b558cc906c8750092100bbb34c5e3 # 7.0.0 + uses: stefanzweifel/git-auto-commit-action@04702edda442b2e678b25b537cec683a1493fcb9 # 7.1.0 if: env.APPLY_FIXES_IF_COMMIT == 'true' with: branch: >- diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e9ef7b2d4e..4d1b7932e2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -93,14 +93,14 @@ jobs: - tests steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # 5.0.0 - - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # 6.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # 6.0.2 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # 6.2.0 with: python-version: "3.13" architecture: x64 - name: Download Coverage Artifacts - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # 6.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # 7.0.0 with: pattern: coverage-* path: ./ @@ -113,7 +113,7 @@ jobs: coverage xml - name: Upload Coverage to Codecov - uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # 5.5.1 + uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # 5.5.2 with: files: coverage.xml fail_ci_if_error: true @@ -127,14 +127,14 @@ jobs: - tests steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # 5.0.0 - - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # 6.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # 6.0.2 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # 6.2.0 with: python-version: "3.13" architecture: x64 - name: Download Results Artifacts - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # 6.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # 7.0.0 with: pattern: results-* path: ./ @@ -166,7 +166,7 @@ jobs: --add-host=host.docker.internal:host-gateway timeout-minutes: 30 steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # 5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # 6.0.2 - name: Fetch git tags run: | @@ -196,7 +196,7 @@ jobs: FORCE_COLOR: "true" - name: Upload Coverage Artifacts - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # 5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # 6.0.0 if: always() with: name: coverage-${{ github.job }}-${{ strategy.job-index }} @@ -206,7 +206,7 @@ jobs: retention-days: 1 - name: Upload Results Artifacts - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # 5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # 6.0.0 if: always() with: name: results-${{ github.job }}-${{ strategy.job-index }} @@ -231,7 +231,7 @@ jobs: --add-host=host.docker.internal:host-gateway timeout-minutes: 30 steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # 5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # 6.0.2 - name: Fetch git tags run: | @@ -261,7 +261,7 @@ jobs: FORCE_COLOR: "true" - name: Upload Coverage Artifacts - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # 5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # 6.0.0 if: always() with: name: coverage-${{ github.job }}-${{ strategy.job-index }} @@ -271,7 +271,7 @@ jobs: retention-days: 1 - name: Upload Results Artifacts - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # 5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # 6.0.0 if: always() with: name: results-${{ github.job }}-${{ strategy.job-index }} @@ -294,14 +294,14 @@ jobs: runs-on: windows-2025 timeout-minutes: 30 steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # 5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # 6.0.2 - name: Fetch git tags run: | git fetch --tags origin - name: Install uv - uses: astral-sh/setup-uv@85856786d1ce8acfbcc2f13a5f3fbd6b938f9f41 # 7.1.2 + uses: astral-sh/setup-uv@eac588ad8def6316056a12d4907a9d4d84ff7a3b # 7.3.0 - name: Install Python run: | @@ -330,7 +330,7 @@ jobs: FORCE_COLOR: "true" - name: Upload Coverage Artifacts - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # 5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # 6.0.0 if: always() with: name: coverage-${{ github.job }}-${{ strategy.job-index }} @@ -340,7 +340,7 @@ jobs: retention-days: 1 - name: Upload Results Artifacts - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # 5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # 6.0.0 if: always() with: name: results-${{ github.job }}-${{ strategy.job-index }} @@ -363,14 +363,14 @@ jobs: runs-on: windows-11-arm timeout-minutes: 30 steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # 5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # 6.0.2 - name: Fetch git tags run: | git fetch --tags origin - name: Install uv - uses: astral-sh/setup-uv@85856786d1ce8acfbcc2f13a5f3fbd6b938f9f41 # 7.1.2 + uses: astral-sh/setup-uv@eac588ad8def6316056a12d4907a9d4d84ff7a3b # 7.3.0 - name: Install Python run: | @@ -403,7 +403,7 @@ jobs: FORCE_COLOR: "true" - name: Upload Coverage Artifacts - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # 5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # 6.0.0 if: always() with: name: coverage-${{ github.job }}-${{ strategy.job-index }} @@ -413,7 +413,7 @@ jobs: retention-days: 1 - name: Upload Results Artifacts - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # 5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # 6.0.0 if: always() with: name: results-${{ github.job }}-${{ strategy.job-index }} @@ -443,7 +443,7 @@ jobs: --add-host=host.docker.internal:host-gateway timeout-minutes: 30 steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # 5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # 6.0.2 - name: Fetch git tags run: | @@ -473,7 +473,7 @@ jobs: FORCE_COLOR: "true" - name: Upload Coverage Artifacts - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # 5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # 6.0.0 if: always() with: name: coverage-${{ github.job }}-${{ strategy.job-index }} @@ -483,7 +483,7 @@ jobs: retention-days: 1 - name: Upload Results Artifacts - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # 5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # 6.0.0 if: always() with: name: results-${{ github.job }}-${{ strategy.job-index }} @@ -526,7 +526,7 @@ jobs: --health-retries 10 steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # 5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # 6.0.2 - name: Fetch git tags run: | @@ -556,7 +556,7 @@ jobs: FORCE_COLOR: "true" - name: Upload Coverage Artifacts - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # 5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # 6.0.0 if: always() with: name: coverage-${{ github.job }}-${{ strategy.job-index }} @@ -566,7 +566,7 @@ jobs: retention-days: 1 - name: Upload Results Artifacts - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # 5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # 6.0.0 if: always() with: name: results-${{ github.job }}-${{ strategy.job-index }} @@ -606,7 +606,7 @@ jobs: --health-retries 5 steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # 5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # 6.0.2 - name: Fetch git tags run: | @@ -636,7 +636,7 @@ jobs: FORCE_COLOR: "true" - name: Upload Coverage Artifacts - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # 5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # 6.0.0 if: always() with: name: coverage-${{ github.job }}-${{ strategy.job-index }} @@ -646,7 +646,7 @@ jobs: retention-days: 1 - name: Upload Results Artifacts - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # 5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # 6.0.0 if: always() with: name: results-${{ github.job }}-${{ strategy.job-index }} @@ -687,7 +687,7 @@ jobs: --health-retries 5 steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # 5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # 6.0.2 - name: Fetch git tags run: | @@ -717,7 +717,7 @@ jobs: FORCE_COLOR: "true" - name: Upload Coverage Artifacts - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # 5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # 6.0.0 if: always() with: name: coverage-${{ github.job }}-${{ strategy.job-index }} @@ -727,7 +727,7 @@ jobs: retention-days: 1 - name: Upload Results Artifacts - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # 5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # 6.0.0 if: always() with: name: results-${{ github.job }}-${{ strategy.job-index }} @@ -772,7 +772,7 @@ jobs: # from every being executed as bash commands. steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # 5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # 6.0.2 - name: Fetch git tags run: | @@ -802,7 +802,7 @@ jobs: FORCE_COLOR: "true" - name: Upload Coverage Artifacts - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # 5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # 6.0.0 if: always() with: name: coverage-${{ github.job }}-${{ strategy.job-index }} @@ -812,7 +812,7 @@ jobs: retention-days: 1 - name: Upload Results Artifacts - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # 5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # 6.0.0 if: always() with: name: results-${{ github.job }}-${{ strategy.job-index }} @@ -837,7 +837,7 @@ jobs: --add-host=host.docker.internal:host-gateway timeout-minutes: 30 steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # 5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # 6.0.2 - name: Fetch git tags run: | @@ -867,7 +867,7 @@ jobs: FORCE_COLOR: "true" - name: Upload Coverage Artifacts - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # 5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # 6.0.0 if: always() with: name: coverage-${{ github.job }}-${{ strategy.job-index }} @@ -877,7 +877,7 @@ jobs: retention-days: 1 - name: Upload Results Artifacts - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # 5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # 6.0.0 if: always() with: name: results-${{ github.job }}-${{ strategy.job-index }} @@ -927,7 +927,7 @@ jobs: KAFKA_CFG_INTER_BROKER_LISTENER_NAME: L3 steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # 5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # 6.0.2 - name: Fetch git tags run: | @@ -957,7 +957,7 @@ jobs: FORCE_COLOR: "true" - name: Upload Coverage Artifacts - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # 5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # 6.0.0 if: always() with: name: coverage-${{ github.job }}-${{ strategy.job-index }} @@ -967,7 +967,7 @@ jobs: retention-days: 1 - name: Upload Results Artifacts - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # 5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # 6.0.0 if: always() with: name: results-${{ github.job }}-${{ strategy.job-index }} @@ -1005,7 +1005,7 @@ jobs: --health-retries 5 steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # 5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # 6.0.2 - name: Fetch git tags run: | @@ -1035,7 +1035,7 @@ jobs: FORCE_COLOR: "true" - name: Upload Coverage Artifacts - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # 5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # 6.0.0 if: always() with: name: coverage-${{ github.job }}-${{ strategy.job-index }} @@ -1045,7 +1045,7 @@ jobs: retention-days: 1 - name: Upload Results Artifacts - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # 5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # 6.0.0 if: always() with: name: results-${{ github.job }}-${{ strategy.job-index }} @@ -1083,7 +1083,7 @@ jobs: --health-retries 5 steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # 5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # 6.0.2 - name: Fetch git tags run: | @@ -1113,7 +1113,7 @@ jobs: FORCE_COLOR: "true" - name: Upload Coverage Artifacts - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # 5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # 6.0.0 if: always() with: name: coverage-${{ github.job }}-${{ strategy.job-index }} @@ -1123,7 +1123,7 @@ jobs: retention-days: 1 - name: Upload Results Artifacts - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # 5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # 6.0.0 if: always() with: name: results-${{ github.job }}-${{ strategy.job-index }} @@ -1161,7 +1161,7 @@ jobs: --health-retries 5 steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # 5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # 6.0.2 - name: Fetch git tags run: | @@ -1191,7 +1191,7 @@ jobs: FORCE_COLOR: "true" - name: Upload Coverage Artifacts - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # 5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # 6.0.0 if: always() with: name: coverage-${{ github.job }}-${{ strategy.job-index }} @@ -1201,7 +1201,7 @@ jobs: retention-days: 1 - name: Upload Results Artifacts - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # 5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # 6.0.0 if: always() with: name: results-${{ github.job }}-${{ strategy.job-index }} @@ -1244,7 +1244,7 @@ jobs: --health-retries 5 steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # 5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # 6.0.2 - name: Fetch git tags run: | @@ -1274,7 +1274,7 @@ jobs: FORCE_COLOR: "true" - name: Upload Coverage Artifacts - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # 5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # 6.0.0 if: always() with: name: coverage-${{ github.job }}-${{ strategy.job-index }} @@ -1284,7 +1284,7 @@ jobs: retention-days: 1 - name: Upload Results Artifacts - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # 5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # 6.0.0 if: always() with: name: results-${{ github.job }}-${{ strategy.job-index }} @@ -1327,7 +1327,7 @@ jobs: --health-retries 5 steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # 5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # 6.0.2 - name: Fetch git tags run: | @@ -1357,7 +1357,7 @@ jobs: FORCE_COLOR: "true" - name: Upload Coverage Artifacts - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # 5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # 6.0.0 if: always() with: name: coverage-${{ github.job }}-${{ strategy.job-index }} @@ -1367,7 +1367,7 @@ jobs: retention-days: 1 - name: Upload Results Artifacts - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # 5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # 6.0.0 if: always() with: name: results-${{ github.job }}-${{ strategy.job-index }} @@ -1406,7 +1406,7 @@ jobs: --health-retries 5 steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # 5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # 6.0.2 - name: Fetch git tags run: | @@ -1436,7 +1436,7 @@ jobs: FORCE_COLOR: "true" - name: Upload Coverage Artifacts - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # 5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # 6.0.0 if: always() with: name: coverage-${{ github.job }}-${{ strategy.job-index }} @@ -1446,7 +1446,7 @@ jobs: retention-days: 1 - name: Upload Results Artifacts - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # 5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # 6.0.0 if: always() with: name: results-${{ github.job }}-${{ strategy.job-index }} @@ -1487,7 +1487,7 @@ jobs: --health-retries 5 steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # 5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # 6.0.2 - name: Fetch git tags run: | @@ -1517,7 +1517,7 @@ jobs: FORCE_COLOR: "true" - name: Upload Coverage Artifacts - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # 5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # 6.0.0 if: always() with: name: coverage-${{ github.job }}-${{ strategy.job-index }} @@ -1527,7 +1527,7 @@ jobs: retention-days: 1 - name: Upload Results Artifacts - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # 5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # 6.0.0 if: always() with: name: results-${{ github.job }}-${{ strategy.job-index }} @@ -1567,7 +1567,7 @@ jobs: --health-retries 5 steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # 5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # 6.0.2 - name: Fetch git tags run: | @@ -1597,7 +1597,7 @@ jobs: FORCE_COLOR: "true" - name: Upload Coverage Artifacts - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # 5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # 6.0.0 if: always() with: name: coverage-${{ github.job }}-${{ strategy.job-index }} @@ -1607,7 +1607,7 @@ jobs: retention-days: 1 - name: Upload Results Artifacts - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # 5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # 6.0.0 if: always() with: name: results-${{ github.job }}-${{ strategy.job-index }} @@ -1647,7 +1647,7 @@ jobs: --health-retries 5 steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # 5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # 6.0.2 - name: Fetch git tags run: | @@ -1677,7 +1677,7 @@ jobs: FORCE_COLOR: "true" - name: Upload Coverage Artifacts - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # 5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # 6.0.0 if: always() with: name: coverage-${{ github.job }}-${{ strategy.job-index }} @@ -1687,7 +1687,7 @@ jobs: retention-days: 1 - name: Upload Results Artifacts - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # 5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # 6.0.0 if: always() with: name: results-${{ github.job }}-${{ strategy.job-index }} @@ -1726,7 +1726,7 @@ jobs: --health-retries 5 steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # 5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # 6.0.2 - name: Fetch git tags run: | @@ -1756,7 +1756,7 @@ jobs: FORCE_COLOR: "true" - name: Upload Coverage Artifacts - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # 5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # 6.0.0 if: always() with: name: coverage-${{ github.job }}-${{ strategy.job-index }} @@ -1766,7 +1766,7 @@ jobs: retention-days: 1 - name: Upload Results Artifacts - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # 5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # 6.0.0 if: always() with: name: results-${{ github.job }}-${{ strategy.job-index }} @@ -1804,7 +1804,7 @@ jobs: --health-retries 5 steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # 5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # 6.0.2 - name: Fetch git tags run: | @@ -1834,7 +1834,7 @@ jobs: FORCE_COLOR: "true" - name: Upload Coverage Artifacts - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # 5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # 6.0.0 if: always() with: name: coverage-${{ github.job }}-${{ strategy.job-index }} @@ -1844,7 +1844,7 @@ jobs: retention-days: 1 - name: Upload Results Artifacts - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # 5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # 6.0.0 if: always() with: name: results-${{ github.job }}-${{ strategy.job-index }} @@ -1923,7 +1923,7 @@ jobs: --add-host=host.docker.internal:host-gateway steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # 5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # 6.0.2 - name: Fetch git tags run: | @@ -1953,7 +1953,7 @@ jobs: FORCE_COLOR: "true" - name: Upload Coverage Artifacts - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # 5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # 6.0.0 if: always() with: name: coverage-${{ github.job }}-${{ strategy.job-index }} @@ -1963,7 +1963,7 @@ jobs: retention-days: 1 - name: Upload Results Artifacts - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # 5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # 6.0.0 if: always() with: name: results-${{ github.job }}-${{ strategy.job-index }} @@ -2003,7 +2003,7 @@ jobs: --health-retries 5 steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # 5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # 6.0.2 - name: Fetch git tags run: | @@ -2033,7 +2033,7 @@ jobs: FORCE_COLOR: "true" - name: Upload Coverage Artifacts - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # 5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # 6.0.0 if: always() with: name: coverage-${{ github.job }}-${{ strategy.job-index }} @@ -2043,7 +2043,7 @@ jobs: retention-days: 1 - name: Upload Results Artifacts - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # 5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # 6.0.0 if: always() with: name: results-${{ github.job }}-${{ strategy.job-index }} @@ -2081,7 +2081,7 @@ jobs: --health-retries 5 steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # 5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # 6.0.2 - name: Fetch git tags run: | @@ -2111,7 +2111,7 @@ jobs: FORCE_COLOR: "true" - name: Upload Coverage Artifacts - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # 5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # 6.0.0 if: always() with: name: coverage-${{ github.job }}-${{ strategy.job-index }} @@ -2121,7 +2121,7 @@ jobs: retention-days: 1 - name: Upload Results Artifacts - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # 5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # 6.0.0 if: always() with: name: results-${{ github.job }}-${{ strategy.job-index }} diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml index c373a38bb1..0a2c1e0e4f 100644 --- a/.github/workflows/trivy.yml +++ b/.github/workflows/trivy.yml @@ -32,14 +32,14 @@ jobs: steps: # Git Checkout - name: Checkout Code - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # 5.0.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # 6.0.2 with: token: ${{ secrets.PAT || secrets.GITHUB_TOKEN }} fetch-depth: 0 - name: Run Trivy vulnerability scanner in repo mode if: ${{ github.event_name == 'pull_request' }} - uses: aquasecurity/trivy-action@b6643a29fecd7f34b3597bc6acb0a98b03d33ff8 # v0.33.1 + uses: aquasecurity/trivy-action@e368e328979b113139d6f9068e03accaed98a518 # v0.34.1 with: scan-type: "fs" ignore-unfixed: true @@ -50,7 +50,7 @@ jobs: - name: Run Trivy vulnerability scanner in repo mode if: ${{ github.event_name == 'schedule' }} - uses: aquasecurity/trivy-action@b6643a29fecd7f34b3597bc6acb0a98b03d33ff8 # v0.33.1 + uses: aquasecurity/trivy-action@e368e328979b113139d6f9068e03accaed98a518 # v0.34.1 with: scan-type: "fs" ignore-unfixed: true @@ -61,6 +61,6 @@ jobs: - name: Upload Trivy scan results to GitHub Security tab if: ${{ github.event_name == 'schedule' }} - uses: github/codeql-action/upload-sarif@0499de31b99561a6d14a36a5f662c2a54f91beee # 4.31.2 + uses: github/codeql-action/upload-sarif@89a39a4e59826350b863aa6b6252a07ad50cf83e # 4.32.4 with: sarif_file: "trivy-results.sarif" diff --git a/newrelic/api/application.py b/newrelic/api/application.py index 9aa6d7b6b8..f3a68413fe 100644 --- a/newrelic/api/application.py +++ b/newrelic/api/application.py @@ -156,11 +156,11 @@ def normalize_name(self, name, rule_type="url"): return self._agent.normalize_name(self._name, name, rule_type) return name, False - def compute_sampled(self): + def compute_sampled(self, full_granularity, section, *args, **kwargs): if not self.active or not self.settings.distributed_tracing.enabled: return False - return self._agent.compute_sampled(self._name) + return self._agent.compute_sampled(self._name, full_granularity, section, *args, **kwargs) def application_instance(name=None, activate=True): diff --git a/newrelic/api/asgi_application.py b/newrelic/api/asgi_application.py index 669d3e6db5..6b9a31130e 100644 --- a/newrelic/api/asgi_application.py +++ b/newrelic/api/asgi_application.py @@ -132,10 +132,20 @@ async def send_inject_browser_agent(self, message): message_type = message["type"] if message_type == "http.response.start" and not self.initial_message: - headers = list(message.get("headers", ())) + # message["headers"] may be a generator, and consuming it via process_response will leave the original + # application with no headers. Fix this by preserving them in a list before consuming them. + if "headers" in message: + message["headers"] = headers = list(message["headers"]) + else: + headers = [] + + # Check if we should insert the HTML snippet based on the headers. + # Currently if there are no headers this will always be False, but call the function + # anyway in case this logic changes in the future. if not self.should_insert_html(headers): await self.abort() return + message["headers"] = headers self.initial_message = message elif message_type == "http.response.body" and self.initial_message: @@ -232,7 +242,13 @@ async def send(self, event): finally: self.__exit__(*sys.exc_info()) elif event["type"] == "http.response.start": - self.process_response(event["status"], event.get("headers", ())) + # event["headers"] may be a generator, and consuming it via process_response will leave the original + # ASGI application with no headers. Fix this by preserving them in a list before consuming them. + if "headers" in event: + event["headers"] = headers = list(event["headers"]) + else: + headers = [] + self.process_response(event["status"], headers) return await self._send(event) diff --git a/newrelic/api/database_trace.py b/newrelic/api/database_trace.py index a2f31ca504..cb449da068 100644 --- a/newrelic/api/database_trace.py +++ b/newrelic/api/database_trace.py @@ -226,6 +226,8 @@ def create_node(self): guid=self.guid, agent_attributes=self.agent_attributes, user_attributes=self.user_attributes, + span_link_events=self.span_link_events, + span_event_events=self.span_event_events, ) diff --git a/newrelic/api/datastore_trace.py b/newrelic/api/datastore_trace.py index 4d3a0db0ad..d9a71acf1c 100644 --- a/newrelic/api/datastore_trace.py +++ b/newrelic/api/datastore_trace.py @@ -123,6 +123,8 @@ def create_node(self): guid=self.guid, agent_attributes=self.agent_attributes, user_attributes=self.user_attributes, + span_link_events=self.span_link_events, + span_event_events=self.span_event_events, ) diff --git a/newrelic/api/error_trace.py b/newrelic/api/error_trace.py index db63c54316..aaa12b50e3 100644 --- a/newrelic/api/error_trace.py +++ b/newrelic/api/error_trace.py @@ -15,6 +15,7 @@ import functools from newrelic.api.time_trace import current_trace, notice_error +from newrelic.common.async_wrapper import async_wrapper as get_async_wrapper from newrelic.common.object_wrapper import FunctionWrapper, wrap_object @@ -43,17 +44,31 @@ def __exit__(self, exc, value, tb): ) -def ErrorTraceWrapper(wrapped, ignore=None, expected=None, status_code=None): - def wrapper(wrapped, instance, args, kwargs): - parent = current_trace() +def ErrorTraceWrapper(wrapped, ignore=None, expected=None, status_code=None, async_wrapper=None): + def literal_wrapper(wrapped, instance, args, kwargs): + # Determine if the wrapped function is async or sync + wrapper = async_wrapper if async_wrapper is not None else get_async_wrapper(wrapped) + # Sync function path + if not wrapper: + parent = current_trace() + if not parent: + # No active tracing context so just call the wrapped function directly + return wrapped(*args, **kwargs) + # Async function path + else: + # For async functions, the async wrapper will handle trace context propagation + parent = None - if parent is None: - return wrapped(*args, **kwargs) + trace = ErrorTrace(ignore, expected, status_code, parent=parent) + + if wrapper: + # The async wrapper handles the context management for us + return wrapper(wrapped, trace)(*args, **kwargs) - with ErrorTrace(ignore, expected, status_code, parent=parent): + with trace: return wrapped(*args, **kwargs) - return FunctionWrapper(wrapped, wrapper) + return FunctionWrapper(wrapped, literal_wrapper) def error_trace(ignore=None, expected=None, status_code=None): diff --git a/newrelic/api/external_trace.py b/newrelic/api/external_trace.py index 372eb2ca09..9d3dff69e4 100644 --- a/newrelic/api/external_trace.py +++ b/newrelic/api/external_trace.py @@ -59,6 +59,8 @@ def create_node(self): guid=self.guid, agent_attributes=self.agent_attributes, user_attributes=self.user_attributes, + span_link_events=self.span_link_events, + span_event_events=self.span_event_events, ) diff --git a/newrelic/api/function_trace.py b/newrelic/api/function_trace.py index a782c1cfac..5f56104d7a 100644 --- a/newrelic/api/function_trace.py +++ b/newrelic/api/function_trace.py @@ -75,6 +75,8 @@ def create_node(self): guid=self.guid, agent_attributes=self.agent_attributes, user_attributes=self.user_attributes, + span_link_events=self.span_link_events, + span_event_events=self.span_event_events, ) diff --git a/newrelic/api/import_hook.py b/newrelic/api/import_hook.py index 15bd3ba992..07e8477aa3 100644 --- a/newrelic/api/import_hook.py +++ b/newrelic/api/import_hook.py @@ -211,3 +211,15 @@ def decorator(wrapped): def import_module(name): __import__(name) return sys.modules[name] + + +def enable_import_hook_finder(): + if sys.meta_path and not isinstance(sys.meta_path[0], ImportHookFinder): + # If we don't hold the first position in sys.meta_path then we need to insert ourselves + # there and remove all other instances of ImportHookFinder. + sys.meta_path.insert(0, ImportHookFinder()) + + # Remove any duplicate instances of ImportHookFinder. + for finder in list(sys.meta_path[1:]): + if isinstance(finder, ImportHookFinder): + sys.meta_path.remove(finder) diff --git a/newrelic/api/memcache_trace.py b/newrelic/api/memcache_trace.py index 810ed621b6..8963c303e0 100644 --- a/newrelic/api/memcache_trace.py +++ b/newrelic/api/memcache_trace.py @@ -48,6 +48,8 @@ def create_node(self): guid=self.guid, agent_attributes=self.agent_attributes, user_attributes=self.user_attributes, + span_link_events=self.span_link_events, + span_event_events=self.span_event_events, ) diff --git a/newrelic/api/message_trace.py b/newrelic/api/message_trace.py index 5f6f9a76d0..f7fd3a601a 100644 --- a/newrelic/api/message_trace.py +++ b/newrelic/api/message_trace.py @@ -84,6 +84,8 @@ def create_node(self): guid=self.guid, agent_attributes=self.agent_attributes, user_attributes=self.user_attributes, + span_link_events=self.span_link_events, + span_event_events=self.span_event_events, ) diff --git a/newrelic/api/opentelemetry.py b/newrelic/api/opentelemetry.py new file mode 100644 index 0000000000..6a2be370d3 --- /dev/null +++ b/newrelic/api/opentelemetry.py @@ -0,0 +1,977 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import logging +import os +import sys +import time +from contextlib import contextmanager + +try: + from opentelemetry import trace as otel_api_trace + from opentelemetry.baggage.propagation import W3CBaggagePropagator + from opentelemetry.propagate import set_global_textmap + from opentelemetry.propagators.composite import CompositePropagator + from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator + from opentelemetry.trace.status import Status, StatusCode +except ImportError: + otel_api_trace = None + W3CBaggagePropagator = None + set_global_textmap = None + CompositePropagator = None + TraceContextTextMapPropagator = None + Status = None + StatusCode = None + +from newrelic.api.application import application_instance +from newrelic.api.background_task import BackgroundTask +from newrelic.api.datastore_trace import DatastoreTrace +from newrelic.api.external_trace import ExternalTrace +from newrelic.api.function_trace import FunctionTrace +from newrelic.api.message_trace import MessageTrace +from newrelic.api.message_transaction import MessageTransaction +from newrelic.api.time_trace import add_custom_span_attribute, current_trace, notice_error +from newrelic.api.transaction import Sentinel, current_transaction +from newrelic.api.web_transaction import WebTransaction, WSGIWebTransaction +from newrelic.core.attribute import sanitize +from newrelic.core.database_utils import ( + _all_literals_re, + _quotes_table, + generate_dynamodb_arn, + get_database_operation_target_from_statement, +) +from newrelic.core.otlp_utils import create_resource + +_logger = logging.getLogger(__name__) + + +class NRTraceContextPropagator(TraceContextTextMapPropagator): + HEADER_KEYS = ("traceparent", "tracestate", "newrelic") + + def extract(self, carrier, context=None, getter=None): + transaction = current_transaction() + # If we are passing into New Relic, traceparent + # and/or tracestate's keys also need to be NR compatible. + + if transaction: + nr_headers = { + header_key: getter.get(carrier, header_key)[0] + for header_key in self.HEADER_KEYS + if getter.get(carrier, header_key) + } + transaction.accept_distributed_trace_headers(nr_headers) + + extracted_context = super().extract(carrier=carrier, context=context, getter=getter) + + return extracted_context + + def inject(self, carrier, context=None, setter=None): + transaction = current_transaction() + if not transaction: + return super().inject(carrier=carrier, context=context, setter=setter) + + nr_headers = [] + transaction.insert_distributed_trace_headers(nr_headers) + for key, value in nr_headers: + setter.set(carrier, key, value) + # Do NOT call super().inject() since we have already + # inserted the headers here. It will not cause harm, + # but it is redundant logic. + + +# Context and Context Propagator Setup +try: + opentelemetry_context_propagator = CompositePropagator( + propagators=[NRTraceContextPropagator(), W3CBaggagePropagator()] + ) + set_global_textmap(opentelemetry_context_propagator) +except: + pass + +# ---------------------------------------------- +# Custom OpenTelemetry Spans and Traces +# ---------------------------------------------- + + +class Span(otel_api_trace.Span): + def __init__( + self, + name=None, + parent=None, # SpanContext + resource=None, + attributes=None, + kind=otel_api_trace.SpanKind.INTERNAL, + record_exception=True, + set_status_on_exception=True, + nr_transaction=None, + nr_trace_type=FunctionTrace, + instrumenting_module=None, + create_nr_trace=True, + links=None, + *args, + **kwargs, + ): + self.name = name + self.opentelemetry_parent = parent + self.attributes = attributes or {} + self.kind = kind + self.nr_transaction = ( + nr_transaction or current_transaction() + ) # This attribute is purely to prevent garbage collection + self.nr_trace = None + self.instrumenting_module = instrumenting_module + self.status = Status(StatusCode.UNSET) + self._record_exception = record_exception + self.set_status_on_exception = set_status_on_exception + self.links = links or [] + self.create_nr_trace = create_nr_trace + + self.nr_parent = None + current_nr_trace = current_trace() + if ( + not self.opentelemetry_parent + or (self.opentelemetry_parent and self.opentelemetry_parent.span_id == int(current_nr_trace.guid, 16)) + or (self.opentelemetry_parent and isinstance(current_nr_trace, Sentinel)) + ): + # Expected to come here if one of three scenarios have occured: + # 1. `start_as_current_span` was used. + # 2. `start_span` was used and the current span was explicitly set + # to the newly created one. + # 3. Only a Sentinel Trace exists so far while still having a + # remote parent. From OpenTelemetry's end, this will be represented + # as a `NonRecordingSpan` (and be seen as `None` at this + # point). This covers cases where span is remote. + self.nr_parent = current_nr_trace + else: + # This should not occur, but if it does, we need to + # log an error and not create a New Relic trace. + _logger.error( + "OpenTelemetry span (%s) and NR trace (%s) do not match nor correspond to a remote span. Open Telemetry span will not be reported to New Relic. Please report this problem to New Relic.", + self.opentelemetry_parent, + current_nr_trace, # NR parent trace + ) + return + + if not self.create_nr_trace: + # Do not create a New Relic trace for this OpenTelemetry span. + # While this OpenTelemetry span exists it will not be explicitly + # translated to a NR trace. This may occur during the + # creation of a Transaction, which will create the root + # span. This may also occur during special cases, such + # as back to back calls to Kafka's queue's consumer. + # If a transaction already exists, we do not want to + # create another transaction or trace, but rather just + # append existing attributes to the existing transaction. + self.nr_trace = current_nr_trace + # Add Instrumentation Scope Attributes + self.nr_trace._add_agent_attribute("otel.scope.name", self.attributes.get("library_name")) + self.nr_trace._add_agent_attribute("otel.scope.version", self.attributes.get("library_version")) + self.nr_trace._add_agent_attribute("otel.library.name", self.attributes.get("library_name")) + self.nr_trace._add_agent_attribute("otel.library.version", self.attributes.get("library_version")) + return + elif nr_trace_type == FunctionTrace: + trace_kwargs = {"name": self.name, "params": self.attributes, "parent": self.nr_parent} + self.nr_trace = nr_trace_type(**trace_kwargs) + elif nr_trace_type == DatastoreTrace: + trace_kwargs = { + "product": self.instrumenting_module, + "target": None, + "operation": self.name, + "parent": self.nr_parent, + } + self.nr_trace = nr_trace_type(**trace_kwargs) + elif nr_trace_type == ExternalTrace: + trace_kwargs = { + "library": self.instrumenting_module, + "url": self.attributes.get("http.url"), + "method": self.attributes.get("http.method") or self.name, + "parent": self.nr_parent, + } + self.nr_trace = nr_trace_type(**trace_kwargs) + elif nr_trace_type == MessageTrace: + trace_kwargs = { + "library": self.instrumenting_module, + "operation": "Produce" if self.kind == otel_api_trace.SpanKind.PRODUCER else "Consume", + "destination_type": "Exchange", + "destination_name": self.name, + "params": self.attributes, + "parent": self.nr_parent, + "terminal": False, + } + self.nr_trace = nr_trace_type(**trace_kwargs) + else: + trace_kwargs = {"name": self.name, "params": self.attributes, "parent": self.nr_parent} + self.nr_trace = nr_trace_type(**trace_kwargs) + + self.nr_trace.__enter__() + + # Add Instrumentation Scope Attributes + self.nr_trace._add_agent_attribute("otel.scope.name", self.attributes.get("library_name")) + self.nr_trace._add_agent_attribute("otel.scope.version", self.attributes.get("library_version")) + self.nr_trace._add_agent_attribute("otel.library.name", self.attributes.get("library_name")) + self.nr_trace._add_agent_attribute("otel.library.version", self.attributes.get("library_version")) + + # Process Links that were passed in upon span creation + for link in self.links: + self.add_link(context=link.context, attributes=link.attributes, timestamp=self.nr_trace.start_time) + + def _remote(self): + """ + Remote span denotes if propagated from a remote parent + """ + return bool(self.opentelemetry_parent and self.opentelemetry_parent.is_remote) + + def get_span_context(self): + if not getattr(self, "nr_trace", False): + return otel_api_trace.INVALID_SPAN_CONTEXT + + return otel_api_trace.SpanContext( + trace_id=int(self.nr_transaction.trace_id, 16), + span_id=int(self.nr_trace.guid, 16), + is_remote=self._remote(), + trace_flags=otel_api_trace.TraceFlags(0x01), + trace_state=otel_api_trace.TraceState(), + ) + + def set_attribute(self, key, value): + self.attributes[key] = value + + def set_attributes(self, attributes): + for key, value in attributes.items(): + self.set_attribute(key, value) + + def _set_attributes_in_nr(self, opentelemetry_attributes=None): + if not opentelemetry_attributes or not getattr(self, "nr_trace", None): + return + + # If these attributes already exist in NR's agent attributes, + # keep the attributes in the OpenTelemetry span, but do not add them + # to NR's user attributes to avoid sending the same data + # multiple times. + for key, value in opentelemetry_attributes.items(): + if key not in self.nr_trace.agent_attributes: + self.nr_trace.add_custom_attribute(key, value) + + def add_event(self, name, attributes=None, timestamp=None): + """Add an event to the current span. + + If timestamp is None, this will get set to the current time. + """ + current_span_context = self.get_span_context() + current_trace_id = f"{current_span_context.trace_id:032x}" + current_span_id = f"{current_span_context.span_id:016x}" + + # Sanitize name, if not already a string. + try: + name = sanitize(name) + except Exception as e: + _logger.error("Invalid event name %s passed to add_event; event will not be created. Error: %s", name, e) + return + + self.nr_trace._add_span_event_event( + span_id=current_span_id, trace_id=current_trace_id, name=name, timestamp=timestamp, attributes=attributes + ) + + def add_link(self, context=None, attributes=None, timestamp=None): + """Add a link to another span. + + NOTE: `timestamp` is not an OpenTelemetry specific value. This is + a Hybrid Agent specific argument that allows us to set the + time of the link based on when the NR trace was created + (if the link was passed in during the span's creation), or + if added later on (the time when the link was added). + """ + if not context or not context.is_valid: + _logger.error("Invalid span context passed to add_link; link will not be created.") + return + + # If timestamp is None, use the current time + if timestamp: + timestamp = int(timestamp * 1e3) + else: + timestamp = int(time.time() * 1e3) + + link_trace_id = f"{context.trace_id:032x}" + link_span_id = f"{context.span_id:016x}" + current_span_context = self.get_span_context() + current_trace_id = f"{current_span_context.trace_id:032x}" + current_span_id = f"{current_span_context.span_id:016x}" + + self.nr_trace._add_span_link_event( + span_id=current_span_id, + trace_id=current_trace_id, + linked_span_id=link_span_id, + linked_trace_id=link_trace_id, + timestamp=timestamp, + attributes=attributes, + ) + + def update_name(self, name): + # NOTE: Sentinel, MessageTrace, DatastoreTrace, and + # ExternalTrace types do not have a name attribute. + self.name = name + if hasattr(self, "nr_trace") and hasattr(self.nr_trace, "name"): + self.nr_trace.name = self.name + + def is_recording(self): + # If the trace has an end time set then it is done recording. Otherwise, + # if it does not have an end time set and the transaction's priority + # has not been set yet or it is set to something other than 0 then it + # is also still recording. + if getattr(self.nr_trace, "end_time", None): + return False + + # If priority is either not set at this point + # or greater than 0, we are recording. + priority = self.nr_transaction.priority + return (priority is None) or (priority > 0) + + def set_status(self, status, description=None): + """ + This code is modeled after the OpenTelemetry SDK's + status implementation: + https://github.com/open-telemetry/opentelemetry-python/blob/main/opentelemetry-sdk/src/opentelemetry/sdk/trace/__init__.py#L979 + + Additional Notes: + 1. Ignore future calls if status is already set to OK + since span should be completed if status is OK. + 2. Similarly, ignore calls to set to StatusCode.UNSET + since this will be either invalid or unnecessary. + """ + if isinstance(status, Status): + if (self.status.status_code is StatusCode.OK) or status.is_unset: + return + if description is not None: + # `description` should only exist if status is StatusCode.ERROR + _logger.warning( + "Description %s ignored. Use either `Status` or `(StatusCode, Description)`", description + ) + self.status = status + elif isinstance(status, StatusCode): + if (self.status.status_code is StatusCode.OK) or (status is StatusCode.UNSET): + return + self.status = Status(status, description) + else: + _logger.warning("Invalid status type %s. Expected Status or StatusCode.", type(status)) + return + + # Add status as attribute + self.set_attribute("status_code", self.status.status_code.name) + self.set_attribute("status_description", self.status.description) + + def record_exception(self, exception, attributes=None, timestamp=None, escaped=False): + error_args = sys.exc_info() if not exception else (type(exception), exception, exception.__traceback__) + + # `escaped` indicates whether the exception has not + # been unhandled by the time the span has ended. + if attributes: + attributes["exception.escaped"] = escaped + else: + attributes = {"exception.escaped": escaped} + + self.set_attributes(attributes) + + notice_error(error_args, attributes=attributes) + + def _obfuscate_query(self, sql, database): + database_to_quote_style_mapping = { + "postgresql": "single+dollar", + "psycopg2": "single+dollar", + "graphql": "single+double", + "mysql": "single+double", + } + + quotes_re, quotes_cleanup_re = _quotes_table.get( + database_to_quote_style_mapping.get(database), _quotes_table.get("single") + ) + sql = quotes_re.sub("?", sql) + sql = _all_literals_re.sub("?", sql) + if quotes_cleanup_re.search(sql): + sql = "?" + + return sql + + def _messagequeue_attribute_mapping(self): + host = self.attributes.get("net.peer.name") or self.attributes.get("server.address") + port = self.attributes.get("net.peer.port") or self.attributes.get("server.port") + name = self.name.split(maxsplit=1)[0] # OpenTelemetry's format for this is "name operation" + + # Logic for Pika/RabbitMQ + span_obj_attrs = { + "library": self.attributes.get("messaging.system"), + "destination_name": name, # OpenTelemetry's format for this is "name operation" + } + + if span_obj_attrs["library"] == "rabbitmq": + # In RabbitMQ, destination_type is always Exchange and + # destination_name is actually stored in the span name. + # messaging.destination stores the task_name (such as + # consumer tag) + span_obj_attrs["destination_type"] = "Exchange" + + agent_attrs = {"host": host, "port": port, "server.address": host, "server.port": port} + + # Kafka Specific Logic + if span_obj_attrs["library"] == "kafka": + span_obj_attrs.update( + { + "transport_type": "Kafka", + "destination_type": "Topic", + "destination_name": name + if (name != "unknown") + else "Default", # OpenTelemetry's format for this is "name operation" + } + ) + if isinstance(self.nr_transaction, MessageTransaction): + self.nr_transaction.transport_type = "Kafka" + self.nr_transaction.destination_type = "Topic" + + if ( + self.nr_transaction.destination_name.startswith("unknown") + and span_obj_attrs["destination_name"] != "unknown" + ): + self.nr_transaction.destination_name = span_obj_attrs["destination_name"] + else: + self.nr_transaction.destination_name = "Default" + + bootstrap_servers = json.loads(self.attributes.get("messaging.url", "[]")) + for server_name in bootstrap_servers: + produce_or_consume = "Produce" if self.kind == otel_api_trace.SpanKind.PRODUCER else "Consume" + self.nr_transaction.record_custom_metric( + f"MessageBroker/kafka/Nodes/{server_name}/{produce_or_consume}/{span_obj_attrs['destination_name']}", + 1, + ) + + # Even if the attribute is set to None, it should rename + # the transaction destination_name attribute as well: + if isinstance(self.nr_transaction, MessageTransaction): + name, group = self.nr_transaction.get_transaction_name( + span_obj_attrs["library"], span_obj_attrs["destination_type"], span_obj_attrs["destination_name"] + ) + self.nr_transaction.set_transaction_name(name, group) + + # We do not want to override any agent attributes + # with `None` if `value` does not exist. + for key, value in span_obj_attrs.items(): + if value: + setattr(self.nr_trace, key, value) + for key, value in agent_attrs.items(): + if value: + self.nr_trace._add_agent_attribute(key, value) + + def _database_attribute_mapping(self): + span_obj_attrs = { + "host": self.attributes.get("net.peer.name") or self.attributes.get("server.address"), + "database_name": self.attributes.get("db.name"), + "port_path_or_id": self.attributes.get("net.peer.port") or self.attributes.get("server.port"), + "product": self.attributes.get("db.system", self.attributes.get("db.system.name")), + } + agent_attrs = {} + + db_statement = self.attributes.pop("db.statement", None) + if db_statement: + if hasattr(db_statement, "string"): + db_statement = db_statement.string + operation, target = get_database_operation_target_from_statement(db_statement) + target = target or self.attributes.get("db.mongodb.collection") + span_obj_attrs.update({"operation": operation, "target": target}) + if self.nr_transaction.application.settings.transaction_tracer.record_sql != "off": + if self.nr_transaction.application.settings.transaction_tracer.record_sql == "obfuscated": + db_statement = self._obfuscate_query(db_statement, span_obj_attrs["product"]) + agent_attrs["db.statement"] = db_statement + elif span_obj_attrs["product"] == "dynamodb": + region = self.attributes.get("cloud.region") + operation = self.attributes.get("db.operation", self.attributes.get("db.operation.name")) + target = self.attributes.get("aws.dynamodb.table_names", [None])[-1] + account_id = self.nr_transaction.settings.cloud.aws.account_id + resource_id = generate_dynamodb_arn(span_obj_attrs["host"], region, account_id, target) + agent_attrs.update( + { + "aws.operation": operation, + "cloud.resource_id": resource_id, + "cloud.region": region, + "aws.requestId": self.attributes.get("aws.request_id"), + "http.statusCode": self.attributes.get("http.status_code"), + "cloud.account.id": account_id, + } + ) + span_obj_attrs.update({"target": target, "operation": operation}) + + # We do not want to override any agent attributes + # with `None` if `value` does not exist. + for key, value in span_obj_attrs.items(): + if value: + setattr(self.nr_trace, key, value) + for key, value in agent_attrs.items(): + if value: + self.nr_trace._add_agent_attribute(key, value) + + def _strawberry_operation_name_parser(self, span_name): + if ": " in span_name: + return span_name.split(": ")[1] + return self.nr_trace.agent_attributes.get("graphql.operation.name") + + def _graphql_attribute_mapping(self): + if self.nr_transaction.application.settings.transaction_tracer.record_sql == "obfuscated": + sql = self.attributes.get("query", "") + if sql: + self.attributes["query"] = self._obfuscate_query(sql, "graphql") + + for key in self.attributes.keys(): + if ("graphql.arg" in key) or ("graphql.param." in key): + self.attributes[key] = "?" + elif self.nr_transaction.application.settings.transaction_tracer.record_sql == "off": + self.attributes.pop("query", None) + for key in self.attributes.keys(): + if ("graphql.arg" in key) or ("graphql.param." in key): + self.attributes[key] = "" + + self.nr_trace._add_agent_attribute( + "graphql.field.path", + self.attributes.get("graphql.path", self.nr_trace.agent_attributes.get("graphql.field.path")), + ) + self.nr_trace._add_agent_attribute( + "graphql.field.parentType", + self.attributes.get("graphql.parentType", self.nr_trace.agent_attributes.get("graphql.field.parentType")), + ) + self.nr_trace._add_agent_attribute( + "graphql.operation.name", + self.attributes.get("graphql.operation.name", self._strawberry_operation_name_parser(self.name)), + ) + self.nr_trace._add_agent_attribute( + "graphql.operation.query", + self.attributes.get("query", self.nr_trace.agent_attributes.get("graphql.operation.query")), + ) + + def end(self, end_time=None, *args, **kwargs): + # We will ignore the end_time parameter and use NR's end_time + + # Check to see if New Relic trace ever existed + # or, if it does, that trace has already ended + if not self.nr_trace or getattr(self.nr_trace, "end_time", None): + return + + # We will need to add specific attributes to the + # NR trace before the node creation because the + # attributes were likely not available at the time + # of the trace's creation but eventually populated + # throughout the span's lifetime. + + # Database/Datastore specific attributes + if self.attributes.get("db.system", self.attributes.get("db.system.name")): + self._database_attribute_mapping() + + # Message specific attributes + if self.attributes.get("messaging.system"): + self._messagequeue_attribute_mapping() + + # External/Web specific attributes + if ("http.status_code" in self.attributes) and (isinstance(self.nr_transaction, WebTransaction)): + response_headers = { + key.split("http.response.header.")[1].replace("_", "-"): value[0] + for key, value in self.attributes.items() + if key.startswith("http.response.header.") + } + self.nr_transaction.process_response(str(self.attributes.get("http.status_code")), response_headers) + + self.nr_trace._add_agent_attribute("http.statusCode", self.attributes.get("http.status_code")) + + # GraphQL specific attributes + if self.attributes.get("component") and self.attributes.get("component").lower() == "graphql": + self._graphql_attribute_mapping() + + # Add OpenTelemetry attributes as custom NR trace attributes + self._set_attributes_in_nr(self.attributes) + + error = sys.exc_info() + self.set_status(StatusCode.OK if not error[0] else StatusCode.ERROR) + + # Only if unhandled exception do we want to abruptly end. + # Otherwise, ensure that the span is the last one to end. + if getattr(self.attributes, "exception.escaped", False) or ( + self.kind in (otel_api_trace.SpanKind.SERVER, otel_api_trace.SpanKind.CONSUMER) + and isinstance(current_trace(), Sentinel) + ): + # We need to end the transaction, which will + # end the sentinel trace as well. + self.nr_transaction.__exit__(*error) + else: + # Just end the existing trace + self.nr_trace.__exit__(*error) + + def __exit__(self, exc_type, exc_val, exc_tb): + """ + Ends context manager and calls `end` on the `Span`. + This is used when span is called as a context manager + i.e. `with tracer.start_span() as span:` + """ + if exc_val and self.is_recording(): + if self._record_exception: + self.record_exception(exception=exc_val, escaped=True) + if self.set_status_on_exception: + self.set_status(Status(status_code=StatusCode.ERROR, description=f"{exc_type.__name__}: {exc_val}")) + + super().__exit__(exc_type, exc_val, exc_tb) + + +class LazySpan(otel_api_trace.NonRecordingSpan, Span): + def __init__(self, context, trace): + super().__init__(context) + self.nr_trace = trace + + def set_attribute(self, key, value): + add_custom_span_attribute(key, value) + + def set_attributes(self, attributes): + for key, value in attributes.items(): + add_custom_span_attribute(key, value) + + add_event = Span.add_event + add_link = Span.add_link + + +class Tracer(otel_api_trace.Tracer): + def __init__( + self, + instrumentation_library=None, + instrumenting_library_version=None, + schema_url=None, + attributes=None, + resource=None, + *args, + **kwargs, + ): + self.instrumentation_library = instrumentation_library.split(".")[-1] + self.instrumenting_library_version = instrumenting_library_version + self.schema_url = schema_url + self.tracer_attributes = attributes or {} + self.resource = resource + + def _create_web_transaction(self, nr_headers=None): + if "nr.wsgi.environ" in self.attributes: + # This is a WSGI request + transaction = WSGIWebTransaction(self.nr_application, environ=self.attributes.pop("nr.wsgi.environ")) + elif "nr.asgi.scope" in self.attributes: + # This is an ASGI request + scope = self.attributes.pop("nr.asgi.scope") + scheme = scope.get("scheme", "http") + server = scope.get("server") or (None, None) + host, port = scope["server"] = tuple(server) + request_method = scope.get("method") + request_path = scope.get("path") + query_string = scope.get("query_string") + headers = scope["headers"] + transaction = WebTransaction( + application=self.nr_application, + name=self.name, + group="Uri", + scheme=scheme, + host=host, + port=port, + request_method=request_method, + request_path=request_path, + query_string=query_string, + headers=headers, + ) + else: + # This is a web request + nr_headers = nr_headers or {} + headers = self.attributes.pop("nr.http.headers", nr_headers) + scheme = self.attributes.get("http.scheme") + host = self.attributes.get("http.server_name") + port = self.attributes.get("net.host.port") + request_method = self.attributes.get("http.method") + request_path = self.attributes.get("http.route") + + transaction = WebTransaction( + self.nr_application, + name=self.name, + scheme=scheme, + host=host, + port=port, + request_method=request_method, + request_path=request_path, + headers=headers, + ) + return transaction + + def start_span( + self, + name, + context=None, # Optional[Context] + kind=otel_api_trace.SpanKind.INTERNAL, + attributes=None, + links=None, + start_time=None, + record_exception=True, + set_status_on_exception=True, + *args, + **kwargs, + ): + nr_trace_type = FunctionTrace + transaction = current_transaction() + self.nr_application = application_instance() + self.attributes = { + **(attributes or {}), + **self.tracer_attributes, + "schema_url": self.schema_url, + "library_name": self.instrumentation_library, + "library_version": self.instrumenting_library_version, + } + self.name = name + self.links = links or [] + + if not self.nr_application.active: + # Force application registration if not already active + self.nr_application.activate() + + self._record_exception = record_exception + self.set_status_on_exception = set_status_on_exception + + if not ( + self.nr_application.settings and self.nr_application.settings.opentelemetry.enabled + ) and not os.environ.get("NEW_RELIC_OPENTELEMETRY_ENABLED"): + return otel_api_trace.INVALID_SPAN + + # Retrieve parent span + parent_span_context = otel_api_trace.get_current_span(context).get_span_context() + + # Set default value for whether the span + # should create an analogous NR trace. + create_nr_trace = True + + if parent_span_context is None or not parent_span_context.is_valid: + parent_span_context = None + + parent_span_trace_id = None + nr_headers = {} + if parent_span_context and self.nr_application.settings.distributed_tracing.enabled: + parent_span_trace_id = parent_span_context.trace_id + if len(parent_span_context.trace_state) > 0: + # If headers did not propagate from an existing transaction due + # to no transaction existing at the time of extraction, the + # traceparent and tracestate will still be available in the context. + nr_headers["tracestate"] = parent_span_context.trace_state.to_header() + parent_span_span_id = parent_span_context.span_id + parent_span_trace_flag = parent_span_context.trace_flags + nr_headers["traceparent"] = ( + f"00-{parent_span_trace_id:032x}-{parent_span_span_id:016x}-{'01' if parent_span_trace_flag else '00'}" + ) + + if not self.nr_application.settings.opentelemetry.traces.enabled: + create_nr_trace = False + + # If remote_parent, transaction must be created, regardless of kind type + # Make sure we transfer DT headers when we are here, if DT is enabled + if parent_span_context and parent_span_context.is_remote: + if kind in (otel_api_trace.SpanKind.SERVER, otel_api_trace.SpanKind.CLIENT): + transaction = self._create_web_transaction(nr_headers) + if not self.nr_application.settings.opentelemetry.traces.enabled: + transaction.ignore_transaction = True + transaction.__enter__() + # If a transaction was already active, we want to create + # an NR trace under the existing transaction. Otherwise, + # do not create a new NR trace, aside from the transaction's + # root span. + if transaction.enabled: + create_nr_trace = False + elif kind in (otel_api_trace.SpanKind.PRODUCER, otel_api_trace.SpanKind.INTERNAL): + transaction = BackgroundTask(self.nr_application, name=self.name) + if not self.nr_application.settings.opentelemetry.traces.enabled: + transaction.ignore_transaction = True + transaction.__enter__() + # If a transaction was already active, we want to create + # an NR trace under the existing transaction. Otherwise, + # do not create a new NR trace, aside from the transaction's + # root span. + if transaction.enabled: + create_nr_trace = False + elif kind == otel_api_trace.SpanKind.CONSUMER: + transaction = MessageTransaction( + library=self.instrumentation_library, + destination_type="Exchange", + destination_name=self.name, + application=self.nr_application, + headers=nr_headers, + ) + if not self.nr_application.settings.opentelemetry.traces.enabled: + transaction.ignore_transaction = True + transaction.__enter__() + # In the case of a Kafka consumer span, we do not want to create + # a trace regardless of whether a transaction already existed. + # This scenario should either create a transaction or use + # the existing transaction and add additional attributes to it. + create_nr_trace = False + + if not transaction.enabled: + # We will reach this if there already was a transaction + # active. The attempt at creating a transaction will + # create one where transaction.enabled == False, so + # we do not want to pass an inactive transaction along. + transaction = current_transaction() + + # If not parent_span_context or not parent_span_context.is_remote + # To simplify calculation logic, we will use DeMorgan's Theorem: + # (!parent_span_context or !parent_span_context.is_remote) + # !!(!parent_span_context or !parent_span_context.is_remote) + # !(parent_span_context and parent_span_context.is_remote) + elif not (parent_span_context and parent_span_context.is_remote): + if kind == otel_api_trace.SpanKind.SERVER: + if transaction: + nr_trace_type = FunctionTrace + elif not transaction: + transaction = self._create_web_transaction(nr_headers) + + transaction._trace_id = ( + f"{parent_span_trace_id:x}" if parent_span_trace_id else transaction.trace_id + ) + if not self.nr_application.settings.opentelemetry.traces.enabled: + transaction.ignore_transaction = True + transaction.__enter__() + create_nr_trace = False + elif kind == otel_api_trace.SpanKind.INTERNAL: + if transaction: + nr_trace_type = FunctionTrace + else: + return otel_api_trace.INVALID_SPAN + elif kind == otel_api_trace.SpanKind.CLIENT: + if transaction: + if self.attributes.get("http.url") or self.attributes.get("http.method"): + nr_trace_type = ExternalTrace + else: + nr_trace_type = DatastoreTrace + else: + return otel_api_trace.INVALID_SPAN + elif kind == otel_api_trace.SpanKind.CONSUMER: + # NOTE for instrumenting a Kafka consumer span: + # If a transaction already exists, do not create a new one + # nor should we create a MessageTrace under it. We do, + # however, want to add additional attributes from this span + # into the existing transaction. + if transaction and ( + getattr(self, "_create_consumer_trace", False) or (self.instrumentation_library != "kafka") + ): + # If transaction already exists and the + # _create_consumer_trace flag is set to True, + # then create a MessageTrace under it. + # Note that for Kafka, this flag will not be + # set, so we will not create a MessageTrace + nr_trace_type = MessageTrace + else: + transaction = MessageTransaction( + library=self.instrumentation_library, + destination_type="Exchange", + destination_name=self.name, + application=self.nr_application, + headers=nr_headers, + ) + if not self.nr_application.settings.opentelemetry.traces.enabled: + transaction.ignore_transaction = True + transaction.__enter__() + # In the case of a Kafka consumer span, we do not want to create + # a trace regardless of whether a transaction already existed. + # This scenario should either create a transaction or use + # the existing transaction and add additional attributes to it. + if (self.instrumentation_library == "kafka") or not getattr(self, "_create_consumer_trace", False): + create_nr_trace = False + + if self.instrumentation_library == "kafka": + # Whether a transaction exists or not, do not create a NR + # trace for the case of a consumer span. + create_nr_trace = False + elif kind == otel_api_trace.SpanKind.PRODUCER: + if transaction: + nr_trace_type = MessageTrace + else: + return otel_api_trace.INVALID_SPAN + + # Start transactions in this method, but start traces + # in Span. Span function will take in some Span args + # as well as info for NR applications/transactions + span = Span( + name=self.name, + parent=parent_span_context, + resource=self.resource, + attributes=self.attributes, + kind=kind, + nr_transaction=transaction, + nr_trace_type=nr_trace_type, + instrumenting_module=self.instrumentation_library, + record_exception=self._record_exception, + set_status_on_exception=self.set_status_on_exception, + create_nr_trace=create_nr_trace, + links=links, + ) + + # Remove the tracer._create_consumer_trace flag since + # the span is created now. + if hasattr(self, "_create_consumer_trace"): + delattr(self, "_create_consumer_trace") + + return span + + @contextmanager + def start_as_current_span( + self, + name=None, + context=None, + kind=otel_api_trace.SpanKind.INTERNAL, + attributes=None, + links=None, + end_on_exit=True, + record_exception=True, + set_status_on_exception=True, + ): + span = self.start_span( + name, + context=context, + kind=kind, + attributes=attributes, + record_exception=record_exception, + set_status_on_exception=set_status_on_exception, + links=links, + ) + + with otel_api_trace.use_span( + span, + end_on_exit=end_on_exit, + record_exception=record_exception, + set_status_on_exception=set_status_on_exception, + ) as current_span: + yield current_span + + +class TracerProvider(otel_api_trace.TracerProvider): + def __init__(self, *args, **kwargs): + self._resource = create_resource(hybrid_bridge=True) + + def get_tracer( + self, + instrumenting_module_name="Default", + instrumenting_library_version=None, + schema_url=None, + attributes=None, + *args, + **kwargs, + ): + return Tracer( + *args, + instrumentation_library=instrumenting_module_name, + instrumenting_library_version=instrumenting_library_version, + schema_url=schema_url, + attributes=attributes, + resource=self._resource, + **kwargs, + ) diff --git a/newrelic/api/time_trace.py b/newrelic/api/time_trace.py index fd0f62fdef..d5ebc07fef 100644 --- a/newrelic/api/time_trace.py +++ b/newrelic/api/time_trace.py @@ -51,6 +51,8 @@ def __init__(self, parent=None, source=None): self.guid = f"{random.getrandbits(64):016x}" self.agent_attributes = {} self.user_attributes = {} + self.span_link_events = [] + self.span_event_events = [] self._source = source @@ -215,6 +217,66 @@ def add_code_level_metrics(self, source): exc, ) + def _add_span_link_event(self, span_id, trace_id, linked_span_id, linked_trace_id, timestamp=None, attributes=None): + settings = self.settings + if not settings: + return + + if not settings.opentelemetry.enabled: + return + + if len(self.span_link_events) >= 100: + self.transaction._record_supportability("Supportability/SpanEvent/Links/Dropped") + return + + if attributes: + attributes = dict(attributes) + else: + attributes = {} + + event = [ + { + "type": "SpanLink", + "timestamp": timestamp or int(time.time() * 1e3), + "id": span_id, + "trace.id": trace_id, + "linkedSpanId": linked_span_id, + "linkedTraceId": linked_trace_id, + }, + attributes, + {}, + ] + + self.span_link_events.append(event) + + def _add_span_event_event(self, name, span_id, trace_id, timestamp=None, attributes=None): + settings = self.settings + if not settings: + return + + if not settings.opentelemetry.enabled: + return + + if len(self.span_event_events) >= 100: + self.transaction._record_supportability("Supportability/SpanEvent/Events/Dropped") + return + + attributes = dict(attributes) or {} + + event = [ + { + "type": "SpanEvent", + "timestamp": timestamp or int(time.time() * 1e3), + "span.id": span_id, + "trace.id": trace_id, + "name": name, + }, + attributes, + {}, + ] + + self.span_event_events.append(event) + def _observe_exception(self, exc_info=None, ignore=None, expected=None, status_code=None): # Bail out if the transaction is not active or # collection of errors not enabled. @@ -362,15 +424,19 @@ def _observe_exception(self, exc_info=None, ignore=None, expected=None, status_c def notice_error(self, error=None, attributes=None, expected=None, ignore=None, status_code=None): attributes = attributes if attributes is not None else {} - # If no exception details provided, use current exception. + # If an exception instance is passed, attempt to unpack it into an exception tuple with traceback + if isinstance(error, BaseException): + error = (type(error), error, getattr(error, "__traceback__", None)) - # Pull from sys.exc_info if no exception is passed - if not error or None in error: + # Use current exception from sys.exc_info() if no exception was passed, + # or if the exception tuple is missing components like the traceback + if not error or (isinstance(error, (tuple, list)) and None in error): error = sys.exc_info() - # If no exception to report, exit - if not error or None in error: - return + # Error should be a tuple or list of 3 elements by this point. + # If it's falsey or missing a component like the traceback, quietly exit early. + if not isinstance(error, (tuple, list)) or len(error) != 3 or None in error: + return exc, value, tb = error diff --git a/newrelic/api/transaction.py b/newrelic/api/transaction.py index b163ff54fd..5d80ca2b83 100644 --- a/newrelic/api/transaction.py +++ b/newrelic/api/transaction.py @@ -285,7 +285,8 @@ def __init__(self, application, enabled=None, source=None): self.tracestate = "" self._priority = None self._sampled = None - self._traceparent_sampled = None + # Remote parent sampled is set from the W3C parent header or the Newrelic header if no W3C parent header is present. + self._remote_parent_sampled = None self._distributed_trace_state = 0 @@ -535,6 +536,8 @@ def __exit__(self, exc, value, tb): path=self.path, trusted_parent_span=self.trusted_parent_span, tracing_vendors=self.tracing_vendors, + span_link_events=root.span_link_events, + span_event_events=root.span_event_events, ) # Add transaction exclusive time to total exclusive time @@ -569,7 +572,13 @@ def __exit__(self, exc, value, tb): if self._settings.distributed_tracing.enabled: # Sampled and priority need to be computed at the end of the # transaction when distributed tracing or span events are enabled. - self._compute_sampled_and_priority() + self._make_sampling_decision() + else: + # If dt is disabled, set sampled=False and priority random number between 0 and 1. + # The priority of the transaction is used for other data like transaction + # events even when span events are not sent. + self._priority = float(f"{random.random():.6f}") # noqa: S311 + self._sampled = False self._cached_path._name = self.path agent_attributes = self.agent_attributes @@ -636,6 +645,7 @@ def __exit__(self, exc, value, tb): trace_id=self.trace_id, loop_time=self._loop_time, root=root_node, + partial_granularity_sampled=getattr(self, "partial_granularity_sampled", False), ) # Clear settings as we are all done and don't need it @@ -1004,35 +1014,149 @@ def _update_agent_attributes(self): def user_attributes(self): return create_attributes(self._custom_params, DST_ALL, self.attribute_filter) - def sampling_algo_compute_sampled_and_priority(self): - if self._priority is None: + def sampling_algo_compute_sampled_and_priority(self, priority, sampled, sampler_kwargs): + # self._priority and self._sampled are set when parsing the W3C tracestate + # or newrelic DT headers and may be overridden in _make_sampling_decision + # based on the configuration. The only time they are set in here is when the + # sampling decision must be made by the adaptive sampling algorithm. + adjust_priority = priority is None or sampled is None + if priority is None: # Truncate priority field to 6 digits past the decimal. - self._priority = float(f"{random.random():.6f}") # noqa: S311 - if self._sampled is None: - self._sampled = self._application.compute_sampled() - if self._sampled: - self._priority += 1 - - def _compute_sampled_and_priority(self): - if self._traceparent_sampled is None: - config = "default" # Use sampling algo. - elif self._traceparent_sampled: - setting_path = "distributed_tracing.sampler.remote_parent_sampled" - config = self.settings.distributed_tracing.sampler.remote_parent_sampled - else: # self._traceparent_sampled is False. - setting_path = "distributed_tracing.sampler.remote_parent_not_sampled" - config = self.settings.distributed_tracing.sampler.remote_parent_not_sampled - + priority = float(f"{random.random():.6f}") # noqa: S311 + if sampled is None: + # _logger.trace("No trusted account id found. Sampling decision will be made by adaptive sampling algorithm.") + sampled = self._application.compute_sampled(**sampler_kwargs) + if adjust_priority and sampled: + # Make sure priority is <1 so we don't end up with priorities >3. + priority = priority - int(priority) + # Increment the priority + 2 for full and + 1 for partial granularity. + priority += 1 + int(sampler_kwargs.get("full_granularity")) + return priority, sampled + + def _compute_sampled_and_priority( + self, + priority, + sampled, + full_granularity, + root_setting, + remote_parent_sampled_setting, + remote_parent_not_sampled_setting, + ): + if self._remote_parent_sampled is None: + section = 0 + # setting_path = f"distributed_tracing.sampler{'' if full_granularity else '.partial_granularity'}.root" + config = root_setting + # _logger.trace( + # "Sampling decision made based on no remote parent sampling decision present and %s=%s.", + # setting_path, + # config, + # ) + elif self._remote_parent_sampled: + section = 1 + # setting_path = ( + # f"distributed_tracing.sampler{'' if full_granularity else '.partial_granularity'}.remote_parent_sampled" + # ) + config = remote_parent_sampled_setting + # _logger.trace( + # "Sampling decision made based on remote_parent_sampled=%s and %s=%s.", + # self._remote_parent_sampled, + # setting_path, + # config, + # ) + else: # self._remote_parent_sampled is False. + section = 2 + # setting_path = f"distributed_tracing.sampler{'' if full_granularity else '.partial_granularity'}.remote_parent_not_sampled" + config = remote_parent_not_sampled_setting + # _logger.trace( + # "Sampling decision made based on remote_parent_sampled=%s and %s=%s.", + # self._remote_parent_sampled, + # setting_path, + # config, + # ) if config == "always_on": - self._sampled = True - self._priority = 2.0 + sampled = True + # priority=3 for full granularity and priority=2 for partial granularity. + priority = 2.0 + int(full_granularity) + return priority, sampled elif config == "always_off": - self._sampled = False - self._priority = 0 - else: - if config != "default": - _logger.warning("%s=%s is not a recognized value. Using 'default' instead.", setting_path, config) - self.sampling_algo_compute_sampled_and_priority() + sampled = False + priority = 0 + return priority, sampled + elif config == "trace_id_ratio_based": + # _logger.trace("Let trace id ratio based sampler algorithm decide based on trace_id = %s.", self._trace_id) + # If the ratio is not set the sampler proxy will fall back on the global adaptive sampler. + priority, sampled = self.sampling_algo_compute_sampled_and_priority( + priority, + None, # The sampled value from the parent is not used in this case and should always be overridden. + { + "full_granularity": full_granularity, + "section": section, + "trace_id": int(self._trace_id.lower().zfill(32), 16), + }, + ) + return priority, sampled + if config not in ("default", "adaptive"): + _logger.warning("%s is not a recognized value for a sampler type. Using 'adaptive' instead.", config) + + # _logger.trace("Let adaptive sampler algorithm decide based on sampled=%s and priority=%s.", sampled, priority) + priority, sampled = self.sampling_algo_compute_sampled_and_priority( + priority, sampled, {"full_granularity": full_granularity, "section": section} + ) + return priority, sampled + + def _make_sampling_decision(self): + # The sampling decision is computed each time a DT header is generated for exit spans as it is needed + # to send the DT headers. Don't recompute the sampling decision multiple times as it is expensive. + if hasattr(self, "_sampling_decision_made"): + return + priority = self._priority + sampled = self._sampled + # Compute sampling decision for full granularity. + if self.settings.distributed_tracing.sampler.full_granularity.enabled: + # _logger.trace( + # "Full granularity tracing is enabled. Asking if full granularity wants to sample. priority=%s, sampled=%s", + # priority, + # sampled, + # ) + computed_priority, computed_sampled = self._compute_sampled_and_priority( + priority, + sampled, + full_granularity=True, + root_setting=self.settings.distributed_tracing.sampler._root, + remote_parent_sampled_setting=self.settings.distributed_tracing.sampler._remote_parent_sampled, + remote_parent_not_sampled_setting=self.settings.distributed_tracing.sampler._remote_parent_not_sampled, + ) + # _logger.trace("Full granularity sampling decision was %s with priority=%s.", sampled, priority) + if computed_sampled or not self.settings.distributed_tracing.sampler.partial_granularity.enabled: + self._priority = computed_priority + self._sampled = computed_sampled + self._sampling_decision_made = True + return + + # If full granularity is not going to sample, let partial granularity decide. + if self.settings.distributed_tracing.sampler.partial_granularity.enabled: + # _logger.trace("Partial granularity tracing is enabled. Asking if partial granularity wants to sample.") + self._priority, self._sampled = self._compute_sampled_and_priority( + priority, + sampled, + full_granularity=False, + root_setting=self.settings.distributed_tracing.sampler.partial_granularity._root, + remote_parent_sampled_setting=self.settings.distributed_tracing.sampler.partial_granularity._remote_parent_sampled, + remote_parent_not_sampled_setting=self.settings.distributed_tracing.sampler.partial_granularity._remote_parent_not_sampled, + ) + # _logger.trace( + # "Partial granularity sampling decision was %s with priority=%s.", self._sampled, self._priority + # ) + self._sampling_decision_made = True + if self._sampled: + self.partial_granularity_sampled = True + return + + # This is only reachable if both full and partial granularity tracing are off. + # Set priority to random number between 0 and 1 and do not sample. This enables + # DT headers to still be sent even if the trace is never sampled. + self._priority = float(f"{random.random():.6f}") # noqa: S311 + self._sampled = False def _freeze_path(self): if self._frozen_path is None: @@ -1101,7 +1225,7 @@ def _create_distributed_trace_data(self): if not (account_id and application_id and trusted_account_key and settings.distributed_tracing.enabled): return - self._compute_sampled_and_priority() + self._make_sampling_decision() data = { "ty": "App", "ac": account_id, @@ -1204,7 +1328,7 @@ def _accept_distributed_trace_payload(self, payload, transport_type="HTTP"): if not any(k in data for k in ("id", "tx")): self._record_supportability("Supportability/DistributedTrace/AcceptPayload/ParseException") return False - + self._remote_parent_sampled = data.get("sa") settings = self._settings account_id = data.get("ac") trusted_account_key = settings.trusted_account_key or ( @@ -1254,10 +1378,8 @@ def _accept_distributed_trace_data(self, data, transport_type): self._trace_id = data.get("tr") - priority = data.get("pr") - if priority is not None: - self._priority = priority - self._sampled = data.get("sa") + self._priority = data.get("pr") + self._sampled = data.get("sa") if "ti" in data: transport_start = data["ti"] / 1000.0 @@ -1297,6 +1419,7 @@ def accept_distributed_trace_headers(self, headers, transport_type="HTTP"): try: traceparent = ensure_str(traceparent).strip() data = W3CTraceParent.decode(traceparent) + self._remote_parent_sampled = data.pop("sa", None) except: data = None @@ -1332,7 +1455,6 @@ def accept_distributed_trace_headers(self, headers, transport_type="HTTP"): else: self._record_supportability("Supportability/TraceContext/TraceState/NoNrEntry") - self._traceparent_sampled = data.get("sa") self._accept_distributed_trace_data(data, transport_type) self._record_supportability("Supportability/TraceContext/Accept/Success") return True diff --git a/newrelic/common/encoding_utils.py b/newrelic/common/encoding_utils.py index 6f7e9d199f..b59992782a 100644 --- a/newrelic/common/encoding_utils.py +++ b/newrelic/common/encoding_utils.py @@ -418,7 +418,9 @@ def text(self): else: guid = f"{random.getrandbits(64):016x}" - return f"00-{self['tr'].lower().zfill(32)}-{guid}-{int(self.get('sa', 0)):02x}" + v = self.get("v", "00") + + return f"{v}-{self['tr'].lower().zfill(32)}-{guid}-{int(self.get('sa', 0)):02x}" @classmethod def decode(cls, payload): @@ -459,7 +461,7 @@ def decode(cls, payload): # Sampled flag sa = bool(int(fields[3], 2) & FLAG_SAMPLED) - return cls(tr=trace_id, id=parent_id, sa=sa) + return cls(tr=trace_id, id=parent_id, sa=sa, v=version) class W3CTraceState(OrderedDict): @@ -483,14 +485,27 @@ def decode(cls, tracestate): class NrTraceState(dict): - FIELDS = ("ty", "ac", "ap", "id", "tx", "sa", "pr") + FIELDS = ("v", "ty", "ac", "ap", "id", "tx", "sa", "pr") + # Fields Key: + # v: version + # ty: type; {"0": "App", "1": "Browser", "2": "Mobile"} + # ac: account ID + # ap: application ID + # id: span ID + # tx: transaction guid + # sa: sampled + # pr: priority + # tr: trace ID + # tk: trusted account key + # ti: time def text(self): pr = self.get("pr", "") if pr: pr = f"{pr:.6f}".rstrip("0").rstrip(".") - - payload = f"0-0-{self['ac']}-{self['ap']}-{self.get('id', '')}-{self.get('tx', '')}-{'1' if self.get('sa') else '0'}-{pr}-{self['ti']!s}" + version = self.get("v", "0") + ty = "0" # Hardcode this as it will always be App. + payload = f"{version}-{ty}-{self['ac']}-{self['ap']}-{self.get('id', '')}-{self.get('tx', '')}-{'1' if self.get('sa') else '0'}-{pr}-{self['ti']!s}" return f"{self.get('tk', self['ac'])}@nr={payload}" @classmethod @@ -504,7 +519,7 @@ def decode(cls, payload, tk): except: return - for name, value in zip(cls.FIELDS, fields[1:]): + for name, value in zip(cls.FIELDS, fields): if value: data[name] = value diff --git a/newrelic/common/llm_utils.py b/newrelic/common/llm_utils.py new file mode 100644 index 0000000000..062ce60f1d --- /dev/null +++ b/newrelic/common/llm_utils.py @@ -0,0 +1,108 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import itertools +import logging + +from newrelic.api.transaction import current_transaction +from newrelic.common.object_wrapper import ObjectProxy + +_logger = logging.getLogger(__name__) + + +def _get_llm_metadata(transaction): + if not transaction: + return {} + try: + # Grab LLM-related custom attributes off of the transaction to store as metadata on LLM events + custom_attrs_dict = getattr(transaction, "_custom_params", {}) + llm_metadata_dict = {key: value for key, value in custom_attrs_dict.items() if key.startswith("llm.")} + llm_context_attrs = getattr(transaction, "_llm_context_attrs", None) + if llm_context_attrs: + llm_metadata_dict.update(llm_context_attrs) + except Exception: + _logger.warning("Unable to capture custom metadata attributes to record on LLM events.") + return {} + + return llm_metadata_dict + + +class GeneratorProxy(ObjectProxy): + def __init__(self, wrapped, on_stop_iteration, on_error): + super().__init__(wrapped) + self._nr_on_stop_iteration = on_stop_iteration + self._nr_on_error = on_error + + def __iter__(self): + self._nr_wrapped_iter = self.__wrapped__.__iter__() + return self + + def __next__(self): + transaction = current_transaction() + if not transaction: + return self._nr_wrapped_iter.__next__() + + return_val = None + try: + return_val = self._nr_wrapped_iter.__next__() + except StopIteration: + self._nr_on_stop_iteration(self, transaction) + raise + except Exception: + self._nr_on_error(self, transaction) + raise + return return_val + + def close(self): + return self.__wrapped__.close() + + def __copy__(self): + # Required to properly interface with itertool.tee, which can be called by LangChain on generators + self.__wrapped__, copy = itertools.tee(self.__wrapped__, 2) + return GeneratorProxy(copy, self._nr_on_stop_iteration, self._nr_on_error) + + +class AsyncGeneratorProxy(ObjectProxy): + def __init__(self, wrapped, on_stop_iteration, on_error): + super().__init__(wrapped) + self._nr_on_stop_iteration = on_stop_iteration + self._nr_on_error = on_error + + def __aiter__(self): + self._nr_wrapped_iter = self.__wrapped__.__aiter__() + return self + + async def __anext__(self): + transaction = current_transaction() + if not transaction: + return await self._nr_wrapped_iter.__anext__() + + return_val = None + try: + return_val = await self._nr_wrapped_iter.__anext__() + except StopAsyncIteration: + self._nr_on_stop_iteration(self, transaction) + raise + except Exception: + self._nr_on_error(self, transaction) + raise + return return_val + + async def aclose(self): + return await self.__wrapped__.aclose() + + def __copy__(self): + # Required to properly interface with itertool.tee, which can be called by LangChain on generators + self.__wrapped__, copy = itertools.tee(self.__wrapped__, n=2) + return AsyncGeneratorProxy(copy, self._nr_on_stop_iteration, self._nr_on_error) diff --git a/newrelic/common/opentelemetry_tracers.py b/newrelic/common/opentelemetry_tracers.py new file mode 100644 index 0000000000..4085fef634 --- /dev/null +++ b/newrelic/common/opentelemetry_tracers.py @@ -0,0 +1,243 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# This is a mapping of the entry_point name of the +# OpenTelemetry instrumentor and the list of targets in the +# NR hooks that need to be disabled if the OpenTelemetry +# instrumentor is to be used instead. +HYBRID_AGENT_DEFAULT_INCLUDED_TRACERS_TO_NR_HOOKS = { + "aio-pika": [], + "aiohttp_client": ["aiohttp.client", "aiohttp.client_reqrep"], + "aiohttp_server": [ + "aiohttp.web", + "aiohttp.wsgi", + "aiohttp.web_reqrep", + "aiohttp.web_response", + "aiohttp.web_urldispatcher", + "aiohttp.protocol", + ], + "aiokafka": [], + "aiopg": [ + "psycopg2", + "psycopg2._psycopg2", + "psycopg2.extensions", + "psycopg2._json", + "psycopg2._range", + "psycopg2.sql", + ], + "ariadne": [ + "ariadne.graphql", + "graphql.graphql", + "graphql.execution.execute", + "graphql.execution.executor", + "graphql.execution.middleware", + "graphql.execution.utils", + "graphql.error.located_error", + "graphql.language.parser", + "graphql.validation.validate", + "graphql.validation.validation", + "graphql.type.schema", + ], + "asyncclick": [], + "asyncpg": ["asyncpg.connect_utils", "asyncpg.protocol"], + # "boto": [], # this is boto3 + # "boto3": [], # this is boto3sqs + # "botocore": [], + "cassandra": ["cassandra", "cassandra.cluster"], + "celery": ["celery.local", "celery.app.trace", "celery.worker", "celery.concurrency.prefork", "billiard.pool"], + "click": [], + "confluent_kafka": [ + "confluent_kafka.cimpl", + "confluent_kafka.serializing_producer", + "confluent_kafka.deserializing_consumer", + ], + "django": [ + "django.core.handlers.base", + "django.core.handlers.asgi", + "django.core.handlers.wsgi", + "django.core.urlresolvers", + "django.template", + "django.template.loader_tags", + "django.core.servers.basehttp", + "django.contrib.staticfiles.views", + "django.contrib.staticfiles.handlers", + "django.views.debug", + "django.http.multipartparser", + "django.core.mail", + "django.core.mail.message", + "django.views.generic.base", + "django.core.management.base", + "django.template.base", + "django.middleware.gzip", + "django.urls.resolvers", + "django.urls.base", + "django.core.handlers.exception", + ], + "elasticsearch": [ + "elasticsearch.client", + "elasticsearch._async.client", + "elasticsearch._sync.client", + "elasticsearch._async.client", + "elasticsearch.client.cat", + "elasticsearch._async.client.cat", + "elasticsearch._sync.client.cat", + "elasticsearch.client.cluster", + "elasticsearch._async.client.cluster", + "elasticsearch._sync.client.cluster", + "elasticsearch.client.indices", + "elasticsearch._async.client.indices", + "elasticsearch._sync.client.indices", + "elasticsearch.client.nodes", + "elasticsearch._async.client.nodes", + "elasticsearch._sync.client.nodes", + "elasticsearch.client.snapshot", + "elasticsearch._async.client.snapshot", + "elasticsearch._sync.client.snapshot", + "elasticsearch.client.tasks", + "elasticsearch._async.client.tasks", + "elasticsearch._sync.client.tasks", + "elasticsearch.client.ingest", + "elasticsearch._async.client.ingest", + "elasticsearch._sync.client.ingest", + "elasticsearch.connection.base", + "elasticsearch._async.http_aiohttp", + "elastic_transport._node._base", + "elastic_transport._node._base_async", + "elasticsearch.transport", + "elasticsearch._async.transport", + "elastic_transport._transport", + "elastic_transport._async_transport", + ], + "falcon": ["falcon.api", "falcon.app", "falcon.routing.util"], + "fastapi": [ + "fastapi.routing", + "starlette.requests", + "starlette.routing", + "starlette.applications", + "starlette.middleware.errors", + "starlette.middleware.exceptions", + "starlette.exceptions", + "starlette.background", + ], + "flask": ["flask.app", "flask.templating", "flask.blueprints", "flask.views"], + "httpx": ["httpx._client", "urllib.request"], + "jinja2": ["jinja2.environment"], + "kafka": ["kafka.consumer.group", "kafka.producer.kafka", "kafka.coordinator.heartbeat"], + "logging": ["logging"], + "mysql": ["mysql.connector"], + "mysqlclient": ["MySQLdb"], + "pika": ["pika.adapters", "pika.channel", "pika.spec"], + "psycopg": ["psycopg", "psycopg.sql"], + "psycopg2": [ + "psycopg2", + "psycopg2._psycopg2", + "psycopg2.extensions", + "psycopg2._json", + "psycopg2._range", + "psycopg2.sql", + ], + "pymemcache": ["pymemcache.client"], + "pymongo": [ + "pymongo.synchronous.pool", + "pymongo.asynchronous.pool", + "pymongo.synchronous.collection", + "pymongo.asynchronous.collection", + "pymongo.synchronous.mongo_client", + "pymongo.asynchronous.mongo_client", + "pymongo.connection", + "pymongo.collection", + "pymongo.mongo_client", + ], + "pymssql": ["pymssql"], + "pymysql": ["pymysql"], + "pyramid": ["pyramid.router", "pyramid.config", "pyramid.config.views", "pyramid.config.tweens"], + "redis": [ + "redis.asyncio.client", + "redis.asyncio.commands", + "redis.asyncio.connection", + "redis.connection", + "redis.client", + "redis.commands.cluster", + "redis.commands.core", + "redis.commands.sentinel", + "redis.commands.json.commands", + "redis.commands.search.commands", + "redis.commands.timeseries.commands", + "redis.commands.bf.commands", + "redis.commands.graph.commands", + "redis.commands.vectorset.commands", + ], + "remoulade": [], + "requests": [ + "requests.sessions", + "requests.api", + "urllib3.connectionpool", + "urllib3.connection", + "requests.packages.urllib3.connection", + "http.client", + "httplib2", + ], + "sqlalchemy": [], + "sqlite3": ["sqlite3", "sqlite3.dbapi2", "pysqlite2", "pysqlite2.dbapi2"], + "starlette": [ + "starlette.requests", + "starlette.routing", + "starlette.applications", + "starlette.middleware.errors", + "starlette.middleware.exceptions", + "starlette.exceptions", + "starlette.background", + # "starlette.concurrency", # This is deliberately excluded + ], + "strawberry-graphql": [ + "strawberry.schema.schema", + "strawberry.schema.schema_converter", + "graphql.graphql", + "graphql.execution.execute", + "graphql.execution.executor", + "graphql.execution.middleware", + "graphql.execution.utils", + "graphql.error.located_error", + "graphql.language.parser", + "graphql.validation.validate", + "graphql.validation.validation", + "graphql.type.schema", + ], + "system_metrics": [], + "threading": [], + "tornado": [ + "tornado.httpserver", + "tornado.httputil", + "tornado.httpclient", + "tornado.routing", + "tornado.web", + "http.client", + ], + "tortoiseorm": [], + "urllib": ["urllib.request", "http.client"], + "urllib3": ["urllib3.connectionpool", "urllib3.connection", "requests.packages.urllib3.connection", "http.client"], +} + + +TEMPORARILY_DISABLED_OPENTELEMETRY_FRAMEWORKS = { + "boto", + "boto3", + "botocore", + "aws-lambda", + "grpc_aio_client", + "grpc_aio_server", + "grpc_client", + "grpc_server", +} diff --git a/newrelic/common/utilization.py b/newrelic/common/utilization.py index 22b158e3ec..b092dc99b8 100644 --- a/newrelic/common/utilization.py +++ b/newrelic/common/utilization.py @@ -233,21 +233,29 @@ class AzureFunctionUtilization(CommonUtilization): HEADERS = {"Metadata": "true"} # noqa: RUF012 VENDOR_NAME = "azurefunction" - @staticmethod - def fetch(): + @classmethod + def fetch(cls): cloud_region = os.environ.get("REGION_NAME") website_owner_name = os.environ.get("WEBSITE_OWNER_NAME") azure_function_app_name = os.environ.get("WEBSITE_SITE_NAME") if all((cloud_region, website_owner_name, azure_function_app_name)): - if website_owner_name.endswith("-Linux"): - resource_group_name = AZURE_RESOURCE_GROUP_NAME_RE.search(website_owner_name).group(1) - else: - resource_group_name = AZURE_RESOURCE_GROUP_NAME_PARTIAL_RE.search(website_owner_name).group(1) - subscription_id = re.search(r"(?:(?!\+).)*", website_owner_name).group(0) - faas_app_name = f"/subscriptions/{subscription_id}/resourceGroups/{resource_group_name}/providers/Microsoft.Web/sites/{azure_function_app_name}" - # Only send if all values are present - return (faas_app_name, cloud_region) + try: + if website_owner_name.endswith("-Linux"): + resource_group_name = AZURE_RESOURCE_GROUP_NAME_RE.search(website_owner_name).group(1) + else: + resource_group_name = AZURE_RESOURCE_GROUP_NAME_PARTIAL_RE.search(website_owner_name).group(1) + subscription_id = re.search(r"(?:(?!\+).)*", website_owner_name).group(0) + faas_app_name = f"/subscriptions/{subscription_id}/resourceGroups/{resource_group_name}/providers/Microsoft.Web/sites/{azure_function_app_name}" + # Only send if all values are present + return (faas_app_name, cloud_region) + except Exception: + _logger.debug( + "Unable to determine Azure Functions subscription id from WEBSITE_OWNER_NAME. %r", + website_owner_name, + ) + + return None @classmethod def get_values(cls, response): diff --git a/newrelic/config.py b/newrelic/config.py index 21ce996f6c..37bbb7b4a6 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -43,6 +43,11 @@ import newrelic.core.config from newrelic.common.log_file import initialize_logging from newrelic.common.object_names import callable_name, expand_builtin_exception_name +from newrelic.common.opentelemetry_tracers import ( + HYBRID_AGENT_DEFAULT_INCLUDED_TRACERS_TO_NR_HOOKS, + TEMPORARILY_DISABLED_OPENTELEMETRY_FRAMEWORKS, +) +from newrelic.common.package_version_utils import get_package_version from newrelic.core import trace_cache from newrelic.core.agent_control_health import ( HealthStatus, @@ -53,6 +58,15 @@ __all__ = ["filter_app_factory", "initialize"] + +# Add trace log level to logging module +def trace(self, message, *args, **kws): + self.log(logging.TRACE, message, args, **kws) + + +logging.TRACE = 5 +logging.addLevelName(logging.TRACE, "TRACE") +logging.Logger.trace = trace _logger = logging.getLogger(__name__) DEPRECATED_MODULES = {"aioredis": datetime(2022, 2, 22, 0, 0, tzinfo=timezone.utc)} @@ -66,7 +80,7 @@ def _map_aws_account_id(s): # triggering of callbacks to monkey patch modules before import # returns them to caller. -sys.meta_path.insert(0, newrelic.api.import_hook.ImportHookFinder()) +newrelic.api.import_hook.enable_import_hook_finder() # The set of valid feature flags that the agent currently uses. # This will be used to validate what is provided and issue warnings @@ -95,7 +109,17 @@ def _map_aws_account_id(s): # modules to look up customised settings defined in the loaded # configuration file. -_config_object = configparser.RawConfigParser() + +def ratio(value): + try: + val = float(value) + if 0 < val <= 1: + return val + except ValueError: + pass + + +_config_object = configparser.RawConfigParser(converters={"ratio": ratio}) # Cache of the parsed global settings found in the configuration # file. We cache these so can dump them out to the log file once @@ -108,7 +132,7 @@ def _map_aws_account_id(s): def _reset_config_parser(): global _config_object global _cache_object - _config_object = configparser.RawConfigParser() + _config_object = configparser.RawConfigParser(converters={"ratio": ratio}) _cache_object = [] @@ -150,6 +174,7 @@ def extra_settings(section, types=None, defaults=None): "WARNING": logging.WARNING, "INFO": logging.INFO, "DEBUG": logging.DEBUG, + "TRACE": logging.TRACE, } _RECORD_SQL = { @@ -319,6 +344,92 @@ def _process_setting(section, option, getter, mapper): _raise_configuration_error(section, option) +def _process_dt_hidden_setting(section, option, getter): + try: + # The type of a value is dictated by the getter + # function supplied. + + value = getattr(_config_object, getter)(section, option) + + # Now need to apply the option from the + # configuration file to the internal settings + # object. Walk the object path and assign it. + + target = _settings + fields = option.split(".", 1) + + if value == "trace_id_ratio_based": + raise configparser.NoOptionError("trace_id_ratio_sampler option can only be set by configuring the ratio") + while True: + if len(fields) == 1: + value = value or "default" + # Store the value at the underscored location so if option is + # distributed_tracing.sampler.full_granularity.remote_parent_sampled + # store it at location + # distributed_tracing.sampler.full_granularity._remote_parent_sampled + setattr(target, f"_{fields[0]}", value) + break + target = getattr(target, fields[0]) + fields = fields[1].split(".", 1) + + # Cache the configuration so can be dumped out to + # log file when whole main configuration has been + # processed. This ensures that the log file and log + # level entries have been set. + + _cache_object.append((option, value)) + + except configparser.NoSectionError: + pass + + except configparser.NoOptionError: + pass + + except Exception: + _raise_configuration_error(section, option) + + +def _process_dt_sampler_setting(section, option, getter): + try: + # The type of a value is dictated by the getter + # function supplied. + + value = getattr(_config_object, getter)(section, option) + + # Now need to apply the option from the + # configuration file to the internal settings + # object. Walk the object path and assign it. + + target = _settings + fields = option.split(".", 1) + + while True: + if len(fields) == 1: + setattr(target, f"{fields[0]}", value) + break + elif fields[0] in ("root", "remote_parent_sampled", "remote_parent_not_sampled"): + sampler = fields[1].split(".", 1)[0] + setattr(target, f"_{fields[0]}", sampler) + target = getattr(target, fields[0]) + fields = fields[1].split(".", 1) + + # Cache the configuration so can be dumped out to + # log file when whole main configuration has been + # processed. This ensures that the log file and log + # level entries have been set. + + _cache_object.append((option, value)) + + except configparser.NoSectionError: + pass + + except configparser.NoOptionError: + pass + + except Exception: + _raise_configuration_error(section, option) + + # Processing of all the settings for specified section except # for log file and log level which are applied separately to # ensure they are set as soon as possible. @@ -404,8 +515,58 @@ def _process_configuration(section): _process_setting(section, "ml_insights_events.enabled", "getboolean", None) _process_setting(section, "distributed_tracing.enabled", "getboolean", None) _process_setting(section, "distributed_tracing.exclude_newrelic_header", "getboolean", None) - _process_setting(section, "distributed_tracing.sampler.remote_parent_sampled", "get", None) - _process_setting(section, "distributed_tracing.sampler.remote_parent_not_sampled", "get", None) + _process_setting(section, "distributed_tracing.sampler.adaptive_sampling_target", "getint", None) + _process_dt_hidden_setting(section, "distributed_tracing.sampler.root", "get") + _process_dt_sampler_setting(section, "distributed_tracing.sampler.root.adaptive.sampling_target", "getint") + _process_dt_sampler_setting(section, "distributed_tracing.sampler.root.trace_id_ratio_based.ratio", "getratio") + _process_dt_hidden_setting(section, "distributed_tracing.sampler.remote_parent_sampled", "get") + _process_dt_sampler_setting( + section, "distributed_tracing.sampler.remote_parent_sampled.adaptive.sampling_target", "getint" + ) + _process_dt_sampler_setting( + section, "distributed_tracing.sampler.remote_parent_sampled.trace_id_ratio_based.ratio", "getratio" + ) + _process_dt_hidden_setting(section, "distributed_tracing.sampler.remote_parent_not_sampled", "get") + _process_dt_sampler_setting( + section, "distributed_tracing.sampler.remote_parent_not_sampled.adaptive.sampling_target", "getint" + ) + _process_dt_sampler_setting( + section, "distributed_tracing.sampler.remote_parent_not_sampled.trace_id_ratio_based.ratio", "getratio" + ) + _process_setting(section, "distributed_tracing.sampler.full_granularity.enabled", "getboolean", None) + _process_setting(section, "distributed_tracing.sampler.partial_granularity.enabled", "getboolean", None) + _process_setting(section, "distributed_tracing.sampler.partial_granularity.type", "get", None) + _process_dt_hidden_setting(section, "distributed_tracing.sampler.partial_granularity.root", "get") + _process_dt_sampler_setting( + section, "distributed_tracing.sampler.partial_granularity.root.adaptive.sampling_target", "getint" + ) + _process_dt_sampler_setting( + section, "distributed_tracing.sampler.partial_granularity.root.trace_id_ratio_based.ratio", "getratio" + ) + _process_dt_hidden_setting(section, "distributed_tracing.sampler.partial_granularity.remote_parent_sampled", "get") + _process_dt_sampler_setting( + section, + "distributed_tracing.sampler.partial_granularity.remote_parent_sampled.adaptive.sampling_target", + "getint", + ) + _process_dt_sampler_setting( + section, + "distributed_tracing.sampler.partial_granularity.remote_parent_sampled.trace_id_ratio_based.ratio", + "getratio", + ) + _process_dt_hidden_setting( + section, "distributed_tracing.sampler.partial_granularity.remote_parent_not_sampled", "get" + ) + _process_dt_sampler_setting( + section, + "distributed_tracing.sampler.partial_granularity.remote_parent_not_sampled.adaptive.sampling_target", + "getint", + ) + _process_dt_sampler_setting( + section, + "distributed_tracing.sampler.partial_granularity.remote_parent_not_sampled.trace_id_ratio_based.ratio", + "getratio", + ) _process_setting(section, "span_events.enabled", "getboolean", None) _process_setting(section, "span_events.max_samples_stored", "getint", None) _process_setting(section, "span_events.attributes.enabled", "getboolean", None) @@ -518,6 +679,8 @@ def _process_configuration(section): _process_setting(section, "instrumentation.middleware.django.enabled", "getboolean", None) _process_setting(section, "instrumentation.middleware.django.exclude", "get", _map_inc_excl_middleware) _process_setting(section, "instrumentation.middleware.django.include", "get", _map_inc_excl_middleware) + _process_setting(section, "opentelemetry.enabled", "getboolean", None) + _process_setting(section, "opentelemetry.traces.enabled", "getboolean", None) # Loading of configuration from specified file and for specified @@ -1986,6 +2149,10 @@ def _process_function_profile_configuration(): _raise_configuration_error(section) +opentelemetry_instrumentation = [] +opentelemetry_entrypoints = [] + + def _process_module_definition(target, module, function="instrument"): enabled = True execute = None @@ -2006,6 +2173,9 @@ def _process_module_definition(target, module, function="instrument"): except Exception: _raise_configuration_error(section) + if target in opentelemetry_instrumentation: + enabled = False + try: if _config_object.has_option(section, "execute"): execute = _config_object.get(section, "execute") @@ -2084,6 +2254,14 @@ def _process_module_builtin_defaults(): "asyncio.base_events", "newrelic.hooks.coroutines_asyncio", "instrument_asyncio_base_events" ) + _process_module_definition("asyncio.events", "newrelic.hooks.coroutines_asyncio", "instrument_asyncio_events") + + _process_module_definition("asyncio.runners", "newrelic.hooks.coroutines_asyncio", "instrument_asyncio_runners") + + _process_module_definition( + "langgraph.prebuilt.tool_node", "newrelic.hooks.mlmodel_langgraph", "instrument_langgraph_prebuilt_tool_node" + ) + _process_module_definition( "langchain_core.runnables.base", "newrelic.hooks.mlmodel_langchain", @@ -2095,13 +2273,19 @@ def _process_module_builtin_defaults(): "instrument_langchain_core_runnables_config", ) _process_module_definition( - "langchain.chains.base", "newrelic.hooks.mlmodel_langchain", "instrument_langchain_chains_base" + "langchain_core.tools.structured", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_core_tools_structured", ) + _process_module_definition( - "langchain_classic.chains.base", "newrelic.hooks.mlmodel_langchain", "instrument_langchain_chains_base" + "langchain.agents.factory", "newrelic.hooks.mlmodel_langchain", "instrument_langchain_agents_factory" + ) + _process_module_definition( + "langchain.chains.base", "newrelic.hooks.mlmodel_langchain", "instrument_langchain_chains_base" ) _process_module_definition( - "langchain_core.callbacks.manager", "newrelic.hooks.mlmodel_langchain", "instrument_langchain_callbacks_manager" + "langchain_classic.chains.base", "newrelic.hooks.mlmodel_langchain", "instrument_langchain_chains_base" ) # VectorStores with similarity_search method @@ -2667,12 +2851,6 @@ def _process_module_builtin_defaults(): "langchain_core.tools", "newrelic.hooks.mlmodel_langchain", "instrument_langchain_core_tools" ) - _process_module_definition( - "langchain_core.callbacks.manager", "newrelic.hooks.mlmodel_langchain", "instrument_langchain_callbacks_manager" - ) - - _process_module_definition("asyncio.events", "newrelic.hooks.coroutines_asyncio", "instrument_asyncio_events") - _process_module_definition("asgiref.sync", "newrelic.hooks.adapter_asgiref", "instrument_asgiref_sync") _process_module_definition( @@ -2946,6 +3124,30 @@ def _process_module_builtin_defaults(): "newrelic.hooks.mlmodel_autogen", "instrument_autogen_agentchat_agents__assistant_agent", ) + _process_module_definition( + "strands.agent.agent", "newrelic.hooks.mlmodel_strands", "instrument_strands_agent_agent" + ) + _process_module_definition( + "strands.multiagent.graph", "newrelic.hooks.mlmodel_strands", "instrument_strands_multiagent_graph" + ) + _process_module_definition( + "strands.multiagent.swarm", "newrelic.hooks.mlmodel_strands", "instrument_strands_multiagent_swarm" + ) + _process_module_definition( + "strands.tools.decorator", "newrelic.hooks.mlmodel_strands", "instrument_strands_tools_decorator" + ) + _process_module_definition( + "strands.tools.executors._executor", + "newrelic.hooks.mlmodel_strands", + "instrument_strands_tools_executors__executor", + ) + _process_module_definition( + "strands.tools.registry", "newrelic.hooks.mlmodel_strands", "instrument_strands_tools_registry" + ) + _process_module_definition( + "strands.models.bedrock", "newrelic.hooks.mlmodel_strands", "instrument_strands_models_bedrock" + ) + _process_module_definition("mcp.client.session", "newrelic.hooks.adapter_mcp", "instrument_mcp_client_session") _process_module_definition( "mcp.server.fastmcp.tools.tool_manager", @@ -4192,6 +4394,33 @@ def _process_module_builtin_defaults(): "pyzeebe.worker.job_executor", "newrelic.hooks.external_pyzeebe", "instrument_pyzeebe_worker_job_executor" ) + # Hybrid Agent Hooks + _process_module_definition( + "opentelemetry.context", "newrelic.hooks.hybridagent_opentelemetry", "instrument_context_api" + ) + + _process_module_definition( + "opentelemetry.instrumentation.propagators", + "newrelic.hooks.hybridagent_opentelemetry", + "instrument_global_propagators_api", + ) + + _process_module_definition( + "opentelemetry.trace", "newrelic.hooks.hybridagent_opentelemetry", "instrument_trace_api" + ) + + _process_module_definition( + "opentelemetry.util.http", "newrelic.hooks.hybridagent_opentelemetry", "instrument_util_http" + ) + + _process_module_definition( + "opentelemetry.instrumentation.utils", "newrelic.hooks.hybridagent_opentelemetry", "instrument_utils" + ) + + _process_module_definition( + "opentelemetry.instrumentation.pika.utils", "newrelic.hooks.hybridagent_opentelemetry", "instrument_pika_utils" + ) + def _process_module_entry_points(): try: @@ -4241,6 +4470,84 @@ def _reset_instrumentation_done(): _instrumentation_done = False +def _is_installed(req): + version = get_package_version(req) + + if version: + return True + return False + + +def _process_opentelemetry_instrumentation_entry_points( + final_include_dict=HYBRID_AGENT_DEFAULT_INCLUDED_TRACERS_TO_NR_HOOKS, +): + if not _settings.opentelemetry.enabled or not _is_installed("opentelemetry-api"): + return + + try: + # importlib.metadata was introduced into the standard library starting in Python 3.8. + from importlib.metadata import entry_points + except ImportError: + try: + # importlib_metadata is a backport library installable from PyPI. + from importlib_metadata import entry_points + except ImportError: + try: + # Fallback to pkg_resources, which is available in older versions of setuptools. + from pkg_resources import iter_entry_points as entry_points + except ImportError: + return + + group = "opentelemetry_instrumentor" + + try: + # group kwarg was only added to importlib.metadata.entry_points in Python 3.10. + _entry_points = entry_points(group=group) + except TypeError: + # Grab entire entry_points dictionary and select group from it. + _entry_points = entry_points().get(group, ()) + + entry_points_generator = ( + entrypoint + for entrypoint in _entry_points + if entrypoint.name in final_include_dict + and entrypoint.name not in TEMPORARILY_DISABLED_OPENTELEMETRY_FRAMEWORKS + ) + + for entrypoint in entry_points_generator: + opentelemetry_entrypoints.append(entrypoint) + opentelemetry_instrumentation.extend(final_include_dict[entrypoint.name]) + + # Check for native installations + # NOTE: This logic will change once enabled and disabled + # functionality is implemented for opentelemetry.traces setting. + # NOTE: elasticsearch is instrumented both with libs and natively. + # To handle this case: If lib is installed, the library itself + # will check for native instrumentation and switch to that on its + # own, but if not, native instrumentation could still be used and + # we would not know. We handle this as we are with strawberry-graphql + # and ariadne where we check to see if opentelemetry-api and the + # specific library are installed on the system. + for lib in ["strawberry-graphql", "ariadne", "elasticsearch"]: + if _is_installed(lib): + opentelemetry_instrumentation.extend(final_include_dict[lib]) + + +def _process_opentelemetry_instrumentors(): + if not _settings.opentelemetry.enabled or not _is_installed("opentelemetry-api"): + return + + tracer_provider = newrelic.core.agent.opentelemetry_tracer_provider() + for entrypoint in opentelemetry_entrypoints: + try: + instrumentor_class = entrypoint.load() + instrumentor = instrumentor_class() + instrumentor.instrument(tracer_provider=tracer_provider) + _logger.debug("Successfully instrumented OpenTelemetry tracer '%s' via entry point.", entrypoint.name) + except Exception as exc: + _logger.warning("Failed to instrument OpenTelemetry tracer '%s' via entry point: %s", entrypoint.name, exc) + + def _setup_instrumentation(): global _instrumentation_done @@ -4251,6 +4558,10 @@ def _setup_instrumentation(): _process_module_configuration() _process_module_entry_points() + # Collection of NR disabled hooks must happen before _process_module_builtin_defaults() + # but the loading of the entrypoints must not happen until after the NR hooks are registered. + _process_opentelemetry_instrumentation_entry_points() + _process_trace_cache_import_hooks() _process_module_builtin_defaults() @@ -4272,6 +4583,8 @@ def _setup_instrumentation(): _process_function_profile_configuration() + _process_opentelemetry_instrumentors() + def _setup_extensions(): try: diff --git a/newrelic/core/agent.py b/newrelic/core/agent.py index 90690a573d..2693b4ee1f 100644 --- a/newrelic/core/agent.py +++ b/newrelic/core/agent.py @@ -120,6 +120,7 @@ class Agent: _instance_lock = threading.Lock() _instance = None + _tracer_provider = None _startup_callables = [] # noqa: RUF012 _registration_callables = {} # noqa: RUF012 @@ -188,6 +189,46 @@ def agent_singleton(): return Agent._instance + @staticmethod + def opentelemetry_tracer_provider(): + """Used by the tracer_provider() function to access/create the + single tracer provider object instance. + + """ + settings = newrelic.core.config.global_settings() + + if not settings.opentelemetry.enabled and not newrelic.core.config._environ_as_bool( + "NEW_RELIC_OPENTELEMETRY_ENABLED" + ): + _logger.debug("OpenTelemetry mode is disabled.") + return + + if Agent._tracer_provider: + return Agent._tracer_provider + + with Agent._instance_lock: + if not Agent._tracer_provider: + try: + from opentelemetry.trace import NoOpTracerProvider + + from newrelic.api.opentelemetry import TracerProvider + + if not settings.opentelemetry.traces.enabled and not newrelic.core.config._environ_as_bool( + "NEW_RELIC_OPENTELEMETRY_TRACES_ENABLED" + ): + # Set this to prevent any potential crashes + _logger.debug("OpenTelemetry traces are disabled.") + Agent._tracer_provider = NoOpTracerProvider() + else: + Agent._tracer_provider = TracerProvider() + except ImportError: + # `opentelemetry-api` is not installed, so tracer provider cannot be created + _logger.warning( + "OpenTelemetry mode has been enabled but `opentelemetry-api` is not installed, so no TracerProvider can be created. Defaulting to New Relic specific monitoring." + ) + + return Agent._tracer_provider + def __init__(self, config): """Initialises the agent and attempt to establish a connection to the core application. Will start the harvest loop running but @@ -581,9 +622,9 @@ def normalize_name(self, app_name, name, rule_type="url"): return application.normalize_name(name, rule_type) - def compute_sampled(self, app_name): + def compute_sampled(self, app_name, full_granularity, section, *args, **kwargs): application = self._applications.get(app_name, None) - return application.compute_sampled() + return application.compute_sampled(full_granularity, section, *args, **kwargs) def _harvest_shutdown_is_set(self): try: @@ -768,6 +809,10 @@ def agent_instance(): return Agent.agent_singleton() +def opentelemetry_tracer_provider(): + return agent_instance().opentelemetry_tracer_provider() + + def shutdown_agent(timeout=None): agent = agent_instance() agent.shutdown_agent(timeout) diff --git a/newrelic/core/agent_control_health.py b/newrelic/core/agent_control_health.py index f78d26a152..4cfe4488c3 100644 --- a/newrelic/core/agent_control_health.py +++ b/newrelic/core/agent_control_health.py @@ -24,6 +24,7 @@ from urllib.parse import urlparse from urllib.request import url2pathname +from newrelic.api.time_trace import get_linking_metadata from newrelic.core.config import _environ_as_bool, _environ_as_int _logger = logging.getLogger(__name__) @@ -217,7 +218,8 @@ def update_to_healthy_status(self, protocol_error=False, collector_error=False): def write_to_health_file(self): status_time_unix_nano = time.time_ns() - + service_metadata = get_linking_metadata() + entity_guid = service_metadata.get("entity.guid", "") if service_metadata else "" try: health_dir_path = self.health_delivery_location if health_dir_path is None: @@ -229,6 +231,7 @@ def write_to_health_file(self): is_healthy = self.is_healthy # Cache property value to avoid multiple calls with health_file_path.open("w") as f: + f.write(f"entity_guid: {entity_guid}\n") f.write(f"healthy: {is_healthy}\n") f.write(f"status: {self.status_message}\n") f.write(f"start_time_unix_nano: {self.start_time_unix_nano}\n") diff --git a/newrelic/core/agent_protocol.py b/newrelic/core/agent_protocol.py index 0657adc547..ad8c677fe0 100644 --- a/newrelic/core/agent_protocol.py +++ b/newrelic/core/agent_protocol.py @@ -297,6 +297,9 @@ def _connect_payload(app_name, linked_applications, environment, settings): connect_settings["browser_monitoring.loader"] = settings["browser_monitoring.loader"] connect_settings["browser_monitoring.debug"] = settings["browser_monitoring.debug"] connect_settings["ai_monitoring.enabled"] = settings["ai_monitoring.enabled"] + connect_settings["distributed_tracing.sampler.adaptive_sampling_target"] = settings[ + "distributed_tracing.sampler.adaptive_sampling_target" + ] security_settings = {} security_settings["capture_params"] = settings["capture_params"] diff --git a/newrelic/core/application.py b/newrelic/core/application.py index 3ba8168d60..81839715ec 100644 --- a/newrelic/core/application.py +++ b/newrelic/core/application.py @@ -23,7 +23,6 @@ from functools import partial from newrelic.common.object_names import callable_name -from newrelic.core.adaptive_sampler import AdaptiveSampler from newrelic.core.agent_control_health import ( HealthStatus, agent_control_health_instance, @@ -37,6 +36,7 @@ from newrelic.core.internal_metrics import InternalTrace, InternalTraceContext, internal_count_metric, internal_metric from newrelic.core.profile_sessions import profile_session_manager from newrelic.core.rules_engine import RulesEngine, SegmentCollapseEngine +from newrelic.core.samplers.sampler_proxy import SamplerProxy from newrelic.core.stats_engine import CustomMetrics, StatsEngine from newrelic.network.exceptions import ( DiscardDataForRequest, @@ -78,7 +78,7 @@ def __init__(self, app_name, linked_applications=None): self._transaction_count = 0 self._last_transaction = 0.0 - self.adaptive_sampler = None + self.sampler = None self._global_events_account = 0 @@ -156,11 +156,23 @@ def configuration(self): def active(self): return self.configuration is not None - def compute_sampled(self): - if self.adaptive_sampler is None: + def compute_sampled(self, full_granularity, section, *args, **kwargs): + if self.sampler is None: return False - return self.adaptive_sampler.compute_sampled() + return self.sampler.compute_sampled(full_granularity, section, *args, **kwargs) + + def _flattened_span_samples(self, spans, flattened_list=None): + if flattened_list is None: + flattened_list = [] + + if isinstance(spans[-1], dict): + flattened_list.append(spans) + elif isinstance(spans[-1], list): + for span in spans: + self._flattened_span_samples(span, flattened_list) + + return flattened_list def dump(self, file): """Dumps details about the application to the file object.""" @@ -501,12 +513,7 @@ def connect_to_data_collector(self, activate_agent): with self._stats_lock: self._stats_engine.reset_stats(configuration, reset_stream=True) - - if configuration.serverless_mode.enabled: - sampling_target_period = 60.0 - else: - sampling_target_period = configuration.sampling_target_period_in_seconds - self.adaptive_sampler = AdaptiveSampler(configuration.sampling_target, sampling_target_period) + self.sampler = SamplerProxy(configuration) active_session.connect_span_stream(self._stats_engine.span_stream, self.record_custom_metric) @@ -591,6 +598,33 @@ def connect_to_data_collector(self, activate_agent): f"Supportability/InfiniteTracing/gRPC/Compression/{'enabled' if infinite_tracing_compression else 'disabled'}", 1, ) + if configuration.distributed_tracing.enabled: + if configuration.distributed_tracing.sampler.full_granularity.enabled: + internal_metric( + f"Supportability/Python/FullGranularity/Root/{configuration.distributed_tracing.sampler._root}", + 1, + ) + internal_metric( + f"Supportability/Python/FullGranularity/RemoteParentSampled/{configuration.distributed_tracing.sampler._remote_parent_sampled}", + 1, + ) + internal_metric( + f"Supportability/Python/FullGranularity/RemoteParentNotSampled/{configuration.distributed_tracing.sampler._remote_parent_not_sampled}", + 1, + ) + if configuration.distributed_tracing.sampler.partial_granularity.enabled: + internal_metric( + f"Supportability/Python/PartialGranularity/Root/{configuration.distributed_tracing.sampler.partial_granularity._root}", + 1, + ) + internal_metric( + f"Supportability/Python/PartialGranularity/RemoteParentSampled/{configuration.distributed_tracing.sampler.partial_granularity._remote_parent_sampled}", + 1, + ) + internal_metric( + f"Supportability/Python/PartialGranularity/RemoteParentNotSampled/{configuration.distributed_tracing.sampler.partial_granularity._remote_parent_not_sampled}", + 1, + ) # Agent Control health check metric if self._agent_control.health_check_enabled: @@ -601,6 +635,10 @@ def connect_to_data_collector(self, activate_agent): if os.environ.get("FUNCTIONS_WORKER_RUNTIME", None): internal_metric("Supportability/Python/AzureFunctionMode/enabled", 1) + # OpenTelemetry Bridge toggle metric + opentelemetry_bridge = "enabled" if configuration.opentelemetry.enabled else "disabled" + internal_metric(f"Supportability/Tracing/Python/OpenTelemetryBridge/{opentelemetry_bridge}", 1) + self._stats_engine.merge_custom_metrics(internal_metrics.metrics()) # Update the active session in this object. This will the @@ -1361,9 +1399,9 @@ def harvest(self, shutdown=False, flexible=False): spans = stats.span_events if spans: if spans.num_samples > 0: - span_samples = list(spans) + span_samples = self._flattened_span_samples(list(spans)) - _logger.debug("Sending span event data for harvest of %r.", self._app_name) + _logger.debug("Sending Span event data for harvest of %r.", self._app_name) self._active_session.send_span_events(spans.sampling_info, span_samples) span_samples = None @@ -1373,7 +1411,6 @@ def harvest(self, shutdown=False, flexible=False): spans_sampled = spans.num_samples internal_count_metric("Supportability/SpanEvent/TotalEventsSeen", spans_seen) internal_count_metric("Supportability/SpanEvent/TotalEventsSent", spans_sampled) - stats.reset_span_events() # Send error events diff --git a/newrelic/core/attribute.py b/newrelic/core/attribute.py index 79b9a56cb2..c516a94e75 100644 --- a/newrelic/core/attribute.py +++ b/newrelic/core/attribute.py @@ -87,6 +87,13 @@ "message.routingKey", "messaging.destination.name", "messaging.system", + "nr.durations", + "nr.ids", + "nr.pg", + "otel.library.name", + "otel.library.version", + "otel.scope.name", + "otel.scope.version", "peer.address", "peer.hostname", "request.headers.accept", @@ -100,6 +107,8 @@ "response.headers.contentType", "response.status", "server.address", + "server.port", + "subcomponent", "zeebe.client.bpmnProcessId", "zeebe.client.messageName", "zeebe.client.correlationKey", @@ -108,6 +117,25 @@ "zeebe.client.resourceFile", } +SPAN_ENTITY_RELATIONSHIP_ATTRIBUTES = { + "cloud.account.id", + "cloud.platform", + "cloud.region", + "cloud.resource_id", + "db.instance", + "db.system", + "http.url", + "messaging.destination.name", + "messaging.system", + "peer.hostname", + "server.address", + "server.port", + "span.kind", +} + +SPAN_ERROR_ATTRIBUTES = {"error.class", "error.message", "error.expected"} + + MAX_NUM_USER_ATTRIBUTES = 128 MAX_ATTRIBUTE_LENGTH = 255 MAX_NUM_ML_USER_ATTRIBUTES = 64 diff --git a/newrelic/core/config.py b/newrelic/core/config.py index 8cfdeda0ae..20181aefb3 100644 --- a/newrelic/core/config.py +++ b/newrelic/core/config.py @@ -334,6 +334,90 @@ class DistributedTracingSettings(Settings): class DistributedTracingSamplerSettings(Settings): + _root = "default" + _remote_parent_sampled = "default" + _remote_parent_not_sampled = "default" + + +class DistributedTracingSamplerFullGranularitySettings(Settings): + pass + + +class DistributedTracingSamplerRootSettings: + pass + + +class DistributedTracingSamplerRootAdaptiveSettings: + pass + + +class DistributedTracingSamplerRootTraceIdRatioBasedSettings: + pass + + +class DistributedTracingSamplerRemoteParentSampledSettings: + pass + + +class DistributedTracingSamplerRemoteParentSampledAdaptiveSettings: + pass + + +class DistributedTracingSamplerRemoteParentSampledTraceIdRatioBasedSettings: + pass + + +class DistributedTracingSamplerRemoteParentNotSampledSettings: + pass + + +class DistributedTracingSamplerRemoteParentNotSampledAdaptiveSettings: + pass + + +class DistributedTracingSamplerRemoteParentNotSampledTraceIdRatioBasedSettings: + pass + + +class DistributedTracingSamplerPartialGranularitySettings(Settings): + _root = "default" + _remote_parent_sampled = "default" + _remote_parent_not_sampled = "default" + + +class DistributedTracingSamplerPartialGranularityRootSettings: + pass + + +class DistributedTracingSamplerPartialGranularityRootAdaptiveSettings: + pass + + +class DistributedTracingSamplerPartialGranularityRootTraceIdRatioBasedSettings: + pass + + +class DistributedTracingSamplerPartialGranularityRemoteParentSampledSettings: + pass + + +class DistributedTracingSamplerPartialGranularityRemoteParentSampledAdaptiveSettings: + pass + + +class DistributedTracingSamplerPartialGranularityRemoteParentSampledTraceIdRatioBasedSettings: + pass + + +class DistributedTracingSamplerPartialGranularityRemoteParentNotSampledSettings: + pass + + +class DistributedTracingSamplerPartialGranularityRemoteParentNotSampledAdaptiveSettings: + pass + + +class DistributedTracingSamplerPartialGranularityRemoteParentNotSampledTraceIdRatioBasedSettings: pass @@ -474,6 +558,14 @@ class EventHarvestConfigHarvestLimitSettings(Settings): nested = True +class OpentelemetrySettings(Settings): + pass + + +class OpentelemetryTracesSettings(Settings): + pass + + _settings = TopLevelSettings() _settings.agent_limits = AgentLimitsSettings() _settings.application_logging = ApplicationLoggingSettings() @@ -507,6 +599,59 @@ class EventHarvestConfigHarvestLimitSettings(Settings): _settings.debug = DebugSettings() _settings.distributed_tracing = DistributedTracingSettings() _settings.distributed_tracing.sampler = DistributedTracingSamplerSettings() +_settings.distributed_tracing.sampler.full_granularity = DistributedTracingSamplerFullGranularitySettings() +_settings.distributed_tracing.sampler.root = DistributedTracingSamplerRootSettings() +_settings.distributed_tracing.sampler.root.adaptive = DistributedTracingSamplerRootAdaptiveSettings() +_settings.distributed_tracing.sampler.root.trace_id_ratio_based = ( + DistributedTracingSamplerRootTraceIdRatioBasedSettings() +) +_settings.distributed_tracing.sampler.remote_parent_sampled = DistributedTracingSamplerRemoteParentSampledSettings() +_settings.distributed_tracing.sampler.remote_parent_sampled.adaptive = ( + DistributedTracingSamplerRemoteParentSampledAdaptiveSettings() +) +_settings.distributed_tracing.sampler.remote_parent_sampled.trace_id_ratio_based = ( + DistributedTracingSamplerRemoteParentSampledTraceIdRatioBasedSettings() +) +_settings.distributed_tracing.sampler.remote_parent_not_sampled = ( + DistributedTracingSamplerRemoteParentNotSampledSettings() +) +_settings.distributed_tracing.sampler.remote_parent_not_sampled.adaptive = ( + DistributedTracingSamplerRemoteParentNotSampledAdaptiveSettings() +) +_settings.distributed_tracing.sampler.remote_parent_not_sampled.trace_id_ratio_based = ( + DistributedTracingSamplerRemoteParentNotSampledTraceIdRatioBasedSettings() +) +_settings.distributed_tracing.sampler.partial_granularity = DistributedTracingSamplerPartialGranularitySettings() +_settings.distributed_tracing.sampler.partial_granularity.root = ( + DistributedTracingSamplerPartialGranularityRootSettings() +) +_settings.distributed_tracing.sampler.partial_granularity.root.adaptive = ( + DistributedTracingSamplerPartialGranularityRootAdaptiveSettings() +) +_settings.distributed_tracing.sampler.partial_granularity.root.trace_id_ratio_based = ( + DistributedTracingSamplerPartialGranularityRootTraceIdRatioBasedSettings() +) +_settings.distributed_tracing.sampler.partial_granularity.remote_parent_sampled = ( + DistributedTracingSamplerPartialGranularityRemoteParentSampledSettings() +) +_settings.distributed_tracing.sampler.partial_granularity.remote_parent_sampled.adaptive = ( + DistributedTracingSamplerPartialGranularityRemoteParentSampledAdaptiveSettings() +) +_settings.distributed_tracing.sampler.partial_granularity.remote_parent_sampled.trace_id_ratio_based = ( + DistributedTracingSamplerPartialGranularityRemoteParentSampledTraceIdRatioBasedSettings() +) +_settings.distributed_tracing.sampler.partial_granularity.remote_parent_sampled.trace_id_ratio_based = ( + DistributedTracingSamplerPartialGranularityRemoteParentSampledAdaptiveSettings() +) +_settings.distributed_tracing.sampler.partial_granularity.remote_parent_not_sampled = ( + DistributedTracingSamplerPartialGranularityRemoteParentNotSampledSettings() +) +_settings.distributed_tracing.sampler.partial_granularity.remote_parent_not_sampled.adaptive = ( + DistributedTracingSamplerPartialGranularityRemoteParentNotSampledAdaptiveSettings() +) +_settings.distributed_tracing.sampler.partial_granularity.remote_parent_not_sampled.trace_id_ratio_based = ( + DistributedTracingSamplerPartialGranularityRemoteParentNotSampledTraceIdRatioBasedSettings() +) _settings.error_collector = ErrorCollectorSettings() _settings.error_collector.attributes = ErrorCollectorAttributesSettings() _settings.event_harvest_config = EventHarvestConfigSettings() @@ -524,6 +669,8 @@ class EventHarvestConfigHarvestLimitSettings(Settings): _settings.instrumentation.middleware = InstrumentationMiddlewareSettings() _settings.instrumentation.middleware.django = InstrumentationDjangoMiddlewareSettings() _settings.message_tracer = MessageTracerSettings() +_settings.opentelemetry = OpentelemetrySettings() +_settings.opentelemetry.traces = OpentelemetryTracesSettings() _settings.process_host = ProcessHostSettings() _settings.rum = RumSettings() _settings.serverless_mode = ServerlessModeSettings() @@ -548,9 +695,19 @@ class EventHarvestConfigHarvestLimitSettings(Settings): _settings.audit_log_file = os.environ.get("NEW_RELIC_AUDIT_LOG", None) +def _environ_as_sampler(name, default): + val = os.environ.get(name, default) + # The trace_id_ratio_based value can only be set by setting the ratio + if val == "trace_id_ratio_based": + return default + return val + + def _environ_as_int(name, default=0): val = os.environ.get(name, default) try: + if default is None and val is None: + return None return int(val) except ValueError: return default @@ -560,11 +717,27 @@ def _environ_as_float(name, default=0.0): val = os.environ.get(name, default) try: + if default is None and val is None: + return None return float(val) except ValueError: return default +def _environ_as_ratio(name, default=0.0): + val = os.environ.get(name, default) + + try: + if default is None and val is None: + return None + f_val = float(val) + if 0 < f_val <= 1: + return f_val + except ValueError: + return default + return default + + def _environ_as_bool(name, default=False): flag = os.environ.get(name, default) if default is None or default: @@ -842,11 +1015,164 @@ def default_otlp_host(host): _settings.ml_insights_events.enabled = False _settings.distributed_tracing.enabled = _environ_as_bool("NEW_RELIC_DISTRIBUTED_TRACING_ENABLED", default=True) -_settings.distributed_tracing.sampler.remote_parent_sampled = os.environ.get( - "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_REMOTE_PARENT_SAMPLED", "default" +_settings.distributed_tracing.sampler.adaptive_sampling_target = _environ_as_int( + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_ADAPTIVE_SAMPLING_TARGET", default=10 +) +_settings.distributed_tracing.sampler.full_granularity.enabled = _environ_as_bool( + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_FULL_GRANULARITY_ENABLED", default=True +) +_settings.distributed_tracing.sampler._root = ( + ( + "trace_id_ratio_based" + if _environ_as_ratio("NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_ROOT_TRACE_ID_RATIO_BASED_RATIO", None) + else None + ) + or ( + "adaptive" + if os.environ.get("NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_ROOT_ADAPTIVE_SAMPLING_TARGET", None) + else None + ) + or _environ_as_sampler("NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_ROOT", "default") +) +_settings.distributed_tracing.sampler.root.adaptive.sampling_target = _environ_as_int( + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_ROOT_ADAPTIVE_SAMPLING_TARGET", None +) +_settings.distributed_tracing.sampler.root.trace_id_ratio_based.ratio = _environ_as_ratio( + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_ROOT_TRACE_ID_RATIO_BASED_RATIO", None +) +_settings.distributed_tracing.sampler._remote_parent_sampled = ( + ( + "trace_id_ratio_based" + if _environ_as_ratio( + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_REMOTE_PARENT_SAMPLED_TRACE_ID_RATIO_BASED_RATIO", None + ) + else None + ) + or ( + "adaptive" + if os.environ.get("NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_REMOTE_PARENT_SAMPLED_ADAPTIVE_SAMPLING_TARGET", None) + else None + ) + or _environ_as_sampler("NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_REMOTE_PARENT_SAMPLED", "default") +) +_settings.distributed_tracing.sampler.remote_parent_sampled.adaptive.sampling_target = _environ_as_int( + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_REMOTE_PARENT_SAMPLED_ADAPTIVE_SAMPLING_TARGET", None +) +_settings.distributed_tracing.sampler.remote_parent_sampled.trace_id_ratio_based.ratio = _environ_as_ratio( + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_REMOTE_PARENT_SAMPLED_TRACE_ID_RATIO_BASED_RATIO", None ) -_settings.distributed_tracing.sampler.remote_parent_not_sampled = os.environ.get( - "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_REMOTE_PARENT_NOT_SAMPLED", "default" +_settings.distributed_tracing.sampler._remote_parent_not_sampled = ( + ( + "trace_id_ratio_based" + if _environ_as_ratio( + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_REMOTE_PARENT_NOT_SAMPLED_TRACE_ID_RATIO_BASED_RATIO", None + ) + else None + ) + or ( + "adaptive" + if os.environ.get( + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_REMOTE_PARENT_NOT_SAMPLED_ADAPTIVE_SAMPLING_TARGET", None + ) + else None + ) + or _environ_as_sampler("NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_REMOTE_PARENT_NOT_SAMPLED", "default") +) +_settings.distributed_tracing.sampler.remote_parent_not_sampled.adaptive.sampling_target = _environ_as_int( + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_REMOTE_PARENT_NOT_SAMPLED_ADAPTIVE_SAMPLING_TARGET", None +) +_settings.distributed_tracing.sampler.remote_parent_not_sampled.trace_id_ratio_based.ratio = _environ_as_ratio( + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_REMOTE_PARENT_NOT_SAMPLED_TRACE_ID_RATIO_BASED_RATIO", None +) +_settings.distributed_tracing.sampler.partial_granularity.enabled = _environ_as_bool( + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_PARTIAL_GRANULARITY_ENABLED", default=False +) +_settings.distributed_tracing.sampler.partial_granularity.type = os.environ.get( + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_PARTIAL_GRANULARITY_TYPE", "essential" +) +_settings.distributed_tracing.sampler.partial_granularity._root = ( + ( + "trace_id_ratio_based" + if _environ_as_ratio( + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_PARTIAL_GRANULARITY_ROOT_TRACE_ID_RATIO_BASED_RATIO", None + ) + else None + ) + or ( + "adaptive" + if os.environ.get( + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_PARTIAL_GRANULARITY_ROOT_ADAPTIVE_SAMPLING_TARGET", None + ) + else None + ) + or _environ_as_sampler("NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_PARTIAL_GRANULARITY_ROOT", "default") +) +_settings.distributed_tracing.sampler.partial_granularity.root.adaptive.sampling_target = _environ_as_int( + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_PARTIAL_GRANULARITY_ROOT_ADAPTIVE_SAMPLING_TARGET", None +) +_settings.distributed_tracing.sampler.partial_granularity.root.trace_id_ratio_based.ratio = _environ_as_ratio( + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_PARTIAL_GRANULARITY_ROOT_TRACE_ID_RATIO_BASED_RATIO", None +) +_settings.distributed_tracing.sampler.partial_granularity._remote_parent_sampled = ( + ( + "trace_id_ratio_based" + if _environ_as_ratio( + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_PARTIAL_GRANULARITY_REMOTE_PARENT_SAMPLED_TRACE_ID_RATIO_BASED_RATIO", + None, + ) + else None + ) + or ( + "adaptive" + if os.environ.get( + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_PARTIAL_GRANULARITY_REMOTE_PARENT_SAMPLED_ADAPTIVE_SAMPLING_TARGET", + None, + ) + else None + ) + or _environ_as_sampler("NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_PARTIAL_GRANULARITY_REMOTE_PARENT_SAMPLED", "default") +) +_settings.distributed_tracing.sampler.partial_granularity.remote_parent_sampled.adaptive.sampling_target = ( + _environ_as_int( + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_PARTIAL_GRANULARITY_REMOTE_PARENT_SAMPLED_ADAPTIVE_SAMPLING_TARGET", None + ) +) +_settings.distributed_tracing.sampler.partial_granularity.remote_parent_sampled.trace_id_ratio_based.ratio = ( + _environ_as_ratio( + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_PARTIAL_GRANULARITY_REMOTE_PARENT_SAMPLED_TRACE_ID_RATIO_BASED_RATIO", + None, + ) +) +_settings.distributed_tracing.sampler.partial_granularity._remote_parent_not_sampled = ( + ( + "trace_id_ratio_based" + if _environ_as_ratio( + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_PARTIAL_GRANULARITY_REMOTE_PARENT_NOT_SAMPLED_TRACE_ID_RATIO_BASED_RATIO", + None, + ) + else None + ) + or ( + "adaptive" + if os.environ.get( + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_PARTIAL_GRANULARITY_REMOTE_PARENT_NOT_SAMPLED_ADAPTIVE_SAMPLING_TARGET", + None, + ) + else None + ) + or _environ_as_sampler( + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_PARTIAL_GRANULARITY_REMOTE_PARENT_NOT_SAMPLED", "default" + ) +) +_settings.distributed_tracing.sampler.partial_granularity.remote_parent_not_sampled.adaptive.sampling_target = ( + _environ_as_int( + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_PARTIAL_GRANULARITY_REMOTE_PARENT_NOT_SAMPLED_ADAPTIVE_SAMPLING_TARGET", + None, + ) +) +_settings.distributed_tracing.sampler.partial_granularity.remote_parent_not_sampled.trace_id_ratio_based.ratio = _environ_as_ratio( + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_PARTIAL_GRANULARITY_REMOTE_PARENT_NOT_SAMPLED_TRACE_ID_RATIO_BASED_RATIO", + None, ) _settings.distributed_tracing.exclude_newrelic_header = False _settings.span_events.enabled = _environ_as_bool("NEW_RELIC_SPAN_EVENTS_ENABLED", default=True) @@ -1088,6 +1414,8 @@ def default_otlp_host(host): _settings.azure_operator.enabled = _environ_as_bool("NEW_RELIC_AZURE_OPERATOR_ENABLED", default=False) _settings.package_reporting.enabled = _environ_as_bool("NEW_RELIC_PACKAGE_REPORTING_ENABLED", default=True) _settings.ml_insights_events.enabled = _environ_as_bool("NEW_RELIC_ML_INSIGHTS_EVENTS_ENABLED", default=False) +_settings.opentelemetry.enabled = _environ_as_bool("NEW_RELIC_OPENTELEMETRY_ENABLED", default=False) +_settings.opentelemetry.traces.enabled = _environ_as_bool("NEW_RELIC_OPENTELEMETRY_TRACES_ENABLED", default=True) def global_settings(): @@ -1363,6 +1691,16 @@ def apply_server_side_settings(server_side_config=None, settings=_settings): min(settings_snapshot.custom_insights_events.max_attribute_value, 4095), ) + # Partial granularity tracing is not available in infinite tracing mode. + if ( + settings_snapshot.infinite_tracing.enabled + and settings_snapshot.distributed_tracing.sampler.partial_granularity.enabled + ): + _logger.warning( + "Improper configuration. Infinite tracing cannot be enabled at the same time as partial granularity tracing. Setting distributed_tracing.sampler.partial_granularity.enabled=False." + ) + apply_config_setting(settings_snapshot, "distributed_tracing.sampler.partial_granularity.enabled", False) + # This will be removed at some future point # Special case for account_id which will be sent instead of # cross_process_id in the future diff --git a/newrelic/core/custom_event.py b/newrelic/core/custom_event.py index 9bf5f75eda..c960a0afa2 100644 --- a/newrelic/core/custom_event.py +++ b/newrelic/core/custom_event.py @@ -141,7 +141,7 @@ def create_custom_event(event_type, params, settings=None, is_ml_event=False): ) return None - intrinsics = {"type": name, "timestamp": int(1000.0 * time.time())} + intrinsics = {"type": name, "timestamp": params.get("timestamp") or int(1000.0 * time.time())} event = [intrinsics, attributes] return event diff --git a/newrelic/core/data_collector.py b/newrelic/core/data_collector.py index e481f1d6e7..244fa4f9f8 100644 --- a/newrelic/core/data_collector.py +++ b/newrelic/core/data_collector.py @@ -117,7 +117,6 @@ def send_ml_events(self, sampling_info, custom_event_data): def send_span_events(self, sampling_info, span_event_data): """Called to submit sample set for span events.""" - payload = (self.agent_run_id, sampling_info, span_event_data) return self._protocol.send("span_event_data", payload) diff --git a/newrelic/core/database_node.py b/newrelic/core/database_node.py index 7c4032c5b9..7629e894d4 100644 --- a/newrelic/core/database_node.py +++ b/newrelic/core/database_node.py @@ -40,6 +40,8 @@ "port_path_or_id", "database_name", "params", + "span_link_events", + "span_event_events", ], ) @@ -81,6 +83,8 @@ def identifier(self): "guid", "agent_attributes", "user_attributes", + "span_link_events", + "span_event_events", ], ) @@ -224,6 +228,8 @@ def slow_sql_node(self, stats, root): port_path_or_id=self.port_path_or_id, database_name=self.database_name, params=params, + span_link_events=self.span_link_events, + span_event_events=self.span_event_events, ) def trace_node(self, stats, root, connections): @@ -279,7 +285,7 @@ def trace_node(self, stats, root, connections): start_time=start_time, end_time=end_time, name=name, params=params, children=children, label=None ) - def span_event(self, *args, **kwargs): + def span_event(self, settings, base_attrs=None, parent_guid=None, attr_class=dict): sql = self.formatted if sql: @@ -288,4 +294,6 @@ def span_event(self, *args, **kwargs): self.agent_attributes["db.statement"] = sql - return super().span_event(*args, **kwargs) + return DatastoreNodeMixin.span_event( + self, settings, base_attrs=base_attrs, parent_guid=parent_guid, attr_class=attr_class + ) diff --git a/newrelic/core/database_utils.py b/newrelic/core/database_utils.py index c37b419a39..ec593c86b3 100644 --- a/newrelic/core/database_utils.py +++ b/newrelic/core/database_utils.py @@ -419,6 +419,12 @@ def _parse_operation(sql): return operation if operation in _operation_table else "" +def _parse_operation_opentelemetry(sql): + match = _parse_operation_re.search(sql) + operation = match and match.group(1).lower() + return operation or "" + + def _parse_target(sql, operation): sql = sql.rstrip(";") parse = _operation_table.get(operation, None) @@ -898,5 +904,23 @@ def sql_statement(sql, dbapi2_module): result = SQLStatement(sql, database) _sql_statements[key] = result - return result + + +def generate_dynamodb_arn(host, region=None, account_id=None, target=None): + # There are 3 different partition options. + # See https://docs.aws.amazon.com/IAM/latest/UserGuide/reference-arns.html for details. + partition = "aws" + if "amazonaws.cn" in host: + partition = "aws-cn" + elif "amazonaws-us-gov.com" in host: + partition = "aws-us-gov" + + if partition and region and account_id and target: + return f"arn:{partition}:dynamodb:{region}:{account_id:012d}:table/{target}" + + +def get_database_operation_target_from_statement(db_statement): + operation = _parse_operation_opentelemetry(db_statement) + target = _parse_target(db_statement, operation) + return operation, target diff --git a/newrelic/core/datastore_node.py b/newrelic/core/datastore_node.py index 68c5254f4d..488ff27cd3 100644 --- a/newrelic/core/datastore_node.py +++ b/newrelic/core/datastore_node.py @@ -36,6 +36,8 @@ "guid", "agent_attributes", "user_attributes", + "span_link_events", + "span_event_events", ], ) diff --git a/newrelic/core/external_node.py b/newrelic/core/external_node.py index 9165d2081f..d066c40f2e 100644 --- a/newrelic/core/external_node.py +++ b/newrelic/core/external_node.py @@ -35,6 +35,8 @@ "guid", "agent_attributes", "user_attributes", + "span_link_events", + "span_event_events", ], ) @@ -169,11 +171,10 @@ def trace_node(self, stats, root, connections): start_time=start_time, end_time=end_time, name=name, params=params, children=children, label=None ) - def span_event(self, *args, **kwargs): + def span_event(self, settings, base_attrs=None, parent_guid=None, attr_class=dict): self.agent_attributes["http.url"] = self.http_url - attrs = super().span_event(*args, **kwargs) - i_attrs = attrs[0] + i_attrs = (base_attrs and base_attrs.copy()) or attr_class() i_attrs["category"] = "http" i_attrs["span.kind"] = "client" _, i_attrs["component"] = attribute.process_user_attribute("component", self.library) @@ -181,4 +182,4 @@ def span_event(self, *args, **kwargs): if self.method: _, i_attrs["http.method"] = attribute.process_user_attribute("http.method", self.method) - return attrs + return i_attrs, attr_class, self.span_link_events, self.span_event_events diff --git a/newrelic/core/function_node.py b/newrelic/core/function_node.py index 809f26742c..5d2726ea84 100644 --- a/newrelic/core/function_node.py +++ b/newrelic/core/function_node.py @@ -34,6 +34,8 @@ "guid", "agent_attributes", "user_attributes", + "span_link_events", + "span_event_events", ], ) @@ -114,10 +116,8 @@ def trace_node(self, stats, root, connections): start_time=start_time, end_time=end_time, name=name, params=params, children=children, label=self.label ) - def span_event(self, *args, **kwargs): - attrs = super().span_event(*args, **kwargs) - i_attrs = attrs[0] - + def span_event(self, settings, base_attrs=None, parent_guid=None, attr_class=dict): + i_attrs = (base_attrs and base_attrs.copy()) or attr_class() i_attrs["name"] = f"{self.group}/{self.name}" - return attrs + return i_attrs, attr_class, self.span_link_events, self.span_event_events diff --git a/newrelic/core/loop_node.py b/newrelic/core/loop_node.py index b9328e7013..3488c47ec2 100644 --- a/newrelic/core/loop_node.py +++ b/newrelic/core/loop_node.py @@ -79,10 +79,8 @@ def trace_node(self, stats, root, connections): start_time=start_time, end_time=end_time, name=name, params=params, children=children, label=None ) - def span_event(self, *args, **kwargs): - attrs = super().span_event(*args, **kwargs) - i_attrs = attrs[0] - + def span_event(self, settings, base_attrs=None, parent_guid=None, attr_class=dict): + i_attrs = (base_attrs and base_attrs.copy()) or attr_class() i_attrs["name"] = f"EventLoop/Wait/{self.name}" - return attrs + return i_attrs, attr_class, None, None diff --git a/newrelic/core/memcache_node.py b/newrelic/core/memcache_node.py index 85641fcff4..b82147fb57 100644 --- a/newrelic/core/memcache_node.py +++ b/newrelic/core/memcache_node.py @@ -30,6 +30,8 @@ "guid", "agent_attributes", "user_attributes", + "span_link_events", + "span_event_events", ], ) @@ -74,3 +76,6 @@ def trace_node(self, stats, root, connections): return newrelic.core.trace_node.TraceNode( start_time=start_time, end_time=end_time, name=name, params=params, children=children, label=None ) + + def span_event(self, settings, base_attrs=None, parent_guid=None, attr_class=dict): + return base_attrs, attr_class, self.span_link_events, self.span_event_events diff --git a/newrelic/core/message_node.py b/newrelic/core/message_node.py index 202e4dca75..ae7f5e107b 100644 --- a/newrelic/core/message_node.py +++ b/newrelic/core/message_node.py @@ -34,6 +34,8 @@ "guid", "agent_attributes", "user_attributes", + "span_link_events", + "span_event_events", ], ) @@ -80,3 +82,6 @@ def trace_node(self, stats, root, connections): return newrelic.core.trace_node.TraceNode( start_time=start_time, end_time=end_time, name=name, params=params, children=children, label=None ) + + def span_event(self, settings, base_attrs=None, parent_guid=None, attr_class=dict): + return base_attrs, attr_class, self.span_link_events, self.span_event_events diff --git a/newrelic/core/node_mixin.py b/newrelic/core/node_mixin.py index 8eedd191d4..f6490ffe1b 100644 --- a/newrelic/core/node_mixin.py +++ b/newrelic/core/node_mixin.py @@ -49,14 +49,17 @@ def get_trace_segment_params(self, settings, params=None): _params["exclusive_duration_millis"] = 1000.0 * self.exclusive return _params - def span_event(self, settings, base_attrs=None, parent_guid=None, attr_class=dict): + def _span_event_full_granularity(self, settings, base_attrs=None, parent_guid=None, attr_class=dict): + base_attrs, attr_class, span_link_events, span_event_events = self.span_event( + settings, base_attrs=base_attrs, parent_guid=parent_guid, attr_class=attr_class + ) i_attrs = (base_attrs and base_attrs.copy()) or attr_class() i_attrs["type"] = "Span" - i_attrs["name"] = self.name + i_attrs["name"] = i_attrs.get("name") or self.name i_attrs["guid"] = self.guid i_attrs["timestamp"] = int(self.start_time * 1000) i_attrs["duration"] = self.duration - i_attrs["category"] = "generic" + i_attrs["category"] = i_attrs.get("category") or "generic" if parent_guid: i_attrs["parentId"] = parent_guid @@ -68,18 +71,286 @@ def span_event(self, settings, base_attrs=None, parent_guid=None, attr_class=dic u_attrs = attribute.resolve_user_attributes( self.processed_user_attributes, settings.attribute_filter, DST_SPAN_EVENTS, attr_class=attr_class ) - # intrinsics, user attrs, agent attrs - return [i_attrs, u_attrs, a_attrs] + base_span_event = [i_attrs, u_attrs, a_attrs] + if span_link_events or span_event_events: + return [base_span_event, span_link_events, span_event_events] + return base_span_event + + def _span_event_partial_granularity_reduced( + self, settings, base_attrs=None, parent_guid=None, attr_class=dict, ct_exit_spans=None + ): + base_attrs, attr_class, span_link_events, span_event_events = self.span_event( + settings, base_attrs=base_attrs, parent_guid=parent_guid, attr_class=attr_class + ) + if ct_exit_spans is None: + ct_exit_spans = {"instrumented": 0, "kept": 0, "dropped_ids": 0} - def span_events(self, settings, base_attrs=None, parent_guid=None, attr_class=dict): - yield self.span_event(settings, base_attrs=base_attrs, parent_guid=parent_guid, attr_class=attr_class) + ct_exit_spans["instrumented"] += 1 + + i_attrs = (base_attrs and base_attrs.copy()) or attr_class() + i_attrs["type"] = "Span" + i_attrs["name"] = i_attrs.get("name") or self.name + i_attrs["guid"] = self.guid + i_attrs["timestamp"] = int(self.start_time * 1000) + i_attrs["duration"] = self.duration + i_attrs["category"] = i_attrs.get("category") or "generic" + + if parent_guid: + i_attrs["parentId"] = parent_guid + + a_attrs = attribute.resolve_agent_attributes( + self.agent_attributes, settings.attribute_filter, DST_SPAN_EVENTS, attr_class=attr_class + ) + u_attrs = attribute.resolve_user_attributes( + self.processed_user_attributes, settings.attribute_filter, DST_SPAN_EVENTS, attr_class=attr_class + ) + + # If this is an entry span, add `nr.pg` to indicate transaction is partial + # granularity sampled. + if i_attrs.get("nr.entryPoint"): + i_attrs["nr.pg"] = True + # If this is the entry node or an LLM span always return it. + if i_attrs.get("nr.entryPoint") or i_attrs["name"].startswith("Llm/"): + ct_exit_spans["kept"] += 1 + base_span_event = [i_attrs, u_attrs, a_attrs] + if span_link_events or span_event_events: + return [base_span_event, span_link_events, span_event_events] + return base_span_event + exit_span_attrs_present = attribute.SPAN_ENTITY_RELATIONSHIP_ATTRIBUTES & set(a_attrs) + # If the span is not an exit span, skip it by returning None. + if not exit_span_attrs_present: + return None + # If the span is an exit span and we are in reduced mode (meaning no attribute dropping), + # just return the exit span as is. + ct_exit_spans["kept"] += 1 + base_span_event = [i_attrs, u_attrs, a_attrs] + if span_link_events or span_event_events: + return [base_span_event, span_link_events, span_event_events] + return base_span_event + + def _span_event_partial_granularity_essential( + self, settings, base_attrs=None, parent_guid=None, attr_class=dict, ct_exit_spans=None + ): + base_attrs, attr_class, span_link_events, span_event_events = self.span_event( + settings, base_attrs=base_attrs, parent_guid=parent_guid, attr_class=attr_class + ) + if ct_exit_spans is None: + ct_exit_spans = {"instrumented": 0, "kept": 0, "dropped_ids": 0} + + ct_exit_spans["instrumented"] += 1 + + i_attrs = (base_attrs and base_attrs.copy()) or attr_class() + i_attrs["type"] = "Span" + i_attrs["name"] = i_attrs.get("name") or self.name + i_attrs["guid"] = self.guid + i_attrs["timestamp"] = int(self.start_time * 1000) + i_attrs["duration"] = self.duration + i_attrs["category"] = i_attrs.get("category") or "generic" + + if parent_guid: + i_attrs["parentId"] = parent_guid + + a_attrs = self.agent_attributes + + a_attrs_set = set(a_attrs) + exit_span_attrs_present = attribute.SPAN_ENTITY_RELATIONSHIP_ATTRIBUTES & a_attrs_set + exit_span_error_attrs_present = attribute.SPAN_ERROR_ATTRIBUTES & a_attrs_set + # If this is an entry span, add `nr.pg` to indicate transaction is partial + # granularity sampled. + if i_attrs.get("nr.entryPoint"): + i_attrs["nr.pg"] = True + # If this is the entry node or an LLM span always return it. + if i_attrs.get("nr.entryPoint") or i_attrs["name"].startswith("Llm/"): + ct_exit_spans["kept"] += 1 + # Only keep entity-synthesis and error agent attributes, and intrinsics. + a_minimized_attrs = attribute.resolve_agent_attributes( + {key: a_attrs[key] for key in exit_span_attrs_present | exit_span_error_attrs_present}, + settings.attribute_filter, + DST_SPAN_EVENTS, + attr_class=attr_class, + ) + base_span_event = [i_attrs, {}, a_minimized_attrs] + if span_link_events or span_event_events: + return [base_span_event, span_link_events, span_event_events] + return base_span_event + # If the span is not an exit span, skip it by returning None. + if not exit_span_attrs_present: + return None + ct_exit_spans["kept"] += 1 + # Only keep entity-synthesis, and error agent attributes, and intrinsics. + a_minimized_attrs = attribute.resolve_agent_attributes( + {key: a_attrs[key] for key in exit_span_attrs_present | exit_span_error_attrs_present}, + settings.attribute_filter, + DST_SPAN_EVENTS, + attr_class=attr_class, + ) + base_span_event = [i_attrs, {}, a_minimized_attrs] + if span_link_events or span_event_events: + return [base_span_event, span_link_events, span_event_events] + return base_span_event + + def _span_event_partial_granularity_compact( + self, settings, base_attrs=None, parent_guid=None, attr_class=dict, ct_exit_spans=None + ): + base_attrs, attr_class, span_link_events, span_event_events = self.span_event( + settings, base_attrs=base_attrs, parent_guid=parent_guid, attr_class=attr_class + ) + if ct_exit_spans is None: + ct_exit_spans = {"instrumented": 0, "kept": 0, "dropped_ids": 0} + + ct_exit_spans["instrumented"] += 1 + + i_attrs = (base_attrs and base_attrs.copy()) or attr_class() + i_attrs["type"] = "Span" + i_attrs["name"] = i_attrs.get("name") or self.name + i_attrs["guid"] = self.guid + i_attrs["timestamp"] = int(self.start_time * 1000) + i_attrs["duration"] = self.duration + i_attrs["category"] = i_attrs.get("category") or "generic" + + if parent_guid: + i_attrs["parentId"] = parent_guid + + a_attrs = self.agent_attributes + + a_attrs_set = set(a_attrs) + exit_span_attrs_present = attribute.SPAN_ENTITY_RELATIONSHIP_ATTRIBUTES & a_attrs_set + exit_span_error_attrs_present = attribute.SPAN_ERROR_ATTRIBUTES & a_attrs_set + # If this is an entry span, add `nr.pg` to indicate transaction is partial + # granularity sampled. + if i_attrs.get("nr.entryPoint"): + i_attrs["nr.pg"] = True + # If this is the entry node or an LLM span always return it. + if i_attrs.get("nr.entryPoint") or i_attrs["name"].startswith("Llm/"): + ct_exit_spans["kept"] += 1 + # Only keep entity-synthesis and error agent attributes, and intrinsics. + a_minimized_attrs = attribute.resolve_agent_attributes( + {key: a_attrs[key] for key in exit_span_attrs_present | exit_span_error_attrs_present}, + settings.attribute_filter, + DST_SPAN_EVENTS, + attr_class=attr_class, + ) + base_span_event = [i_attrs, {}, a_minimized_attrs] + if span_link_events or span_event_events: + return [base_span_event, span_link_events, span_event_events] + return base_span_event + # If the span is not an exit span, skip it by returning None. + if not exit_span_attrs_present: + return None + a_minimized_attrs = attribute.resolve_agent_attributes( + {key: a_attrs[key] for key in exit_span_attrs_present | exit_span_error_attrs_present}, + settings.attribute_filter, + DST_SPAN_EVENTS, + attr_class=attr_class, + ) + + # If the span is an exit span but span compression (compact) is enabled, + # we need to check for uniqueness before returning it. + # Combine all the entity relationship attr values into a frozenset of tuples to be + # used as the hash to check for uniqueness. + span_attrs_hash = hash( + frozenset((key, a_minimized_attrs[key]) for key in exit_span_attrs_present if key in a_minimized_attrs) + ) + # If this is a new exit span, add it to the known ct_exit_spans and + # return it. + if span_attrs_hash not in ct_exit_spans: + # nr.ids is the list of span guids that share this unqiue exit span. + i_attrs["nr.ids"] = [] + i_attrs["nr.durations"] = self.duration + ct_exit_spans[span_attrs_hash] = [i_attrs, a_minimized_attrs] + ct_exit_spans["kept"] += 1 + # Only keep entity-synthesis, and error agent attributes, and intrinsics. + base_span_event = [i_attrs, {}, a_minimized_attrs] + if span_link_events or span_event_events: + return [base_span_event, span_link_events, span_event_events] + return base_span_event + # If this is an exit span we've already seen, add the error attributes + # (last occurring error takes precedence), add it's guid to the list + # of ids on the seen span, compute the new duration & start time, and + # return None. + exit_span = ct_exit_spans[span_attrs_hash] + exit_span[1].update( + attr_class( + {key: a_minimized_attrs[key] for key in exit_span_error_attrs_present if key in a_minimized_attrs} + ) + ) + # Max size for `nr.ids` = 1024. Max length = 63 (each span id is 16 bytes + 8 bytes for list type). + if len(exit_span[0]["nr.ids"]) < 63: + exit_span[0]["nr.ids"].append(self.guid) + else: + ct_exit_spans["dropped_ids"] += 1 + + # Compute the new start and end time for all compressed spans and use + # that to set the duration for all compressed spans. + current_start_time = exit_span[0]["timestamp"] + current_end_time = exit_span[0]["timestamp"] / 1000 + exit_span[0]["nr.durations"] + new_start_time = i_attrs["timestamp"] + new_end_time = i_attrs["timestamp"] / 1000 + i_attrs["duration"] + set_start_time = min(new_start_time, current_start_time) + # If the new span starts after the old span's end time or the new span + # ends before the current span starts; add the durations. + if current_end_time < new_start_time / 1000 or new_end_time < current_start_time / 1000: + set_duration = exit_span[0]["nr.durations"] + i_attrs["duration"] + # Otherwise, if the new and old span's overlap in time, use the newest + # end time and subtract the start time from it to calculate the new + # duration. + else: + set_duration = max(current_end_time, new_end_time) - set_start_time / 1000 + exit_span[0]["timestamp"] = set_start_time + exit_span[0]["nr.durations"] = set_duration + + PARTIAL_GRANULARITY_SPAN_EVENT_METHODS = { # noqa: RUF012 + "reduced": _span_event_partial_granularity_reduced, + "essential": _span_event_partial_granularity_essential, + "compact": _span_event_partial_granularity_compact, + } + + def span_event(self, settings, base_attrs=None, parent_guid=None, attr_class=dict): + return base_attrs, attr_class, None, None + + def span_events_full_granularity(self, settings, base_attrs=None, parent_guid=None, attr_class=dict): + yield self._span_event_full_granularity( + settings, base_attrs=base_attrs, parent_guid=parent_guid, attr_class=attr_class + ) for child in self.children: - for event in child.span_events( # noqa: UP028 + yield from child.span_events_full_granularity( settings, base_attrs=base_attrs, parent_guid=self.guid, attr_class=attr_class + ) + + def span_events_partial_granularity( + self, settings, span_event_method, base_attrs=None, parent_guid=None, attr_class=dict, ct_exit_spans=None + ): + span = span_event_method( + self=self, + settings=settings, + base_attrs=base_attrs, + parent_guid=parent_guid, + attr_class=attr_class, + ct_exit_spans=ct_exit_spans, + ) + parent_id = parent_guid + # In partial granularity tracing, span will be None if the span is an inprocess span or repeated exit span. + if span: + yield span + # Compressed spans are always reparented onto the entry span. + if settings.distributed_tracing.sampler.partial_granularity.type != "compact" or span[0].get( + "nr.entryPoint" ): - yield event + parent_id = self.guid + for child in self.children: + for event in child.span_events_partial_granularity( + settings, + span_event_method, + base_attrs=base_attrs, + parent_guid=parent_id, + attr_class=attr_class, + ct_exit_spans=ct_exit_spans, + ): + # In partial granularity tracing, event will be None if the span is an inprocess span or repeated exit span. + if event: + yield event class DatastoreNodeMixin(GenericNodeMixin): @@ -108,11 +379,10 @@ def db_instance(self): self._db_instance = db_instance_attr return db_instance_attr - def span_event(self, *args, **kwargs): - self.agent_attributes["db.instance"] = self.db_instance - attrs = super().span_event(*args, **kwargs) - i_attrs = attrs[0] - a_attrs = attrs[2] + def span_event(self, settings, base_attrs=None, parent_guid=None, attr_class=dict): + a_attrs = self.agent_attributes + a_attrs["db.instance"] = self.db_instance + i_attrs = (base_attrs and base_attrs.copy()) or attr_class() i_attrs["category"] = "datastore" i_attrs["span.kind"] = "client" @@ -141,4 +411,4 @@ def span_event(self, *args, **kwargs): except Exception: pass - return attrs + return i_attrs, attr_class, self.span_link_events, self.span_event_events diff --git a/newrelic/core/otlp_utils.py b/newrelic/core/otlp_utils.py index e34ba3e6c2..e9753eb247 100644 --- a/newrelic/core/otlp_utils.py +++ b/newrelic/core/otlp_utils.py @@ -116,8 +116,9 @@ def create_key_values_from_iterable(iterable): return list(filter(lambda i: i is not None, (create_key_value(key, value) for key, value in iterable))) -def create_resource(attributes=None, attach_apm_entity=True): - attributes = attributes or {"instrumentation.provider": "newrelic-opentelemetry-python-ml"} +def create_resource(attributes=None, attach_apm_entity=True, hybrid_bridge=False): + instrumentation_provider = "newrelic-opentelemetry-bridge" if hybrid_bridge else "newrelic-opentelemetry-python-ml" + attributes = attributes or {"instrumentation.provider": instrumentation_provider} if attach_apm_entity: metadata = get_service_linking_metadata() attributes.update(metadata) diff --git a/newrelic/core/root_node.py b/newrelic/core/root_node.py index 1591afa3ad..fabe2c35e8 100644 --- a/newrelic/core/root_node.py +++ b/newrelic/core/root_node.py @@ -32,21 +32,23 @@ "path", "trusted_parent_span", "tracing_vendors", + "span_link_events", + "span_event_events", ], ) class RootNode(_RootNode, GenericNodeMixin): - def span_event(self, *args, **kwargs): - span = super().span_event(*args, **kwargs) - i_attrs = span[0] + def span_event(self, settings, base_attrs=None, parent_guid=None, attr_class=dict): + i_attrs = (base_attrs and base_attrs.copy()) or attr_class() i_attrs["transaction.name"] = self.path i_attrs["nr.entryPoint"] = True if self.trusted_parent_span: i_attrs["trustedParentId"] = self.trusted_parent_span if self.tracing_vendors: i_attrs["tracingVendors"] = self.tracing_vendors - return span + + return i_attrs, attr_class, self.span_link_events, self.span_event_events def trace_node(self, stats, root, connections): name = self.path diff --git a/newrelic/core/samplers/__init__.py b/newrelic/core/samplers/__init__.py new file mode 100644 index 0000000000..bfe7af1430 --- /dev/null +++ b/newrelic/core/samplers/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/newrelic/core/adaptive_sampler.py b/newrelic/core/samplers/adaptive_sampler.py similarity index 100% rename from newrelic/core/adaptive_sampler.py rename to newrelic/core/samplers/adaptive_sampler.py diff --git a/newrelic/core/samplers/sampler_proxy.py b/newrelic/core/samplers/sampler_proxy.py new file mode 100644 index 0000000000..c4c8ed667f --- /dev/null +++ b/newrelic/core/samplers/sampler_proxy.py @@ -0,0 +1,170 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging + +from newrelic.core.samplers.adaptive_sampler import AdaptiveSampler +from newrelic.core.samplers.trace_id_ratio_based_sampler import TraceIdRatioBasedSampler + +_logger = logging.getLogger(__name__) + + +class SamplerProxy: + def __init__(self, settings): + if settings.serverless_mode.enabled: + sampling_target_period = 60.0 + else: + sampling_target_period = settings.sampling_target_period_in_seconds + adaptive_sampler = AdaptiveSampler(settings.sampling_target, sampling_target_period) + self._samplers = {"global": adaptive_sampler} + + full_gran_root_ratio = None + full_gran_parent_sampled_ratio = None + full_gran_parent_not_sampled_ratio = None + # Add sampler instances for each config section if configured. + if settings.distributed_tracing.sampler.full_granularity.enabled: + # If the ratio is not defined fallback to adaptive sampler. + if ( + settings.distributed_tracing.sampler._root == "trace_id_ratio_based" + and settings.distributed_tracing.sampler.root.trace_id_ratio_based.ratio + ): + full_gran_root_ratio = settings.distributed_tracing.sampler.root.trace_id_ratio_based.ratio + self.add_trace_id_ratio_based_sampler((True, 0), full_gran_root_ratio) + else: + self.add_adaptive_sampler( + (True, 0), + settings.distributed_tracing.sampler.root.adaptive.sampling_target, + sampling_target_period, + ) + # If the ratio is not defined fallback to adaptive sampler. + if ( + settings.distributed_tracing.sampler._remote_parent_sampled == "trace_id_ratio_based" + and settings.distributed_tracing.sampler.remote_parent_sampled.trace_id_ratio_based.ratio + ): + full_gran_parent_sampled_ratio = ( + settings.distributed_tracing.sampler.remote_parent_sampled.trace_id_ratio_based.ratio + ) + self.add_trace_id_ratio_based_sampler((True, 1), full_gran_parent_sampled_ratio) + else: + self.add_adaptive_sampler( + (True, 1), + settings.distributed_tracing.sampler.remote_parent_sampled.adaptive.sampling_target, + sampling_target_period, + ) + # If the ratio is not defined fallback to adaptive sampler. + if ( + settings.distributed_tracing.sampler._remote_parent_not_sampled == "trace_id_ratio_based" + and settings.distributed_tracing.sampler.remote_parent_not_sampled.trace_id_ratio_based.ratio + ): + full_gran_parent_not_sampled_ratio = ( + settings.distributed_tracing.sampler.remote_parent_not_sampled.trace_id_ratio_based.ratio + ) + self.add_trace_id_ratio_based_sampler((True, 2), full_gran_parent_not_sampled_ratio) + else: + self.add_adaptive_sampler( + (True, 2), + settings.distributed_tracing.sampler.remote_parent_not_sampled.adaptive.sampling_target, + sampling_target_period, + ) + if settings.distributed_tracing.sampler.partial_granularity.enabled: + # If the ratio is not defined fallback to adaptive sampler. + if ( + settings.distributed_tracing.sampler.partial_granularity._root == "trace_id_ratio_based" + and settings.distributed_tracing.sampler.partial_granularity.root.trace_id_ratio_based.ratio + ): + # If both full and partial are set to use the trace id ratio based sampler, + # set partial granularity ratio = full ratio + partial ratio. + ratio = settings.distributed_tracing.sampler.partial_granularity.root.trace_id_ratio_based.ratio + if full_gran_root_ratio: + ratio = min(ratio + full_gran_root_ratio, 1) + self.add_trace_id_ratio_based_sampler((False, 0), ratio) + else: + self.add_adaptive_sampler( + (False, 0), + settings.distributed_tracing.sampler.partial_granularity.root.adaptive.sampling_target, + sampling_target_period, + ) + # If the ratio is not defined fallback to adaptive sampler. + if ( + settings.distributed_tracing.sampler.partial_granularity._remote_parent_sampled + == "trace_id_ratio_based" + and settings.distributed_tracing.sampler.partial_granularity.remote_parent_sampled.trace_id_ratio_based.ratio + ): + # If both full and partial are set to use the trace id ratio based sampler, + # set partial granularity ratio = full ratio + partial ratio. + ratio = settings.distributed_tracing.sampler.partial_granularity.remote_parent_sampled.trace_id_ratio_based.ratio + if full_gran_parent_sampled_ratio: + ratio = min(ratio + full_gran_parent_sampled_ratio, 1) + self.add_trace_id_ratio_based_sampler((False, 1), ratio) + else: + self.add_adaptive_sampler( + (False, 1), + settings.distributed_tracing.sampler.partial_granularity.remote_parent_sampled.adaptive.sampling_target, + sampling_target_period, + ) + # If the ratio is not defined fallback to adaptive sampler. + if ( + settings.distributed_tracing.sampler.partial_granularity._remote_parent_not_sampled + == "trace_id_ratio_based" + and settings.distributed_tracing.sampler.partial_granularity.remote_parent_not_sampled.trace_id_ratio_based.ratio + ): + # If both full and partial are set to use the trace id ratio based sampler, + # set partial granularity ratio = full ratio + partial ratio. + ratio = settings.distributed_tracing.sampler.partial_granularity.remote_parent_not_sampled.trace_id_ratio_based.ratio + if full_gran_parent_not_sampled_ratio: + ratio = min(ratio + full_gran_parent_not_sampled_ratio, 1) + self.add_trace_id_ratio_based_sampler((False, 2), ratio) + else: + self.add_adaptive_sampler( + (False, 2), + settings.distributed_tracing.sampler.partial_granularity.remote_parent_not_sampled.adaptive.sampling_target, + sampling_target_period, + ) + + def add_trace_id_ratio_based_sampler(self, key, ratio): + """ + Add a trace id ratio based sampler instance to self._samplers. + """ + ratio_sampler = TraceIdRatioBasedSampler(ratio) + self._samplers[key] = ratio_sampler + + def add_adaptive_sampler(self, key, sampling_target, sampling_target_period): + """ + Add an adaptive sampler instance to self._samplers if the sampling_target is specified. + """ + if sampling_target: + adaptive_sampler = AdaptiveSampler(sampling_target, sampling_target_period) + self._samplers[key] = adaptive_sampler + + def get_sampler(self, full_granularity, section): + # Return the sampler instance for the given config section. + # If no instance is present, return the global adaptive sampler instance instead. + return self._samplers.get((full_granularity, section)) or self._samplers["global"] + + def compute_sampled(self, full_granularity, section, *args, **kwargs): + """ + full_granularity: True is full granularity, False is partial granularity + section: 0-root, 1-remote_parent_sampled, 2-remote_parent_not_sampled + """ + try: + return self.get_sampler(full_granularity, section).compute_sampled(*args, **kwargs) + except Exception: + # This happens when there is a mismatch in the settings used to create the + # samplers vs request a sampler inside a transaction. While this shouldn't + # ever happen this is a safety guard. + _logger.warning( + "Attempted to access sampler (%s, %s) but encountered an error. Falling back on global adaptive sampler.", + full_granularity, + section, + ) + return self._samplers["global"].compute_sampled() diff --git a/newrelic/core/samplers/trace_id_ratio_based_sampler.py b/newrelic/core/samplers/trace_id_ratio_based_sampler.py new file mode 100644 index 0000000000..457347cb10 --- /dev/null +++ b/newrelic/core/samplers/trace_id_ratio_based_sampler.py @@ -0,0 +1,33 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# For compatibility with 64 bit trace IDs, the sampler checks the 64 +# low-order bits of the trace ID to decide whether to sample a given trace. +TRACE_ID_LIMIT = (1 << 64) - 1 + + +class TraceIdRatioBasedSampler: + """ + This replicates behavior of TraceIdRatioBased sampler in + https://github.com/open-telemetry/opentelemetry-python/blob/main/opentelemetry-sdk/src/opentelemetry/sdk/trace/sampling.py. + """ + + def __init__(self, ratio): + self.ratio = ratio + self.bound = round(ratio * (TRACE_ID_LIMIT + 1)) + + def compute_sampled(self, trace_id): + if trace_id & TRACE_ID_LIMIT < self.bound: + return True + return False diff --git a/newrelic/core/stats_engine.py b/newrelic/core/stats_engine.py index f44f82fe13..7c998076cc 100644 --- a/newrelic/core/stats_engine.py +++ b/newrelic/core/stats_engine.py @@ -678,7 +678,6 @@ def record_time_metrics(self, metrics): def notice_error(self, error=None, attributes=None, expected=None, ignore=None, status_code=None): attributes = attributes if attributes is not None else {} settings = self.__settings - if not settings: return @@ -690,13 +689,19 @@ def notice_error(self, error=None, attributes=None, expected=None, ignore=None, if not settings.collect_errors and not settings.collect_error_events: return - # Pull from sys.exc_info if no exception is passed - if not error or None in error: + # If an exception instance is passed, attempt to unpack it into an exception tuple with traceback + if isinstance(error, BaseException): + error = (type(error), error, getattr(error, "__traceback__", None)) + + # Use current exception from sys.exc_info() if no exception was passed, + # or if the exception tuple is missing components like the traceback + if not error or (isinstance(error, (tuple, list)) and None in error): error = sys.exc_info() - # If no exception to report, exit - if not error or None in error: - return + # Error should be a tuple or list of 3 elements by this point. + # If it's falsey or missing a component like the traceback, quietly exit early. + if not isinstance(error, (tuple, list)) or len(error) != 3 or None in error: + return exc, value, tb = error @@ -1188,8 +1193,47 @@ def record_transaction(self, transaction): for event in transaction.span_protos(settings): self._span_stream.put(event) elif transaction.sampled: - for event in transaction.span_events(self.__settings): - self._span_events.add(event, priority=transaction.priority) + opentelemetry_enabled = settings.opentelemetry.enabled + if not opentelemetry_enabled: + # When opentelemetry is not enabled, the event will not contain SpanLinks or SpanEvents, + # so we can add the spans directly without filtering. + for event in transaction.span_events(self.__settings): + self._span_events.add(event, priority=transaction.priority) + else: + for event in transaction.span_events(self.__settings): + # When opentelemetry is enabled, the event may contain + # SpanLinks and/or SpanEvents. + if isinstance(event[-1], dict): + # No SpanLinks or SpanEvents to consider, add spans directly + self._span_events.add(event, priority=transaction.priority) + else: + # SpanLinks or SpanEvents are possible, one or both may also be empty lists. + # A filter is used to remove any empty lists. + new_event = list(filter(bool, event)) + self._span_events.add(new_event, priority=transaction.priority) + + if transaction.partial_granularity_sampled: + partial_gran_type = settings.distributed_tracing.sampler.partial_granularity.type + self.record_custom_metric( + f"Supportability/Python/PartialGranularity/{partial_gran_type}", {"count": 1} + ) + instrumented = getattr(transaction, "spans_instrumented", 0) + if instrumented: + self.record_custom_metric( + f"Supportability/DistributedTrace/PartialGranularity/{partial_gran_type}/Span/Instrumented", + {"count": instrumented}, + ) + kept = getattr(transaction, "spans_kept", 0) + if instrumented: + self.record_custom_metric( + f"Supportability/DistributedTrace/PartialGranularity/{partial_gran_type}/Span/Kept", + {"count": kept}, + ) + dropped_ids = getattr(transaction, "partial_granularity_dropped_ids", 0) + if dropped_ids: + self.record_custom_metric( + "Supportability/Python/PartialGranularity/NrIds/Dropped", {"count": dropped_ids} + ) # Merge in log events diff --git a/newrelic/core/transaction_node.py b/newrelic/core/transaction_node.py index 34871d8b21..6bad1cc077 100644 --- a/newrelic/core/transaction_node.py +++ b/newrelic/core/transaction_node.py @@ -98,6 +98,7 @@ "root_span_guid", "trace_id", "loop_time", + "partial_granularity_sampled", ], ) @@ -633,5 +634,47 @@ def span_events(self, settings, attr_class=dict): ("priority", self.priority), ) ) - - yield from self.root.span_events(settings, base_attrs, parent_guid=self.parent_span, attr_class=attr_class) + if not self.partial_granularity_sampled: + yield from self.root.span_events_full_granularity( + settings, base_attrs, parent_guid=self.parent_span, attr_class=attr_class + ) + else: + ct_exit_spans = {"instrumented": 0, "kept": 0, "dropped_ids": 0} + partial_type = settings.distributed_tracing.sampler.partial_granularity.type + # Get the appropriate span_event method for the partial granularity type. + # If the type does not exist fallback on the default "essential". + span_event_method = self.root.PARTIAL_GRANULARITY_SPAN_EVENT_METHODS.get( + partial_type, self.root.PARTIAL_GRANULARITY_SPAN_EVENT_METHODS["essential"] + ) + # In corner case scenarios where there is a harvest while spans are being added + # to the reservoir, a compact span may be sent before its agent attributes have + # been updated. This is solved by cacheing all spans in compact mode and not + # adding them to the reservoir until all spans are touched. + if partial_type == "compact": + events = list( + self.root.span_events_partial_granularity( + settings, + span_event_method, + base_attrs, + parent_guid=self.parent_span, + attr_class=attr_class, + ct_exit_spans=ct_exit_spans, + ) + ) + yield from events + else: + yield from self.root.span_events_partial_granularity( + settings, + span_event_method, + base_attrs, + parent_guid=self.parent_span, + attr_class=attr_class, + ct_exit_spans=ct_exit_spans, + ) + # If this transaction is partial granularity sampled, record the number of spans + # instrumented and the number of spans kept to monitor cost savings of partial + # granularity tracing. + # Also record the number of span ids dropped (fragmentation) in compact mode. + self.spans_instrumented = ct_exit_spans["instrumented"] + self.spans_kept = ct_exit_spans["kept"] + self.partial_granularity_dropped_ids = ct_exit_spans["dropped_ids"] diff --git a/newrelic/hooks/adapter_mcp.py b/newrelic/hooks/adapter_mcp.py index bcc8ae0a39..e891df0325 100644 --- a/newrelic/hooks/adapter_mcp.py +++ b/newrelic/hooks/adapter_mcp.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json import logging from newrelic.api.function_trace import FunctionTrace @@ -37,8 +38,10 @@ async def wrap_call_tool(wrapped, instance, args, kwargs): bound_args = bind_args(wrapped, args, kwargs) tool_name = bound_args.get("name") or "tool" function_trace_name = f"{func_name}/{tool_name}" + agentic_subcomponent_data = {"type": "APM-AI_TOOL", "name": tool_name} - with FunctionTrace(name=function_trace_name, group="Llm/tool/MCP", source=wrapped): + with FunctionTrace(name=function_trace_name, group="Llm/tool/MCP", source=wrapped) as ft: + ft._add_agent_attribute("subcomponent", json.dumps(agentic_subcomponent_data)) return await wrapped(*args, **kwargs) diff --git a/newrelic/hooks/coroutines_asyncio.py b/newrelic/hooks/coroutines_asyncio.py index 41fc776595..6f862d52dd 100644 --- a/newrelic/hooks/coroutines_asyncio.py +++ b/newrelic/hooks/coroutines_asyncio.py @@ -16,36 +16,73 @@ from newrelic.core.trace_cache import trace_cache -def remove_from_cache(task): +def remove_from_cache_callback(task): cache = trace_cache() cache.task_stop(task) -def propagate_task_context(task): +def wrap_create_task(task): trace_cache().task_start(task) - task.add_done_callback(remove_from_cache) + task.add_done_callback(remove_from_cache_callback) return task -def _bind_loop(loop, *args, **kwargs): +def _instrument_event_loop(loop): + if loop and hasattr(loop, "create_task") and not hasattr(loop.create_task, "__wrapped__"): + wrap_out_function(loop, "create_task", wrap_create_task) + + +def _bind_set_event_loop(loop, *args, **kwargs): return loop -def wrap_create_task(wrapped, instance, args, kwargs): - loop = _bind_loop(*args, **kwargs) +def wrap_set_event_loop(wrapped, instance, args, kwargs): + loop = _bind_set_event_loop(*args, **kwargs) - if loop and not hasattr(loop.create_task, "__wrapped__"): - wrap_out_function(loop, "create_task", propagate_task_context) + _instrument_event_loop(loop) return wrapped(*args, **kwargs) +def wrap__lazy_init(wrapped, instance, args, kwargs): + result = wrapped(*args, **kwargs) + # This logic can be used for uvloop, but should + # work for any valid custom loop factory. + + # A custom loop_factory will be used to create + # a new event loop instance. It will then run + # the main() coroutine on this event loop. Once + # this coroutine is complete, the event loop will + # be stopped and closed. + + # The new loop that is created and set as the + # running loop of the duration of the run() call. + # When the coroutine starts, it runs in the context + # that was active when run() was called. Any tasks + # created within this coroutine on this new event + # loop will inherit that context. + + # Note: The loop created by loop_factory is never + # set as the global current loop for the thread, + # even while it is running. + loop = instance._loop + _instrument_event_loop(loop) + + return result + + def instrument_asyncio_base_events(module): - wrap_out_function(module, "BaseEventLoop.create_task", propagate_task_context) + wrap_out_function(module, "BaseEventLoop.create_task", wrap_create_task) def instrument_asyncio_events(module): if hasattr(module, "_BaseDefaultEventLoopPolicy"): # Python >= 3.14 - wrap_function_wrapper(module, "_BaseDefaultEventLoopPolicy.set_event_loop", wrap_create_task) - else: # Python <= 3.13 - wrap_function_wrapper(module, "BaseDefaultEventLoopPolicy.set_event_loop", wrap_create_task) + wrap_function_wrapper(module, "_BaseDefaultEventLoopPolicy.set_event_loop", wrap_set_event_loop) + elif hasattr(module, "BaseDefaultEventLoopPolicy"): # Python <= 3.13 + wrap_function_wrapper(module, "BaseDefaultEventLoopPolicy.set_event_loop", wrap_set_event_loop) + + +# For Python >= 3.11 +def instrument_asyncio_runners(module): + if hasattr(module, "Runner") and hasattr(module.Runner, "_lazy_init"): + wrap_function_wrapper(module, "Runner._lazy_init", wrap__lazy_init) diff --git a/newrelic/hooks/database_oracledb.py b/newrelic/hooks/database_oracledb.py index b2888de464..20f6730e58 100644 --- a/newrelic/hooks/database_oracledb.py +++ b/newrelic/hooks/database_oracledb.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from newrelic.api.database_trace import register_database_client +from newrelic.api.database_trace import DatabaseTrace, register_database_client from newrelic.common.object_wrapper import wrap_object from newrelic.hooks.database_dbapi2 import ConnectionFactory as DBAPI2ConnectionFactory from newrelic.hooks.database_dbapi2 import ConnectionWrapper as DBAPI2ConnectionWrapper @@ -27,6 +27,16 @@ def __enter__(self): self.__wrapped__.__enter__() return self + # Signature differs from DBAPI 2.0 spec + def callproc(self, name, parameters=None, keyword_parameters=None): + with DatabaseTrace( + sql=f"CALL {name}", + dbapi2_module=self._nr_dbapi2_module, + connect_params=self._nr_connect_params, + source=self.__wrapped__.callproc, + ): + return self.__wrapped__.callproc(name=name, parameters=parameters, keyword_parameters=keyword_parameters) + class ConnectionWrapper(DBAPI2ConnectionWrapper): __cursor_wrapper__ = CursorWrapper @@ -45,6 +55,18 @@ async def __aenter__(self): await self.__wrapped__.__aenter__() return self + # Signature differs from DBAPI 2.0 spec + async def callproc(self, name, parameters=None, keyword_parameters=None): + with DatabaseTrace( + sql=f"CALL {name}", + dbapi2_module=self._nr_dbapi2_module, + connect_params=self._nr_connect_params, + source=self.__wrapped__.callproc, + ): + return await self.__wrapped__.callproc( + name=name, parameters=parameters, keyword_parameters=keyword_parameters + ) + class AsyncConnectionWrapper(DBAPI2AsyncConnectionWrapper): __cursor_wrapper__ = AsyncCursorWrapper diff --git a/newrelic/hooks/datastore_redis.py b/newrelic/hooks/datastore_redis.py index 0888f4a4b3..3b1a63910d 100644 --- a/newrelic/hooks/datastore_redis.py +++ b/newrelic/hooks/datastore_redis.py @@ -28,10 +28,10 @@ "blmpop", "bzmpop", "client", - "command", "command_docs", "command_getkeysandflags", "command_info", + "command", "debug_segfault", "expiretime", "failover", @@ -41,6 +41,10 @@ "hexpiretime", "hgetdel", "hgetex", + "hotkeys_get", + "hotkeys_reset", + "hotkeys_start", + "hotkeys_stop", "hpersist", "hpexpire", "hpexpireat", @@ -77,8 +81,8 @@ "sentinel_set", "sentinel_slaves", "shutdown", - "sort", "sort_ro", + "sort", "spop", "srandmember", "unwatch", @@ -90,10 +94,12 @@ "vinfo", "vlinks", "vrandmember", + "vrange", "vrem", "vsetattr", "vsim", "watch", + "xcfgset", "zlexcount", "zrevrangebyscore", } @@ -278,6 +284,7 @@ "hsetnx", "hstrlen", "hvals", + "hybrid_search", "incr", "incrby", "incrbyfloat", @@ -325,6 +332,7 @@ "mrange", "mrevrange", "mset", + "msetex", "msetnx", "numincrby", "object_encoding", diff --git a/newrelic/hooks/external_aiobotocore.py b/newrelic/hooks/external_aiobotocore.py index ddb9d4d056..1dbb2f2816 100644 --- a/newrelic/hooks/external_aiobotocore.py +++ b/newrelic/hooks/external_aiobotocore.py @@ -98,6 +98,7 @@ async def wrap_client__make_api_call(wrapped, instance, args, kwargs): response_extractor = getattr(instance, "_nr_response_extractor", None) stream_extractor = getattr(instance, "_nr_stream_extractor", None) response_streaming = getattr(instance, "_nr_response_streaming", False) + request_timestamp = getattr(instance, "_nr_request_timestamp", None) is_converse = getattr(instance, "_nr_is_converse", False) ft = getattr(instance, "_nr_ft", None) @@ -125,6 +126,7 @@ async def wrap_client__make_api_call(wrapped, instance, args, kwargs): transaction, bedrock_args, is_converse, + request_timestamp, ) raise @@ -149,6 +151,17 @@ async def wrap_client__make_api_call(wrapped, instance, args, kwargs): bedrock_attrs = extract_bedrock_converse_attrs( args[1], response, response_headers, model, span_id, trace_id ) + + if response_streaming: + # Wrap EventStream object here to intercept __iter__ method instead of instrumenting class. + # This class is used in numerous other services in botocore, and would cause conflicts. + response["stream"] = stream = AsyncEventStreamWrapper(response["stream"]) + stream._nr_ft = ft or None + stream._nr_bedrock_attrs = bedrock_attrs or {} + stream._nr_model_extractor = stream_extractor or None + stream._nr_is_converse = True + return response + else: bedrock_attrs = { "request_id": response_headers.get("x-amzn-requestid"), @@ -176,7 +189,9 @@ async def wrap_client__make_api_call(wrapped, instance, args, kwargs): if ft: ft.__exit__(None, None, None) bedrock_attrs["duration"] = ft.duration * 1000 - run_bedrock_response_extractor(response_extractor, response_body, bedrock_attrs, is_embedding, transaction) + run_bedrock_response_extractor( + response_extractor, response_body, bedrock_attrs, is_embedding, transaction, request_timestamp + ) except Exception: _logger.warning(RESPONSE_PROCESSING_FAILURE_LOG_MESSAGE, exc_info=True) diff --git a/newrelic/hooks/external_botocore.py b/newrelic/hooks/external_botocore.py index 39317ea752..6895d958a6 100644 --- a/newrelic/hooks/external_botocore.py +++ b/newrelic/hooks/external_botocore.py @@ -17,6 +17,7 @@ import logging import re import sys +import time import uuid from io import BytesIO @@ -33,6 +34,7 @@ from newrelic.common.package_version_utils import get_package_version from newrelic.common.signature import bind_args from newrelic.core.config import global_settings +from newrelic.core.database_utils import generate_dynamodb_arn QUEUE_URL_PATTERN = re.compile(r"https://sqs.([\w\d-]+).amazonaws.com/(\d+)/([^/]+)") BOTOCORE_VERSION = get_package_version("botocore") @@ -192,7 +194,9 @@ def create_chat_completion_message_event( request_model, request_id, llm_metadata_dict, + all_token_counts, response_id=None, + request_timestamp=None, ): if not transaction: return @@ -224,9 +228,13 @@ def create_chat_completion_message_event( "vendor": "bedrock", "ingest_source": "Python", } + if all_token_counts: + chat_completion_message_dict["token_count"] = 0 if settings.ai_monitoring.record_content.enabled: chat_completion_message_dict["content"] = content + if request_timestamp: + chat_completion_message_dict["timestamp"] = request_timestamp chat_completion_message_dict.update(llm_metadata_dict) @@ -263,6 +271,8 @@ def create_chat_completion_message_event( "ingest_source": "Python", "is_response": True, } + if all_token_counts: + chat_completion_message_dict["token_count"] = 0 if settings.ai_monitoring.record_content.enabled: chat_completion_message_dict["content"] = content @@ -272,24 +282,21 @@ def create_chat_completion_message_event( transaction.record_custom_event("LlmChatCompletionMessage", chat_completion_message_dict) -def extract_bedrock_titan_text_model_request(request_body, bedrock_attrs): +def extract_bedrock_titan_embedding_model_request(request_body, bedrock_attrs): request_body = json.loads(request_body) - request_config = request_body.get("textGenerationConfig", {}) - input_message_list = [{"role": "user", "content": request_body.get("inputText")}] - - bedrock_attrs["input_message_list"] = input_message_list - bedrock_attrs["request.max_tokens"] = request_config.get("maxTokenCount") - bedrock_attrs["request.temperature"] = request_config.get("temperature") + bedrock_attrs["input"] = request_body.get("inputText") return bedrock_attrs -def extract_bedrock_mistral_text_model_request(request_body, bedrock_attrs): - request_body = json.loads(request_body) - bedrock_attrs["input_message_list"] = [{"role": "user", "content": request_body.get("prompt")}] - bedrock_attrs["request.max_tokens"] = request_body.get("max_tokens") - bedrock_attrs["request.temperature"] = request_body.get("temperature") +def extract_bedrock_titan_embedding_model_response(response_body, bedrock_attrs): + if response_body: + response_body = json.loads(response_body) + + input_tokens = response_body.get("inputTextTokenCount", 0) + bedrock_attrs["response.usage.total_tokens"] = input_tokens + return bedrock_attrs @@ -297,16 +304,31 @@ def extract_bedrock_titan_text_model_response(response_body, bedrock_attrs): if response_body: response_body = json.loads(response_body) + input_tokens = response_body.get("inputTextTokenCount", 0) + completion_tokens = sum(result.get("tokenCount", 0) for result in response_body.get("results", [])) + total_tokens = input_tokens + completion_tokens + output_message_list = [ - {"role": "assistant", "content": result["outputText"]} for result in response_body.get("results", []) + {"role": "assistant", "content": result.get("outputText")} for result in response_body.get("results", []) ] bedrock_attrs["response.choices.finish_reason"] = response_body["results"][0]["completionReason"] + bedrock_attrs["response.usage.completion_tokens"] = completion_tokens + bedrock_attrs["response.usage.prompt_tokens"] = input_tokens + bedrock_attrs["response.usage.total_tokens"] = total_tokens bedrock_attrs["output_message_list"] = output_message_list return bedrock_attrs +def extract_bedrock_mistral_text_model_request(request_body, bedrock_attrs): + request_body = json.loads(request_body) + bedrock_attrs["input_message_list"] = [{"role": "user", "content": request_body.get("prompt")}] + bedrock_attrs["request.max_tokens"] = request_body.get("max_tokens") + bedrock_attrs["request.temperature"] = request_body.get("temperature") + return bedrock_attrs + + def extract_bedrock_mistral_text_model_response(response_body, bedrock_attrs): if response_body: response_body = json.loads(response_body) @@ -319,17 +341,6 @@ def extract_bedrock_mistral_text_model_response(response_body, bedrock_attrs): return bedrock_attrs -def extract_bedrock_titan_text_model_streaming_response(response_body, bedrock_attrs): - if response_body: - if "outputText" in response_body: - bedrock_attrs["output_message_list"] = messages = bedrock_attrs.get("output_message_list", []) - messages.append({"role": "assistant", "content": response_body["outputText"]}) - - bedrock_attrs["response.choices.finish_reason"] = response_body.get("completionReason", None) - - return bedrock_attrs - - def extract_bedrock_mistral_text_model_streaming_response(response_body, bedrock_attrs): if response_body: outputs = response_body.get("outputs") @@ -338,14 +349,46 @@ def extract_bedrock_mistral_text_model_streaming_response(response_body, bedrock "output_message_list", [{"role": "assistant", "content": ""}] ) bedrock_attrs["output_message_list"][0]["content"] += outputs[0].get("text", "") - bedrock_attrs["response.choices.finish_reason"] = outputs[0].get("stop_reason", None) + bedrock_attrs["response.choices.finish_reason"] = outputs[0].get("stop_reason") return bedrock_attrs -def extract_bedrock_titan_embedding_model_request(request_body, bedrock_attrs): +def extract_bedrock_titan_text_model_request(request_body, bedrock_attrs): request_body = json.loads(request_body) + request_config = request_body.get("textGenerationConfig", {}) - bedrock_attrs["input"] = request_body.get("inputText") + input_message_list = [{"role": "user", "content": request_body.get("inputText")}] + + bedrock_attrs["input_message_list"] = input_message_list + bedrock_attrs["request.max_tokens"] = request_config.get("maxTokenCount") + bedrock_attrs["request.temperature"] = request_config.get("temperature") + + return bedrock_attrs + + +def extract_bedrock_titan_text_model_streaming_response(response_body, bedrock_attrs): + if response_body: + if "outputText" in response_body: + bedrock_attrs["output_message_list"] = messages = bedrock_attrs.get("output_message_list", []) + messages.append({"role": "assistant", "content": response_body["outputText"]}) + + bedrock_attrs["response.choices.finish_reason"] = response_body.get("completionReason") + + # Extract token information + invocation_metrics = response_body.get("amazon-bedrock-invocationMetrics", {}) + prompt_tokens = invocation_metrics.get("inputTokenCount", 0) + completion_tokens = invocation_metrics.get("outputTokenCount", 0) + total_tokens = prompt_tokens + completion_tokens + + bedrock_attrs["response.usage.completion_tokens"] = ( + bedrock_attrs.get("response.usage.completion_tokens", 0) + completion_tokens + ) + bedrock_attrs["response.usage.prompt_tokens"] = ( + bedrock_attrs.get("response.usage.prompt_tokens", 0) + prompt_tokens + ) + bedrock_attrs["response.usage.total_tokens"] = ( + bedrock_attrs.get("response.usage.total_tokens", 0) + total_tokens + ) return bedrock_attrs @@ -415,6 +458,17 @@ def extract_bedrock_claude_model_response(response_body, bedrock_attrs): bedrock_attrs["response.choices.finish_reason"] = response_body.get("stop_reason") bedrock_attrs["output_message_list"] = output_message_list + bedrock_attrs[""] = str(response_body.get("id")) + + # Extract token information + token_usage = response_body.get("usage", {}) + if token_usage: + prompt_tokens = token_usage.get("input_tokens", 0) + completion_tokens = token_usage.get("output_tokens", 0) + total_tokens = prompt_tokens + completion_tokens + bedrock_attrs["response.usage.prompt_tokens"] = prompt_tokens + bedrock_attrs["response.usage.completion_tokens"] = completion_tokens + bedrock_attrs["response.usage.total_tokens"] = total_tokens return bedrock_attrs @@ -427,6 +481,22 @@ def extract_bedrock_claude_model_streaming_response(response_body, bedrock_attrs bedrock_attrs["output_message_list"][0]["content"] += content bedrock_attrs["response.choices.finish_reason"] = response_body.get("stop_reason") + # Extract token information + invocation_metrics = response_body.get("amazon-bedrock-invocationMetrics", {}) + prompt_tokens = invocation_metrics.get("inputTokenCount", 0) + completion_tokens = invocation_metrics.get("outputTokenCount", 0) + total_tokens = prompt_tokens + completion_tokens + + bedrock_attrs["response.usage.completion_tokens"] = ( + bedrock_attrs.get("response.usage.completion_tokens", 0) + completion_tokens + ) + bedrock_attrs["response.usage.prompt_tokens"] = ( + bedrock_attrs.get("response.usage.prompt_tokens", 0) + prompt_tokens + ) + bedrock_attrs["response.usage.total_tokens"] = ( + bedrock_attrs.get("response.usage.total_tokens", 0) + total_tokens + ) + return bedrock_attrs @@ -447,6 +517,13 @@ def extract_bedrock_llama_model_response(response_body, bedrock_attrs): response_body = json.loads(response_body) output_message_list = [{"role": "assistant", "content": response_body.get("generation")}] + prompt_tokens = response_body.get("prompt_token_count", 0) + completion_tokens = response_body.get("generation_token_count", 0) + total_tokens = prompt_tokens + completion_tokens + + bedrock_attrs["response.usage.completion_tokens"] = completion_tokens + bedrock_attrs["response.usage.prompt_tokens"] = prompt_tokens + bedrock_attrs["response.usage.total_tokens"] = total_tokens bedrock_attrs["response.choices.finish_reason"] = response_body.get("stop_reason") bedrock_attrs["output_message_list"] = output_message_list @@ -460,6 +537,22 @@ def extract_bedrock_llama_model_streaming_response(response_body, bedrock_attrs) bedrock_attrs["output_message_list"] = [{"role": "assistant", "content": ""}] bedrock_attrs["output_message_list"][0]["content"] += content bedrock_attrs["response.choices.finish_reason"] = response_body.get("stop_reason") + + # Extract token information + invocation_metrics = response_body.get("amazon-bedrock-invocationMetrics", {}) + prompt_tokens = invocation_metrics.get("inputTokenCount", 0) + completion_tokens = invocation_metrics.get("outputTokenCount", 0) + total_tokens = prompt_tokens + completion_tokens + + bedrock_attrs["response.usage.completion_tokens"] = ( + bedrock_attrs.get("response.usage.completion_tokens", 0) + completion_tokens + ) + bedrock_attrs["response.usage.prompt_tokens"] = ( + bedrock_attrs.get("response.usage.prompt_tokens", 0) + prompt_tokens + ) + bedrock_attrs["response.usage.total_tokens"] = ( + bedrock_attrs.get("response.usage.total_tokens", 0) + total_tokens + ) return bedrock_attrs @@ -500,12 +593,33 @@ def extract_bedrock_cohere_model_streaming_response(response_body, bedrock_attrs bedrock_attrs["response.choices.finish_reason"] = response_body["generations"][0]["finish_reason"] bedrock_attrs["response_id"] = str(response_body.get("id")) + # Extract token information + invocation_metrics = response_body.get("amazon-bedrock-invocationMetrics", {}) + prompt_tokens = invocation_metrics.get("inputTokenCount", 0) + completion_tokens = invocation_metrics.get("outputTokenCount", 0) + total_tokens = prompt_tokens + completion_tokens + + bedrock_attrs["response.usage.completion_tokens"] = ( + bedrock_attrs.get("response.usage.completion_tokens", 0) + completion_tokens + ) + bedrock_attrs["response.usage.prompt_tokens"] = ( + bedrock_attrs.get("response.usage.prompt_tokens", 0) + prompt_tokens + ) + bedrock_attrs["response.usage.total_tokens"] = ( + bedrock_attrs.get("response.usage.total_tokens", 0) + total_tokens + ) + return bedrock_attrs NULL_EXTRACTOR = lambda *args: {} # noqa: E731 # Empty extractor that returns nothing MODEL_EXTRACTORS = [ # Order is important here, avoiding dictionaries - ("amazon.titan-embed", extract_bedrock_titan_embedding_model_request, NULL_EXTRACTOR, NULL_EXTRACTOR), + ( + "amazon.titan-embed", + extract_bedrock_titan_embedding_model_request, + extract_bedrock_titan_embedding_model_response, + NULL_EXTRACTOR, + ), ("cohere.embed", extract_bedrock_cohere_embedding_model_request, NULL_EXTRACTOR, NULL_EXTRACTOR), ( "amazon.titan", @@ -542,10 +656,22 @@ def extract_bedrock_cohere_model_streaming_response(response_body, bedrock_attrs def handle_bedrock_exception( - exc, is_embedding, model, span_id, trace_id, request_extractor, request_body, ft, transaction, kwargs, is_converse + exc, + is_embedding, + model, + span_id, + trace_id, + request_extractor, + request_body, + ft, + transaction, + kwargs, + is_converse, + request_timestamp=None, ): try: bedrock_attrs = {"model": model, "span_id": span_id, "trace_id": trace_id} + if is_converse: try: input_message_list = [ @@ -557,8 +683,8 @@ def handle_bedrock_exception( input_message_list = [] bedrock_attrs["input_message_list"] = input_message_list - bedrock_attrs["request.max_tokens"] = kwargs.get("inferenceConfig", {}).get("maxTokens", None) - bedrock_attrs["request.temperature"] = kwargs.get("inferenceConfig", {}).get("temperature", None) + bedrock_attrs["request.max_tokens"] = kwargs.get("inferenceConfig", {}).get("maxTokens") + bedrock_attrs["request.temperature"] = kwargs.get("inferenceConfig", {}).get("temperature") try: request_extractor(request_body, bedrock_attrs) @@ -576,9 +702,9 @@ def handle_bedrock_exception( } if is_embedding: - notice_error_attributes.update({"embedding_id": str(uuid.uuid4())}) + notice_error_attributes["embedding_id"] = str(uuid.uuid4()) else: - notice_error_attributes.update({"completion_id": str(uuid.uuid4())}) + notice_error_attributes["completion_id"] = str(uuid.uuid4()) if ft: ft.notice_error(attributes=notice_error_attributes) @@ -589,12 +715,14 @@ def handle_bedrock_exception( if is_embedding: handle_embedding_event(transaction, error_attributes) else: - handle_chat_completion_event(transaction, error_attributes) + handle_chat_completion_event(transaction, error_attributes, request_timestamp) except Exception: _logger.warning(EXCEPTION_HANDLING_FAILURE_LOG_MESSAGE, exc_info=True) -def run_bedrock_response_extractor(response_extractor, response_body, bedrock_attrs, is_embedding, transaction): +def run_bedrock_response_extractor( + response_extractor, response_body, bedrock_attrs, is_embedding, transaction, request_timestamp=None +): # Run response extractor for non-streaming responses try: response_extractor(response_body, bedrock_attrs) @@ -604,7 +732,7 @@ def run_bedrock_response_extractor(response_extractor, response_body, bedrock_at if is_embedding: handle_embedding_event(transaction, bedrock_attrs) else: - handle_chat_completion_event(transaction, bedrock_attrs) + handle_chat_completion_event(transaction, bedrock_attrs, request_timestamp) def run_bedrock_request_extractor(request_extractor, request_body, bedrock_attrs): @@ -628,6 +756,8 @@ def _wrap_bedrock_runtime_invoke_model(wrapped, instance, args, kwargs): if not settings.ai_monitoring.enabled: return wrapped(*args, **kwargs) + request_timestamp = int(1000.0 * time.time()) + transaction.add_ml_model_info("Bedrock", BOTOCORE_VERSION) transaction._add_agent_attribute("llm", True) @@ -683,6 +813,7 @@ def _wrap_bedrock_runtime_invoke_model(wrapped, instance, args, kwargs): instance._nr_ft = ft instance._nr_response_streaming = response_streaming instance._nr_settings = settings + instance._nr_request_timestamp = request_timestamp # Add a bedrock flag to instance so we can determine when make_api_call instrumentation is hit from non-Bedrock paths and bypass it if so instance._nr_is_bedrock = True @@ -703,6 +834,7 @@ def _wrap_bedrock_runtime_invoke_model(wrapped, instance, args, kwargs): transaction, kwargs, is_converse=False, + request_timestamp=request_timestamp, ) raise @@ -733,6 +865,8 @@ def _wrap_bedrock_runtime_invoke_model(wrapped, instance, args, kwargs): run_bedrock_request_extractor(request_extractor, request_body, bedrock_attrs) try: + bedrock_attrs.pop("timestamp", None) # The request timestamp is only needed for request extraction + if response_streaming: # Wrap EventStream object here to intercept __iter__ method instead of instrumenting class. # This class is used in numerous other services in botocore, and would cause conflicts. @@ -748,7 +882,14 @@ def _wrap_bedrock_runtime_invoke_model(wrapped, instance, args, kwargs): bedrock_attrs["duration"] = ft.duration * 1000 response["body"] = StreamingBody(BytesIO(response_body), len(response_body)) - run_bedrock_response_extractor(response_extractor, response_body, bedrock_attrs, is_embedding, transaction) + run_bedrock_response_extractor( + response_extractor, + response_body, + bedrock_attrs, + is_embedding, + transaction, + request_timestamp=request_timestamp, + ) except Exception: _logger.warning(RESPONSE_PROCESSING_FAILURE_LOG_MESSAGE, exc_info=True) @@ -766,10 +907,12 @@ def _wrap_bedrock_runtime_converse(wrapped, instance, args, kwargs): if not transaction: return wrapped(*args, **kwargs) - settings = transaction.settings or global_settings + settings = transaction.settings or global_settings() if not settings.ai_monitoring.enabled: return wrapped(*args, **kwargs) + request_timestamp = int(1000.0 * time.time()) + transaction.add_ml_model_info("Bedrock", BOTOCORE_VERSION) transaction._add_agent_attribute("llm", True) @@ -800,6 +943,7 @@ def _wrap_bedrock_runtime_converse(wrapped, instance, args, kwargs): instance._nr_ft = ft instance._nr_response_streaming = response_streaming instance._nr_settings = settings + instance._nr_request_timestamp = request_timestamp instance._nr_is_converse = True # Add a bedrock flag to instance so we can determine when make_api_call instrumentation is hit from non-Bedrock paths and bypass it if so @@ -808,9 +952,21 @@ def _wrap_bedrock_runtime_converse(wrapped, instance, args, kwargs): try: # For aioboto3 clients, this will call make_api_call instrumentation in external_aiobotocore response = wrapped(*args, **kwargs) + except Exception as exc: handle_bedrock_exception( - exc, False, model, span_id, trace_id, request_extractor, {}, ft, transaction, kwargs, is_converse=True + exc, + False, + model, + span_id, + trace_id, + request_extractor, + {}, + ft, + transaction, + kwargs, + is_converse=True, + request_timestamp=request_timestamp, ) raise @@ -824,11 +980,24 @@ def _wrap_bedrock_runtime_converse(wrapped, instance, args, kwargs): response_headers = response.get("ResponseMetadata", {}).get("HTTPHeaders") or {} bedrock_attrs = extract_bedrock_converse_attrs(kwargs, response, response_headers, model, span_id, trace_id) + bedrock_attrs["timestamp"] = request_timestamp try: + if response_streaming: + # Wrap EventStream object here to intercept __iter__ method instead of instrumenting class. + # This class is used in numerous other services in botocore, and would cause conflicts. + response["stream"] = stream = EventStreamWrapper(response["stream"]) + stream._nr_ft = ft + stream._nr_bedrock_attrs = bedrock_attrs + stream._nr_model_extractor = stream_extractor + stream._nr_is_converse = True + return response + ft.__exit__(None, None, None) bedrock_attrs["duration"] = ft.duration * 1000 - run_bedrock_response_extractor(response_extractor, {}, bedrock_attrs, False, transaction) + run_bedrock_response_extractor( + response_extractor, {}, bedrock_attrs, False, transaction, request_timestamp=request_timestamp + ) except Exception: _logger.warning(RESPONSE_PROCESSING_FAILURE_LOG_MESSAGE, exc_info=True) @@ -840,20 +1009,44 @@ def _wrap_bedrock_runtime_converse(wrapped, instance, args, kwargs): def extract_bedrock_converse_attrs(kwargs, response, response_headers, model, span_id, trace_id): input_message_list = [] - # If a system message is supplied, it is under its own key in kwargs rather than with the other input messages - if "system" in kwargs.keys(): - input_message_list.extend({"role": "system", "content": result["text"]} for result in kwargs.get("system", [])) - - # kwargs["messages"] can hold multiple requests and responses to maintain conversation history - # We grab the last message (the newest request) in the list each time, so we don't duplicate recorded data - input_message_list.extend( - [{"role": "user", "content": result["text"]} for result in kwargs["messages"][-1].get("content", [])] - ) + try: + # If a system message is supplied, it is under its own key in kwargs rather than with the other input messages + if "system" in kwargs.keys(): + input_message_list.extend( + {"role": "system", "content": result["text"]} for result in kwargs.get("system", []) if "text" in result + ) + + # kwargs["messages"] can hold multiple requests and responses to maintain conversation history + # We grab the last message (the newest request) in the list each time, so we don't duplicate recorded data + _input_messages = kwargs.get("messages", []) + _input_messages = _input_messages and (_input_messages[-1] or {}) + _input_messages = _input_messages.get("content", []) + input_message_list.extend( + [{"role": "user", "content": result["text"]} for result in _input_messages if "text" in result] + ) + except Exception: + _logger.warning( + "Exception occurred in botocore instrumentation for AWS Bedrock: Failed to extract input messages from Converse request. Report this issue to New Relic Support.", + exc_info=True, + ) + + output_message_list = None + try: + if "output" in response: + output_message_list = [ + {"role": "assistant", "content": result["text"]} + for result in response.get("output").get("message").get("content", []) + if "text" in result + ] + except Exception: + _logger.warning( + "Exception occurred in botocore instrumentation for AWS Bedrock: Failed to extract output messages from onverse response. Report this issue to New Relic Support.", + exc_info=True, + ) - output_message_list = [ - {"role": "assistant", "content": result["text"]} - for result in response.get("output").get("message").get("content", []) - ] + response_prompt_tokens = response.get("usage", {}).get("inputTokens") if response else None + response_completion_tokens = response.get("usage", {}).get("outputTokens") if response else None + response_total_tokens = response.get("usage", {}).get("totalTokens") if response else None bedrock_attrs = { "request_id": response_headers.get("x-amzn-requestid"), @@ -861,26 +1054,118 @@ def extract_bedrock_converse_attrs(kwargs, response, response_headers, model, sp "span_id": span_id, "trace_id": trace_id, "response.choices.finish_reason": response.get("stopReason"), - "output_message_list": output_message_list, - "request.max_tokens": kwargs.get("inferenceConfig", {}).get("maxTokens", None), - "request.temperature": kwargs.get("inferenceConfig", {}).get("temperature", None), + "request.max_tokens": kwargs.get("inferenceConfig", {}).get("maxTokens"), + "request.temperature": kwargs.get("inferenceConfig", {}).get("temperature"), "input_message_list": input_message_list, + "response.usage.prompt_tokens": response_prompt_tokens, + "response.usage.completion_tokens": response_completion_tokens, + "response.usage.total_tokens": response_total_tokens, } + + if output_message_list is not None: + bedrock_attrs["output_message_list"] = output_message_list + return bedrock_attrs +class BedrockRecordEventMixin: + def record_events_on_stop_iteration(self, transaction, request_timestamp=None): + if hasattr(self, "_nr_ft"): + bedrock_attrs = getattr(self, "_nr_bedrock_attrs", {}) + self._nr_ft.__exit__(None, None, None) + + # If there are no bedrock attrs exit early as there's no data to record. + if not bedrock_attrs: + return + + try: + bedrock_attrs["duration"] = self._nr_ft.duration * 1000 + handle_chat_completion_event(transaction, bedrock_attrs, request_timestamp) + except Exception: + _logger.warning(RESPONSE_PROCESSING_FAILURE_LOG_MESSAGE, exc_info=True) + + # Clear cached data as this can be very large. + self._nr_bedrock_attrs.clear() + + def record_error(self, transaction, exc, request_timestamp=None): + if hasattr(self, "_nr_ft"): + try: + ft = self._nr_ft + error_attributes = getattr(self, "_nr_bedrock_attrs", {}) + + # If there are no bedrock attrs exit early as there's no data to record. + if not error_attributes: + return + + error_attributes = bedrock_error_attributes(exc, error_attributes) + notice_error_attributes = { + "http.statusCode": error_attributes.get("http.statusCode"), + "error.message": error_attributes.get("error.message"), + "error.code": error_attributes.get("error.code"), + } + notice_error_attributes["completion_id"] = str(uuid.uuid4()) + + ft.notice_error(attributes=notice_error_attributes) + + ft.__exit__(*sys.exc_info()) + error_attributes["duration"] = ft.duration * 1000 + + handle_chat_completion_event(transaction, error_attributes, request_timestamp) + + # Clear cached data as this can be very large. + error_attributes.clear() + except Exception: + _logger.warning(EXCEPTION_HANDLING_FAILURE_LOG_MESSAGE, exc_info=True) + + def record_stream_chunk(self, event, transaction, request_timestamp=None): + if event: + try: + if getattr(self, "_nr_is_converse", False): + return self.converse_record_stream_chunk(event, transaction) + else: + return self.invoke_record_stream_chunk(event, transaction, request_timestamp) + except Exception: + _logger.warning(RESPONSE_EXTRACTOR_FAILURE_LOG_MESSAGE, exc_info=True) + + def invoke_record_stream_chunk(self, event, transaction, request_timestamp=None): + bedrock_attrs = getattr(self, "_nr_bedrock_attrs", {}) + chunk = json.loads(event["chunk"]["bytes"].decode("utf-8")) + self._nr_model_extractor(chunk, bedrock_attrs) + # In Langchain, the bedrock iterator exits early if type is "content_block_stop". + # So we need to call the record events here since stop iteration will not be raised. + _type = chunk.get("type") + if _type == "content_block_stop": + self.record_events_on_stop_iteration(transaction, request_timestamp) + + def converse_record_stream_chunk(self, event, transaction): + bedrock_attrs = getattr(self, "_nr_bedrock_attrs", {}) + if "contentBlockDelta" in event: + if not bedrock_attrs: + return + + content = ((event.get("contentBlockDelta") or {}).get("delta") or {}).get("text", "") + if "output_message_list" not in bedrock_attrs: + bedrock_attrs["output_message_list"] = [{"role": "assistant", "content": ""}] + bedrock_attrs["output_message_list"][0]["content"] += content + + if "messageStop" in event: + bedrock_attrs["response.choices.finish_reason"] = (event.get("messageStop") or {}).get("stopReason", "") + + class EventStreamWrapper(ObjectProxy): def __iter__(self): g = GeneratorProxy(self.__wrapped__.__iter__()) g._nr_ft = getattr(self, "_nr_ft", None) g._nr_bedrock_attrs = getattr(self, "_nr_bedrock_attrs", {}) g._nr_model_extractor = getattr(self, "_nr_model_extractor", NULL_EXTRACTOR) + g._nr_is_converse = getattr(self, "_nr_is_converse", False) return g -class GeneratorProxy(ObjectProxy): +class GeneratorProxy(BedrockRecordEventMixin, ObjectProxy): def __init__(self, wrapped): super().__init__(wrapped) + self._nr_request_timestamp = int(1000.0 * time.time()) def __iter__(self): return self @@ -893,17 +1178,17 @@ def __next__(self): return_val = None try: return_val = self.__wrapped__.__next__() - record_stream_chunk(self, return_val, transaction) + self.record_stream_chunk(return_val, transaction, self._nr_request_timestamp) except StopIteration: - record_events_on_stop_iteration(self, transaction) + self.record_events_on_stop_iteration(transaction, self._nr_request_timestamp) raise except Exception as exc: - record_error(self, transaction, exc) + self.record_error(transaction, exc, self._nr_request_timestamp) raise return return_val def close(self): - return super().close() + return self.__wrapped__.close() class AsyncEventStreamWrapper(ObjectProxy): @@ -912,12 +1197,14 @@ def __aiter__(self): g._nr_ft = getattr(self, "_nr_ft", None) g._nr_bedrock_attrs = getattr(self, "_nr_bedrock_attrs", {}) g._nr_model_extractor = getattr(self, "_nr_model_extractor", NULL_EXTRACTOR) + g._nr_is_converse = getattr(self, "_nr_is_converse", False) return g -class AsyncGeneratorProxy(ObjectProxy): +class AsyncGeneratorProxy(BedrockRecordEventMixin, ObjectProxy): def __init__(self, wrapped): super().__init__(wrapped) + self._nr_request_timestamp = int(1000.0 * time.time()) def __aiter__(self): return self @@ -929,81 +1216,17 @@ async def __anext__(self): return_val = None try: return_val = await self.__wrapped__.__anext__() - record_stream_chunk(self, return_val, transaction) + self.record_stream_chunk(return_val, transaction, self._nr_request_timestamp) except StopAsyncIteration: - record_events_on_stop_iteration(self, transaction) + self.record_events_on_stop_iteration(transaction, self._nr_request_timestamp) raise except Exception as exc: - record_error(self, transaction, exc) + self.record_error(transaction, exc, self._nr_request_timestamp) raise return return_val async def aclose(self): - return await super().aclose() - - -def record_stream_chunk(self, return_val, transaction): - if return_val: - try: - chunk = json.loads(return_val["chunk"]["bytes"].decode("utf-8")) - self._nr_model_extractor(chunk, self._nr_bedrock_attrs) - # In Langchain, the bedrock iterator exits early if type is "content_block_stop". - # So we need to call the record events here since stop iteration will not be raised. - _type = chunk.get("type") - if _type == "content_block_stop": - record_events_on_stop_iteration(self, transaction) - except Exception: - _logger.warning(RESPONSE_EXTRACTOR_FAILURE_LOG_MESSAGE, exc_info=True) - - -def record_events_on_stop_iteration(self, transaction): - if hasattr(self, "_nr_ft"): - bedrock_attrs = getattr(self, "_nr_bedrock_attrs", {}) - self._nr_ft.__exit__(None, None, None) - - # If there are no bedrock attrs exit early as there's no data to record. - if not bedrock_attrs: - return - - try: - bedrock_attrs["duration"] = self._nr_ft.duration * 1000 - handle_chat_completion_event(transaction, bedrock_attrs) - except Exception: - _logger.warning(RESPONSE_PROCESSING_FAILURE_LOG_MESSAGE, exc_info=True) - - # Clear cached data as this can be very large. - self._nr_bedrock_attrs.clear() - - -def record_error(self, transaction, exc): - if hasattr(self, "_nr_ft"): - try: - ft = self._nr_ft - error_attributes = getattr(self, "_nr_bedrock_attrs", {}) - - # If there are no bedrock attrs exit early as there's no data to record. - if not error_attributes: - return - - error_attributes = bedrock_error_attributes(exc, error_attributes) - notice_error_attributes = { - "http.statusCode": error_attributes.get("http.statusCode"), - "error.message": error_attributes.get("error.message"), - "error.code": error_attributes.get("error.code"), - } - notice_error_attributes.update({"completion_id": str(uuid.uuid4())}) - - ft.notice_error(attributes=notice_error_attributes) - - ft.__exit__(*sys.exc_info()) - error_attributes["duration"] = ft.duration * 1000 - - handle_chat_completion_event(transaction, error_attributes) - - # Clear cached data as this can be very large. - error_attributes.clear() - except Exception: - _logger.warning(EXCEPTION_HANDLING_FAILURE_LOG_MESSAGE, exc_info=True) + return await self.__wrapped__.aclose() def handle_embedding_event(transaction, bedrock_attrs): @@ -1015,29 +1238,34 @@ def handle_embedding_event(transaction, bedrock_attrs): custom_attrs_dict = transaction._custom_params llm_metadata_dict = {key: value for key, value in custom_attrs_dict.items() if key.startswith("llm.")} - span_id = bedrock_attrs.get("span_id", None) - trace_id = bedrock_attrs.get("trace_id", None) - request_id = bedrock_attrs.get("request_id", None) - model = bedrock_attrs.get("model", None) + span_id = bedrock_attrs.get("span_id") + trace_id = bedrock_attrs.get("trace_id") + request_id = bedrock_attrs.get("request_id") + model = bedrock_attrs.get("model") input_ = bedrock_attrs.get("input") + response_total_tokens = bedrock_attrs.get("response.usage.total_tokens") + + total_tokens = ( + settings.ai_monitoring.llm_token_count_callback(model, input_) + if settings.ai_monitoring.llm_token_count_callback and input_ + else response_total_tokens + ) + embedding_dict = { "vendor": "bedrock", "ingest_source": "Python", "id": embedding_id, "span_id": span_id, "trace_id": trace_id, - "token_count": ( - settings.ai_monitoring.llm_token_count_callback(model, input_) - if settings.ai_monitoring.llm_token_count_callback - else None - ), "request_id": request_id, - "duration": bedrock_attrs.get("duration", None), + "duration": bedrock_attrs.get("duration"), "request.model": model, "response.model": model, - "error": bedrock_attrs.get("error", None), + "response.usage.total_tokens": total_tokens, + "error": bedrock_attrs.get("error"), } + embedding_dict.update(llm_metadata_dict) if settings.ai_monitoring.record_content.enabled: @@ -1047,7 +1275,8 @@ def handle_embedding_event(transaction, bedrock_attrs): transaction.record_custom_event("LlmEmbedding", embedding_dict) -def handle_chat_completion_event(transaction, bedrock_attrs): +def handle_chat_completion_event(transaction, bedrock_attrs, request_timestamp=None): + settings = transaction.settings or global_settings() chat_completion_id = str(uuid.uuid4()) # Grab LLM-related custom attributes off of the transaction to store as metadata on LLM events custom_attrs_dict = transaction._custom_params @@ -1056,11 +1285,15 @@ def handle_chat_completion_event(transaction, bedrock_attrs): llm_context_attrs = getattr(transaction, "_llm_context_attrs", None) if llm_context_attrs: llm_metadata_dict.update(llm_context_attrs) - span_id = bedrock_attrs.get("span_id", None) - trace_id = bedrock_attrs.get("trace_id", None) - request_id = bedrock_attrs.get("request_id", None) - response_id = bedrock_attrs.get("response_id", None) - model = bedrock_attrs.get("model", None) + span_id = bedrock_attrs.get("span_id") + trace_id = bedrock_attrs.get("trace_id") + request_id = bedrock_attrs.get("request_id") + response_id = bedrock_attrs.get("response_id") + model = bedrock_attrs.get("model") + + response_prompt_tokens = bedrock_attrs.get("response.usage.prompt_tokens") + response_completion_tokens = bedrock_attrs.get("response.usage.completion_tokens") + response_total_tokens = bedrock_attrs.get("response.usage.total_tokens") input_message_list = bedrock_attrs.get("input_message_list", []) output_message_list = bedrock_attrs.get("output_message_list", []) @@ -1075,6 +1308,25 @@ def handle_chat_completion_event(transaction, bedrock_attrs): len(input_message_list) + len(output_message_list) ) or None # If 0, attribute will be set to None and removed + input_message_content = " ".join([msg.get("content") for msg in input_message_list if msg.get("content")]) + prompt_tokens = ( + settings.ai_monitoring.llm_token_count_callback(model, input_message_content) + if settings.ai_monitoring.llm_token_count_callback and input_message_content + else response_prompt_tokens + ) + + output_message_content = " ".join([msg.get("content") for msg in output_message_list if msg.get("content")]) + completion_tokens = ( + settings.ai_monitoring.llm_token_count_callback(model, output_message_content) + if settings.ai_monitoring.llm_token_count_callback and output_message_content + else response_completion_tokens + ) + total_tokens = ( + prompt_tokens + completion_tokens if all([prompt_tokens, completion_tokens]) else response_total_tokens + ) + + all_token_counts = bool(prompt_tokens and completion_tokens and total_tokens) + chat_completion_summary_dict = { "vendor": "bedrock", "ingest_source": "Python", @@ -1083,15 +1335,22 @@ def handle_chat_completion_event(transaction, bedrock_attrs): "trace_id": trace_id, "request_id": request_id, "response_id": response_id, - "duration": bedrock_attrs.get("duration", None), - "request.max_tokens": bedrock_attrs.get("request.max_tokens", None), - "request.temperature": bedrock_attrs.get("request.temperature", None), + "duration": bedrock_attrs.get("duration"), + "request.max_tokens": bedrock_attrs.get("request.max_tokens"), + "request.temperature": bedrock_attrs.get("request.temperature"), "request.model": model, "response.model": model, # Duplicate data required by the UI "response.number_of_messages": number_of_messages, "response.choices.finish_reason": bedrock_attrs.get("response.choices.finish_reason", None), "error": bedrock_attrs.get("error", None), + "timestamp": request_timestamp or None, } + + if all_token_counts: + chat_completion_summary_dict["response.usage.prompt_tokens"] = prompt_tokens + chat_completion_summary_dict["response.usage.completion_tokens"] = completion_tokens + chat_completion_summary_dict["response.usage.total_tokens"] = total_tokens + chat_completion_summary_dict.update(llm_metadata_dict) chat_completion_summary_dict = {k: v for k, v in chat_completion_summary_dict.items() if v is not None} transaction.record_custom_event("LlmChatCompletionSummary", chat_completion_summary_dict) @@ -1106,7 +1365,9 @@ def handle_chat_completion_event(transaction, bedrock_attrs): request_model=model, request_id=request_id, llm_metadata_dict=llm_metadata_dict, + all_token_counts=all_token_counts, response_id=response_id, + request_timestamp=request_timestamp, ) @@ -1186,21 +1447,10 @@ def _nr_dynamodb_datastore_trace_wrapper_(wrapped, instance, args, kwargs): settings = transaction.settings if transaction.settings else global_settings() account_id = settings.cloud.aws.account_id if settings and settings.cloud.aws.account_id else None - # There are 3 different partition options. - # See https://docs.aws.amazon.com/IAM/latest/UserGuide/reference-arns.html for details. - partition = None - if hasattr(instance, "_endpoint") and hasattr(instance._endpoint, "host"): - _db_host = instance._endpoint.host - partition = "aws" - if "amazonaws.cn" in _db_host: - partition = "aws-cn" - elif "amazonaws-us-gov.com" in _db_host: - partition = "aws-us-gov" - - if partition and region and account_id and _target: - agent_attrs["cloud.resource_id"] = ( - f"arn:{partition}:dynamodb:{region}:{account_id:012d}:table/{_target}" - ) + _db_host = getattr(getattr(instance, "_endpoint", None), "host", None) + resource_id = generate_dynamodb_arn(_db_host, region, account_id, _target) + if resource_id: + agent_attrs["cloud.resource_id"] = resource_id except Exception: _logger.debug("Failed to capture AWS DynamoDB info.", exc_info=True) @@ -1551,6 +1801,7 @@ def wrap_serialize_to_request(wrapped, instance, args, kwargs): response_streaming=True ), ("bedrock-runtime", "converse"): wrap_bedrock_runtime_converse(response_streaming=False), + ("bedrock-runtime", "converse_stream"): wrap_bedrock_runtime_converse(response_streaming=True), } diff --git a/newrelic/hooks/framework_sanic.py b/newrelic/hooks/framework_sanic.py index 14077eb6d9..74d8ab678e 100644 --- a/newrelic/hooks/framework_sanic.py +++ b/newrelic/hooks/framework_sanic.py @@ -183,7 +183,7 @@ async def _nr_sanic_response_send(wrapped, instance, args, kwargs): transaction = current_transaction() result = wrapped(*args, **kwargs) if isawaitable(result): - await result + result = await result if transaction is None: return result diff --git a/newrelic/hooks/hybridagent_opentelemetry.py b/newrelic/hooks/hybridagent_opentelemetry.py new file mode 100644 index 0000000000..aee8a35a41 --- /dev/null +++ b/newrelic/hooks/hybridagent_opentelemetry.py @@ -0,0 +1,280 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from newrelic.api.application import application_instance +from newrelic.api.time_trace import current_trace +from newrelic.api.transaction import current_transaction +from newrelic.common.object_wrapper import wrap_function_wrapper +from newrelic.common.signature import bind_args +from newrelic.core.config import global_settings + +########################################### +# Context Instrumentation +########################################### + + +def wrap__load_runtime_context(wrapped, instance, args, kwargs): + application = application_instance(activate=False) + settings = global_settings() if not application else application.settings + if not settings.opentelemetry.enabled: + return wrapped(*args, **kwargs) + + from opentelemetry.context.contextvars_context import ContextVarsRuntimeContext + + context = ContextVarsRuntimeContext() + return context + + +def wrap_get_global_response_propagator(wrapped, instance, args, kwargs): + from newrelic.api.opentelemetry import opentelemetry_context_propagator + + application = application_instance() + if not application.active: + # Force application registration if not already active + application.activate() + + settings = global_settings() + + if not (settings and settings.opentelemetry.enabled) and not os.environ.get("NEW_RELIC_OPENTELEMETRY_ENABLED"): + return wrapped(*args, **kwargs) + + from opentelemetry.instrumentation.propagators import set_global_response_propagator + + set_global_response_propagator(opentelemetry_context_propagator) + + return opentelemetry_context_propagator + + +def instrument_context_api(module): + if hasattr(module, "_load_runtime_context"): + wrap_function_wrapper(module, "_load_runtime_context", wrap__load_runtime_context) + + +def instrument_global_propagators_api(module): + if hasattr(module, "get_global_response_propagator"): + wrap_function_wrapper(module, "get_global_response_propagator", wrap_get_global_response_propagator) + + +########################################### +# Trace Instrumentation +########################################### + + +def wrap_set_tracer_provider(wrapped, instance, args, kwargs): + # This needs to act as a singleton, like the agent instance. + # We should initialize the agent here as well, if there is + # not an instance already. + + application = application_instance() + if not application.active: + # Force application registration if not already active + application.activate() + + settings = global_settings() + + if not (settings and settings.opentelemetry.enabled) and not os.environ.get("NEW_RELIC_OPENTELEMETRY_ENABLED"): + return wrapped(*args, **kwargs) + + nr_tracer_provider = application._agent.opentelemetry_tracer_provider() + return wrapped(nr_tracer_provider) + + +def wrap_get_tracer_provider(wrapped, instance, args, kwargs): + # This needs to act as a singleton, like the agent instance. + # We should initialize the agent here as well, if there is + # not an instance already. + + application = application_instance() + if not application.active: + # Force application registration if not already active + application.activate() + + settings = global_settings() + + if not (settings and settings.opentelemetry.enabled) and not os.environ.get("NEW_RELIC_OPENTELEMETRY_ENABLED"): + return wrapped(*args, **kwargs) + + return application._agent.opentelemetry_tracer_provider() + + +def wrap_get_custom_headers(wrapped, instance, args, kwargs): + # Capture all headers now and let New Relic handle + # filtering, either through attribute filtering or + # settings like HSM. + + capture_header_env_vars = [ + "OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE", + "OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_CLIENT_RESPONSE", + ] + + bound_args = bind_args(wrapped, args, kwargs) + env_var = bound_args.get("env_var") + if env_var and (env_var in capture_header_env_vars): + return [".*"] + + return wrapped(*args, **kwargs) + + +def wrap_get_current_span(wrapped, instance, args, kwargs): + transaction = current_transaction() + trace = current_trace() + span = wrapped(*args, **kwargs) + + if not transaction: + return span + + # Do not allow the wrapper to continue if + # the Hybrid Agent setting is not enabled + application = application_instance(activate=False) + settings = global_settings() if not application else application.settings + + if not settings.opentelemetry.enabled: + return span + + # If a NR trace does exist, check to see if the current + # OpenTelemetry span corresponds to the current NR trace. If so, + # return the original function's result. + if span.get_span_context().span_id == int(trace.guid, 16): + return span + + # If the current OpenTelemetry span does not match the current NR + # trace, this means that a NR trace was created either + # manually or through the NR agent. Either way, the OpenTelemetry + # API was not used to create a span object. The Hybrid + # Agent's Span object creates a NR trace but since the NR + # trace has already been created, we just need a symbolic + # OpenTelemetry span to represent it the span object. A LazySpan + # will be created. It will effectively be a NonRecordingSpan + # with the ability to add custom attributes. + + from opentelemetry import trace as otel_api_trace + + from newrelic.api.opentelemetry import LazySpan + + span_context = otel_api_trace.SpanContext( + trace_id=int(transaction.trace_id, 16), + span_id=int(trace.guid, 16), + is_remote=span.get_span_context().is_remote, + trace_flags=otel_api_trace.TraceFlags(0x01), + trace_state=otel_api_trace.TraceState(), + ) + + return LazySpan(span_context, trace) + + +def wrap_start_internal_or_server_span(wrapped, instance, args, kwargs): + # We want to take the NR version of the context_carrier + # and put that into the attributes. Keep the original + # context_carrier intact. + + # Do not allow the wrapper to continue if + # the Hybrid Agent setting is not enabled + application = application_instance(activate=False) + settings = global_settings() if not application else application.settings + + if not settings.opentelemetry.enabled: + return wrapped(*args, **kwargs) + + bound_args = bind_args(wrapped, args, kwargs) + context_carrier = bound_args.get("context_carrier") + attributes = bound_args.get("attributes", {}) + + if context_carrier: + if ("HTTP_HOST" in context_carrier) or ("http_version" in context_carrier): + # This is an HTTP request (WSGI, ASGI, or otherwise) + if "wsgi.version" in context_carrier: + attributes["nr.wsgi.environ"] = context_carrier + elif "asgi" in context_carrier: + attributes["nr.asgi.scope"] = context_carrier + else: + attributes["nr.http.headers"] = context_carrier + else: + attributes["nr.nonhttp.headers"] = context_carrier + + bound_args["attributes"] = attributes + + return wrapped(**bound_args) + + +def wrap__get_span(wrapped, instance, args, kwargs): + # Do not allow the wrapper to continue if + # the Hybrid Agent setting is not enabled + application = application_instance(activate=False) + settings = global_settings() if not application else application.settings + + if not settings.opentelemetry.enabled: + return wrapped(*args, **kwargs) + + bound_args = bind_args(wrapped, args, kwargs) + channel = bound_args.get("channel") + properties = bound_args.get("properties") + span_kind = bound_args.get("span_kind") + task_name = bound_args.get("task_name") + tracer = bound_args.get("tracer") + + properties_to_extract = ("correlation_id", "reply_to", "headers") + + if span_kind == span_kind.PRODUCER: + # Do nothing special for producer spans + pass + elif channel: + # This is a callback related consumer call + # if transaction already exists, create trace + # for callback; else, do not do anything + tracer._create_consumer_trace = True + elif not channel: + # This is a consumer generator call + # Create a new transaction only. + # if transaction already exists, a new one + # will not be created and nothing will occur. + # This is the current behavior that Kafka has + tracer._create_consumer_trace = False + + params = {"task_name": task_name} + for _property in properties_to_extract: + value = getattr(properties, _property, None) + if properties and value: + params[_property] = value + + span = wrapped(*args, **kwargs) + span.set_attributes(params) + + return span + + +def instrument_trace_api(module): + if hasattr(module, "set_tracer_provider"): + wrap_function_wrapper(module, "set_tracer_provider", wrap_set_tracer_provider) + + if hasattr(module, "get_tracer_provider"): + wrap_function_wrapper(module, "get_tracer_provider", wrap_get_tracer_provider) + + if hasattr(module, "get_current_span"): + wrap_function_wrapper(module, "get_current_span", wrap_get_current_span) + + +def instrument_utils(module): + if hasattr(module, "_start_internal_or_server_span"): + wrap_function_wrapper(module, "_start_internal_or_server_span", wrap_start_internal_or_server_span) + + +def instrument_pika_utils(module): + if hasattr(module, "_get_span"): + wrap_function_wrapper(module, "_get_span", wrap__get_span) + + +def instrument_util_http(module): + wrap_function_wrapper(module, "get_custom_headers", wrap_get_custom_headers) diff --git a/newrelic/hooks/mlmodel_gemini.py b/newrelic/hooks/mlmodel_gemini.py index 8aeb1355d0..710db9b7f3 100644 --- a/newrelic/hooks/mlmodel_gemini.py +++ b/newrelic/hooks/mlmodel_gemini.py @@ -14,6 +14,7 @@ import logging import sys +import time import uuid import google @@ -175,20 +176,24 @@ def _record_embedding_success(transaction, embedding_id, linking_metadata, kwarg embedding_content = str(embedding_content) request_model = kwargs.get("model") + embedding_token_count = ( + settings.ai_monitoring.llm_token_count_callback(request_model, embedding_content) + if settings.ai_monitoring.llm_token_count_callback + else None + ) + full_embedding_response_dict = { "id": embedding_id, "span_id": span_id, "trace_id": trace_id, - "token_count": ( - settings.ai_monitoring.llm_token_count_callback(request_model, embedding_content) - if settings.ai_monitoring.llm_token_count_callback - else None - ), "request.model": request_model, "duration": ft.duration * 1000, "vendor": "gemini", "ingest_source": "Python", } + if embedding_token_count: + full_embedding_response_dict["response.usage.total_tokens"] = embedding_token_count + if settings.ai_monitoring.record_content.enabled: full_embedding_response_dict["input"] = embedding_content @@ -226,6 +231,7 @@ def wrap_generate_content_sync(wrapped, instance, args, kwargs): transaction._add_agent_attribute("llm", True) completion_id = str(uuid.uuid4()) + request_timestamp = int(1000.0 * time.time()) ft = FunctionTrace(name=wrapped.__name__, group="Llm/completion/Gemini") ft.__enter__() @@ -236,12 +242,12 @@ def wrap_generate_content_sync(wrapped, instance, args, kwargs): except Exception as exc: # In error cases, exit the function trace in _record_generation_error before recording the LLM error event so # that the duration is calculated correctly. - _record_generation_error(transaction, linking_metadata, completion_id, kwargs, ft, exc) + _record_generation_error(transaction, linking_metadata, completion_id, kwargs, ft, exc, request_timestamp) raise ft.__exit__(None, None, None) - _handle_generation_success(transaction, linking_metadata, completion_id, kwargs, ft, return_val) + _handle_generation_success(transaction, linking_metadata, completion_id, kwargs, ft, return_val, request_timestamp) return return_val @@ -260,6 +266,7 @@ async def wrap_generate_content_async(wrapped, instance, args, kwargs): transaction._add_agent_attribute("llm", True) completion_id = str(uuid.uuid4()) + request_timestamp = int(1000.0 * time.time()) ft = FunctionTrace(name=wrapped.__name__, group="Llm/completion/Gemini") ft.__enter__() @@ -269,17 +276,17 @@ async def wrap_generate_content_async(wrapped, instance, args, kwargs): except Exception as exc: # In error cases, exit the function trace in _record_generation_error before recording the LLM error event so # that the duration is calculated correctly. - _record_generation_error(transaction, linking_metadata, completion_id, kwargs, ft, exc) + _record_generation_error(transaction, linking_metadata, completion_id, kwargs, ft, exc, request_timestamp) raise ft.__exit__(None, None, None) - _handle_generation_success(transaction, linking_metadata, completion_id, kwargs, ft, return_val) + _handle_generation_success(transaction, linking_metadata, completion_id, kwargs, ft, return_val, request_timestamp) return return_val -def _record_generation_error(transaction, linking_metadata, completion_id, kwargs, ft, exc): +def _record_generation_error(transaction, linking_metadata, completion_id, kwargs, ft, exc, request_timestamp=None): span_id = linking_metadata.get("span.id") trace_id = linking_metadata.get("trace.id") @@ -300,15 +307,13 @@ def _record_generation_error(transaction, linking_metadata, completion_id, kwarg "Unable to parse input message to Gemini LLM. Message content and role will be omitted from " "corresponding LlmChatCompletionMessage event. " ) + # Extract the input message content and role from the input message if it exists + input_message_content, input_role = _parse_input_message(input_message) if input_message else (None, None) - generation_config = kwargs.get("config") - if generation_config: - request_temperature = getattr(generation_config, "temperature", None) - request_max_tokens = getattr(generation_config, "max_output_tokens", None) - else: - request_temperature = None - request_max_tokens = None + # Extract data from generation config object + request_temperature, request_max_tokens = _extract_generation_config(kwargs) + # Prepare error attributes notice_error_attributes = { "http.statusCode": getattr(exc, "code", None), "error.message": getattr(exc, "message", None), @@ -339,6 +344,7 @@ def _record_generation_error(transaction, linking_metadata, completion_id, kwarg "ingest_source": "Python", "duration": ft.duration * 1000, "error": True, + "timestamp": request_timestamp, } llm_metadata = _get_llm_attributes(transaction) error_chat_completion_dict.update(llm_metadata) @@ -348,21 +354,26 @@ def _record_generation_error(transaction, linking_metadata, completion_id, kwarg create_chat_completion_message_event( transaction, - input_message, + input_message_content, + input_role, completion_id, span_id, trace_id, # Passing the request model as the response model here since we do not have access to a response model request_model, - request_model, llm_metadata, output_message_list, + # We do not record token counts in error cases, so set all_token_counts to True so the pipeline tokenizer does not run + True, + request_timestamp, ) except Exception: _logger.warning(RECORD_EVENTS_FAILURE_LOG_MESSAGE, exc_info=True) -def _handle_generation_success(transaction, linking_metadata, completion_id, kwargs, ft, return_val): +def _handle_generation_success( + transaction, linking_metadata, completion_id, kwargs, ft, return_val, request_timestamp=None +): if not return_val: return @@ -370,13 +381,18 @@ def _handle_generation_success(transaction, linking_metadata, completion_id, kwa # Response objects are pydantic models so this function call converts the response into a dict response = return_val.model_dump() if hasattr(return_val, "model_dump") else return_val - _record_generation_success(transaction, linking_metadata, completion_id, kwargs, ft, response) + _record_generation_success( + transaction, linking_metadata, completion_id, kwargs, ft, response, request_timestamp + ) except Exception: _logger.warning(RECORD_EVENTS_FAILURE_LOG_MESSAGE, exc_info=True) -def _record_generation_success(transaction, linking_metadata, completion_id, kwargs, ft, response): +def _record_generation_success( + transaction, linking_metadata, completion_id, kwargs, ft, response, request_timestamp=None +): + settings = transaction.settings or global_settings() span_id = linking_metadata.get("span.id") trace_id = linking_metadata.get("trace.id") try: @@ -385,12 +401,14 @@ def _record_generation_success(transaction, linking_metadata, completion_id, kwa # finish_reason is an enum, so grab just the stringified value from it to report finish_reason = response.get("candidates")[0].get("finish_reason").value output_message_list = [response.get("candidates")[0].get("content")] + token_usage = response.get("usage_metadata") or {} else: # Set all values to NoneTypes since we cannot access them through kwargs or another method that doesn't # require the response object response_model = None output_message_list = [] finish_reason = None + token_usage = {} request_model = kwargs.get("model") @@ -412,13 +430,44 @@ def _record_generation_success(transaction, linking_metadata, completion_id, kwa "corresponding LlmChatCompletionMessage event. " ) - generation_config = kwargs.get("config") - if generation_config: - request_temperature = getattr(generation_config, "temperature", None) - request_max_tokens = getattr(generation_config, "max_output_tokens", None) + input_message_content, input_role = _parse_input_message(input_message) if input_message else (None, None) + + # Parse output message content + # This list should have a length of 1 to represent the output message + # Parse the message text out to pass to any registered token counting callback + output_message_content = output_message_list[0].get("parts")[0].get("text") if output_message_list else None + + # Extract token counts from response object + if token_usage: + response_prompt_tokens = token_usage.get("prompt_token_count") + response_completion_tokens = token_usage.get("candidates_token_count") + response_total_tokens = token_usage.get("total_token_count") + else: - request_temperature = None - request_max_tokens = None + response_prompt_tokens = None + response_completion_tokens = None + response_total_tokens = None + + # Calculate token counts by checking if a callback is registered and if we have the necessary content to pass + # to it. If not, then we use the token counts provided in the response object + prompt_tokens = ( + settings.ai_monitoring.llm_token_count_callback(request_model, input_message_content) + if settings.ai_monitoring.llm_token_count_callback and input_message_content + else response_prompt_tokens + ) + completion_tokens = ( + settings.ai_monitoring.llm_token_count_callback(response_model, output_message_content) + if settings.ai_monitoring.llm_token_count_callback and output_message_content + else response_completion_tokens + ) + total_tokens = ( + prompt_tokens + completion_tokens if all([prompt_tokens, completion_tokens]) else response_total_tokens + ) + + all_token_counts = bool(prompt_tokens and completion_tokens and total_tokens) + + # Extract generation config + request_temperature, request_max_tokens = _extract_generation_config(kwargs) full_chat_completion_summary_dict = { "id": completion_id, @@ -436,68 +485,83 @@ def _record_generation_success(transaction, linking_metadata, completion_id, kwa # message This value should be 2 in almost all cases since we will report a summary event for each # separate request (every input and output from the LLM) "response.number_of_messages": 1 + len(output_message_list), + "timestamp": request_timestamp, } + if all_token_counts: + full_chat_completion_summary_dict["response.usage.prompt_tokens"] = prompt_tokens + full_chat_completion_summary_dict["response.usage.completion_tokens"] = completion_tokens + full_chat_completion_summary_dict["response.usage.total_tokens"] = total_tokens + llm_metadata = _get_llm_attributes(transaction) full_chat_completion_summary_dict.update(llm_metadata) transaction.record_custom_event("LlmChatCompletionSummary", full_chat_completion_summary_dict) create_chat_completion_message_event( transaction, - input_message, + input_message_content, + input_role, completion_id, span_id, trace_id, response_model, - request_model, llm_metadata, output_message_list, + all_token_counts, + request_timestamp, ) except Exception: _logger.warning(RECORD_EVENTS_FAILURE_LOG_MESSAGE, exc_info=True) +def _parse_input_message(input_message): + # The input_message will be a string if generate_content was called directly. In this case, we don't have + # access to the role, so we default to user since this was an input message + if isinstance(input_message, str): + return input_message, "user" + # The input_message will be a Google Content type if send_message was called, so we parse out the message + # text and role (which should be "user") + elif isinstance(input_message, google.genai.types.Content): + return input_message.parts[0].text, input_message.role + else: + return None, None + + +def _extract_generation_config(kwargs): + generation_config = kwargs.get("config") + if generation_config: + request_temperature = getattr(generation_config, "temperature", None) + request_max_tokens = getattr(generation_config, "max_output_tokens", None) + else: + request_temperature = None + request_max_tokens = None + + return request_temperature, request_max_tokens + + def create_chat_completion_message_event( transaction, - input_message, + input_message_content, + input_role, chat_completion_id, span_id, trace_id, response_model, - request_model, llm_metadata, output_message_list, + all_token_counts, + request_timestamp=None, ): try: settings = transaction.settings or global_settings() - if input_message: - # The input_message will be a string if generate_content was called directly. In this case, we don't have - # access to the role, so we default to user since this was an input message - if isinstance(input_message, str): - input_message_content = input_message - input_role = "user" - # The input_message will be a Google Content type if send_message was called, so we parse out the message - # text and role (which should be "user") - elif isinstance(input_message, google.genai.types.Content): - input_message_content = input_message.parts[0].text - input_role = input_message.role - # Set input data to NoneTypes to ensure token_count callback is not called - else: - input_message_content = None - input_role = None - + if input_message_content: message_id = str(uuid.uuid4()) chat_completion_input_message_dict = { "id": message_id, "span_id": span_id, "trace_id": trace_id, - "token_count": ( - settings.ai_monitoring.llm_token_count_callback(request_model, input_message_content) - if settings.ai_monitoring.llm_token_count_callback and input_message_content - else None - ), "role": input_role, "completion_id": chat_completion_id, # The input message will always be the first message in our request/ response sequence so this will @@ -507,10 +571,15 @@ def create_chat_completion_message_event( "vendor": "gemini", "ingest_source": "Python", } + if all_token_counts: + chat_completion_input_message_dict["token_count"] = 0 if settings.ai_monitoring.record_content.enabled: chat_completion_input_message_dict["content"] = input_message_content + if request_timestamp: + chat_completion_input_message_dict["timestamp"] = request_timestamp + chat_completion_input_message_dict.update(llm_metadata) transaction.record_custom_event("LlmChatCompletionMessage", chat_completion_input_message_dict) @@ -523,7 +592,7 @@ def create_chat_completion_message_event( # Add one to the index to account for the single input message so our sequence value is accurate for # the output message - if input_message: + if input_message_content: index += 1 message_id = str(uuid.uuid4()) @@ -532,11 +601,6 @@ def create_chat_completion_message_event( "id": message_id, "span_id": span_id, "trace_id": trace_id, - "token_count": ( - settings.ai_monitoring.llm_token_count_callback(response_model, message_content) - if settings.ai_monitoring.llm_token_count_callback - else None - ), "role": message.get("role"), "completion_id": chat_completion_id, "sequence": index, @@ -546,6 +610,9 @@ def create_chat_completion_message_event( "is_response": True, } + if all_token_counts: + chat_completion_output_message_dict["token_count"] = 0 + if settings.ai_monitoring.record_content.enabled: chat_completion_output_message_dict["content"] = message_content diff --git a/newrelic/hooks/mlmodel_langchain.py b/newrelic/hooks/mlmodel_langchain.py index cfcc031e9d..a1c22e331f 100644 --- a/newrelic/hooks/mlmodel_langchain.py +++ b/newrelic/hooks/mlmodel_langchain.py @@ -12,19 +12,22 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json import logging import sys +import time import traceback import uuid from newrelic.api.function_trace import FunctionTrace from newrelic.api.time_trace import current_trace, get_trace_linking_metadata from newrelic.api.transaction import current_transaction -from newrelic.common.object_wrapper import wrap_function_wrapper +from newrelic.common.llm_utils import AsyncGeneratorProxy, GeneratorProxy, _get_llm_metadata +from newrelic.common.object_wrapper import ObjectProxy, wrap_function_wrapper from newrelic.common.package_version_utils import get_package_version from newrelic.common.signature import bind_args from newrelic.core.config import global_settings -from newrelic.core.context import context_wrapper +from newrelic.core.context import ContextOf, context_wrapper _logger = logging.getLogger(__name__) LANGCHAIN_VERSION = get_package_version("langchain") @@ -129,6 +132,218 @@ } +def _construct_base_agent_event_dict(agent_name, agent_id, transaction): + try: + linking_metadata = get_trace_linking_metadata() + + agent_event_dict = { + "id": agent_id, + "name": agent_name, + "span_id": linking_metadata.get("span.id"), + "trace_id": linking_metadata.get("trace.id"), + "vendor": "langchain", + "ingest_source": "Python", + } + agent_event_dict.update(_get_llm_metadata(transaction)) + except Exception: + agent_event_dict = {} + _logger.warning(RECORD_EVENTS_FAILURE_LOG_MESSAGE, exc_info=True) + + return agent_event_dict + + +class AgentObjectProxy(ObjectProxy): + def invoke(self, *args, **kwargs): + transaction = current_transaction() + if not transaction: + return self.__wrapped__.invoke(*args, **kwargs) + + agent_name = getattr(self.__wrapped__, "name", "agent") + agent_id = str(uuid.uuid4()) + agent_event_dict = _construct_base_agent_event_dict(agent_name, agent_id, transaction) + function_trace_name = f"invoke/{agent_name}" + agentic_subcomponent_data = {"type": "APM-AI_AGENT", "name": agent_name} + + ft = FunctionTrace(name=function_trace_name, group="Llm/agent/LangChain") + ft.__enter__() + ft._add_agent_attribute("subcomponent", json.dumps(agentic_subcomponent_data)) + try: + return_val = self.__wrapped__.invoke(*args, **kwargs) + except Exception: + ft.notice_error(attributes={"agent_id": agent_id}) + ft.__exit__(*sys.exc_info()) + # If we hit an exception, append the error attribute and duration from the exited function trace + agent_event_dict.update({"duration": ft.duration * 1000, "error": True}) + transaction.record_custom_event("LlmAgent", agent_event_dict) + raise + + ft.__exit__(None, None, None) + agent_event_dict.update({"duration": ft.duration * 1000}) + transaction.record_custom_event("LlmAgent", agent_event_dict) + + return return_val + + async def ainvoke(self, *args, **kwargs): + transaction = current_transaction() + if not transaction: + return await self.__wrapped__.ainvoke(*args, **kwargs) + + agent_name = getattr(self.__wrapped__, "name", "agent") + agent_id = str(uuid.uuid4()) + agent_event_dict = _construct_base_agent_event_dict(agent_name, agent_id, transaction) + function_trace_name = f"ainvoke/{agent_name}" + agentic_subcomponent_data = {"type": "APM-AI_AGENT", "name": agent_name} + + ft = FunctionTrace(name=function_trace_name, group="Llm/agent/LangChain") + ft.__enter__() + ft._add_agent_attribute("subcomponent", json.dumps(agentic_subcomponent_data)) + try: + return_val = await self.__wrapped__.ainvoke(*args, **kwargs) + except Exception: + ft.notice_error(attributes={"agent_id": agent_id}) + ft.__exit__(*sys.exc_info()) + # If we hit an exception, append the error attribute and duration from the exited function trace + agent_event_dict.update({"duration": ft.duration * 1000, "error": True}) + transaction.record_custom_event("LlmAgent", agent_event_dict) + raise + + ft.__exit__(None, None, None) + agent_event_dict.update({"duration": ft.duration * 1000}) + transaction.record_custom_event("LlmAgent", agent_event_dict) + + return return_val + + def stream(self, *args, **kwargs): + transaction = current_transaction() + if not transaction: + return self.__wrapped__.stream(*args, **kwargs) + + agent_name = getattr(self.__wrapped__, "name", "agent") + agent_id = str(uuid.uuid4()) + agent_event_dict = _construct_base_agent_event_dict(agent_name, agent_id, transaction) + function_trace_name = f"stream/{agent_name}" + agentic_subcomponent_data = {"type": "APM-AI_AGENT", "name": agent_name} + + ft = FunctionTrace(name=function_trace_name, group="Llm/agent/LangChain") + ft.__enter__() + ft._add_agent_attribute("subcomponent", json.dumps(agentic_subcomponent_data)) + try: + return_val = self.__wrapped__.stream(*args, **kwargs) + return_val = GeneratorProxy( + return_val, + on_stop_iteration=self._nr_on_stop_iteration(ft, agent_event_dict), + on_error=self._nr_on_error(ft, agent_event_dict, agent_id), + ) + except Exception: + self._nr_on_error(ft, agent_event_dict, agent_id)(transaction) + raise + + return return_val + + def astream(self, *args, **kwargs): + transaction = current_transaction() + if not transaction: + return self.__wrapped__.astream(*args, **kwargs) + + agent_name = getattr(self.__wrapped__, "name", "agent") + agent_id = str(uuid.uuid4()) + agent_event_dict = _construct_base_agent_event_dict(agent_name, agent_id, transaction) + function_trace_name = f"astream/{agent_name}" + agentic_subcomponent_data = {"type": "APM-AI_AGENT", "name": agent_name} + + ft = FunctionTrace(name=function_trace_name, group="Llm/agent/LangChain") + ft.__enter__() + ft._add_agent_attribute("subcomponent", json.dumps(agentic_subcomponent_data)) + try: + return_val = self.__wrapped__.astream(*args, **kwargs) + return_val = AsyncGeneratorProxy( + return_val, + on_stop_iteration=self._nr_on_stop_iteration(ft, agent_event_dict), + on_error=self._nr_on_error(ft, agent_event_dict, agent_id), + ) + except Exception: + self._nr_on_error(ft, agent_event_dict, agent_id)(transaction) + raise + + return return_val + + def transform(self, *args, **kwargs): + transaction = current_transaction() + if not transaction: + return self.__wrapped__.transform(*args, **kwargs) + + agent_name = getattr(self.__wrapped__, "name", "agent") + agent_id = str(uuid.uuid4()) + agent_event_dict = _construct_base_agent_event_dict(agent_name, agent_id, transaction) + function_trace_name = f"stream/{agent_name}" + agentic_subcomponent_data = {"type": "APM-AI_AGENT", "name": agent_name} + + ft = FunctionTrace(name=function_trace_name, group="Llm/agent/LangChain") + ft.__enter__() + ft._add_agent_attribute("subcomponent", json.dumps(agentic_subcomponent_data)) + try: + return_val = self.__wrapped__.transform(*args, **kwargs) + return_val = GeneratorProxy( + return_val, + on_stop_iteration=self._nr_on_stop_iteration(ft, agent_event_dict), + on_error=self._nr_on_error(ft, agent_event_dict, agent_id), + ) + except Exception: + self._nr_on_error(ft, agent_event_dict, agent_id)(transaction) + raise + + return return_val + + def atransform(self, *args, **kwargs): + transaction = current_transaction() + if not transaction: + return self.__wrapped__.atransform(*args, **kwargs) + + agent_name = getattr(self.__wrapped__, "name", "agent") + agent_id = str(uuid.uuid4()) + agent_event_dict = _construct_base_agent_event_dict(agent_name, agent_id, transaction) + function_trace_name = f"astream/{agent_name}" + agentic_subcomponent_data = {"type": "APM-AI_AGENT", "name": agent_name} + + ft = FunctionTrace(name=function_trace_name, group="Llm/agent/LangChain") + ft.__enter__() + ft._add_agent_attribute("subcomponent", json.dumps(agentic_subcomponent_data)) + try: + return_val = self.__wrapped__.atransform(*args, **kwargs) + return_val = AsyncGeneratorProxy( + return_val, + on_stop_iteration=self._nr_on_stop_iteration(ft, agent_event_dict), + on_error=self._nr_on_error(ft, agent_event_dict, agent_id), + ) + except Exception: + self._nr_on_error(ft, agent_event_dict, agent_id)(transaction) + raise + + return return_val + + def _nr_on_stop_iteration(self, ft, agent_event_dict): + def _on_stop_iteration(proxy, transaction): + ft.__exit__(None, None, None) + if agent_event_dict: + agent_event_dict.update({"duration": ft.duration * 1000}) + transaction.record_custom_event("LlmAgent", agent_event_dict) + agent_event_dict.clear() + + return _on_stop_iteration + + def _nr_on_error(self, ft, agent_event_dict, agent_id): + def _on_error(proxy, transaction): + ft.notice_error(attributes={"agent_id": agent_id}) + ft.__exit__(*sys.exc_info()) + if agent_event_dict: + # If we hit an exception, append the error attribute and duration from the exited function trace + agent_event_dict.update({"duration": ft.duration * 1000, "error": True}) + transaction.record_custom_event("LlmAgent", agent_event_dict) + agent_event_dict.clear() + + return _on_error + + def bind_submit(func, *args, **kwargs): return {"func": func, "args": args, "kwargs": kwargs} @@ -300,27 +515,35 @@ def wrap_tool_sync_run(wrapped, instance, args, kwargs): transaction.add_ml_model_info("LangChain", LANGCHAIN_VERSION) transaction._add_agent_attribute("llm", True) - tool_id, metadata, tags, tool_input, tool_name, tool_description, run_args = _capture_tool_info( + tool_id, agent_name, tool_input, tool_name, tool_run_id, run_args = _capture_tool_info( instance, wrapped, args, kwargs ) - ft = FunctionTrace(name=wrapped.__name__, group="Llm/tool/LangChain") + # Filter out injected State or ToolRuntime arguments that would clog up the input + try: + filtered_tool_input = instance._filter_injected_args(tool_input) + except Exception: + filtered_tool_input = tool_input + + agentic_subcomponent_data = {"type": "APM-AI_TOOL", "name": tool_name} + + ft = FunctionTrace(name=f"{wrapped.__name__}/{tool_name}", group="Llm/tool/LangChain") ft.__enter__() + ft._add_agent_attribute("subcomponent", json.dumps(agentic_subcomponent_data)) linking_metadata = get_trace_linking_metadata() try: return_val = wrapped(**run_args) except Exception: _record_tool_error( - instance, - transaction, - linking_metadata, - tags, - metadata, - tool_id, - tool_input, - tool_name, - tool_description, - ft, + instance=instance, + transaction=transaction, + linking_metadata=linking_metadata, + agent_name=agent_name, + tool_id=tool_id, + tool_input=filtered_tool_input, + tool_name=tool_name, + tool_run_id=tool_run_id, + ft=ft, ) raise ft.__exit__(None, None, None) @@ -329,17 +552,16 @@ def wrap_tool_sync_run(wrapped, instance, args, kwargs): return return_val _record_tool_success( - instance, - transaction, - linking_metadata, - tags, - metadata, - tool_id, - tool_input, - tool_name, - tool_description, - ft, - return_val, + instance=instance, + transaction=transaction, + linking_metadata=linking_metadata, + agent_name=agent_name, + tool_id=tool_id, + tool_input=filtered_tool_input, + tool_name=tool_name, + tool_run_id=tool_run_id, + ft=ft, + response=return_val, ) return return_val @@ -357,27 +579,35 @@ async def wrap_tool_async_run(wrapped, instance, args, kwargs): transaction.add_ml_model_info("LangChain", LANGCHAIN_VERSION) transaction._add_agent_attribute("llm", True) - tool_id, metadata, tags, tool_input, tool_name, tool_description, run_args = _capture_tool_info( + tool_id, agent_name, tool_input, tool_name, tool_run_id, run_args = _capture_tool_info( instance, wrapped, args, kwargs ) - ft = FunctionTrace(name=wrapped.__name__, group="Llm/tool/LangChain") + # Filter out injected State or ToolRuntime arguments that would clog up the input + try: + filtered_tool_input = instance._filter_injected_args(tool_input) + except Exception: + filtered_tool_input = tool_input + + agentic_subcomponent_data = {"type": "APM-AI_TOOL", "name": tool_name} + + ft = FunctionTrace(name=f"{wrapped.__name__}/{tool_name}", group="Llm/tool/LangChain") ft.__enter__() + ft._add_agent_attribute("subcomponent", json.dumps(agentic_subcomponent_data)) linking_metadata = get_trace_linking_metadata() try: return_val = await wrapped(**run_args) except Exception: _record_tool_error( - instance, - transaction, - linking_metadata, - tags, - metadata, - tool_id, - tool_input, - tool_name, - tool_description, - ft, + instance=instance, + transaction=transaction, + linking_metadata=linking_metadata, + agent_name=agent_name, + tool_id=tool_id, + tool_input=filtered_tool_input, + tool_name=tool_name, + tool_run_id=tool_run_id, + ft=ft, ) raise ft.__exit__(None, None, None) @@ -386,17 +616,16 @@ async def wrap_tool_async_run(wrapped, instance, args, kwargs): return return_val _record_tool_success( - instance, - transaction, - linking_metadata, - tags, - metadata, - tool_id, - tool_input, - tool_name, - tool_description, - ft, - return_val, + instance=instance, + transaction=transaction, + linking_metadata=linking_metadata, + agent_name=agent_name, + tool_id=tool_id, + tool_input=filtered_tool_input, + tool_name=tool_name, + tool_run_id=tool_run_id, + ft=ft, + response=return_val, ) return return_val @@ -406,51 +635,36 @@ def _capture_tool_info(instance, wrapped, args, kwargs): tool_id = str(uuid.uuid4()) metadata = run_args.get("metadata") or {} - metadata["nr_tool_id"] = tool_id - run_args["metadata"] = metadata - tags = run_args.get("tags") or [] + # lc_agent_name was added to metadata in LangChain 1.2.4 + agent_name = metadata.pop("_nr_agent_name", None) or metadata.get("lc_agent_name", None) tool_input = run_args.get("tool_input") tool_name = getattr(instance, "name", None) - tool_description = getattr(instance, "description", None) - return tool_id, metadata, tags, tool_input, tool_name, tool_description, run_args + # Checking multiple places for an acceptable tool run ID, fallback to creating our own. + tool_run_id = run_args.get("run_id", None) or run_args.get("tool_call_id", None) or str(uuid.uuid4()) + + return tool_id, agent_name, tool_input, tool_name, tool_run_id, run_args def _record_tool_success( - instance, - transaction, - linking_metadata, - tags, - metadata, - tool_id, - tool_input, - tool_name, - tool_description, - ft, - response, + instance, transaction, linking_metadata, agent_name, tool_id, tool_input, tool_name, tool_run_id, ft, response ): settings = transaction.settings if transaction.settings is not None else global_settings() - run_id = getattr(transaction, "_nr_tool_run_ids", {}).pop(tool_id, None) - # Update tags and metadata previously obtained from run_args with instance values - metadata.update(getattr(instance, "metadata", None) or {}) - tags.extend(getattr(instance, "tags", None) or []) - full_tool_event_dict = {f"metadata.{key}": value for key, value in metadata.items() if key != "nr_tool_id"} - full_tool_event_dict.update( - { - "id": tool_id, - "run_id": run_id, - "name": tool_name, - "description": tool_description, - "span_id": linking_metadata.get("span.id"), - "trace_id": linking_metadata.get("trace.id"), - "vendor": "langchain", - "ingest_source": "Python", - "duration": ft.duration * 1000, - "tags": tags or None, - } - ) + + full_tool_event_dict = { + "id": tool_id, + "run_id": tool_run_id, + "name": tool_name, + "agent_name": agent_name, + "span_id": linking_metadata.get("span.id"), + "trace_id": linking_metadata.get("trace.id"), + "vendor": "langchain", + "ingest_source": "Python", + "duration": ft.duration * 1000, + } + result = None try: - result = str(response) + result = str(response.content) if hasattr(response, "content") else str(response) except Exception: _logger.debug("Failed to convert tool response into a string.\n%s", traceback.format_exception(*sys.exc_info())) if settings.ai_monitoring.record_content.enabled: @@ -460,79 +674,31 @@ def _record_tool_success( def _record_tool_error( - instance, transaction, linking_metadata, tags, metadata, tool_id, tool_input, tool_name, tool_description, ft + instance, transaction, linking_metadata, agent_name, tool_id, tool_input, tool_name, tool_run_id, ft ): settings = transaction.settings if transaction.settings is not None else global_settings() ft.notice_error(attributes={"tool_id": tool_id}) ft.__exit__(*sys.exc_info()) - run_id = getattr(transaction, "_nr_tool_run_ids", {}).pop(tool_id, None) - # Update tags and metadata previously obtained from run_args with instance values - metadata.update(getattr(instance, "metadata", None) or {}) - tags.extend(getattr(instance, "tags", None) or []) # Make sure the builtin attributes take precedence over metadata attributes. - error_tool_event_dict = {f"metadata.{key}": value for key, value in metadata.items() if key != "nr_tool_id"} - error_tool_event_dict.update( - { - "id": tool_id, - "run_id": run_id, - "name": tool_name, - "description": tool_description, - "span_id": linking_metadata.get("span.id"), - "trace_id": linking_metadata.get("trace.id"), - "vendor": "langchain", - "ingest_source": "Python", - "duration": ft.duration * 1000, - "tags": tags or None, - "error": True, - } - ) + error_tool_event_dict = { + "id": tool_id, + "run_id": tool_run_id, + "name": tool_name, + "agent_name": agent_name, + "span_id": linking_metadata.get("span.id"), + "trace_id": linking_metadata.get("trace.id"), + "vendor": "langchain", + "ingest_source": "Python", + "duration": ft.duration * 1000, + "error": True, + } + if settings.ai_monitoring.record_content.enabled: error_tool_event_dict["input"] = tool_input error_tool_event_dict.update(_get_llm_metadata(transaction)) - transaction.record_custom_event("LlmTool", error_tool_event_dict) - - -def wrap_on_tool_start_sync(wrapped, instance, args, kwargs): - transaction = current_transaction() - if not transaction: - return wrapped(*args, **kwargs) - - settings = transaction.settings if transaction.settings is not None else global_settings() - if not settings.ai_monitoring.enabled: - return wrapped(*args, **kwargs) - - tool_id = _get_tool_id(instance) - run_manager = wrapped(*args, **kwargs) - _capture_tool_run_id(transaction, run_manager, tool_id) - return run_manager - - -async def wrap_on_tool_start_async(wrapped, instance, args, kwargs): - transaction = current_transaction() - if not transaction: - return await wrapped(*args, **kwargs) - - settings = transaction.settings if transaction.settings is not None else global_settings() - if not settings.ai_monitoring.enabled: - return await wrapped(*args, **kwargs) - - tool_id = _get_tool_id(instance) - run_manager = await wrapped(*args, **kwargs) - _capture_tool_run_id(transaction, run_manager, tool_id) - return run_manager - - -def _get_tool_id(instance): - return (getattr(instance, "metadata", None) or {}).pop("nr_tool_id", None) - -def _capture_tool_run_id(transaction, run_manager, tool_id): - if tool_id: - if not hasattr(transaction, "_nr_tool_run_ids"): - transaction._nr_tool_run_ids = {} - if tool_id not in transaction._nr_tool_run_ids: - transaction._nr_tool_run_ids[tool_id] = getattr(run_manager, "run_id", None) + transaction.record_custom_event("LlmTool", error_tool_event_dict) async def wrap_chain_async_run(wrapped, instance, args, kwargs): @@ -549,6 +715,7 @@ async def wrap_chain_async_run(wrapped, instance, args, kwargs): transaction._add_agent_attribute("llm", True) run_args = bind_args(wrapped, args, kwargs) + run_args["timestamp"] = int(1000.0 * time.time()) completion_id = str(uuid.uuid4()) add_nr_completion_id(run_args, completion_id) # Check to see if launched from agent or directly from chain. @@ -565,7 +732,12 @@ async def wrap_chain_async_run(wrapped, instance, args, kwargs): ft.notice_error(attributes={"completion_id": completion_id}) ft.__exit__(*sys.exc_info()) _create_error_chain_run_events( - transaction, instance, run_args, completion_id, linking_metadata, ft.duration * 1000 + transaction=transaction, + instance=instance, + run_args=run_args, + completion_id=completion_id, + linking_metadata=linking_metadata, + duration=ft.duration * 1000, ) raise ft.__exit__(None, None, None) @@ -574,7 +746,13 @@ async def wrap_chain_async_run(wrapped, instance, args, kwargs): return response _create_successful_chain_run_events( - transaction, instance, run_args, completion_id, response, linking_metadata, ft.duration * 1000 + transaction=transaction, + instance=instance, + run_args=run_args, + completion_id=completion_id, + response=response, + linking_metadata=linking_metadata, + duration=ft.duration * 1000, ) return response @@ -593,6 +771,7 @@ def wrap_chain_sync_run(wrapped, instance, args, kwargs): transaction._add_agent_attribute("llm", True) run_args = bind_args(wrapped, args, kwargs) + run_args["timestamp"] = int(1000.0 * time.time()) completion_id = str(uuid.uuid4()) add_nr_completion_id(run_args, completion_id) # Check to see if launched from agent or directly from chain. @@ -609,7 +788,12 @@ def wrap_chain_sync_run(wrapped, instance, args, kwargs): ft.notice_error(attributes={"completion_id": completion_id}) ft.__exit__(*sys.exc_info()) _create_error_chain_run_events( - transaction, instance, run_args, completion_id, linking_metadata, ft.duration * 1000 + transaction=transaction, + instance=instance, + run_args=run_args, + completion_id=completion_id, + linking_metadata=linking_metadata, + duration=ft.duration * 1000, ) raise ft.__exit__(None, None, None) @@ -618,11 +802,157 @@ def wrap_chain_sync_run(wrapped, instance, args, kwargs): return response _create_successful_chain_run_events( - transaction, instance, run_args, completion_id, response, linking_metadata, ft.duration * 1000 + transaction=transaction, + instance=instance, + run_args=run_args, + completion_id=completion_id, + response=response, + linking_metadata=linking_metadata, + duration=ft.duration * 1000, ) return response +def wrap_RunnableSequence_stream(wrapped, instance, args, kwargs): + transaction = current_transaction() + if not transaction: + return wrapped(*args, **kwargs) + + settings = transaction.settings if transaction.settings is not None else global_settings() + if not settings.ai_monitoring.enabled: + return wrapped(*args, **kwargs) + + # Framework metric also used for entity tagging in the UI + transaction.add_ml_model_info("LangChain", LANGCHAIN_VERSION) + transaction._add_agent_attribute("llm", True) + + run_args = bind_args(wrapped, args, kwargs) + run_args["timestamp"] = int(1000.0 * time.time()) + completion_id = str(uuid.uuid4()) + add_nr_completion_id(run_args, completion_id) + # Check to see if launched from agent or directly from chain. + # The trace group will reflect from where it has started. + # The AgentExecutor class has an attribute "agent" that does + # not exist within the Chain class + group_name = "Llm/agent/LangChain" if hasattr(instance, "agent") else "Llm/chain/LangChain" + ft = FunctionTrace(name=wrapped.__name__, group=group_name) + ft.__enter__() + linking_metadata = get_trace_linking_metadata() + try: + return_val = wrapped(input=run_args["input"], config=run_args["config"], **run_args.get("kwargs", {})) + return_val = GeneratorProxy( + return_val, + on_stop_iteration=_on_chain_stop_iteration( + ft=ft, + instance=instance, + run_args=run_args, + completion_id=completion_id, + response=[], + linking_metadata=linking_metadata, + ), + on_error=_on_chain_error( + ft=ft, + instance=instance, + run_args=run_args, + completion_id=completion_id, + linking_metadata=linking_metadata, + ), + ) + except Exception: + _on_chain_error( + ft=ft, instance=instance, run_args=run_args, completion_id=completion_id, linking_metadata=linking_metadata + )(transaction) + raise + + return return_val + + +def wrap_RunnableSequence_astream(wrapped, instance, args, kwargs): + transaction = current_transaction() + if not transaction: + return wrapped(*args, **kwargs) + + settings = transaction.settings if transaction.settings is not None else global_settings() + if not settings.ai_monitoring.enabled: + return wrapped(*args, **kwargs) + + # Framework metric also used for entity tagging in the UI + transaction.add_ml_model_info("LangChain", LANGCHAIN_VERSION) + transaction._add_agent_attribute("llm", True) + + run_args = bind_args(wrapped, args, kwargs) + run_args["timestamp"] = int(1000.0 * time.time()) + completion_id = str(uuid.uuid4()) + add_nr_completion_id(run_args, completion_id) + # Check to see if launched from agent or directly from chain. + # The trace group will reflect from where it has started. + # The AgentExecutor class has an attribute "agent" that does + # not exist within the Chain class + group_name = "Llm/agent/LangChain" if hasattr(instance, "agent") else "Llm/chain/LangChain" + ft = FunctionTrace(name=wrapped.__name__, group=group_name) + ft.__enter__() + linking_metadata = get_trace_linking_metadata() + try: + return_val = wrapped(input=run_args["input"], config=run_args["config"], **run_args.get("kwargs", {})) + return_val = AsyncGeneratorProxy( + return_val, + on_stop_iteration=_on_chain_stop_iteration( + ft=ft, + instance=instance, + run_args=run_args, + completion_id=completion_id, + response=[], + linking_metadata=linking_metadata, + ), + on_error=_on_chain_error( + ft=ft, + instance=instance, + run_args=run_args, + completion_id=completion_id, + linking_metadata=linking_metadata, + ), + ) + except Exception: + _on_chain_error( + ft=ft, instance=instance, run_args=run_args, completion_id=completion_id, linking_metadata=linking_metadata + )(transaction) + raise + + return return_val + + +def _on_chain_stop_iteration(ft, instance, run_args, completion_id, response, linking_metadata): + def _on_stop_iteration(proxy, transaction): + ft.__exit__(None, None, None) + _create_successful_chain_run_events( + transaction=transaction, + instance=instance, + run_args=run_args, + completion_id=completion_id, + response=response, + linking_metadata=linking_metadata, + duration=ft.duration * 1000, + ) + + return _on_stop_iteration + + +def _on_chain_error(ft, instance, run_args, completion_id, linking_metadata): + def _on_error(proxy, transaction): + ft.notice_error(attributes={"completion_id": completion_id}) + ft.__exit__(*sys.exc_info()) + _create_error_chain_run_events( + transaction=transaction, + instance=instance, + run_args=run_args, + completion_id=completion_id, + linking_metadata=linking_metadata, + duration=ft.duration * 1000, + ) + + return _on_error + + def add_nr_completion_id(run_args, completion_id): # invoke has an argument named "config" that contains metadata and tags. # Add the nr_completion_id into the metadata to be used as the function call @@ -658,12 +988,21 @@ def _create_error_chain_run_events(transaction, instance, run_args, completion_i "response.number_of_messages": len(input_message_list), "tags": tags, "error": True, + "timestamp": run_args.get("timestamp") or None, } ) full_chat_completion_summary_dict.update(llm_metadata_dict) transaction.record_custom_event("LlmChatCompletionSummary", full_chat_completion_summary_dict) create_chat_completion_message_event( - transaction, input_message_list, completion_id, span_id, trace_id, run_id, llm_metadata_dict, [] + transaction, + input_message_list, + completion_id, + span_id, + trace_id, + run_id, + llm_metadata_dict, + [], + run_args["timestamp"] or None, ) @@ -679,17 +1018,6 @@ def _get_run_manager_info(transaction, run_args, instance, completion_id): return run_id, metadata, tags or None -def _get_llm_metadata(transaction): - # Grab LLM-related custom attributes off of the transaction to store as metadata on LLM events - custom_attrs_dict = transaction._custom_params - llm_metadata_dict = {key: value for key, value in custom_attrs_dict.items() if key.startswith("llm.")} - llm_context_attrs = getattr(transaction, "_llm_context_attrs", None) - if llm_context_attrs: - llm_metadata_dict.update(llm_context_attrs) - - return llm_metadata_dict - - def _create_successful_chain_run_events( transaction, instance, run_args, completion_id, response, linking_metadata, duration ): @@ -728,8 +1056,13 @@ def _create_successful_chain_run_events( "duration": duration, "response.number_of_messages": len(input_message_list) + len(output_message_list), "tags": tags, + "timestamp": run_args.get("timestamp") or None, } ) + + if run_args.get("timestamp"): + full_chat_completion_summary_dict["timestamp"] = run_args.get("timestamp") + full_chat_completion_summary_dict.update(llm_metadata_dict) transaction.record_custom_event("LlmChatCompletionSummary", full_chat_completion_summary_dict) create_chat_completion_message_event( @@ -741,6 +1074,7 @@ def _create_successful_chain_run_events( run_id, llm_metadata_dict, output_message_list, + run_args["timestamp"] or None, ) @@ -753,6 +1087,7 @@ def create_chat_completion_message_event( run_id, llm_metadata_dict, output_message_list, + request_timestamp=None, ): settings = transaction.settings if transaction.settings is not None else global_settings() @@ -768,9 +1103,12 @@ def create_chat_completion_message_event( "vendor": "langchain", "ingest_source": "Python", "virtual_llm": True, + "role": "user", # default role for input messages, overridden by values in llm_metadata_dict } if settings.ai_monitoring.record_content.enabled: chat_completion_input_message_dict["content"] = message + if request_timestamp: + chat_completion_input_message_dict["timestamp"] = request_timestamp chat_completion_input_message_dict.update(llm_metadata_dict) transaction.record_custom_event("LlmChatCompletionMessage", chat_completion_input_message_dict) @@ -791,54 +1129,63 @@ def create_chat_completion_message_event( "ingest_source": "Python", "is_response": True, "virtual_llm": True, + "role": "assistant", # default role for output messages, overridden by values in llm_metadata_dict } if settings.ai_monitoring.record_content.enabled: chat_completion_output_message_dict["content"] = message + chat_completion_output_message_dict.update(llm_metadata_dict) transaction.record_custom_event("LlmChatCompletionMessage", chat_completion_output_message_dict) -def wrap_on_chain_start(wrapped, instance, args, kwargs): +def wrap_create_agent(wrapped, instance, args, kwargs): transaction = current_transaction() if not transaction: return wrapped(*args, **kwargs) - settings = transaction.settings if transaction.settings is not None else global_settings() + settings = transaction.settings or global_settings() if not settings.ai_monitoring.enabled: return wrapped(*args, **kwargs) - completion_id = _get_completion_id(instance) - run_manager = wrapped(*args, **kwargs) - _capture_chain_run_id(transaction, run_manager, completion_id) - return run_manager + # Framework metric also used for entity tagging in the UI + transaction.add_ml_model_info("LangChain", LANGCHAIN_VERSION) + transaction._add_agent_attribute("llm", True) + return_val = wrapped(*args, **kwargs) -async def wrap_async_on_chain_start(wrapped, instance, args, kwargs): - transaction = current_transaction() - if not transaction: - return await wrapped(*args, **kwargs) + return AgentObjectProxy(return_val) - settings = transaction.settings if transaction.settings is not None else global_settings() - if not settings.ai_monitoring.enabled: - return await wrapped(*args, **kwargs) - completion_id = _get_completion_id(instance) - run_manager = await wrapped(*args, **kwargs) - _capture_chain_run_id(transaction, run_manager, completion_id) - return run_manager +def wrap_StructuredTool_invoke(wrapped, instance, args, kwargs): + """If StructuredTool.invoke is being run inside a ThreadPoolExecutor, propagate context from StructuredTool.ainvoke.""" + trace = current_trace() + if trace: + return wrapped(*args, **kwargs) + + metadata = bind_args(wrapped, args, kwargs).get("config", {}).get("metadata", {}) + # Delete the reference after grabbing it to avoid it ending up in LangChain attributes + trace = metadata.pop("_nr_trace", None) + if not trace: + return wrapped(*args, **kwargs) + with ContextOf(trace=trace): + return wrapped(*args, **kwargs) -def _get_completion_id(instance): - return (getattr(instance, "metadata", None) or {}).pop("nr_completion_id", None) +async def wrap_StructuredTool_ainvoke(wrapped, instance, args, kwargs): + """Save a copy of the current trace if we're about to run StructuredTool.invoke inside a ThreadPoolExecutor.""" + trace = current_trace() + # We only need to propagate for synchronous calls with an active trace + if not trace or instance.coroutine: + return await wrapped(*args, **kwargs) + + metadata = bind_args(wrapped, args, kwargs).get("config", {}).get("metadata", {}) + metadata["_nr_trace"] = trace -def _capture_chain_run_id(transaction, run_manager, completion_id): - if completion_id: - if not hasattr(transaction, "_nr_chain_run_ids"): - transaction._nr_chain_run_ids = {} - # Only capture the first run_id. - if completion_id not in transaction._nr_chain_run_ids: - transaction._nr_chain_run_ids[completion_id] = getattr(run_manager, "run_id", "") + try: + return await wrapped(*args, **kwargs) + finally: + metadata.pop("_nr_trace", None) def instrument_langchain_runnables_chains_base(module): @@ -846,6 +1193,10 @@ def instrument_langchain_runnables_chains_base(module): wrap_function_wrapper(module, "RunnableSequence.invoke", wrap_chain_sync_run) if hasattr(module.RunnableSequence, "ainvoke"): wrap_function_wrapper(module, "RunnableSequence.ainvoke", wrap_chain_async_run) + if hasattr(module.RunnableSequence, "stream"): + wrap_function_wrapper(module, "RunnableSequence.stream", wrap_RunnableSequence_stream) + if hasattr(module.RunnableSequence, "astream"): + wrap_function_wrapper(module, "RunnableSequence.astream", wrap_RunnableSequence_astream) def instrument_langchain_chains_base(module): @@ -879,17 +1230,19 @@ def instrument_langchain_core_tools(module): wrap_function_wrapper(module, "BaseTool.arun", wrap_tool_async_run) -def instrument_langchain_callbacks_manager(module): - if hasattr(module.CallbackManager, "on_tool_start"): - wrap_function_wrapper(module, "CallbackManager.on_tool_start", wrap_on_tool_start_sync) - if hasattr(module.AsyncCallbackManager, "on_tool_start"): - wrap_function_wrapper(module, "AsyncCallbackManager.on_tool_start", wrap_on_tool_start_async) - if hasattr(module.CallbackManager, "on_chain_start"): - wrap_function_wrapper(module, "CallbackManager.on_chain_start", wrap_on_chain_start) - if hasattr(module.AsyncCallbackManager, "on_chain_start"): - wrap_function_wrapper(module, "AsyncCallbackManager.on_chain_start", wrap_async_on_chain_start) - - def instrument_langchain_core_runnables_config(module): if hasattr(module, "ContextThreadPoolExecutor"): wrap_function_wrapper(module, "ContextThreadPoolExecutor.submit", wrap_ContextThreadPoolExecutor_submit) + + +def instrument_langchain_core_tools_structured(module): + if hasattr(module, "StructuredTool"): + if hasattr(module.StructuredTool, "invoke"): + wrap_function_wrapper(module, "StructuredTool.invoke", wrap_StructuredTool_invoke) + if hasattr(module.StructuredTool, "ainvoke"): + wrap_function_wrapper(module, "StructuredTool.ainvoke", wrap_StructuredTool_ainvoke) + + +def instrument_langchain_agents_factory(module): + if hasattr(module, "create_agent"): + wrap_function_wrapper(module, "create_agent", wrap_create_agent) diff --git a/newrelic/hooks/mlmodel_langgraph.py b/newrelic/hooks/mlmodel_langgraph.py new file mode 100644 index 0000000000..6644b80e1b --- /dev/null +++ b/newrelic/hooks/mlmodel_langgraph.py @@ -0,0 +1,57 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from newrelic.api.transaction import current_transaction +from newrelic.common.object_wrapper import wrap_function_wrapper +from newrelic.common.signature import bind_args + + +def wrap_ToolNode__execute_tool_sync(wrapped, instance, args, kwargs): + if not current_transaction(): + return wrapped(*args, **kwargs) + + try: + bound_args = bind_args(wrapped, args, kwargs) + agent_name = bound_args["request"].state["messages"][-1].name + if agent_name: + metadata = bound_args["config"]["metadata"] + metadata["_nr_agent_name"] = agent_name + except Exception: + pass + + return wrapped(*args, **kwargs) + + +async def wrap_ToolNode__execute_tool_async(wrapped, instance, args, kwargs): + if not current_transaction(): + return await wrapped(*args, **kwargs) + + try: + bound_args = bind_args(wrapped, args, kwargs) + agent_name = bound_args["request"].state["messages"][-1].name + if agent_name: + metadata = bound_args["config"]["metadata"] + metadata["_nr_agent_name"] = agent_name + except Exception: + pass + + return await wrapped(*args, **kwargs) + + +def instrument_langgraph_prebuilt_tool_node(module): + if hasattr(module, "ToolNode"): + if hasattr(module.ToolNode, "_execute_tool_sync"): + wrap_function_wrapper(module, "ToolNode._execute_tool_sync", wrap_ToolNode__execute_tool_sync) + if hasattr(module.ToolNode, "_execute_tool_async"): + wrap_function_wrapper(module, "ToolNode._execute_tool_async", wrap_ToolNode__execute_tool_async) diff --git a/newrelic/hooks/mlmodel_openai.py b/newrelic/hooks/mlmodel_openai.py index c3f7960b6e..6335e9ceff 100644 --- a/newrelic/hooks/mlmodel_openai.py +++ b/newrelic/hooks/mlmodel_openai.py @@ -15,6 +15,7 @@ import json import logging import sys +import time import traceback import uuid @@ -84,6 +85,8 @@ def wrap_chat_completion_sync(wrapped, instance, args, kwargs): if (kwargs.get("extra_headers") or {}).get("X-Stainless-Raw-Response") == "stream": return wrapped(*args, **kwargs) + request_timestamp = int(1000.0 * time.time()) + settings = transaction.settings if transaction.settings is not None else global_settings() if not settings.ai_monitoring.enabled: return wrapped(*args, **kwargs) @@ -100,9 +103,10 @@ def wrap_chat_completion_sync(wrapped, instance, args, kwargs): try: return_val = wrapped(*args, **kwargs) except Exception as exc: - _record_completion_error(transaction, linking_metadata, completion_id, kwargs, ft, exc) + _record_completion_error(transaction, linking_metadata, completion_id, kwargs, ft, exc, request_timestamp) raise - _handle_completion_success(transaction, linking_metadata, completion_id, kwargs, ft, return_val) + + _handle_completion_success(transaction, linking_metadata, completion_id, kwargs, ft, return_val, request_timestamp) return return_val @@ -129,11 +133,12 @@ def create_chat_completion_message_event( span_id, trace_id, response_model, - request_model, response_id, request_id, llm_metadata, output_message_list, + all_token_counts, + request_timestamp=None, ): settings = transaction.settings if transaction.settings is not None else global_settings() @@ -153,11 +158,6 @@ def create_chat_completion_message_event( "request_id": request_id, "span_id": span_id, "trace_id": trace_id, - "token_count": ( - settings.ai_monitoring.llm_token_count_callback(request_model, message_content) - if settings.ai_monitoring.llm_token_count_callback - else None - ), "role": message.get("role"), "completion_id": chat_completion_id, "sequence": index, @@ -166,9 +166,15 @@ def create_chat_completion_message_event( "ingest_source": "Python", } - if settings.ai_monitoring.record_content.enabled: + if all_token_counts: + chat_completion_input_message_dict["token_count"] = 0 + + if settings.ai_monitoring.record_content.enabled and message_content: chat_completion_input_message_dict["content"] = message_content + if request_timestamp: + chat_completion_input_message_dict["timestamp"] = request_timestamp + chat_completion_input_message_dict.update(llm_metadata) transaction.record_custom_event("LlmChatCompletionMessage", chat_completion_input_message_dict) @@ -193,11 +199,6 @@ def create_chat_completion_message_event( "request_id": request_id, "span_id": span_id, "trace_id": trace_id, - "token_count": ( - settings.ai_monitoring.llm_token_count_callback(response_model, message_content) - if settings.ai_monitoring.llm_token_count_callback - else None - ), "role": message.get("role"), "completion_id": chat_completion_id, "sequence": index, @@ -207,7 +208,10 @@ def create_chat_completion_message_event( "is_response": True, } - if settings.ai_monitoring.record_content.enabled: + if all_token_counts: + chat_completion_output_message_dict["token_count"] = 0 + + if settings.ai_monitoring.record_content.enabled and message_content: chat_completion_output_message_dict["content"] = message_content chat_completion_output_message_dict.update(llm_metadata) @@ -280,21 +284,24 @@ def _record_embedding_success(transaction, embedding_id, linking_metadata, kwarg else getattr(attribute_response, "organization", None) ) + response_total_tokens = attribute_response.get("usage", {}).get("total_tokens") if response else None + + total_tokens = ( + settings.ai_monitoring.llm_token_count_callback(response_model, input_) + if settings.ai_monitoring.llm_token_count_callback and input_ + else response_total_tokens + ) + full_embedding_response_dict = { "id": embedding_id, "span_id": span_id, "trace_id": trace_id, - "token_count": ( - settings.ai_monitoring.llm_token_count_callback(response_model, input_) - if settings.ai_monitoring.llm_token_count_callback - else None - ), "request.model": kwargs.get("model") or kwargs.get("engine"), "request_id": request_id, "duration": ft.duration * 1000, "response.model": response_model, "response.organization": organization, - "response.headers.llmVersion": response_headers.get("openai-version"), + "response.headers.llmVersion": response_headers.get("openai-version") or None, "response.headers.ratelimitLimitRequests": check_rate_limit_header( response_headers, "x-ratelimit-limit-requests", True ), @@ -313,6 +320,7 @@ def _record_embedding_success(transaction, embedding_id, linking_metadata, kwarg "response.headers.ratelimitRemainingRequests": check_rate_limit_header( response_headers, "x-ratelimit-remaining-requests", True ), + "response.usage.total_tokens": total_tokens, "vendor": "openai", "ingest_source": "Python", } @@ -403,6 +411,8 @@ async def wrap_chat_completion_async(wrapped, instance, args, kwargs): if (kwargs.get("extra_headers") or {}).get("X-Stainless-Raw-Response") == "stream": return await wrapped(*args, **kwargs) + request_timestamp = int(1000.0 * time.time()) + settings = transaction.settings if transaction.settings is not None else global_settings() if not settings.ai_monitoring.enabled: return await wrapped(*args, **kwargs) @@ -419,14 +429,16 @@ async def wrap_chat_completion_async(wrapped, instance, args, kwargs): try: return_val = await wrapped(*args, **kwargs) except Exception as exc: - _record_completion_error(transaction, linking_metadata, completion_id, kwargs, ft, exc) + _record_completion_error(transaction, linking_metadata, completion_id, kwargs, ft, exc, request_timestamp) raise - _handle_completion_success(transaction, linking_metadata, completion_id, kwargs, ft, return_val) + _handle_completion_success(transaction, linking_metadata, completion_id, kwargs, ft, return_val, request_timestamp) return return_val -def _handle_completion_success(transaction, linking_metadata, completion_id, kwargs, ft, return_val): +def _handle_completion_success( + transaction, linking_metadata, completion_id, kwargs, ft, return_val, request_timestamp=None +): settings = transaction.settings if transaction.settings is not None else global_settings() stream = kwargs.get("stream", False) # Only if streaming and streaming monitoring is enabled and the response is not empty @@ -446,7 +458,7 @@ def _handle_completion_success(transaction, linking_metadata, completion_id, kwa return_val._nr_openai_attrs = getattr(return_val, "_nr_openai_attrs", {}) return_val._nr_openai_attrs["messages"] = kwargs.get("messages", []) return_val._nr_openai_attrs["temperature"] = kwargs.get("temperature") - return_val._nr_openai_attrs["max_tokens"] = kwargs.get("max_tokens") + return_val._nr_openai_attrs["max_tokens"] = kwargs.get("max_tokens") or kwargs.get("max_completion_tokens") return_val._nr_openai_attrs["model"] = kwargs.get("model") or kwargs.get("engine") return except Exception: @@ -469,18 +481,25 @@ def _handle_completion_success(transaction, linking_metadata, completion_id, kwa # openai._legacy_response.LegacyAPIResponse response = json.loads(response.http_response.text.strip()) - _record_completion_success(transaction, linking_metadata, completion_id, kwargs, ft, response_headers, response) + _record_completion_success( + transaction, linking_metadata, completion_id, kwargs, ft, response_headers, response, request_timestamp + ) except Exception: _logger.warning(RECORD_EVENTS_FAILURE_LOG_MESSAGE, traceback.format_exception(*sys.exc_info())) -def _record_completion_success(transaction, linking_metadata, completion_id, kwargs, ft, response_headers, response): +def _record_completion_success( + transaction, linking_metadata, completion_id, kwargs, ft, response_headers, response, request_timestamp=None +): + settings = transaction.settings if transaction.settings is not None else global_settings() span_id = linking_metadata.get("span.id") trace_id = linking_metadata.get("trace.id") + try: if response: response_model = response.get("model") response_id = response.get("id") + token_usage = response.get("usage") or {} output_message_list = [] finish_reason = None choices = response.get("choices") or [] @@ -494,6 +513,7 @@ def _record_completion_success(transaction, linking_metadata, completion_id, kwa else: response_model = kwargs.get("response.model") response_id = kwargs.get("id") + token_usage = {} output_message_list = [] finish_reason = kwargs.get("finish_reason") if "content" in kwargs: @@ -505,17 +525,52 @@ def _record_completion_success(transaction, linking_metadata, completion_id, kwa output_message_list = [] request_model = kwargs.get("model") or kwargs.get("engine") - request_id = response_headers.get("x-request-id") - organization = response_headers.get("openai-organization") or getattr(response, "organization", None) messages = kwargs.get("messages") or [{"content": kwargs.get("prompt"), "role": "user"}] input_message_list = list(messages) + + # Extract token counts from response object + if token_usage: + response_prompt_tokens = token_usage.get("prompt_tokens") + response_completion_tokens = token_usage.get("completion_tokens") + response_total_tokens = token_usage.get("total_tokens") + + else: + response_prompt_tokens = None + response_completion_tokens = None + response_total_tokens = None + + # Calculate token counts by checking if a callback is registered and if we have the necessary content to pass + # to it. If not, then we use the token counts provided in the response object + input_message_content = " ".join([msg.get("content", "") for msg in input_message_list if msg.get("content")]) + prompt_tokens = ( + settings.ai_monitoring.llm_token_count_callback(request_model, input_message_content) + if settings.ai_monitoring.llm_token_count_callback and input_message_content + else response_prompt_tokens + ) + output_message_content = " ".join([msg.get("content", "") for msg in output_message_list if msg.get("content")]) + completion_tokens = ( + settings.ai_monitoring.llm_token_count_callback(response_model, output_message_content) + if settings.ai_monitoring.llm_token_count_callback and output_message_content + else response_completion_tokens + ) + + total_tokens = ( + prompt_tokens + completion_tokens if all([prompt_tokens, completion_tokens]) else response_total_tokens + ) + + all_token_counts = bool(prompt_tokens and completion_tokens and total_tokens) + + request_id = response_headers.get("x-request-id") + organization = response_headers.get("openai-organization") or getattr(response, "organization", None) + full_chat_completion_summary_dict = { "id": completion_id, "span_id": span_id, "trace_id": trace_id, "request.model": request_model, "request.temperature": kwargs.get("temperature"), - "request.max_tokens": kwargs.get("max_tokens"), + # Later gpt models may use "max_completion_tokens" instead of "max_tokens" + "request.max_tokens": kwargs.get("max_tokens") or kwargs.get("max_completion_tokens"), "vendor": "openai", "ingest_source": "Python", "request_id": request_id, @@ -552,7 +607,14 @@ def _record_completion_success(transaction, linking_metadata, completion_id, kwa response_headers, "x-ratelimit-remaining-tokens_usage_based", True ), "response.number_of_messages": len(input_message_list) + len(output_message_list), + "timestamp": request_timestamp, } + + if all_token_counts: + full_chat_completion_summary_dict["response.usage.prompt_tokens"] = prompt_tokens + full_chat_completion_summary_dict["response.usage.completion_tokens"] = completion_tokens + full_chat_completion_summary_dict["response.usage.total_tokens"] = total_tokens + llm_metadata = _get_llm_attributes(transaction) full_chat_completion_summary_dict.update(llm_metadata) transaction.record_custom_event("LlmChatCompletionSummary", full_chat_completion_summary_dict) @@ -564,28 +626,30 @@ def _record_completion_success(transaction, linking_metadata, completion_id, kwa span_id, trace_id, response_model, - request_model, response_id, request_id, llm_metadata, output_message_list, + all_token_counts, + request_timestamp, ) except Exception: _logger.warning(RECORD_EVENTS_FAILURE_LOG_MESSAGE, traceback.format_exception(*sys.exc_info())) -def _record_completion_error(transaction, linking_metadata, completion_id, kwargs, ft, exc): +def _record_completion_error(transaction, linking_metadata, completion_id, kwargs, ft, exc, request_timestamp=None): span_id = linking_metadata.get("span.id") trace_id = linking_metadata.get("trace.id") request_message_list = kwargs.get("messages", None) or [] notice_error_attributes = {} + try: if OPENAI_V1: response = getattr(exc, "response", None) response_headers = getattr(response, "headers", None) or {} exc_organization = response_headers.get("openai-organization") # There appears to be a bug here in openai v1 where despite having code, - # param, etc in the error response, they are not populated on the exception + # param, etc. in the error response, they are not populated on the exception # object so grab them from the response body object instead. body = getattr(exc, "body", None) or {} notice_error_attributes = { @@ -629,12 +693,13 @@ def _record_completion_error(transaction, linking_metadata, completion_id, kwarg "response.number_of_messages": len(request_message_list), "request.model": request_model, "request.temperature": kwargs.get("temperature"), - "request.max_tokens": kwargs.get("max_tokens"), + "request.max_tokens": kwargs.get("max_tokens") or kwargs.get("max_completion_tokens"), "vendor": "openai", "ingest_source": "Python", "response.organization": exc_organization, "duration": ft.duration * 1000, "error": True, + "timestamp": request_timestamp, } llm_metadata = _get_llm_attributes(transaction) error_chat_completion_dict.update(llm_metadata) @@ -643,6 +708,7 @@ def _record_completion_error(transaction, linking_metadata, completion_id, kwarg output_message_list = [] if "content" in kwargs: output_message_list = [{"content": kwargs.get("content"), "role": kwargs.get("role")}] + create_chat_completion_message_event( transaction, request_message_list, @@ -650,11 +716,13 @@ def _record_completion_error(transaction, linking_metadata, completion_id, kwarg span_id, trace_id, kwargs.get("response.model"), - request_model, response_id, request_id, llm_metadata, output_message_list, + # We do not record token counts in error cases, so set all_token_counts to True so the pipeline tokenizer does not run + True, + request_timestamp, ) except Exception: _logger.warning(RECORD_EVENTS_FAILURE_LOG_MESSAGE, traceback.format_exception(*sys.exc_info())) @@ -719,6 +787,7 @@ async def wrap_base_client_process_response_async(wrapped, instance, args, kwarg class GeneratorProxy(ObjectProxy): def __init__(self, wrapped): super().__init__(wrapped) + self._nr_request_timestamp = int(1000.0 * time.time()) def __iter__(self): return self @@ -733,15 +802,15 @@ def __next__(self): return_val = self.__wrapped__.__next__() _record_stream_chunk(self, return_val) except StopIteration: - _record_events_on_stop_iteration(self, transaction) + _record_events_on_stop_iteration(self, transaction, self._nr_request_timestamp) raise except Exception as exc: - _handle_streaming_completion_error(self, transaction, exc) + _handle_streaming_completion_error(self, transaction, exc, self._nr_request_timestamp) raise return return_val def close(self): - return super().close() + return self.__wrapped__.close() def _record_stream_chunk(self, return_val): @@ -770,7 +839,7 @@ def _record_stream_chunk(self, return_val): _logger.warning(STREAM_PARSING_FAILURE_LOG_MESSAGE, traceback.format_exception(*sys.exc_info())) -def _record_events_on_stop_iteration(self, transaction): +def _record_events_on_stop_iteration(self, transaction, request_timestamp=None): if hasattr(self, "_nr_ft"): # We first check for our saved linking metadata before making a new call to get_trace_linking_metadata # Directly calling get_trace_linking_metadata() causes the incorrect span ID to be captured and associated with the LLM call @@ -787,7 +856,14 @@ def _record_events_on_stop_iteration(self, transaction): completion_id = str(uuid.uuid4()) response_headers = openai_attrs.get("response_headers") or {} _record_completion_success( - transaction, linking_metadata, completion_id, openai_attrs, self._nr_ft, response_headers, None + transaction, + linking_metadata, + completion_id, + openai_attrs, + self._nr_ft, + response_headers, + None, + request_timestamp, ) except Exception: _logger.warning(RECORD_EVENTS_FAILURE_LOG_MESSAGE, traceback.format_exception(*sys.exc_info())) @@ -802,7 +878,7 @@ def _record_events_on_stop_iteration(self, transaction): self._nr_openai_attrs.clear() -def _handle_streaming_completion_error(self, transaction, exc): +def _handle_streaming_completion_error(self, transaction, exc, request_timestamp=None): if hasattr(self, "_nr_ft"): openai_attrs = getattr(self, "_nr_openai_attrs", {}) @@ -812,12 +888,15 @@ def _handle_streaming_completion_error(self, transaction, exc): return linking_metadata = get_trace_linking_metadata() completion_id = str(uuid.uuid4()) - _record_completion_error(transaction, linking_metadata, completion_id, openai_attrs, self._nr_ft, exc) + _record_completion_error( + transaction, linking_metadata, completion_id, openai_attrs, self._nr_ft, exc, request_timestamp + ) class AsyncGeneratorProxy(ObjectProxy): def __init__(self, wrapped): super().__init__(wrapped) + self._nr_request_timestamp = int(1000.0 * time.time()) def __aiter__(self): self._nr_wrapped_iter = self.__wrapped__.__aiter__() @@ -833,15 +912,15 @@ async def __anext__(self): return_val = await self._nr_wrapped_iter.__anext__() _record_stream_chunk(self, return_val) except StopAsyncIteration: - _record_events_on_stop_iteration(self, transaction) + _record_events_on_stop_iteration(self, transaction, self._nr_request_timestamp) raise except Exception as exc: - _handle_streaming_completion_error(self, transaction, exc) + _handle_streaming_completion_error(self, transaction, exc, self._nr_request_timestamp) raise return return_val async def aclose(self): - return await super().aclose() + return await self.__wrapped__.aclose() def wrap_stream_iter_events_sync(wrapped, instance, args, kwargs): diff --git a/newrelic/hooks/mlmodel_strands.py b/newrelic/hooks/mlmodel_strands.py new file mode 100644 index 0000000000..bc045df190 --- /dev/null +++ b/newrelic/hooks/mlmodel_strands.py @@ -0,0 +1,524 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import logging +import sys +import uuid + +from newrelic.api.error_trace import ErrorTraceWrapper +from newrelic.api.function_trace import FunctionTrace +from newrelic.api.time_trace import current_trace, get_trace_linking_metadata +from newrelic.api.transaction import current_transaction +from newrelic.common.llm_utils import AsyncGeneratorProxy, _get_llm_metadata +from newrelic.common.object_names import callable_name +from newrelic.common.object_wrapper import wrap_function_wrapper +from newrelic.common.package_version_utils import get_package_version +from newrelic.common.signature import bind_args +from newrelic.core.config import global_settings +from newrelic.core.context import ContextOf + +_logger = logging.getLogger(__name__) +STRANDS_VERSION = get_package_version("strands-agents") + +RECORD_EVENTS_FAILURE_LOG_MESSAGE = "Exception occurred in Strands instrumentation: Failed to record LLM events. Please report this issue to New Relic Support." +TOOL_OUTPUT_FAILURE_LOG_MESSAGE = "Exception occurred in Strands instrumentation: Failed to record output of tool call. Please report this issue to New Relic Support." +AGENT_EVENT_FAILURE_LOG_MESSAGE = "Exception occurred in Strands instrumentation: Failed to record agent data. Please report this issue to New Relic Support." +TOOL_EXTRACTOR_FAILURE_LOG_MESSAGE = "Exception occurred in Strands instrumentation: Failed to extract tool information. If the issue persists, report this issue to New Relic support.\n" +DECORATOR_IMPORT_FAILURE_LOG_MESSAGE = "Exception occurred in Strands instrumentation: Failed to import DecoratedFunctionTool from strands.tools.decorator. Please report this issue to New Relic Support." + + +def wrap_agent__call__(wrapped, instance, args, kwargs): + trace = current_trace() + if not trace: + return wrapped(*args, **kwargs) + + try: + bound_args = bind_args(wrapped, args, kwargs) + # Make a copy of the invocation state before we mutate it + if "invocation_state" in bound_args: + invocation_state = bound_args["invocation_state"] = dict(bound_args["invocation_state"] or {}) + + # Attempt to save the current transaction context into the invocation state dictionary + invocation_state["_nr_transaction"] = trace + except Exception: + return wrapped(*args, **kwargs) + else: + return wrapped(**bound_args) + + +async def wrap_agent_invoke_async(wrapped, instance, args, kwargs): + # If there's already a transaction, don't propagate anything here + if current_transaction(): + return await wrapped(*args, **kwargs) + + try: + # Grab the trace context we should be running under and pass it to ContextOf + bound_args = bind_args(wrapped, args, kwargs) + invocation_state = bound_args["invocation_state"] or {} + trace = invocation_state.pop("_nr_transaction", None) + except Exception: + return await wrapped(*args, **kwargs) + + # If we find a transaction to propagate, use it. Otherwise, just call wrapped. + if trace: + with ContextOf(trace=trace): + return await wrapped(*args, **kwargs) + else: + return await wrapped(*args, **kwargs) + + +def wrap_stream_async(wrapped, instance, args, kwargs): + transaction = current_transaction() + if not transaction: + return wrapped(*args, **kwargs) + + settings = transaction.settings or global_settings() + if not settings.ai_monitoring.enabled: + return wrapped(*args, **kwargs) + + # Framework metric also used for entity tagging in the UI + transaction.add_ml_model_info("Strands", STRANDS_VERSION) + transaction._add_agent_attribute("llm", True) + + func_name = callable_name(wrapped) + agent_name = getattr(instance, "name", "agent") + function_trace_name = f"{func_name}/{agent_name}" + agentic_subcomponent_data = {"type": "APM-AI_AGENT", "name": agent_name} + + ft = FunctionTrace(name=function_trace_name, group="Llm/agent/Strands") + ft.__enter__() + ft._add_agent_attribute("subcomponent", json.dumps(agentic_subcomponent_data)) + + linking_metadata = get_trace_linking_metadata() + agent_id = str(uuid.uuid4()) + + try: + return_val = wrapped(*args, **kwargs) + except Exception: + raise + + try: + # For streaming responses, wrap with proxy and attach metadata + proxied_return_val = AsyncGeneratorProxy( + return_val, _record_agent_event_on_stop_iteration, _handle_agent_streaming_completion_error + ) + proxied_return_val._nr_ft = ft + proxied_return_val._nr_metadata = linking_metadata + proxied_return_val._nr_strands_attrs = {"agent_name": agent_name, "agent_id": agent_id} + return proxied_return_val + except Exception: + # If proxy creation fails, clean up the function trace and return original value + ft.__exit__(*sys.exc_info()) + return return_val + + +def _record_agent_event_on_stop_iteration(self, transaction): + if hasattr(self, "_nr_ft"): + # Use saved linking metadata to maintain correct span association + linking_metadata = self._nr_metadata or get_trace_linking_metadata() + self._nr_ft.__exit__(None, None, None) + try: + strands_attrs = getattr(self, "_nr_strands_attrs", {}) + + # If there are no strands attrs exit early as there's no data to record. + if not strands_attrs: + return + + agent_name = strands_attrs.get("agent_name", "agent") + agent_id = strands_attrs.get("agent_id") + agent_event_dict = _construct_base_agent_event_dict(agent_name, agent_id, transaction, linking_metadata) + agent_event_dict["duration"] = self._nr_ft.duration * 1000 + transaction.record_custom_event("LlmAgent", agent_event_dict) + + except Exception: + _logger.warning(RECORD_EVENTS_FAILURE_LOG_MESSAGE, exc_info=True) + finally: + # Clear cached data to prevent memory leaks and duplicate reporting + if hasattr(self, "_nr_strands_attrs"): + self._nr_strands_attrs.clear() + + +def _record_tool_event_on_stop_iteration(self, transaction): + if hasattr(self, "_nr_ft"): + # Use saved linking metadata to maintain correct span association + linking_metadata = self._nr_metadata or get_trace_linking_metadata() + self._nr_ft.__exit__(None, None, None) + + try: + strands_attrs = getattr(self, "_nr_strands_attrs", {}) + + # If there are no strands attrs exit early as there's no data to record. + if not strands_attrs: + return + + try: + tool_results = strands_attrs.get("tool_results", []) + except Exception: + tool_results = None + _logger.warning(TOOL_OUTPUT_FAILURE_LOG_MESSAGE, exc_info=True) + + tool_event_dict = _construct_base_tool_event_dict( + strands_attrs, tool_results, transaction, linking_metadata + ) + tool_event_dict["duration"] = self._nr_ft.duration * 1000 + transaction.record_custom_event("LlmTool", tool_event_dict) + + except Exception: + _logger.warning(RECORD_EVENTS_FAILURE_LOG_MESSAGE, exc_info=True) + finally: + # Clear cached data to prevent memory leaks and duplicate reporting + if hasattr(self, "_nr_strands_attrs"): + self._nr_strands_attrs.clear() + + +def _construct_base_tool_event_dict(strands_attrs, tool_results, transaction, linking_metadata): + try: + try: + tool_output = tool_results[-1]["content"][0] if tool_results else None + error = tool_results[-1]["status"] == "error" + except Exception: + tool_output = None + error = False + _logger.warning(TOOL_OUTPUT_FAILURE_LOG_MESSAGE, exc_info=True) + + tool_name = strands_attrs.get("tool_name", "tool") + tool_id = strands_attrs.get("tool_id") + run_id = strands_attrs.get("run_id") + tool_input = strands_attrs.get("tool_input") + agent_name = strands_attrs.get("agent_name", "agent") + settings = transaction.settings or global_settings() + + tool_event_dict = { + "id": tool_id, + "run_id": run_id, + "name": tool_name, + "span_id": linking_metadata.get("span.id"), + "trace_id": linking_metadata.get("trace.id"), + "agent_name": agent_name, + "vendor": "strands", + "ingest_source": "Python", + } + # Set error flag if the status shows an error was caught, + # it will be reported further down in the instrumentation. + if error: + tool_event_dict["error"] = True + + if settings.ai_monitoring.record_content.enabled: + tool_event_dict["input"] = tool_input + # In error cases, the output will hold the error message + tool_event_dict["output"] = tool_output + tool_event_dict.update(_get_llm_metadata(transaction)) + except Exception: + tool_event_dict = {} + _logger.warning(RECORD_EVENTS_FAILURE_LOG_MESSAGE, exc_info=True) + + return tool_event_dict + + +def _construct_base_agent_event_dict(agent_name, agent_id, transaction, linking_metadata): + try: + agent_event_dict = { + "id": agent_id, + "name": agent_name, + "span_id": linking_metadata.get("span.id"), + "trace_id": linking_metadata.get("trace.id"), + "vendor": "strands", + "ingest_source": "Python", + } + agent_event_dict.update(_get_llm_metadata(transaction)) + except Exception: + _logger.warning(AGENT_EVENT_FAILURE_LOG_MESSAGE, exc_info=True) + agent_event_dict = {} + + return agent_event_dict + + +def _handle_agent_streaming_completion_error(self, transaction): + if hasattr(self, "_nr_ft"): + strands_attrs = getattr(self, "_nr_strands_attrs", {}) + + # If there are no strands attrs exit early as there's no data to record. + if not strands_attrs: + self._nr_ft.__exit__(*sys.exc_info()) + return + + # Use saved linking metadata to maintain correct span association + linking_metadata = self._nr_metadata or get_trace_linking_metadata() + + try: + agent_name = strands_attrs.get("agent_name", "agent") + agent_id = strands_attrs.get("agent_id") + + # Notice the error on the function trace + self._nr_ft.notice_error(attributes={"agent_id": agent_id}) + self._nr_ft.__exit__(*sys.exc_info()) + + # Create error event + agent_event_dict = _construct_base_agent_event_dict(agent_name, agent_id, transaction, linking_metadata) + agent_event_dict.update({"duration": self._nr_ft.duration * 1000, "error": True}) + transaction.record_custom_event("LlmAgent", agent_event_dict) + + except Exception: + _logger.warning(RECORD_EVENTS_FAILURE_LOG_MESSAGE, exc_info=True) + finally: + # Clear cached data to prevent memory leaks + if hasattr(self, "_nr_strands_attrs"): + self._nr_strands_attrs.clear() + + +def _handle_tool_streaming_completion_error(self, transaction): + if hasattr(self, "_nr_ft"): + strands_attrs = getattr(self, "_nr_strands_attrs", {}) + + # If there are no strands attrs exit early as there's no data to record. + if not strands_attrs: + self._nr_ft.__exit__(*sys.exc_info()) + return + + # Use saved linking metadata to maintain correct span association + linking_metadata = self._nr_metadata or get_trace_linking_metadata() + + try: + tool_id = strands_attrs.get("tool_id") + + # We expect this to never have any output since this is an error case, + # but if it does we will report it. + try: + tool_results = strands_attrs.get("tool_results", []) + except Exception: + tool_results = None + _logger.warning(TOOL_OUTPUT_FAILURE_LOG_MESSAGE, exc_info=True) + + # Notice the error on the function trace + self._nr_ft.notice_error(attributes={"tool_id": tool_id}) + self._nr_ft.__exit__(*sys.exc_info()) + + # Create error event + tool_event_dict = _construct_base_tool_event_dict( + strands_attrs, tool_results, transaction, linking_metadata + ) + tool_event_dict["duration"] = self._nr_ft.duration * 1000 + # Ensure error flag is set to True in case the tool_results did not indicate an error + if "error" not in tool_event_dict: + tool_event_dict["error"] = True + + transaction.record_custom_event("LlmTool", tool_event_dict) + + except Exception: + _logger.warning(RECORD_EVENTS_FAILURE_LOG_MESSAGE, exc_info=True) + finally: + # Clear cached data to prevent memory leaks + if hasattr(self, "_nr_strands_attrs"): + self._nr_strands_attrs.clear() + + +def wrap_tool_executor__stream(wrapped, instance, args, kwargs): + transaction = current_transaction() + if not transaction: + return wrapped(*args, **kwargs) + + settings = transaction.settings or global_settings() + if not settings.ai_monitoring.enabled: + return wrapped(*args, **kwargs) + + # Framework metric also used for entity tagging in the UI + transaction.add_ml_model_info("Strands", STRANDS_VERSION) + transaction._add_agent_attribute("llm", True) + + # Grab tool data + try: + bound_args = bind_args(wrapped, args, kwargs) + agent_name = getattr(bound_args.get("agent"), "name", "agent") + tool_use = bound_args.get("tool_use", {}) + + run_id = tool_use.get("toolUseId", "") + tool_name = tool_use.get("name", "tool") + _input = tool_use.get("input") + tool_input = str(_input) if _input else None + tool_results = bound_args.get("tool_results", []) + except Exception: + tool_name = "tool" + _logger.warning(TOOL_EXTRACTOR_FAILURE_LOG_MESSAGE, exc_info=True) + + func_name = callable_name(wrapped) + function_trace_name = f"{func_name}/{tool_name}" + agentic_subcomponent_data = {"type": "APM-AI_TOOL", "name": tool_name} + + ft = FunctionTrace(name=function_trace_name, group="Llm/tool/Strands") + ft.__enter__() + ft._add_agent_attribute("subcomponent", json.dumps(agentic_subcomponent_data)) + linking_metadata = get_trace_linking_metadata() + tool_id = str(uuid.uuid4()) + + try: + return_val = wrapped(*args, **kwargs) + except Exception: + raise + + try: + # Wrap return value with proxy and attach metadata for later access + proxied_return_val = AsyncGeneratorProxy( + return_val, _record_tool_event_on_stop_iteration, _handle_tool_streaming_completion_error + ) + proxied_return_val._nr_ft = ft + proxied_return_val._nr_metadata = linking_metadata + proxied_return_val._nr_strands_attrs = { + "tool_results": tool_results, + "tool_name": tool_name, + "tool_id": tool_id, + "run_id": run_id, + "tool_input": tool_input, + "agent_name": agent_name, + } + return proxied_return_val + except Exception: + # If proxy creation fails, clean up the function trace and return original value + ft.__exit__(*sys.exc_info()) + return return_val + + +def wrap_ToolRegister_register_tool(wrapped, instance, args, kwargs): + try: + from strands.tools.decorator import DecoratedFunctionTool + except ImportError: + _logger.exception(DECORATOR_IMPORT_FAILURE_LOG_MESSAGE) + # If we can't import this to check for double wrapping, return early + return wrapped(*args, **kwargs) + + try: + bound_args = bind_args(wrapped, args, kwargs) + except Exception: + return wrapped(*args, **kwargs) + + tool = bound_args.get("tool") + + # Ensure we don't double capture exceptions by not touching DecoratedFunctionTool instances here. + # Those should be captured with specific instrumentation that properly handles the thread boundaries. + if hasattr(tool, "stream") and not isinstance(tool, DecoratedFunctionTool): + tool.stream = ErrorTraceWrapper(tool.stream) + + return wrapped(*args, **kwargs) + + +def wrap_bedrock_model_stream(wrapped, instance, args, kwargs): + """Stores trace context on the messages argument to be retrieved by the _stream() instrumentation.""" + trace = current_trace() + if not trace: + return wrapped(*args, **kwargs) + + settings = trace.settings or global_settings() + if not settings.ai_monitoring.enabled: + return wrapped(*args, **kwargs) + + try: + bound_args = bind_args(wrapped, args, kwargs) + except Exception: + return wrapped(*args, **kwargs) + + if "messages" in bound_args and isinstance(bound_args["messages"], list): + bound_args["messages"].append({"newrelic_trace": trace}) + + return wrapped(*args, **kwargs) + + +def wrap_bedrock_model__stream(wrapped, instance, args, kwargs): + """Retrieves trace context stored on the messages argument and propagates it to the new thread.""" + try: + bound_args = bind_args(wrapped, args, kwargs) + except Exception: + return wrapped(*args, **kwargs) + + if ( + "messages" in bound_args + and isinstance(bound_args["messages"], list) + and bound_args["messages"] # non-empty list + and "newrelic_trace" in bound_args["messages"][-1] + ): + trace_message = bound_args["messages"].pop() + with ContextOf(trace=trace_message["newrelic_trace"]): + return wrapped(*args, **kwargs) + + return wrapped(*args, **kwargs) + + +def wrap_decorated_function_tool__wrap_tool_result(wrapped, instance, args, kwargs): + transaction = current_transaction() + if transaction: + exc = sys.exc_info() + try: + if exc: + bound_args = bind_args(wrapped, args, kwargs) + tool_id = bound_args.get("tool_id") + transaction.notice_error(exc, attributes={"tool_id": tool_id}) + finally: + # Delete exc to avoid reference cycles + del exc + + return wrapped(*args, **kwargs) + + +def instrument_strands_agent_agent(module): + if hasattr(module, "Agent"): + if hasattr(module.Agent, "__call__"): # noqa: B004 + wrap_function_wrapper(module, "Agent.__call__", wrap_agent__call__) + if hasattr(module.Agent, "invoke_async"): + wrap_function_wrapper(module, "Agent.invoke_async", wrap_agent_invoke_async) + if hasattr(module.Agent, "stream_async"): + wrap_function_wrapper(module, "Agent.stream_async", wrap_stream_async) + + +def instrument_strands_multiagent_graph(module): + if hasattr(module, "Graph"): + if hasattr(module.Graph, "__call__"): # noqa: B004 + wrap_function_wrapper(module, "Graph.__call__", wrap_agent__call__) + if hasattr(module.Graph, "invoke_async"): + wrap_function_wrapper(module, "Graph.invoke_async", wrap_agent_invoke_async) + + +def instrument_strands_multiagent_swarm(module): + if hasattr(module, "Swarm"): + if hasattr(module.Swarm, "__call__"): # noqa: B004 + wrap_function_wrapper(module, "Swarm.__call__", wrap_agent__call__) + if hasattr(module.Swarm, "invoke_async"): + wrap_function_wrapper(module, "Swarm.invoke_async", wrap_agent_invoke_async) + + +def instrument_strands_tools_decorator(module): + # This instrumentation only exists to pass trace context due to bedrock models using a separate thread. + if hasattr(module, "DecoratedFunctionTool") and hasattr(module.DecoratedFunctionTool, "_wrap_tool_result"): + wrap_function_wrapper( + module, "DecoratedFunctionTool._wrap_tool_result", wrap_decorated_function_tool__wrap_tool_result + ) + + +def instrument_strands_tools_executors__executor(module): + if hasattr(module, "ToolExecutor"): + if hasattr(module.ToolExecutor, "_stream"): + wrap_function_wrapper(module, "ToolExecutor._stream", wrap_tool_executor__stream) + + +def instrument_strands_tools_registry(module): + if hasattr(module, "ToolRegistry"): + if hasattr(module.ToolRegistry, "register_tool"): + wrap_function_wrapper(module, "ToolRegistry.register_tool", wrap_ToolRegister_register_tool) + + +def instrument_strands_models_bedrock(module): + # This instrumentation only exists to pass trace context due to bedrock models using a separate thread. + if hasattr(module, "BedrockModel"): + if hasattr(module.BedrockModel, "stream"): + wrap_function_wrapper(module, "BedrockModel.stream", wrap_bedrock_model_stream) + if hasattr(module.BedrockModel, "_stream"): + wrap_function_wrapper(module, "BedrockModel._stream", wrap_bedrock_model__stream) diff --git a/pyproject.toml b/pyproject.toml index 2dbdb34837..4498ff620b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,6 +71,7 @@ packages = [ "newrelic.bootstrap", "newrelic.common", "newrelic.core", + "newrelic.core.samplers", "newrelic.extras", "newrelic.extras.framework_django", "newrelic.extras.framework_django.templatetags", diff --git a/tests/adapter_mcp/test_mcp.py b/tests/adapter_mcp/test_mcp.py index 5ba6a81074..5424b57ca7 100644 --- a/tests/adapter_mcp/test_mcp.py +++ b/tests/adapter_mcp/test_mcp.py @@ -19,6 +19,7 @@ from mcp.server.fastmcp.tools import ToolManager from testing_support.ml_testing_utils import disabled_ai_monitoring_settings from testing_support.validators.validate_function_not_called import validate_function_not_called +from testing_support.validators.validate_span_events import validate_span_events from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics from newrelic.api.background_task import background_task @@ -57,6 +58,7 @@ def echo_prompt(message: str): rollup_metrics=[("Llm/tool/MCP/mcp.client.session:ClientSession.call_tool/add_exclamation", 1)], background_task=True, ) +@validate_span_events(count=1, exact_agents={"subcomponent": '{"type": "APM-AI_TOOL", "name": "add_exclamation"}'}) @background_task() def test_tool_tracing_via_client_session(loop, fastmcp_server): async def _test(): @@ -75,6 +77,7 @@ async def _test(): rollup_metrics=[("Llm/tool/MCP/mcp.server.fastmcp.tools.tool_manager:ToolManager.call_tool/add_exclamation", 1)], background_task=True, ) +@validate_span_events(count=1, exact_agents={"subcomponent": '{"type": "APM-AI_TOOL", "name": "add_exclamation"}'}) @background_task() def test_tool_tracing_via_tool_manager(loop): async def _test(): diff --git a/tests/adapter_uvicorn/test_uvicorn.py b/tests/adapter_uvicorn/test_uvicorn.py index 0084be3e46..d5db2d6ca6 100644 --- a/tests/adapter_uvicorn/test_uvicorn.py +++ b/tests/adapter_uvicorn/test_uvicorn.py @@ -56,8 +56,8 @@ def app(request): return request.param -@pytest.fixture -def port(app): +@pytest.fixture(params=["asyncio", "uvloop", "none"], ids=["asyncio", "uvloop", "none"]) +def port(app, request): port = get_open_port() loops = [] @@ -72,7 +72,7 @@ def on_tick_sync(): async def on_tick(): on_tick_sync() - config = Config(app, host="127.0.0.1", port=port, loop="asyncio") + config = Config(app, host="127.0.0.1", port=port, loop=request.param) config.callback_notify = on_tick config.log_config = {"version": 1} config.disable_lifespan = True diff --git a/tests/agent_features/test_agent_control_health_check.py b/tests/agent_features/test_agent_control_health_check.py index e12f3a07f0..6adfa6f366 100644 --- a/tests/agent_features/test_agent_control_health_check.py +++ b/tests/agent_features/test_agent_control_health_check.py @@ -22,6 +22,7 @@ from testing_support.fixtures import initialize_agent from testing_support.http_client_recorder import HttpClientRecorder +from newrelic.common.object_wrapper import transient_function_wrapper from newrelic.config import _reset_configuration_done, initialize from newrelic.core.agent_control_health import HealthStatus, agent_control_health_instance from newrelic.core.agent_protocol import AgentProtocol @@ -30,6 +31,18 @@ from newrelic.network.exceptions import DiscardDataForRequest +@transient_function_wrapper("newrelic.api.time_trace", "get_service_linking_metadata") +def _wrap_get_service_linking_metadata(wrapped, instance, args, kwargs): + metadata = {"entity.type": "SERVICE"} + + # Set hardcoded values for testing so we can verify the correct entity guid was written to the health file + metadata["entity.name"] = "test-app" + metadata["entity.guid"] = "mock-entity-guid-12345" + metadata["hostname"] = "test-hostname" + + return metadata + + def get_health_file_contents(tmp_path): # Grab the file we just wrote to and read its contents health_file = list(Path(tmp_path).iterdir())[0] @@ -38,7 +51,7 @@ def get_health_file_contents(tmp_path): return contents -@pytest.fixture(scope="module", autouse=True) +@pytest.fixture(autouse=True) def restore_settings_fixture(): # Backup settings from before this test file runs original_settings = global_settings() @@ -51,6 +64,10 @@ def restore_settings_fixture(): original_settings.__dict__.clear() original_settings.__dict__.update(backup) + # Re-initialize the agent to restore the settings + _reset_configuration_done() + initialize() + @pytest.mark.parametrize("file_uri", ["", "file://", "/test/dir", "foo:/test/dir"]) def test_invalid_file_directory_supplied(monkeypatch, file_uri): @@ -86,6 +103,7 @@ def test_agent_control_not_enabled(monkeypatch, tmp_path): assert not agent_control_health_instance().health_check_enabled +@_wrap_get_service_linking_metadata def test_write_to_file_healthy_status(monkeypatch, tmp_path): # Setup expected env vars to run agent control health check monkeypatch.setenv("NEW_RELIC_AGENT_CONTROL_ENABLED", "True") @@ -100,12 +118,14 @@ def test_write_to_file_healthy_status(monkeypatch, tmp_path): contents = get_health_file_contents(tmp_path) # Assert on contents of health file - assert len(contents) == 4 - assert contents[0] == "healthy: True\n" - assert contents[1] == "status: Healthy\n" - assert int(re.search(r"status_time_unix_nano: (\d+)", contents[3]).group(1)) > 0 + assert len(contents) == 5 + assert contents[0] == "entity_guid: mock-entity-guid-12345\n" + assert contents[1] == "healthy: True\n" + assert contents[2] == "status: Healthy\n" + assert int(re.search(r"status_time_unix_nano: (\d+)", contents[4]).group(1)) > 0 +@_wrap_get_service_linking_metadata def test_write_to_file_unhealthy_status(monkeypatch, tmp_path): # Setup expected env vars to run agent control health check monkeypatch.setenv("NEW_RELIC_AGENT_CONTROL_ENABLED", "True") @@ -122,14 +142,16 @@ def test_write_to_file_unhealthy_status(monkeypatch, tmp_path): contents = get_health_file_contents(tmp_path) # Assert on contents of health file - assert len(contents) == 5 - assert contents[0] == "healthy: False\n" - assert contents[1] == "status: Invalid license key (HTTP status code 401)\n" - assert contents[2] == "start_time_unix_nano: 1234567890\n" - assert int(re.search(r"status_time_unix_nano: (\d+)", contents[3]).group(1)) > 0 - assert contents[4] == "last_error: NR-APM-001\n" + assert len(contents) == 6 + assert contents[0] == "entity_guid: mock-entity-guid-12345\n" + assert contents[1] == "healthy: False\n" + assert contents[2] == "status: Invalid license key (HTTP status code 401)\n" + assert contents[3] == "start_time_unix_nano: 1234567890\n" + assert int(re.search(r"status_time_unix_nano: (\d+)", contents[4]).group(1)) > 0 + assert contents[5] == "last_error: NR-APM-001\n" +@_wrap_get_service_linking_metadata def test_no_override_on_unhealthy_shutdown(monkeypatch, tmp_path): # Setup expected env vars to run agent control health check monkeypatch.setenv("NEW_RELIC_AGENT_CONTROL_ENABLED", "True") @@ -148,17 +170,26 @@ def test_no_override_on_unhealthy_shutdown(monkeypatch, tmp_path): contents = get_health_file_contents(tmp_path) # Assert on contents of health file - assert len(contents) == 5 - assert contents[0] == "healthy: False\n" - assert contents[1] == "status: Invalid license key (HTTP status code 401)\n" - assert contents[4] == "last_error: NR-APM-001\n" + assert len(contents) == 6 + assert contents[0] == "entity_guid: mock-entity-guid-12345\n" + assert contents[1] == "healthy: False\n" + assert contents[2] == "status: Invalid license key (HTTP status code 401)\n" + assert contents[5] == "last_error: NR-APM-001\n" def test_health_check_running_threads(monkeypatch, tmp_path): - running_threads = threading.enumerate() - # Only the main thread should be running since not agent control env vars are set - assert len(running_threads) == 1 + # If the Activate-Session thread is still active, give it time to close before we proceed + timeout = 30.0 + while len(threading.enumerate()) != 1 and timeout > 0: + time.sleep(0.1) + timeout -= 0.1 + + # Only the main thread should be running since no agent control env vars are set + assert len(threading.enumerate()) == 1, ( + f"Expected only the main thread to be running before the test starts. Got: {threading.enumerate()}" + ) + # Setup expected env vars to run agent control health check monkeypatch.setenv("NEW_RELIC_AGENT_CONTROL_ENABLED", "True") file_path = tmp_path.as_uri() monkeypatch.setenv("NEW_RELIC_AGENT_CONTROL_HEALTH_DELIVERY_LOCATION", file_path) @@ -174,12 +205,14 @@ def test_health_check_running_threads(monkeypatch, tmp_path): assert running_threads[1].name == "Agent-Control-Health-Main-Thread" +@_wrap_get_service_linking_metadata def test_proxy_error_status(monkeypatch, tmp_path): # Setup expected env vars to run agent control health check monkeypatch.setenv("NEW_RELIC_AGENT_CONTROL_ENABLED", "True") file_path = tmp_path.as_uri() monkeypatch.setenv("NEW_RELIC_AGENT_CONTROL_HEALTH_DELIVERY_LOCATION", file_path) + # Re-initialize the agent to allow the health check thread to start _reset_configuration_done() initialize() @@ -197,10 +230,11 @@ def test_proxy_error_status(monkeypatch, tmp_path): contents = get_health_file_contents(tmp_path) # Assert on contents of health file - assert len(contents) == 5 - assert contents[0] == "healthy: False\n" - assert contents[1] == "status: HTTP Proxy configuration error; response code 407\n" - assert contents[4] == "last_error: NR-APM-007\n" + assert len(contents) == 6 + assert contents[0] == "entity_guid: mock-entity-guid-12345\n" + assert contents[1] == "healthy: False\n" + assert contents[2] == "status: HTTP Proxy configuration error; response code 407\n" + assert contents[5] == "last_error: NR-APM-007\n" def test_multiple_activations_running_threads(monkeypatch, tmp_path): @@ -209,6 +243,7 @@ def test_multiple_activations_running_threads(monkeypatch, tmp_path): file_path = tmp_path.as_uri() monkeypatch.setenv("NEW_RELIC_AGENT_CONTROL_HEALTH_DELIVERY_LOCATION", file_path) + # Re-initialize the agent to allow the health check thread to start and assert that it did _reset_configuration_done() initialize() @@ -252,10 +287,11 @@ def test_update_to_healthy(monkeypatch, tmp_path): contents = get_health_file_contents(tmp_path) # Assert on contents of health file - assert contents[0] == "healthy: True\n" - assert contents[1] == "status: Healthy\n" + assert contents[1] == "healthy: True\n" + assert contents[2] == "status: Healthy\n" +@_wrap_get_service_linking_metadata def test_max_app_name_status(monkeypatch, tmp_path): # Setup expected env vars to run agent control health check monkeypatch.setenv("NEW_RELIC_AGENT_CONTROL_ENABLED", "True") @@ -271,7 +307,8 @@ def test_max_app_name_status(monkeypatch, tmp_path): contents = get_health_file_contents(tmp_path) # Assert on contents of health file - assert len(contents) == 5 - assert contents[0] == "healthy: False\n" - assert contents[1] == "status: The maximum number of configured app names (3) exceeded\n" - assert contents[4] == "last_error: NR-APM-006\n" + assert len(contents) == 6 + assert contents[0] == "entity_guid: mock-entity-guid-12345\n" + assert contents[1] == "healthy: False\n" + assert contents[2] == "status: The maximum number of configured app names (3) exceeded\n" + assert contents[5] == "last_error: NR-APM-006\n" diff --git a/tests/agent_features/test_asgi_transaction.py b/tests/agent_features/test_asgi_transaction.py index e70ec95901..ac774689bd 100644 --- a/tests/agent_features/test_asgi_transaction.py +++ b/tests/agent_features/test_asgi_transaction.py @@ -19,6 +19,7 @@ from testing_support.fixtures import override_application_settings from testing_support.sample_asgi_applications import ( AppWithDescriptor, + asgi_application_generator_headers, simple_app_v2, simple_app_v2_init_exc, simple_app_v2_raw, @@ -37,6 +38,7 @@ simple_app_v3_wrapped = AsgiTest(simple_app_v3) simple_app_v2_wrapped = AsgiTest(simple_app_v2) simple_app_v2_init_exc = AsgiTest(simple_app_v2_init_exc) +asgi_application_generator_headers = AsgiTest(asgi_application_generator_headers) # Test naming scheme logic and ASGIApplicationWrapper for a single callable @@ -85,6 +87,28 @@ def test_double_callable_raw(): assert response.body == b"" +# Ensure headers object is preserved +@pytest.mark.parametrize("browser_monitoring", [True, False]) +@validate_transaction_metrics(name="", group="Uri") +def test_generator_headers(browser_monitoring): + """ + Both ASGIApplicationWrapper and ASGIBrowserMiddleware can cause headers to be lost if generators are + not handled properly. + + Ensure neither destroys headers by testing with and without the ASGIBrowserMiddleware, to make sure whichever + receives headers first properly preserves them in a list. + """ + + @override_application_settings({"browser_monitoring.enabled": browser_monitoring}) + def _test(): + response = asgi_application_generator_headers.make_request("GET", "/") + assert response.status == 200 + assert response.headers == {"x-my-header": "myvalue"} + assert response.body == b"" + + _test() + + # Test asgi_application decorator with parameters passed in on a single callable @pytest.mark.parametrize("name, group", ((None, "group"), ("name", "group"), ("", "group"))) def test_asgi_application_decorator_single_callable(name, group): diff --git a/tests/agent_features/test_distributed_tracing.py b/tests/agent_features/test_distributed_tracing.py index 36261d97e2..b7ee3896c3 100644 --- a/tests/agent_features/test_distributed_tracing.py +++ b/tests/agent_features/test_distributed_tracing.py @@ -12,8 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +import asyncio import copy import json +import random +import time import pytest import webtest @@ -23,6 +26,21 @@ from testing_support.validators.validate_function_not_called import validate_function_not_called from testing_support.validators.validate_transaction_event_attributes import validate_transaction_event_attributes from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics +from testing_support.validators.validate_transaction_object_attributes import validate_transaction_object_attributes + +from newrelic.api.application import application_instance +from newrelic.api.function_trace import function_trace +from newrelic.common.object_names import callable_name +from newrelic.common.object_wrapper import function_wrapper, transient_function_wrapper + +try: + from newrelic.core.infinite_tracing_pb2 import AttributeValue, Span +except: + AttributeValue = None + Span = None + +from testing_support.mock_external_http_server import MockExternalHTTPHResponseHeadersServer +from testing_support.validators.validate_span_events import check_value_equals, validate_span_events from newrelic.api.application import application_instance from newrelic.api.background_task import BackgroundTask, background_task @@ -39,6 +57,7 @@ from newrelic.api.wsgi_application import wsgi_application from newrelic.core.attribute import Attribute +# ruff: noqa: UP031 distributed_trace_intrinsics = ["guid", "traceId", "priority", "sampled"] inbound_payload_intrinsics = [ "parent.type", @@ -71,6 +90,110 @@ } +def validate_compact_span_event( + name, compressed_span_count, expected_nr_durations_low_bound, expected_nr_durations_high_bound +): + @function_wrapper + def _validate_wrapper(wrapped, instance, args, kwargs): + record_transaction_called = [] + recorded_span_events = [] + + @transient_function_wrapper("newrelic.core.stats_engine", "StatsEngine.record_transaction") + def capture_span_events(wrapped, instance, args, kwargs): + events = [] + + @transient_function_wrapper("newrelic.common.streaming_utils", "StreamBuffer.put") + def stream_capture(wrapped, instance, args, kwargs): + event = args[0] + events.append(event) + return wrapped(*args, **kwargs) + + record_transaction_called.append(True) + try: + result = stream_capture(wrapped)(*args, **kwargs) + except: + raise + else: + if not instance.settings.infinite_tracing.enabled: + events = [event for priority, seen_at, event in instance.span_events.pq] + + recorded_span_events.append(events) + + return result + + _new_wrapper = capture_span_events(wrapped) + val = _new_wrapper(*args, **kwargs) + assert record_transaction_called + captured_events = recorded_span_events.pop(-1) + + mismatches = [] + matching_span_events = 0 + + def _span_details(): + details = [ + f"matching_span_events={matching_span_events}", + f"mismatches={mismatches}", + f"captured_events={captured_events}", + ] + return "\n".join(details) + + for captured_event in captured_events: + if Span and isinstance(captured_event, Span): + intrinsics = captured_event.intrinsics + user_attrs = captured_event.user_attributes + agent_attrs = captured_event.agent_attributes + else: + intrinsics, _, _ = captured_event + + # Find the span by name. + if not check_value_equals(intrinsics, "name", name): + continue + assert check_value_length(intrinsics, "nr.ids", compressed_span_count - 1, mismatches), _span_details() + assert check_value_between( + intrinsics, + "nr.durations", + expected_nr_durations_low_bound, + expected_nr_durations_high_bound, + mismatches, + ), _span_details() + matching_span_events += 1 + + assert matching_span_events == 1, _span_details() + return val + + return _validate_wrapper + + +def check_value_between(dictionary, key, expected_min, expected_max, mismatches): + value = dictionary.get(key) + if AttributeValue and isinstance(value, AttributeValue): + for _, val in value.ListFields(): + if not (expected_min < val < expected_max): + mismatches.append(f"key: {key}, not {expected_min} < {val} < {expected_max}") + return False + return True + else: + if not (expected_min < value < expected_max): + mismatches.append(f"key: {key}, not {expected_min} < {value} < {expected_max}") + return False + return True + + +def check_value_length(dictionary, key, expected_length, mismatches): + value = dictionary.get(key) + if AttributeValue and isinstance(value, AttributeValue): + for _, val in value.ListFields(): + if len(val) != expected_length: + mismatches.append(f"key: {key}, not len({val}) == {expected_length}") + return False + return True + else: + if len(value) != expected_length: + mismatches.append(f"key: {key}, not len({value}) == {expected_length}") + return False + return True + + @wsgi_application() def target_wsgi_application(environ, start_response): status = "200 OK" @@ -419,24 +542,283 @@ def _test_inbound_dt_payload_acceptance(): @pytest.mark.parametrize( - "sampled,remote_parent_sampled,remote_parent_not_sampled,expected_sampled,expected_priority,expected_adaptive_sampling_algo_called", + "newrelic_header,traceparent_sampled,newrelic_sampled,root_setting,remote_parent_sampled_setting,remote_parent_not_sampled_setting,expected_sampled,expected_priority,expected_adaptive_sampling_algo_called", + ( + (False, None, None, "default", "default", "default", None, None, True), # Uses adaptive sampling algo. + (False, None, None, "always_on", "default", "default", True, 3, False), # Always sampled. + (False, None, None, "always_off", "default", "default", False, 0, False), # Never sampled. + (True, True, None, "default", "default", "default", None, None, True), # Uses adaptive sampling algo. + (True, True, None, "default", "always_on", "default", True, 3, False), # Always sampled. + (True, True, None, "default", "always_off", "default", False, 0, False), # Never sampled. + (True, False, None, "default", "default", "default", None, None, True), # Uses adaptive sampling algo. + (True, False, None, "default", "always_on", "default", None, None, True), # Uses adaptive sampling alog. + (True, False, None, "default", "always_off", "default", None, None, True), # Uses adaptive sampling algo. + (True, True, None, "default", "default", "always_on", None, None, True), # Uses adaptive sampling algo. + (True, True, None, "default", "default", "always_off", None, None, True), # Uses adaptive sampling algo. + (True, False, None, "default", "default", "always_on", True, 3, False), # Always sampled. + (True, False, None, "default", "default", "always_off", False, 0, False), # Never sampled. + ( + True, + True, + True, + "default", + "default", + "default", + True, + 1.23456, + False, + ), # Uses sampling decision in W3C TraceState header. + ( + True, + True, + False, + "default", + "default", + "default", + False, + 1.23456, + False, + ), # Uses sampling decision in W3C TraceState header. + ( + True, + False, + False, + "default", + "default", + "default", + False, + 1.23456, + False, + ), # Uses sampling decision in W3C TraceState header. + (True, True, False, "default", "always_on", "default", True, 3, False), # Always sampled. + (True, True, True, "default", "always_off", "default", False, 0, False), # Never sampled. + (True, False, False, "default", "default", "always_on", True, 3, False), # Always sampled. + (True, False, True, "default", "default", "always_off", False, 0, False), # Never sampled. + ( + True, + None, + True, + "default", + "default", + "default", + True, + 0.1234, + False, + ), # Uses sampling and priority from newrelic header. + (True, None, True, "default", "always_on", "default", True, 3, False), # Always sampled. + (True, None, True, "default", "always_off", "default", False, 0, False), # Never sampled. + ( + True, + None, + False, + "default", + "default", + "default", + False, + 0.1234, + False, + ), # Uses sampling and priority from newrelic header. + ( + True, + None, + False, + "default", + "always_on", + "default", + False, + 0.1234, + False, + ), # Uses sampling and priority from newrelic header. + ( + True, + None, + True, + "default", + "default", + "always_on", + True, + 0.1234, + False, + ), # Uses sampling and priority from newrelic header. + (True, None, False, "default", "default", "always_on", True, 3, False), # Always sampled. + (True, None, False, "default", "default", "always_off", False, 0, False), # Never sampled. + (True, None, None, "default", "default", "default", None, None, True), # Uses adaptive sampling algo. + ), +) +def test_distributed_trace_remote_parent_sampling_decision_full_granularity( + newrelic_header, + traceparent_sampled, + newrelic_sampled, + root_setting, + remote_parent_sampled_setting, + remote_parent_not_sampled_setting, + expected_sampled, + expected_priority, + expected_adaptive_sampling_algo_called, +): + required_intrinsics = [] + if expected_sampled is not None: + required_intrinsics.append(Attribute(name="sampled", value=expected_sampled, destinations=0b110)) + if expected_priority is not None: + required_intrinsics.append(Attribute(name="priority", value=expected_priority, destinations=0b110)) + + test_settings = _override_settings.copy() + test_settings.update( + { + "distributed_tracing.sampler._root": root_setting, + "distributed_tracing.sampler._remote_parent_sampled": remote_parent_sampled_setting, + "distributed_tracing.sampler._remote_parent_not_sampled": remote_parent_not_sampled_setting, + "span_events.enabled": True, + } + ) + if expected_adaptive_sampling_algo_called: + function_called_decorator = validate_function_called( + "newrelic.core.samplers.adaptive_sampler", "AdaptiveSampler.compute_sampled" + ) + else: + function_called_decorator = validate_function_not_called( + "newrelic.core.samplers.adaptive_sampler", "AdaptiveSampler.compute_sampled" + ) + + @function_called_decorator + @override_application_settings(test_settings) + @validate_attributes_complete("intrinsic", required_intrinsics) + @background_task(name="test_distributed_trace_attributes") + def _test(): + txn = current_transaction() + + headers = {} + if traceparent_sampled is not None: + headers = { + "traceparent": f"00-0af7651916cd43dd8448eb211c80319c-00f067aa0ba902b7-{int(traceparent_sampled):02x}", + "newrelic": '{"v":[0,1],"d":{"ty":"Mobile","ac":"123","ap":"51424","id":"5f474d64b9cc9b2a","tr":"6e2fea0b173fdad0","pr":0.1234,"sa":true,"ti":1482959525577,"tx":"27856f70d3d314b7"}}', # This header should be ignored. + } + if newrelic_sampled is not None: + headers["tracestate"] = ( + f"1@nr=0-0-1-2827902-0af7651916cd43dd-00f067aa0ba902b7-{int(newrelic_sampled)}-1.23456-1518469636035" + ) + elif newrelic_header: + headers = { + "newrelic": '{"v":[0,1],"d":{"ty":"Mobile","ac":"1","ap":"51424","id":"00f067aa0ba902b7","tr":"0af7651916cd43dd8448eb211c80319c","pr":0.1234,"sa":%s,"ti":1482959525577,"tx":"0af7651916cd43dd"}}' + % (str(newrelic_sampled).lower()) + } + if headers: + accept_distributed_trace_headers(headers) + + _test() + + +@pytest.mark.parametrize( + "newrelic_header,traceparent_sampled,newrelic_sampled,root_setting,remote_parent_sampled_setting,remote_parent_not_sampled_setting,expected_sampled,expected_priority,expected_adaptive_sampling_algo_called", ( - (True, "default", "default", None, None, True), # Uses sampling algo. - (True, "always_on", "default", True, 2, False), # Always sampled. - (True, "always_off", "default", False, 0, False), # Never sampled. - (False, "default", "default", None, None, True), # Uses sampling algo. - (False, "always_on", "default", None, None, True), # Uses sampling alog. - (False, "always_off", "default", None, None, True), # Uses sampling algo. - (True, "default", "always_on", None, None, True), # Uses sampling algo. - (True, "default", "always_off", None, None, True), # Uses sampling algo. - (False, "default", "always_on", True, 2, False), # Always sampled. - (False, "default", "always_off", False, 0, False), # Never sampled. + (False, None, None, "default", "default", "default", None, None, True), # Uses adaptive sampling algo. + (False, None, None, "always_on", "default", "default", True, 2, False), # Always sampled. + (False, None, None, "always_off", "default", "default", False, 0, False), # Never sampled. + (True, True, None, "default", "default", "default", None, None, True), # Uses adaptive sampling algo. + (True, True, None, "default", "always_on", "default", True, 2, False), # Always sampled. + (True, True, None, "default", "always_off", "default", False, 0, False), # Never sampled. + (True, False, None, "default", "default", "default", None, None, True), # Uses adaptive sampling algo. + (True, False, None, "default", "always_on", "default", None, None, True), # Uses adaptive sampling alog. + (True, False, None, "default", "always_off", "default", None, None, True), # Uses adaptive sampling algo. + (True, True, None, "default", "default", "always_on", None, None, True), # Uses adaptive sampling algo. + (True, True, None, "default", "default", "always_off", None, None, True), # Uses adaptive sampling algo. + (True, False, None, "default", "default", "always_on", True, 2, False), # Always sampled. + (True, False, None, "default", "default", "always_off", False, 0, False), # Never sampled. + ( + True, + True, + True, + "default", + "default", + "default", + True, + 1.23456, + False, + ), # Uses sampling decision in W3C TraceState header. + ( + True, + True, + False, + "default", + "default", + "default", + False, + 1.23456, + False, + ), # Uses sampling decision in W3C TraceState header. + ( + True, + False, + False, + "default", + "default", + "default", + False, + 1.23456, + False, + ), # Uses sampling decision in W3C TraceState header. + (True, True, False, "default", "always_on", "default", True, 2, False), # Always sampled. + (True, True, True, "default", "always_off", "default", False, 0, False), # Never sampled. + (True, False, False, "default", "default", "always_on", True, 2, False), # Always sampled. + (True, False, True, "default", "default", "always_off", False, 0, False), # Never sampled. + ( + True, + None, + True, + "default", + "default", + "default", + True, + 0.1234, + False, + ), # Uses sampling and priority from newrelic header. + (True, None, True, "default", "always_on", "default", True, 2, False), # Always sampled. + (True, None, True, "default", "always_off", "default", False, 0, False), # Never sampled. + ( + True, + None, + False, + "default", + "default", + "default", + False, + 0.1234, + False, + ), # Uses sampling and priority from newrelic header. + ( + True, + None, + False, + "default", + "always_on", + "default", + False, + 0.1234, + False, + ), # Uses sampling and priority from newrelic header. + ( + True, + None, + True, + "default", + "default", + "always_on", + True, + 0.1234, + False, + ), # Uses sampling and priority from newrelic header. + (True, None, False, "default", "default", "always_on", True, 2, False), # Always sampled. + (True, None, False, "default", "default", "always_off", False, 0, False), # Never sampled. + (True, None, None, "default", "default", "default", None, None, True), # Uses adaptive sampling algo. ), ) -def test_distributed_trace_w3cparent_sampling_decision( - sampled, - remote_parent_sampled, - remote_parent_not_sampled, +def test_distributed_trace_remote_parent_sampling_decision_partial_granularity( + newrelic_header, + traceparent_sampled, + newrelic_sampled, + root_setting, + remote_parent_sampled_setting, + remote_parent_not_sampled_setting, expected_sampled, expected_priority, expected_adaptive_sampling_algo_called, @@ -450,18 +832,21 @@ def test_distributed_trace_w3cparent_sampling_decision( test_settings = _override_settings.copy() test_settings.update( { - "distributed_tracing.sampler.remote_parent_sampled": remote_parent_sampled, - "distributed_tracing.sampler.remote_parent_not_sampled": remote_parent_not_sampled, + "distributed_tracing.sampler.full_granularity.enabled": False, + "distributed_tracing.sampler.partial_granularity.enabled": True, + "distributed_tracing.sampler.partial_granularity._root": root_setting, + "distributed_tracing.sampler.partial_granularity._remote_parent_sampled": remote_parent_sampled_setting, + "distributed_tracing.sampler.partial_granularity._remote_parent_not_sampled": remote_parent_not_sampled_setting, "span_events.enabled": True, } ) if expected_adaptive_sampling_algo_called: function_called_decorator = validate_function_called( - "newrelic.api.transaction", "Transaction.sampling_algo_compute_sampled_and_priority" + "newrelic.core.samplers.adaptive_sampler", "AdaptiveSampler.compute_sampled" ) else: function_called_decorator = validate_function_not_called( - "newrelic.api.transaction", "Transaction.sampling_algo_compute_sampled_and_priority" + "newrelic.core.samplers.adaptive_sampler", "AdaptiveSampler.compute_sampled" ) @function_called_decorator @@ -471,10 +856,836 @@ def test_distributed_trace_w3cparent_sampling_decision( def _test(): txn = current_transaction() + headers = {} + if traceparent_sampled is not None: + headers = { + "traceparent": f"00-0af7651916cd43dd8448eb211c80319c-00f067aa0ba902b7-{int(traceparent_sampled):02x}", + "newrelic": '{"v":[0,1],"d":{"ty":"Mobile","ac":"123","ap":"51424","id":"5f474d64b9cc9b2a","tr":"6e2fea0b173fdad0","pr":0.1234,"sa":true,"ti":1482959525577,"tx":"27856f70d3d314b7"}}', # This header should be ignored. + } + if newrelic_sampled is not None: + headers["tracestate"] = ( + f"1@nr=0-0-1-2827902-0af7651916cd43dd-00f067aa0ba902b7-{int(newrelic_sampled)}-1.23456-1518469636035" + ) + elif newrelic_header: + headers = { + "newrelic": '{"v":[0,1],"d":{"ty":"Mobile","ac":"1","ap":"51424","id":"00f067aa0ba902b7","tr":"0af7651916cd43dd8448eb211c80319c","pr":0.1234,"sa":%s,"ti":1482959525577,"tx":"0af7651916cd43dd"}}' + % (str(newrelic_sampled).lower()) + } + if headers: + accept_distributed_trace_headers(headers) + + _test() + + +@pytest.mark.parametrize( + "full_granularity_enabled,full_granularity_remote_parent_sampled_setting,partial_granularity_enabled,partial_granularity_remote_parent_sampled_setting,expected_sampled,expected_priority,expected_adaptive_sampling_algo_called", + ( + (True, "always_off", True, "adaptive", None, None, True), # Uses adaptive sampling algo. + (True, "always_on", True, "adaptive", True, 3, False), # Always samples. + ), +) +def test_distributed_trace_remote_parent_sampling_decision_between_full_and_partial_granularity( + full_granularity_enabled, + full_granularity_remote_parent_sampled_setting, + partial_granularity_enabled, + partial_granularity_remote_parent_sampled_setting, + expected_sampled, + expected_priority, + expected_adaptive_sampling_algo_called, +): + required_intrinsics = [] + if expected_sampled is not None: + required_intrinsics.append(Attribute(name="sampled", value=expected_sampled, destinations=0b110)) + if expected_priority is not None: + required_intrinsics.append(Attribute(name="priority", value=expected_priority, destinations=0b110)) + + test_settings = _override_settings.copy() + test_settings.update( + { + "distributed_tracing.sampler.full_granularity.enabled": full_granularity_enabled, + "distributed_tracing.sampler.partial_granularity.enabled": partial_granularity_enabled, + "distributed_tracing.sampler._remote_parent_sampled": full_granularity_remote_parent_sampled_setting, + "distributed_tracing.sampler.partial_granularity._remote_parent_sampled": partial_granularity_remote_parent_sampled_setting, + "span_events.enabled": True, + } + ) + if expected_adaptive_sampling_algo_called: + function_called_decorator = validate_function_called( + "newrelic.core.samplers.adaptive_sampler", "AdaptiveSampler.compute_sampled" + ) + else: + function_called_decorator = validate_function_not_called( + "newrelic.core.samplers.adaptive_sampler", "AdaptiveSampler.compute_sampled" + ) + + @function_called_decorator + @override_application_settings(test_settings) + @validate_attributes_complete("intrinsic", required_intrinsics) + @background_task(name="test_distributed_trace_attributes") + def _test(): + headers = {"traceparent": "00-0af7651916cd43dd8448eb211c80319c-00f067aa0ba902b7-01"} + accept_distributed_trace_headers(headers) + + _test() + + +def test_partial_granularity_entity_synthesis_attr_none_in_compact(): + """ + Tests no crash happens when an entity synthesis attribute is set to None. + """ + + @validate_span_events( + count=1, # Entry span. + exact_intrinsics={ + "name": "Function/test_distributed_tracing:test_partial_granularity_entity_synthesis_attr_none_in_compact.._test", + "nr.pg": True, + }, + expected_intrinsics=["duration", "timestamp"], + ) + @validate_span_events( + count=1, # 1 external compressed span. + exact_intrinsics={"name": "External/localhost:3000/requests/GET"}, + expected_intrinsics=["nr.durations", "nr.ids"], + exact_agents={"http.url": "http://localhost:3000/"}, + unexpected_agents=["db.instance"], + ) + @background_task() + def _test(): + headers = {"traceparent": "00-0af7651916cd43dd8448eb211c80319c-00f067aa0ba902b7-01"} + accept_distributed_trace_headers(headers) + with ExternalTrace("requests", "http://localhost:3000/", method="GET") as trace: + trace._add_agent_attribute("db.instance", None) + time.sleep(0.1) + + _test = override_application_settings( + { + "distributed_tracing.sampler.full_granularity.enabled": False, + "distributed_tracing.sampler.partial_granularity.enabled": True, + "distributed_tracing.sampler.partial_granularity.type": "compact", + "distributed_tracing.sampler.partial_granularity._remote_parent_sampled": "always_on", + "span_events.enabled": True, + } + )(_test) + + _test() + + +def test_partial_granularity_max_compressed_spans(): + """ + Tests `nr.ids` does not exceed 1024 byte limit. + """ + + async def test(index): + with ExternalTrace("requests", "http://localhost:3000/", method="GET") as trace: + time.sleep(0.1) + + @function_trace() + async def call_tests(): + tasks = [test(i) for i in range(65)] + await asyncio.gather(*tasks) + + @validate_span_events( + count=1, # Entry span. + exact_intrinsics={ + "name": "Function/test_distributed_tracing:test_partial_granularity_max_compressed_spans.._test", + "nr.pg": True, + }, + expected_intrinsics=["duration", "timestamp"], + ) + @validate_span_events( + count=1, # 1 external compressed span. + exact_intrinsics={"name": "External/localhost:3000/requests/GET"}, + expected_intrinsics=["nr.durations", "nr.ids"], + exact_agents={"http.url": "http://localhost:3000/"}, + ) + @validate_compact_span_event( + name="External/localhost:3000/requests/GET", + # `nr.ids` can only hold 63 ids but duration reflects all compressed spans. + compressed_span_count=64, + expected_nr_durations_low_bound=6.5, + expected_nr_durations_high_bound=8, # 64 of these + add extra overhead. + ) + @background_task() + def _test(): + headers = {"traceparent": "00-0af7651916cd43dd8448eb211c80319c-00f067aa0ba902b7-01"} + accept_distributed_trace_headers(headers) + asyncio.run(call_tests()) + + _test = override_application_settings( + { + "distributed_tracing.sampler.full_granularity.enabled": False, + "distributed_tracing.sampler.partial_granularity.enabled": True, + "distributed_tracing.sampler.partial_granularity.type": "compact", + "distributed_tracing.sampler.partial_granularity._remote_parent_sampled": "always_on", + "span_events.enabled": True, + } + )(_test) + + _test() + + +def test_partial_granularity_compressed_span_attributes_in_series(): + """ + Tests compressed span attributes when compressed span times are serial. + Aka: each span ends before the next compressed span begins. + """ + + async def test(index): + with ExternalTrace("requests", "http://localhost:3000/", method="GET") as trace: + time.sleep(0.1) + + @function_trace() + async def call_tests(): + tasks = [test(i) for i in range(3)] + await asyncio.gather(*tasks) + + @validate_span_events( + count=1, # Entry span. + exact_intrinsics={ + "name": "Function/test_distributed_tracing:test_partial_granularity_compressed_span_attributes_in_series.._test", + "nr.pg": True, + }, + expected_intrinsics=["duration", "timestamp"], + ) + @validate_span_events( + count=1, # 1 external compressed span. + exact_intrinsics={"name": "External/localhost:3000/requests/GET"}, + expected_intrinsics=["nr.durations", "nr.ids"], + exact_agents={"http.url": "http://localhost:3000/"}, + ) + @validate_compact_span_event( + name="External/localhost:3000/requests/GET", + compressed_span_count=3, + expected_nr_durations_low_bound=0.3, + expected_nr_durations_high_bound=0.4, + ) + @background_task() + def _test(): + headers = {"traceparent": "00-0af7651916cd43dd8448eb211c80319c-00f067aa0ba902b7-01"} + accept_distributed_trace_headers(headers) + asyncio.run(call_tests()) + + _test = override_application_settings( + { + "distributed_tracing.sampler.full_granularity.enabled": False, + "distributed_tracing.sampler.partial_granularity.enabled": True, + "distributed_tracing.sampler.partial_granularity.type": "compact", + "distributed_tracing.sampler.partial_granularity._remote_parent_sampled": "always_on", + "span_events.enabled": True, + } + )(_test) + + _test() + + +def test_partial_granularity_compressed_span_attributes_overlapping(): + """ + Tests compressed span attributes when compressed span times overlap. + Aka: the next span begins in the middle of the first span. + """ + + @validate_span_events( + count=1, # Entry span. + exact_intrinsics={ + "name": "Function/test_distributed_tracing:test_partial_granularity_compressed_span_attributes_overlapping.._test", + "nr.pg": True, + }, + expected_intrinsics=["duration", "timestamp"], + ) + @validate_span_events( + count=1, # 1 external compressed span. + exact_intrinsics={"name": "External/localhost:3000/requests/GET"}, + expected_intrinsics=["nr.durations", "nr.ids"], + exact_agents={"http.url": "http://localhost:3000/"}, + ) + @validate_compact_span_event( + name="External/localhost:3000/requests/GET", + compressed_span_count=2, + expected_nr_durations_low_bound=0.1, + expected_nr_durations_high_bound=0.2, + ) + @background_task() + def _test(): + headers = {"traceparent": "00-0af7651916cd43dd8448eb211c80319c-00f067aa0ba902b7-01"} + accept_distributed_trace_headers(headers) + with ExternalTrace("requests", "http://localhost:3000/", method="GET") as trace1: + # Override terminal_node so we can create a nested exit span. + trace1.terminal_node = lambda: False + trace2 = ExternalTrace("requests", "http://localhost:3000/", method="GET") + trace2.__enter__() + time.sleep(0.1) + trace2.__exit__(None, None, None) + + _test = override_application_settings( + { + "distributed_tracing.sampler.full_granularity.enabled": False, + "distributed_tracing.sampler.partial_granularity.enabled": True, + "distributed_tracing.sampler.partial_granularity.type": "compact", + "distributed_tracing.sampler.partial_granularity._remote_parent_sampled": "always_on", + "span_events.enabled": True, + } + )(_test) + + _test() + + +def test_partial_granularity_reduced_span_attributes(): + """ + In reduced mode, only inprocess spans are dropped. + """ + + @function_trace() + def foo(): + with ExternalTrace("requests", "http://localhost:3000/", method="GET") as trace: + trace.add_custom_attribute("custom", "bar") + + @validate_span_events( + count=1, # Entry span. + exact_intrinsics={ + "name": "Function/test_distributed_tracing:test_partial_granularity_reduced_span_attributes.._test", + "nr.pg": True, + }, + expected_intrinsics=["duration", "timestamp"], + expected_agents=["code.function", "code.lineno", "code.namespace"], + ) + @validate_span_events( + count=0, # Function foo span should not be present. + exact_intrinsics={ + "name": "Function/test_distributed_tracing:test_partial_granularity_reduced_span_attributes..foo" + }, + expected_intrinsics=["duration", "timestamp"], + ) + @validate_span_events( + count=2, # 2 external spans. + exact_intrinsics={"name": "External/localhost:3000/requests/GET"}, + exact_agents={"http.url": "http://localhost:3000/"}, + exact_users={"custom": "bar"}, + ) + @background_task() + def _test(): + headers = {"traceparent": "00-0af7651916cd43dd8448eb211c80319c-00f067aa0ba902b7-01"} + accept_distributed_trace_headers(headers) + with ExternalTrace("requests", "http://localhost:3000/", method="GET") as trace: + # Override terminal_node so we can create a nested exit span. + trace.terminal_node = lambda: False + trace.add_custom_attribute("custom", "bar") + foo() + + _test = override_application_settings( + { + "distributed_tracing.sampler.full_granularity.enabled": False, + "distributed_tracing.sampler.partial_granularity.enabled": True, + "distributed_tracing.sampler.partial_granularity.type": "reduced", + "distributed_tracing.sampler.partial_granularity._remote_parent_sampled": "always_on", + "span_events.enabled": True, + } + )(_test) + + _test() + + +def test_partial_granularity_essential_span_attributes(): + """ + In essential mode, inprocess spans are dropped and non-entity synthesis attributes. + """ + + @function_trace() + def foo(): + with ExternalTrace("requests", "http://localhost:3000/", method="GET") as trace: + trace.add_custom_attribute("custom", "bar") + + @validate_span_events( + count=1, # Entry span. + exact_intrinsics={ + "name": "Function/test_distributed_tracing:test_partial_granularity_essential_span_attributes.._test", + "nr.pg": True, + }, + expected_intrinsics=["duration", "timestamp"], + unexpected_agents=["code.function", "code.lineno", "code.namespace"], + ) + @validate_span_events( + count=0, # Function foo span should not be present. + exact_intrinsics={ + "name": "Function/test_distributed_tracing:test_partial_granularity_essential_span_attributes..foo" + }, + expected_intrinsics=["duration", "timestamp"], + ) + @validate_span_events( + count=2, # 2 external spans. + exact_intrinsics={"name": "External/localhost:3000/requests/GET"}, + exact_agents={"http.url": "http://localhost:3000/"}, + unexpected_users=["custom"], + ) + @background_task() + def _test(): + headers = {"traceparent": "00-0af7651916cd43dd8448eb211c80319c-00f067aa0ba902b7-01"} + accept_distributed_trace_headers(headers) + with ExternalTrace("requests", "http://localhost:3000/", method="GET") as trace: + # Override terminal_node so we can create a nested exit span. + trace.terminal_node = lambda: False + trace.add_custom_attribute("custom", "bar") + foo() + + _test = override_application_settings( + { + "distributed_tracing.sampler.full_granularity.enabled": False, + "distributed_tracing.sampler.partial_granularity.enabled": True, + "distributed_tracing.sampler.partial_granularity.type": "essential", + "distributed_tracing.sampler.partial_granularity._remote_parent_sampled": "always_on", + "span_events.enabled": True, + } + )(_test) + + _test() + + +def test_partial_granularity_unknown_type_falls_back_on_essential(): + """ + In essential mode, inprocess spans are dropped and non-entity synthesis attributes. + """ + + @function_trace() + def foo(): + with ExternalTrace("requests", "http://localhost:3000/", method="GET") as trace: + trace.add_custom_attribute("custom", "bar") + + @validate_span_events( + count=1, # Entry span. + exact_intrinsics={ + "name": "Function/test_distributed_tracing:test_partial_granularity_unknown_type_falls_back_on_essential.._test", + "nr.pg": True, + }, + expected_intrinsics=["duration", "timestamp"], + unexpected_agents=["code.function", "code.lineno", "code.namespace"], + ) + @validate_span_events( + count=0, # Function foo span should not be present. + exact_intrinsics={ + "name": "Function/test_distributed_tracing:test_partial_granularity_unknown_type_falls_back_on_essential..foo" + }, + expected_intrinsics=["duration", "timestamp"], + ) + @validate_span_events( + count=2, # 2 external spans. + exact_intrinsics={"name": "External/localhost:3000/requests/GET"}, + exact_agents={"http.url": "http://localhost:3000/"}, + unexpected_users=["custom"], + ) + @background_task() + def _test(): + headers = {"traceparent": "00-0af7651916cd43dd8448eb211c80319c-00f067aa0ba902b7-01"} + accept_distributed_trace_headers(headers) + with ExternalTrace("requests", "http://localhost:3000/", method="GET") as trace: + # Override terminal_node so we can create a nested exit span. + trace.terminal_node = lambda: False + trace.add_custom_attribute("custom", "bar") + foo() + + _test = override_application_settings( + { + "distributed_tracing.sampler.full_granularity.enabled": False, + "distributed_tracing.sampler.partial_granularity.enabled": True, + "distributed_tracing.sampler.partial_granularity.type": "unknown", + "distributed_tracing.sampler.partial_granularity._remote_parent_sampled": "always_on", + "span_events.enabled": True, + } + )(_test) + + _test() + + +@pytest.mark.parametrize( + "dt_settings,dt_headers,expected_sampling_instance_called,expected_adaptive_computed_count,expected_adaptive_sampled_count,expected_adaptive_sampling_target", + ( + ( + { + "distributed_tracing.sampler.full_granularity.enabled": False, + "distributed_tracing.sampler.partial_granularity.enabled": True, + "distributed_tracing.sampler.partial_granularity._root": "default", + "distributed_tracing.sampler.partial_granularity.root.adaptive.sampling_target": 5, + }, + {}, + (False, 0), + 1, + 1, + 5, + ), + ( + { + "distributed_tracing.sampler.full_granularity.enabled": False, + "distributed_tracing.sampler.partial_granularity.enabled": True, + "distributed_tracing.sampler.partial_granularity._remote_parent_sampled": "default", + "distributed_tracing.sampler.partial_granularity.remote_parent_sampled.adaptive.sampling_target": 5, + "distributed_tracing.sampler.partial_granularity._remote_parent_not_sampled": "default", + "distributed_tracing.sampler.partial_granularity.remote_parent_not_sampled.adaptive.sampling_target": 6, + }, + {"traceparent": "00-0af7651916cd43dd8448eb211c80319c-00f067aa0ba902b7-01"}, + (False, 1), + 1, + 1, + 5, + ), + ( + { + "distributed_tracing.sampler.full_granularity.enabled": False, + "distributed_tracing.sampler.partial_granularity.enabled": True, + "distributed_tracing.sampler.partial_granularity._remote_parent_sampled": "default", + "distributed_tracing.sampler.partial_granularity.remote_parent_sampled.adaptive.sampling_target": 5, + "distributed_tracing.sampler.partial_granularity._remote_parent_not_sampled": "default", + "distributed_tracing.sampler.partial_granularity.remote_parent_not_sampled.adaptive.sampling_target": 6, + }, + {"traceparent": "00-0af7651916cd43dd8448eb211c80319c-00f067aa0ba902b7-00"}, + (False, 2), + 1, + 1, + 6, + ), + ( + { + "distributed_tracing.sampler.full_granularity.enabled": True, + "distributed_tracing.sampler.partial_granularity.enabled": False, + "distributed_tracing.sampler._root": "default", + "distributed_tracing.sampler.root.adaptive.sampling_target": 5, + }, + {}, + (True, 0), + 1, + 1, + 5, + ), + ( + { + "distributed_tracing.sampler.full_granularity.enabled": True, + "distributed_tracing.sampler.partial_granularity.enabled": False, + "distributed_tracing.sampler._remote_parent_sampled": "default", + "distributed_tracing.sampler.remote_parent_sampled.adaptive.sampling_target": 5, + "distributed_tracing.sampler._remote_parent_not_sampled": "default", + "distributed_tracing.sampler.remote_parent_not_sampled.adaptive.sampling_target": 6, + }, + {"traceparent": "00-0af7651916cd43dd8448eb211c80319c-00f067aa0ba902b7-01"}, + (True, 1), + 1, + 1, + 5, + ), + ( + { + "distributed_tracing.sampler.full_granularity.enabled": True, + "distributed_tracing.sampler.partial_granularity.enabled": False, + "distributed_tracing.sampler._remote_parent_sampled": "default", + "distributed_tracing.sampler.remote_parent_sampled.adaptive.sampling_target": 5, + "distributed_tracing.sampler._remote_parent_not_sampled": "default", + "distributed_tracing.sampler.remote_parent_not_sampled.adaptive.sampling_target": 6, + }, + {"traceparent": "00-0af7651916cd43dd8448eb211c80319c-00f067aa0ba902b7-00"}, + (True, 2), + 1, + 1, + 6, + ), + ), +) +def test_distributed_trace_uses_adaptive_sampling_instance( + dt_settings, + dt_headers, + expected_sampling_instance_called, + expected_adaptive_computed_count, + expected_adaptive_sampled_count, + expected_adaptive_sampling_target, +): + test_settings = _override_settings.copy() + test_settings.update(dt_settings) + function_called_decorator = validate_function_called( + "newrelic.core.samplers.adaptive_sampler", "AdaptiveSampler.compute_sampled" + ) + + @function_called_decorator + @override_application_settings(test_settings) + @background_task(name="test_distributed_trace_attributes") + def _test(): + txn = current_transaction() + application = txn._application._agent._applications.get(txn.settings.app_name) + # Re-initialize sampler proxy after overriding settings. + application.sampler.__init__(txn.settings) + + accept_distributed_trace_headers(dt_headers) + # Explicitly call this so we can assert sampling decision during the transaction + # as opposed to after it ends and we lose the application context. + txn._make_sampling_decision() + + assert ( + application.sampler._samplers[expected_sampling_instance_called].computed_count + == expected_adaptive_computed_count + ) + assert ( + application.sampler._samplers[expected_sampling_instance_called].sampled_count + == expected_adaptive_sampled_count + ) + assert ( + application.sampler._samplers[expected_sampling_instance_called].sampling_target + == expected_adaptive_sampling_target + ) + + _test() + + +@pytest.mark.parametrize( + "dt_settings,dt_headers,expected_sampling_instance_called,expected_ratio", + ( + ( # Ratio for partial granularity does not exceed 1. + { + "distributed_tracing.sampler.full_granularity.enabled": True, + "distributed_tracing.sampler.partial_granularity.enabled": True, + "distributed_tracing.sampler.remote_parent_sampled.trace_id_ratio_based.ratio": 0.5, + "distributed_tracing.sampler._remote_parent_sampled": "trace_id_ratio_based", + "distributed_tracing.sampler.partial_granularity.remote_parent_sampled.trace_id_ratio_based.ratio": 0.7, + "distributed_tracing.sampler.partial_granularity._remote_parent_sampled": "trace_id_ratio_based", + }, + {"traceparent": "00-0af7651916cd43dd8448eb211c80319c-00f067aa0ba902b7-01"}, + (False, 1), + 1, + ), + ( # Partial granularity ratio = full ratio + partial ratio. + { + "distributed_tracing.sampler.full_granularity.enabled": True, + "distributed_tracing.sampler.partial_granularity.enabled": True, + "distributed_tracing.sampler.remote_parent_sampled.trace_id_ratio_based.ratio": 0.5, + "distributed_tracing.sampler._remote_parent_sampled": "trace_id_ratio_based", + "distributed_tracing.sampler.partial_granularity.remote_parent_sampled.trace_id_ratio_based.ratio": 0.5, + "distributed_tracing.sampler.partial_granularity._remote_parent_sampled": "trace_id_ratio_based", + }, + {"traceparent": "00-0af7651916cd43dd8448eb211c80319c-00f067aa0ba902b7-01"}, + (False, 1), + 1, + ), + ( # Trace ID ratio sampler is called for full granularity. + { + "distributed_tracing.sampler.full_granularity.enabled": True, + "distributed_tracing.sampler.partial_granularity.enabled": False, + "distributed_tracing.sampler.remote_parent_sampled.trace_id_ratio_based.ratio": 1, + "distributed_tracing.sampler._remote_parent_sampled": "trace_id_ratio_based", + }, + {"traceparent": "00-0af7651916cd43dd8448eb211c80319c-00f067aa0ba902b7-01"}, + (True, 1), + 1, + ), + ), +) +def test_distributed_trace_uses_ratio_sampling_instance( + dt_settings, dt_headers, expected_sampling_instance_called, expected_ratio +): + test_settings = _override_settings.copy() + test_settings.update(dt_settings) + function_called_decorator = validate_function_called( + "newrelic.core.samplers.trace_id_ratio_based_sampler", "TraceIdRatioBasedSampler.compute_sampled" + ) + + @function_called_decorator + @override_application_settings(test_settings) + @background_task(name="test_distributed_trace_attributes") + def _test(): + txn = current_transaction() + application = txn._application._agent._applications.get(txn.settings.app_name) + # Re-initialize sampler proxy after overriding settings. + application.sampler.__init__(txn.settings) + + accept_distributed_trace_headers(dt_headers) + # Explicitly call this so we can assert sampling decision during the transaction + # as opposed to after it ends and we lose the application context. + txn._make_sampling_decision() + + assert application.sampler._samplers[expected_sampling_instance_called].ratio == expected_ratio + + _test() + + +@pytest.mark.parametrize( + "dt_settings,expected_priority,expected_sampled", + ( + ( # When dt is enabled but full and partial are disabled. + { + "distributed_tracing.sampler.full_granularity.enabled": False, + "distributed_tracing.sampler.partial_granularity.enabled": False, + }, + 0.123, # random + False, + ), + ( # When dt is disabled. + {"distributed_tracing.enabled": False}, + 0.123, # random + False, + ), + ( # Verify when full granularity sampled +2 is added to the priority. + {"distributed_tracing.sampler.root.trace_id_ratio_based.ratio": 1}, + 2.123, # random + 2 + True, + ), + ( # Verify when partial granularity sampled +1 is added to the priority. + { + "distributed_tracing.sampler.full_granularity.enabled": False, + "distributed_tracing.sampler.partial_granularity.enabled": True, + "distributed_tracing.sampler.partial_granularity.root.trace_id_ratio_based.ratio": 1, + }, + 1.123, # random + 1 + True, + ), + ), +) +def test_distributed_trace_enabled_settings_set_correct_sampled_priority( + dt_settings, expected_priority, expected_sampled, monkeypatch +): + monkeypatch.setattr(random, "random", lambda *args, **kwargs: 0.123) + + test_settings = _override_settings.copy() + test_settings.update(dt_settings) + + @override_application_settings(test_settings) + @validate_transaction_object_attributes({"sampled": expected_sampled, "priority": expected_priority}) + @background_task(name="test_distributed_trace_attributes") + def _test(): + pass + + _test() + + +def test_distributed_trace_priority_set_when_only_sampled_set_in_tracestate_header(monkeypatch): + monkeypatch.setattr(random, "random", lambda *args, **kwargs: 0.123) + + @override_application_settings(_override_settings) + @validate_transaction_object_attributes({"sampled": True, "priority": 2.123}) + @background_task() + def _test(): headers = { - "traceparent": f"00-0af7651916cd43dd8448eb211c80319c-00f067aa0ba902b7-{int(sampled):02x}", - "tracestate": "rojo=f06a0ba902b7,congo=t61rcWkgMzE", + "traceparent": "00-0af7651916cd43dd8448eb211c80319c-00f067aa0ba902b7-01", + "tracestate": "1@nr=0-0-1-2827902-0af7651916cd43dd-00f067aa0ba902b7-1--1518469636035", + } + accept_distributed_trace_headers(headers) + + _test() + + +def test_partial_granularity_errors_on_compressed_spans(): + @function_trace() + def call_tests(): + with ExternalTrace("requests", "http://localhost:3000/", method="GET") as trace: + time.sleep(0.1) + transaction = current_transaction() + try: + raise Exception("Exception 1") + except: + transaction.notice_error() + with ExternalTrace("requests", "http://localhost:3000/", method="GET") as trace: + time.sleep(0.1) + with ExternalTrace("requests", "http://localhost:3000/", method="GET") as trace: + time.sleep(0.1) + transaction = current_transaction() + try: + raise Exception("Exception 2") + except: + transaction.notice_error(expected=True) + + @validate_span_events( + count=1, # Entry span. + exact_intrinsics={ + "name": "Function/test_distributed_tracing:test_partial_granularity_errors_on_compressed_spans.._test", + "nr.pg": True, + }, + expected_intrinsics=["duration", "timestamp"], + ) + @validate_span_events( + count=1, # 1 external compressed span. + exact_intrinsics={"name": "External/localhost:3000/requests/GET"}, + expected_intrinsics=["nr.durations", "nr.ids"], + exact_agents={ + "http.url": "http://localhost:3000/", + "error.class": callable_name(Exception), + "error.message": "Exception 2", + "error.expected": True, + }, + ) + @validate_compact_span_event( + name="External/localhost:3000/requests/GET", + compressed_span_count=3, + expected_nr_durations_low_bound=0.3, + expected_nr_durations_high_bound=0.4, + ) + @background_task() + def _test(): + headers = {"traceparent": "00-0af7651916cd43dd8448eb211c80319c-00f067aa0ba902b7-01"} + accept_distributed_trace_headers(headers) + call_tests() + + _test = override_application_settings( + { + "distributed_tracing.sampler.full_granularity.enabled": False, + "distributed_tracing.sampler.partial_granularity.enabled": True, + "distributed_tracing.sampler.partial_granularity.type": "compact", + "distributed_tracing.sampler.partial_granularity._remote_parent_sampled": "always_on", + "span_events.enabled": True, } + )(_test) + + _test() + + +def test_partial_granularity_errors_on_compressed_spans_status_overriden(): + @function_trace() + def call_tests(): + transaction = current_transaction() + with ExternalTrace("requests", "http://localhost:3000/", method="GET") as trace: + time.sleep(0.1) + try: + raise Exception("Exception 1") + except: + transaction.notice_error(expected=True) + with ExternalTrace("requests", "http://localhost:3000/", method="GET") as trace: + time.sleep(0.1) + with ExternalTrace("requests", "http://localhost:3000/", method="GET") as trace: + time.sleep(0.1) + try: + raise Exception("Exception 2") + except: + transaction.notice_error() + + @validate_span_events( + count=1, # Entry span. + exact_intrinsics={ + "name": "Function/test_distributed_tracing:test_partial_granularity_errors_on_compressed_spans_status_overriden.._test", + "nr.pg": True, + }, + expected_intrinsics=["duration", "timestamp"], + ) + @validate_span_events( + count=1, # 1 external compressed span. + exact_intrinsics={"name": "External/localhost:3000/requests/GET"}, + expected_intrinsics=["nr.durations", "nr.ids"], + exact_agents={ + "http.url": "http://localhost:3000/", + "error.class": callable_name(Exception), + "error.message": "Exception 2", + "error.expected": False, + }, + ) + @validate_compact_span_event( + name="External/localhost:3000/requests/GET", + compressed_span_count=3, + expected_nr_durations_low_bound=0.3, + expected_nr_durations_high_bound=0.4, + ) + @background_task() + def _test(): + headers = {"traceparent": "00-0af7651916cd43dd8448eb211c80319c-00f067aa0ba902b7-01"} accept_distributed_trace_headers(headers) + call_tests() + + _test = override_application_settings( + { + "distributed_tracing.sampler.full_granularity.enabled": False, + "distributed_tracing.sampler.partial_granularity.enabled": True, + "distributed_tracing.sampler.partial_granularity.type": "compact", + "distributed_tracing.sampler.partial_granularity._remote_parent_sampled": "always_on", + "span_events.enabled": True, + } + )(_test) _test() diff --git a/tests/agent_features/test_event_loop_wait_time.py b/tests/agent_features/test_event_loop_wait_time.py index cad4679600..37e2bf9f33 100644 --- a/tests/agent_features/test_event_loop_wait_time.py +++ b/tests/agent_features/test_event_loop_wait_time.py @@ -27,6 +27,16 @@ from newrelic.core.trace_cache import trace_cache +@pytest.fixture +def event_loop(): + # Redefined fixture with function scope instead of session. + from asyncio import new_event_loop, set_event_loop + + loop = new_event_loop() + set_event_loop(loop) + return loop + + @background_task(name="block") async def block_loop(ready, done, blocking_transaction_active, times=1): for _ in range(times): @@ -64,8 +74,6 @@ async def wait_for_loop(ready, done, times=1): "blocking_transaction_active,event_loop_visibility_enabled", ((True, True), (False, True), (False, False)) ) def test_record_event_loop_wait(event_loop, blocking_transaction_active, event_loop_visibility_enabled): - # import asyncio - metric_count = 2 if event_loop_visibility_enabled else None execute_attributes = {"intrinsic": ("eventLoopTime",), "agent": (), "user": ()} wait_attributes = {"intrinsic": ("eventLoopWait",), "agent": (), "user": ()} @@ -143,8 +151,6 @@ def test_blocking_task_on_different_loop(): def test_record_event_loop_wait_on_different_task(event_loop): - # import asyncio - async def recorder(ready, wait): ready.set() await wait.wait() diff --git a/tests/agent_features/test_notice_error.py b/tests/agent_features/test_notice_error.py index e698dee7be..60e617f9de 100644 --- a/tests/agent_features/test_notice_error.py +++ b/tests/agent_features/test_notice_error.py @@ -39,10 +39,8 @@ # =============== Test errors during a transaction =============== -_test_notice_error_sys_exc_info = [(_runtime_error_name, "one")] - -@validate_transaction_errors(errors=_test_notice_error_sys_exc_info) +@validate_transaction_errors(errors=[(_runtime_error_name, "one")]) @background_task() def test_notice_error_sys_exc_info(): try: @@ -51,10 +49,7 @@ def test_notice_error_sys_exc_info(): notice_error(sys.exc_info()) -_test_notice_error_no_exc_info = [(_runtime_error_name, "one")] - - -@validate_transaction_errors(errors=_test_notice_error_no_exc_info) +@validate_transaction_errors(errors=[(_runtime_error_name, "one")]) @background_task() def test_notice_error_no_exc_info(): try: @@ -63,10 +58,44 @@ def test_notice_error_no_exc_info(): notice_error() -_test_notice_error_custom_params = [(_runtime_error_name, "one")] +@validate_transaction_errors(errors=[(_runtime_error_name, "one")]) +@background_task() +def test_notice_error_exception_instance(): + """Test that notice_error works when passed an exception object directly""" + try: + raise RuntimeError("one") + except RuntimeError as e: + exc = e # Reassign name to ensure scope isn't lost + + # Call notice_error outside of try/except block to ensure it's not pulling from sys.exc_info() + notice_error(exc) + + +@validate_transaction_errors(errors=[(_runtime_error_name, "one"), (_type_error_name, "two")]) +@background_task() +def test_notice_error_exception_instance_multiple_exceptions(): + """Test that notice_error reports the passed exception object even when a different exception is active.""" + try: + raise RuntimeError("one") + except RuntimeError as e: + exc1 = e # Reassign name to ensure scope isn't lost + + try: + raise TypeError("two") + except TypeError as exc2: + notice_error(exc1) + notice_error(exc2) + + +@validate_transaction_error_event_count(0) +@background_task() +def test_notice_error_exception_instance_no_traceback(): + """Test that notice_error does not report an exception if it has not been raised as it has no __traceback__""" + exc = RuntimeError("one") + notice_error(exc) # Try once with no active exception -@validate_transaction_errors(errors=_test_notice_error_custom_params, required_params=[("key", "value")]) +@validate_transaction_errors(errors=[(_runtime_error_name, "one")], required_params=[("key", "value")]) @background_task() def test_notice_error_custom_params(): try: @@ -75,10 +104,7 @@ def test_notice_error_custom_params(): notice_error(sys.exc_info(), attributes={"key": "value"}) -_test_notice_error_multiple_different_type = [(_runtime_error_name, "one"), (_type_error_name, "two")] - - -@validate_transaction_errors(errors=_test_notice_error_multiple_different_type) +@validate_transaction_errors(errors=[(_runtime_error_name, "one"), (_type_error_name, "two")]) @background_task() def test_notice_error_multiple_different_type(): try: @@ -92,10 +118,7 @@ def test_notice_error_multiple_different_type(): notice_error() -_test_notice_error_multiple_same_type = [(_runtime_error_name, "one"), (_runtime_error_name, "two")] - - -@validate_transaction_errors(errors=_test_notice_error_multiple_same_type) +@validate_transaction_errors(errors=[(_runtime_error_name, "one"), (_runtime_error_name, "two")]) @background_task() def test_notice_error_multiple_same_type(): try: @@ -111,11 +134,9 @@ def test_notice_error_multiple_same_type(): # =============== Test errors outside a transaction =============== -_test_application_exception = [(_runtime_error_name, "one")] - @reset_core_stats_engine() -@validate_application_errors(errors=_test_application_exception) +@validate_application_errors(errors=[(_runtime_error_name, "one")]) def test_application_exception(): try: raise RuntimeError("one") @@ -124,11 +145,8 @@ def test_application_exception(): notice_error(application=application_instance) -_test_application_exception_sys_exc_info = [(_runtime_error_name, "one")] - - @reset_core_stats_engine() -@validate_application_errors(errors=_test_application_exception_sys_exc_info) +@validate_application_errors(errors=[(_runtime_error_name, "one")]) def test_application_exception_sys_exec_info(): try: raise RuntimeError("one") @@ -137,11 +155,8 @@ def test_application_exception_sys_exec_info(): notice_error(sys.exc_info(), application=application_instance) -_test_application_exception_custom_params = [(_runtime_error_name, "one")] - - @reset_core_stats_engine() -@validate_application_errors(errors=_test_application_exception_custom_params, required_params=[("key", "value")]) +@validate_application_errors(errors=[(_runtime_error_name, "one")], required_params=[("key", "value")]) def test_application_exception_custom_params(): try: raise RuntimeError("one") @@ -150,11 +165,8 @@ def test_application_exception_custom_params(): notice_error(attributes={"key": "value"}, application=application_instance) -_test_application_exception_multiple = [(_runtime_error_name, "one"), (_runtime_error_name, "one")] - - @reset_core_stats_engine() -@validate_application_errors(errors=_test_application_exception_multiple) +@validate_application_errors(errors=[(_runtime_error_name, "one"), (_runtime_error_name, "one")]) @background_task() def test_application_exception_multiple(): """Exceptions submitted straight to the stats engine doesn't check for @@ -174,12 +186,11 @@ def test_application_exception_multiple(): # =============== Test exception message stripping/allowlisting =============== -_test_notice_error_strip_message_disabled = [(_runtime_error_name, "one")] _strip_message_disabled_settings = {"strip_exception_messages.enabled": False} -@validate_transaction_errors(errors=_test_notice_error_strip_message_disabled) +@validate_transaction_errors(errors=[(_runtime_error_name, "one")]) @override_application_settings(_strip_message_disabled_settings) @background_task() def test_notice_error_strip_message_disabled(): @@ -215,12 +226,10 @@ def test_notice_error_strip_message_disabled_outside_transaction(): assert my_error.message == ErrorOne.message -_test_notice_error_strip_message_enabled = [(_runtime_error_name, STRIP_EXCEPTION_MESSAGE)] - _strip_message_enabled_settings = {"strip_exception_messages.enabled": True} -@validate_transaction_errors(errors=_test_notice_error_strip_message_enabled) +@validate_transaction_errors(errors=[(_runtime_error_name, STRIP_EXCEPTION_MESSAGE)]) @override_application_settings(_strip_message_enabled_settings) @background_task() def test_notice_error_strip_message_enabled(): @@ -256,15 +265,13 @@ def test_notice_error_strip_message_enabled_outside_transaction(): assert my_error.message == STRIP_EXCEPTION_MESSAGE -_test_notice_error_strip_message_in_allowlist = [(_runtime_error_name, "original error message")] - _strip_message_in_allowlist_settings = { "strip_exception_messages.enabled": True, "strip_exception_messages.allowlist": [_runtime_error_name], } -@validate_transaction_errors(errors=_test_notice_error_strip_message_in_allowlist) +@validate_transaction_errors(errors=[(_runtime_error_name, "original error message")]) @override_application_settings(_strip_message_in_allowlist_settings) @background_task() def test_notice_error_strip_message_in_allowlist(): @@ -307,15 +314,13 @@ def test_notice_error_strip_message_in_allowlist_outside_transaction(): assert my_error.message == ErrorThree.message -_test_notice_error_strip_message_not_in_allowlist = [(_runtime_error_name, STRIP_EXCEPTION_MESSAGE)] - _strip_message_not_in_allowlist_settings = { "strip_exception_messages.enabled": True, "strip_exception_messages.allowlist": ["FooError", "BarError"], } -@validate_transaction_errors(errors=_test_notice_error_strip_message_not_in_allowlist) +@validate_transaction_errors(errors=[(_runtime_error_name, STRIP_EXCEPTION_MESSAGE)]) @override_application_settings(_strip_message_not_in_allowlist_settings) @background_task() def test_notice_error_strip_message_not_in_allowlist(): diff --git a/tests/agent_unittests/test_agent_connect.py b/tests/agent_unittests/test_agent_connect.py index a783faddcf..9a37f4ffc5 100644 --- a/tests/agent_unittests/test_agent_connect.py +++ b/tests/agent_unittests/test_agent_connect.py @@ -76,6 +76,26 @@ def test_ml_streaming_disabled_supportability_metrics(): assert app._active_session +@override_generic_settings( + SETTINGS, {"developer_mode": True, "distributed_tracing.sampler.partial_granularity.enabled": True} +) +@validate_internal_metrics( + [ + ("Supportability/Python/FullGranularity/Root/default", 1), + ("Supportability/Python/FullGranularity/RemoteParentSampled/default", 1), + ("Supportability/Python/FullGranularity/RemoteParentNotSampled/default", 1), + ("Supportability/Python/PartialGranularity/Root/default", 1), + ("Supportability/Python/PartialGranularity/RemoteParentSampled/default", 1), + ("Supportability/Python/PartialGranularity/RemoteParentNotSampled/default", 1), + ] +) +def test_sampler_supportability_metrics(): + app = Application("Python Agent Test (agent_unittests-connect)") + app.connect_to_data_collector(None) + + assert app._active_session + + @override_generic_settings(SETTINGS, {"developer_mode": True}) @validate_internal_metrics([("Supportability/AgentControl/Health/enabled", 1)]) def test_agent_control_health_supportability_metric(monkeypatch, tmp_path): diff --git a/tests/agent_unittests/test_agent_protocol.py b/tests/agent_unittests/test_agent_protocol.py index e6f0a04af3..8d9e353978 100644 --- a/tests/agent_unittests/test_agent_protocol.py +++ b/tests/agent_unittests/test_agent_protocol.py @@ -278,10 +278,11 @@ def connect_payload_asserts( assert len(payload_data["security_settings"]) == 2 assert payload_data["security_settings"]["capture_params"] == CAPTURE_PARAMS assert payload_data["security_settings"]["transaction_tracer"] == {"record_sql": RECORD_SQL} - assert len(payload_data["settings"]) == 3 + assert len(payload_data["settings"]) == 4 assert payload_data["settings"]["browser_monitoring.loader"] == (BROWSER_MONITORING_LOADER) assert payload_data["settings"]["browser_monitoring.debug"] == (BROWSER_MONITORING_DEBUG) assert payload_data["settings"]["ai_monitoring.enabled"] is False + assert payload_data["settings"]["distributed_tracing.sampler.adaptive_sampling_target"] == 10 utilization_len = 5 diff --git a/tests/agent_unittests/test_distributed_tracing_settings.py b/tests/agent_unittests/test_distributed_tracing_settings.py index a1c99da58d..0a65d566d5 100644 --- a/tests/agent_unittests/test_distributed_tracing_settings.py +++ b/tests/agent_unittests/test_distributed_tracing_settings.py @@ -14,6 +14,8 @@ import pytest +from newrelic.core.config import finalize_application_settings + INI_FILE_EMPTY = b""" [newrelic] """ @@ -24,9 +26,348 @@ distributed_tracing.exclude_newrelic_header = true """ +INI_FILE_FULL_GRAN_CONFLICTS_ADAPTIVE = b""" +[newrelic] +distributed_tracing.sampler.remote_parent_sampled = always_on +distributed_tracing.sampler.remote_parent_not_sampled = always_off +distributed_tracing.sampler.root.adaptive.sampling_target = 5 +distributed_tracing.sampler.remote_parent_sampled.adaptive.sampling_target = 10 +distributed_tracing.sampler.remote_parent_not_sampled.adaptive.sampling_target = 20 +""" + +INI_FILE_FULL_GRAN_CONFLICTS_RATIO = b""" +[newrelic] +distributed_tracing.sampler.root = always_on +distributed_tracing.sampler.remote_parent_sampled = always_on +distributed_tracing.sampler.remote_parent_not_sampled = always_off +distributed_tracing.sampler.root.trace_id_ratio_based.ratio = .5 +distributed_tracing.sampler.remote_parent_sampled.trace_id_ratio_based.ratio = .1 +distributed_tracing.sampler.remote_parent_not_sampled.trace_id_ratio_based.ratio = .2 +""" + +INI_FILE_FULL_GRAN_MULTIPLE_SAMPLERS_INVALID_RATIO = b""" +[newrelic] +distributed_tracing.sampler.root.adaptive.sampling_target = 5 +distributed_tracing.sampler.remote_parent_sampled.adaptive.sampling_target = 10 +distributed_tracing.sampler.remote_parent_not_sampled.adaptive.sampling_target = 20 +distributed_tracing.sampler.root.trace_id_ratio_based.sampling_target = 5 +distributed_tracing.sampler.remote_parent_sampled.trace_id_ratio_based.sampling_target = 10 +distributed_tracing.sampler.remote_parent_not_sampled.trace_id_ratio_based.sampling_target = 20 +""" + +INI_FILE_FULL_GRAN_NO_RATIO = b""" +[newrelic] +distributed_tracing.sampler.root = trace_id_ratio_based +distributed_tracing.sampler.remote_parent_sampled = trace_id_ratio_based +distributed_tracing.sampler.remote_parent_not_sampled = trace_id_ratio_based +""" + +INI_FILE_FULL_GRAN_MULTIPLE_VALID_SAMPLERS = b""" +[newrelic] +distributed_tracing.sampler.root.adaptive.sampling_target = 5 +distributed_tracing.sampler.remote_parent_sampled.adaptive.sampling_target = 10 +distributed_tracing.sampler.remote_parent_not_sampled.adaptive.sampling_target = 20 +distributed_tracing.sampler.root.trace_id_ratio_based.ratio = .5 +distributed_tracing.sampler.remote_parent_sampled.trace_id_ratio_based.ratio = .1 +distributed_tracing.sampler.remote_parent_not_sampled.trace_id_ratio_based.ratio = .2 +""" + +INI_FILE_PARTIAL_GRAN_NO_RATIO = b""" +[newrelic] +distributed_tracing.sampler.partial_granularity.root = trace_id_ratio_based +distributed_tracing.sampler.partial_granularity.remote_parent_sampled = trace_id_ratio_based +distributed_tracing.sampler.partial_granularity.remote_parent_not_sampled = trace_id_ratio_based +""" + +INI_FILE_PARTIAL_GRAN_CONFLICTS_ADAPTIVE = b""" +[newrelic] +distributed_tracing.sampler.partial_granularity.remote_parent_sampled = always_on +distributed_tracing.sampler.partial_granularity.remote_parent_not_sampled = always_off +distributed_tracing.sampler.partial_granularity.root.adaptive.sampling_target = 5 +distributed_tracing.sampler.partial_granularity.remote_parent_sampled.adaptive.sampling_target = 10 +distributed_tracing.sampler.partial_granularity.remote_parent_not_sampled.adaptive.sampling_target = 20 +""" + +INI_FILE_PARTIAL_GRAN_CONFLICTS_RATIO = b""" +[newrelic] +distributed_tracing.sampler.partial_granularity.remote_parent_sampled = always_on +distributed_tracing.sampler.partial_granularity.remote_parent_not_sampled = always_off +distributed_tracing.sampler.partial_granularity.root.trace_id_ratio_based.ratio = .5 +distributed_tracing.sampler.partial_granularity.remote_parent_sampled.trace_id_ratio_based.ratio = .1 +distributed_tracing.sampler.partial_granularity.remote_parent_not_sampled.trace_id_ratio_based.ratio = .2 +""" + +INI_FILE_PARTIAL_GRAN_MULTIPLE_SAMPLERS = b""" +[newrelic] +distributed_tracing.sampler.partial_granularity.root.adaptive.sampling_target = 5 +distributed_tracing.sampler.partial_granularity.remote_parent_sampled.adaptive.sampling_target = 10 +distributed_tracing.sampler.partial_granularity.remote_parent_not_sampled.adaptive.sampling_target = 20 +distributed_tracing.sampler.partial_granularity.root.trace_id_ratio_based.sampling_target = 5 +distributed_tracing.sampler.partial_granularity.remote_parent_sampled.trace_id_ratio_based.sampling_target = 10 +distributed_tracing.sampler.partial_granularity.remote_parent_not_sampled.trace_id_ratio_based.sampling_target = 20 +""" + # Tests for loading settings and testing for values precedence @pytest.mark.parametrize("ini,env,expected_format", ((INI_FILE_EMPTY, {}, False), (INI_FILE_W3C, {}, True))) def test_distributed_trace_setings(ini, env, expected_format, global_settings): settings = global_settings() assert settings.distributed_tracing.exclude_newrelic_header == expected_format + + +@pytest.mark.parametrize( + "ini,env,expected", + ( + ( # Defaults to adaptive (default) sampler. + INI_FILE_EMPTY, + {}, + ("default", "default", "default", None, None, None), + ), + ( # More specific sampler path overrides less specific path in ini file. + INI_FILE_FULL_GRAN_CONFLICTS_ADAPTIVE, + {}, + ("adaptive", "adaptive", "adaptive", 5, 10, 20), + ), + ( # More specific sampler path overrides less specific path in ini file. + INI_FILE_FULL_GRAN_CONFLICTS_RATIO, + {}, + ("trace_id_ratio_based", "trace_id_ratio_based", "trace_id_ratio_based", 0.5, 0.1, 0.2), + ), + ( # ini file configuration takes precedence over env vars. + INI_FILE_FULL_GRAN_CONFLICTS_ADAPTIVE, + { + "NEW_RELIC_ENABLED": "true", + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_ROOT_ADAPTIVE_SAMPLING_TARGET": "50", + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_REMOTE_PARENT_SAMPLED_ADAPTIVE_SAMPLING_TARGET": "50", + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_REMOTE_PARENT_NOT_SAMPLED_ADAPTIVE_SAMPLING_TARGET": "30", + }, + ("adaptive", "adaptive", "adaptive", 5, 10, 20), + ), + ( # Simple configuration works. + INI_FILE_EMPTY, + { + "NEW_RELIC_ENABLED": "true", + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_ROOT": "always_on", + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_REMOTE_PARENT_SAMPLED": "always_on", + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_REMOTE_PARENT_NOT_SAMPLED": "always_off", + }, + ("always_on", "always_on", "always_off", None, None, None), + ), + ( # More specific sampler path overrides less specific path in env vars. + INI_FILE_EMPTY, + { + "NEW_RELIC_ENABLED": "true", + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_ROOT": "always_on", + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_REMOTE_PARENT_SAMPLED": "always_on", + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_REMOTE_PARENT_NOT_SAMPLED": "always_off", + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_ROOT_ADAPTIVE_SAMPLING_TARGET": "20", + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_REMOTE_PARENT_SAMPLED_ADAPTIVE_SAMPLING_TARGET": "20", + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_REMOTE_PARENT_NOT_SAMPLED_ADAPTIVE_SAMPLING_TARGET": "20", + }, + ("adaptive", "adaptive", "adaptive", 20, 20, 20), + ), + ( # Ratio takes precendence over adaptive in env vars. + INI_FILE_EMPTY, + { + "NEW_RELIC_ENABLED": "true", + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_ROOT": "always_on", + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_REMOTE_PARENT_SAMPLED": "always_on", + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_REMOTE_PARENT_NOT_SAMPLED": "always_off", + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_ROOT_ADAPTIVE_SAMPLING_TARGET": "20", + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_REMOTE_PARENT_SAMPLED_ADAPTIVE_SAMPLING_TARGET": "20", + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_REMOTE_PARENT_NOT_SAMPLED_ADAPTIVE_SAMPLING_TARGET": "20", + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_ROOT_TRACE_ID_RATIO_BASED_RATIO": ".5", + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_REMOTE_PARENT_SAMPLED_TRACE_ID_RATIO_BASED_RATIO": ".1", + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_REMOTE_PARENT_NOT_SAMPLED_TRACE_ID_RATIO_BASED_RATIO": ".2", + }, + ("trace_id_ratio_based", "trace_id_ratio_based", "trace_id_ratio_based", 0.5, 0.1, 0.2), + ), + ( # Falls back on adaptive when invalid ratio. + INI_FILE_EMPTY, + { + "NEW_RELIC_ENABLED": "true", + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_ROOT_TRACE_ID_RATIO_BASED_RATIO": "5", + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_REMOTE_PARENT_SAMPLED_TRACE_ID_RATIO_BASED_RATIO": "10", + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_REMOTE_PARENT_NOT_SAMPLED_TRACE_ID_RATIO_BASED_RATIO": "0", + }, + ("default", "default", "default", None, None, None), + ), + ( # Ignores ratio sampler when invalid ratio path is provided. + INI_FILE_FULL_GRAN_MULTIPLE_SAMPLERS_INVALID_RATIO, + {}, + ("adaptive", "adaptive", "adaptive", 5, 10, 20), + ), + ( # Ignores ratio sampler when ratio is not defined. + INI_FILE_FULL_GRAN_NO_RATIO, + {}, + ("default", "default", "default", None, None, None), + ), + ( # Ratio takes precedence over adaptive. + INI_FILE_FULL_GRAN_MULTIPLE_VALID_SAMPLERS, + {}, + ("trace_id_ratio_based", "trace_id_ratio_based", "trace_id_ratio_based", 0.5, 0.1, 0.2), + ), + ), +) +def test_full_granularity_precedence(ini, env, global_settings, expected): + settings = global_settings() + + app_settings = finalize_application_settings(settings=settings) + + assert app_settings.distributed_tracing.sampler._root == expected[0] + assert app_settings.distributed_tracing.sampler._remote_parent_sampled == expected[1] + assert app_settings.distributed_tracing.sampler._remote_parent_not_sampled == expected[2] + if expected[0] == "trace_id_ratio_based": + assert app_settings.distributed_tracing.sampler.root.trace_id_ratio_based.ratio == expected[3] + else: + assert app_settings.distributed_tracing.sampler.root.adaptive.sampling_target == expected[3] + if expected[1] == "trace_id_ratio_based": + assert app_settings.distributed_tracing.sampler.remote_parent_sampled.trace_id_ratio_based.ratio == expected[4] + else: + assert app_settings.distributed_tracing.sampler.remote_parent_sampled.adaptive.sampling_target == expected[4] + if expected[2] == "trace_id_ratio_based": + assert ( + app_settings.distributed_tracing.sampler.remote_parent_not_sampled.trace_id_ratio_based.ratio == expected[5] + ) + else: + assert ( + app_settings.distributed_tracing.sampler.remote_parent_not_sampled.adaptive.sampling_target == expected[5] + ) + + +@pytest.mark.parametrize( + "ini,env,expected", + ( + ( # Defaults to adaptive (default) sampler. + INI_FILE_EMPTY, + {}, + ("default", "default", "default", None, None, None), + ), + ( # More specific sampler path overrides less specific path in ini file. + INI_FILE_PARTIAL_GRAN_CONFLICTS_ADAPTIVE, + {}, + ("adaptive", "adaptive", "adaptive", 5, 10, 20), + ), + ( # More specific sampler path overrides less specific path in ini file. + INI_FILE_PARTIAL_GRAN_CONFLICTS_RATIO, + {}, + ("trace_id_ratio_based", "trace_id_ratio_based", "trace_id_ratio_based", 0.5, 0.1, 0.2), + ), + ( # ini config takes precedence over env vars. + INI_FILE_PARTIAL_GRAN_CONFLICTS_ADAPTIVE, + { + "NEW_RELIC_ENABLED": "true", + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_PARTIAL_GRANULARITY_REMOTE_PARENT_SAMPLED": "always_on", + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_PARTIAL_GRANULARITY_REMOTE_PARENT_NOT_SAMPLED": "always_off", + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_PARTIAL_GRANULARITY_REMOTE_PARENT_SAMPLED_ADAPTIVE_SAMPLING_TARGET": "20", + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_PARTIAL_GRANULARITY_REMOTE_PARENT_NOT_SAMPLED_ADAPTIVE_SAMPLING_TARGET": "30", + }, + ("adaptive", "adaptive", "adaptive", 5, 10, 20), + ), + ( # Simple configuration works. + INI_FILE_EMPTY, + { + "NEW_RELIC_ENABLED": "true", + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_PARTIAL_GRANULARITY_ROOT": "always_on", + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_PARTIAL_GRANULARITY_REMOTE_PARENT_SAMPLED": "always_on", + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_PARTIAL_GRANULARITY_REMOTE_PARENT_NOT_SAMPLED": "always_off", + }, + ("always_on", "always_on", "always_off", None, None, None), + ), + ( # Ignores ratio if ratio is not defined. + INI_FILE_EMPTY, + { + "NEW_RELIC_ENABLED": "true", + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_PARTIAL_GRANULARITY_ROOT": "trace_id_ratio_based", + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_PARTIAL_GRANULARITY_REMOTE_PARENT_SAMPLED": "trace_id_ratio_based", + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_PARTIAL_GRANULARITY_REMOTE_PARENT_NOT_SAMPLED": "trace_id_ratio_based", + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_PARTIAL_GRANULARITY_REMOTE_PARENT_NOT_SAMPLED_TRACE_ID_RATIO_BASED_RATIO": ".1", + }, + ("default", "default", "trace_id_ratio_based", None, None, 0.1), + ), + ( # More specific sampler path overrides less specific path in env vars. + INI_FILE_EMPTY, + { + "NEW_RELIC_ENABLED": "true", + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_PARTIAL_GRANULARITY_ROOT": "always_on", + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_PARTIAL_GRANULARITY_REMOTE_PARENT_SAMPLED": "always_on", + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_PARTIAL_GRANULARITY_REMOTE_PARENT_NOT_SAMPLED": "always_off", + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_PARTIAL_GRANULARITY_ROOT_ADAPTIVE_SAMPLING_TARGET": "5", + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_PARTIAL_GRANULARITY_REMOTE_PARENT_SAMPLED_ADAPTIVE_SAMPLING_TARGET": "10", + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_PARTIAL_GRANULARITY_REMOTE_PARENT_NOT_SAMPLED_ADAPTIVE_SAMPLING_TARGET": "20", + }, + ("adaptive", "adaptive", "adaptive", 5, 10, 20), + ), + ( # Ratio takes precedence over adaptive in env vars. + INI_FILE_EMPTY, + { + "NEW_RELIC_ENABLED": "true", + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_PARTIAL_GRANULARITY_ROOT": "always_on", + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_PARTIAL_GRANULARITY_REMOTE_PARENT_SAMPLED": "always_on", + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_PARTIAL_GRANULARITY_REMOTE_PARENT_NOT_SAMPLED": "always_off", + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_PARTIAL_GRANULARITY_ROOT_ADAPTIVE_SAMPLING_TARGET": "5", + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_PARTIAL_GRANULARITY_REMOTE_PARENT_SAMPLED_ADAPTIVE_SAMPLING_TARGET": "10", + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_PARTIAL_GRANULARITY_REMOTE_PARENT_NOT_SAMPLED_ADAPTIVE_SAMPLING_TARGET": "20", + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_PARTIAL_GRANULARITY_ROOT_TRACE_ID_RATIO_BASED_RATIO": ".5", + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_PARTIAL_GRANULARITY_REMOTE_PARENT_SAMPLED_TRACE_ID_RATIO_BASED_RATIO": ".1", + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_PARTIAL_GRANULARITY_REMOTE_PARENT_NOT_SAMPLED_TRACE_ID_RATIO_BASED_RATIO": ".2", + }, + ("trace_id_ratio_based", "trace_id_ratio_based", "trace_id_ratio_based", 0.5, 0.1, 0.2), + ), + ( # Ignores other unknown samplers. + INI_FILE_PARTIAL_GRAN_MULTIPLE_SAMPLERS, + {}, + ("adaptive", "adaptive", "adaptive", 5, 10, 20), + ), + ( # Ignores ratio sampler when ratio is not defined. + INI_FILE_PARTIAL_GRAN_NO_RATIO, + {}, + ("default", "default", "default", None, None, None), + ), + ( # Falls back on adaptive when invalid ratio. + INI_FILE_EMPTY, + { + "NEW_RELIC_ENABLED": "true", + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_PARTIAL_GRANULARITY_ROOT_TRACE_ID_RATIO_BASED_RATIO": "5", + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_PARTIAL_GRANULARITY_REMOTE_PARENT_SAMPLED_TRACE_ID_RATIO_BASED_RATIO": "10", + "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_PARTIAL_GRANULARITY_REMOTE_PARENT_NOT_SAMPLED_TRACE_ID_RATIO_BASED_RATIO": "0", + }, + ("default", "default", "default", None, None, None), + ), + ), +) +def test_partial_granularity_precedence(ini, env, global_settings, expected): + settings = global_settings() + + app_settings = finalize_application_settings(settings=settings) + + assert app_settings.distributed_tracing.sampler.partial_granularity._root == expected[0] + assert app_settings.distributed_tracing.sampler.partial_granularity._remote_parent_sampled == expected[1] + assert app_settings.distributed_tracing.sampler.partial_granularity._remote_parent_not_sampled == expected[2] + if expected[0] == "trace_id_ratio_based": + assert ( + app_settings.distributed_tracing.sampler.partial_granularity.root.trace_id_ratio_based.ratio == expected[3] + ) + else: + assert app_settings.distributed_tracing.sampler.partial_granularity.root.adaptive.sampling_target == expected[3] + + if expected[1] == "trace_id_ratio_based": + assert ( + app_settings.distributed_tracing.sampler.partial_granularity.remote_parent_sampled.trace_id_ratio_based.ratio + == expected[4] + ) + else: + assert ( + app_settings.distributed_tracing.sampler.partial_granularity.remote_parent_sampled.adaptive.sampling_target + == expected[4] + ) + if expected[2] == "trace_id_ratio_based": + assert ( + app_settings.distributed_tracing.sampler.partial_granularity.remote_parent_not_sampled.trace_id_ratio_based.ratio + == expected[5] + ) + else: + assert ( + app_settings.distributed_tracing.sampler.partial_granularity.remote_parent_not_sampled.adaptive.sampling_target + == expected[5] + ) diff --git a/tests/agent_unittests/test_harvest_loop.py b/tests/agent_unittests/test_harvest_loop.py index 9717e956ba..e90cacf9d2 100644 --- a/tests/agent_unittests/test_harvest_loop.py +++ b/tests/agent_unittests/test_harvest_loop.py @@ -27,6 +27,7 @@ from newrelic.core.config import finalize_application_settings, global_settings from newrelic.core.custom_event import create_custom_event from newrelic.core.error_node import ErrorNode +from newrelic.core.external_node import ExternalNode from newrelic.core.function_node import FunctionNode from newrelic.core.log_event_node import LogEventNode from newrelic.core.root_node import RootNode @@ -39,135 +40,162 @@ @pytest.fixture(scope="module") def transaction_node(request): - default_capacity = SampledDataSet().capacity - num_events = default_capacity + 1 - - custom_events = SampledDataSet(capacity=num_events) - for _ in range(num_events): - event = create_custom_event("Custom", {}) - custom_events.add(event) - - ml_events = SampledDataSet(capacity=num_events) - for _ in range(num_events): - event = create_custom_event("Custom", {}) - ml_events.add(event) - - log_events = SampledDataSet(capacity=num_events) - for _ in range(num_events): - event = LogEventNode(1653609717, "WARNING", "A", {}) - log_events.add(event) - - error = ErrorNode( - timestamp=0, - type="foo:bar", - message="oh no! your foo had a bar", - expected=False, - span_id=None, - stack_trace="", - error_group_name=None, - custom_params={}, - source=None, - ) - - errors = tuple(error for _ in range(num_events)) - - function = FunctionNode( - group="Function", - name="foo", - children=(), - start_time=0, - end_time=1, - duration=1, - exclusive=1, - label=None, - params=None, - rollup=None, - guid="GUID", - agent_attributes={}, - user_attributes={}, - ) - - children = tuple(function for _ in range(num_events)) - - root = RootNode( - name="Function/main", - children=children, - start_time=1524764430.0, - end_time=1524764430.1, - duration=0.1, - exclusive=0.1, - guid=None, - agent_attributes={}, - user_attributes={}, - path="OtherTransaction/Function/main", - trusted_parent_span=None, - tracing_vendors=None, - ) - - node = TransactionNode( - settings=finalize_application_settings({"agent_run_id": "1234567"}), - path="OtherTransaction/Function/main", - type="OtherTransaction", - group="Function", - base_name="main", - name_for_metric="Function/main", - port=None, - request_uri=None, - queue_start=0.0, - start_time=1524764430.0, - end_time=1524764430.1, - last_byte_time=0.0, - total_time=0.1, - response_time=0.1, - duration=0.1, - exclusive=0.1, - root=root, - errors=errors, - slow_sql=(), - custom_events=custom_events, - ml_events=ml_events, - log_events=log_events, - apdex_t=0.5, - suppress_apdex=False, - custom_metrics=CustomMetrics(), - dimensional_metrics=DimensionalMetrics(), - guid="4485b89db608aece", - cpu_time=0.0, - suppress_transaction_trace=False, - client_cross_process_id=None, - referring_transaction_guid=None, - record_tt=False, - synthetics_resource_id=None, - synthetics_job_id=None, - synthetics_monitor_id=None, - synthetics_header=None, - synthetics_type=None, - synthetics_initiator=None, - synthetics_attributes=None, - synthetics_info_header=None, - is_part_of_cat=False, - trip_id="4485b89db608aece", - path_hash=None, - referring_path_hash=None, - alternate_path_hashes=[], - trace_intrinsics={}, - distributed_trace_intrinsics={}, - agent_attributes=[], - user_attributes=[], - priority=1.0, - parent_transport_duration=None, - parent_span=None, - parent_type=None, - parent_account=None, - parent_app=None, - parent_tx=None, - parent_transport_type=None, - sampled=True, - root_span_guid=None, - trace_id="4485b89db608aece", - loop_time=0.0, - ) - return node + def _transaction_node(partial_granularity=False): + default_capacity = SampledDataSet().capacity + num_events = default_capacity + 1 + + custom_events = SampledDataSet(capacity=num_events) + for _ in range(num_events): + event = create_custom_event("Custom", {}) + custom_events.add(event) + + ml_events = SampledDataSet(capacity=num_events) + for _ in range(num_events): + event = create_custom_event("Custom", {}) + ml_events.add(event) + + log_events = SampledDataSet(capacity=num_events) + for _ in range(num_events): + event = LogEventNode(1653609717, "WARNING", "A", {}) + log_events.add(event) + + error = ErrorNode( + timestamp=0, + type="foo:bar", + message="oh no! your foo had a bar", + expected=False, + span_id=None, + stack_trace="", + error_group_name=None, + custom_params={}, + source=None, + ) + + errors = tuple(error for _ in range(num_events)) + + function = FunctionNode( + group="Function", + name="foo", + children=(), + start_time=0, + end_time=1, + duration=1, + exclusive=1, + label=None, + params=None, + rollup=None, + guid="GUID", + agent_attributes={}, + user_attributes={}, + span_link_events=None, + span_event_events=None, + ) + + children = [function for _ in range(num_events)] + + function = ExternalNode( + library="requests", + url="http:localhost:3000", + method="GET", + children=(), + start_time=0, + end_time=1, + duration=1, + exclusive=1, + params={}, + guid="GUID", + agent_attributes={}, + user_attributes={}, + span_link_events=None, + span_event_events=None, + ) + + children.extend([function for _ in range(num_events)]) + + root = RootNode( + name="Function/main", + children=children, + start_time=1524764430.0, + end_time=1524764430.1, + duration=0.1, + exclusive=0.1, + guid=None, + agent_attributes={}, + user_attributes={}, + path="OtherTransaction/Function/main", + trusted_parent_span=None, + tracing_vendors=None, + span_link_events=None, + span_event_events=None, + ) + + node = TransactionNode( + settings=finalize_application_settings({"agent_run_id": "1234567"}), + path="OtherTransaction/Function/main", + type="OtherTransaction", + group="Function", + base_name="main", + name_for_metric="Function/main", + port=None, + request_uri=None, + queue_start=0.0, + start_time=1524764430.0, + end_time=1524764430.1, + last_byte_time=0.0, + total_time=0.1, + response_time=0.1, + duration=0.1, + exclusive=0.1, + root=root, + errors=errors, + slow_sql=(), + custom_events=custom_events, + ml_events=ml_events, + log_events=log_events, + apdex_t=0.5, + suppress_apdex=False, + custom_metrics=CustomMetrics(), + dimensional_metrics=DimensionalMetrics(), + guid="4485b89db608aece", + cpu_time=0.0, + suppress_transaction_trace=False, + client_cross_process_id=None, + referring_transaction_guid=None, + record_tt=False, + synthetics_resource_id=None, + synthetics_job_id=None, + synthetics_monitor_id=None, + synthetics_header=None, + synthetics_type=None, + synthetics_initiator=None, + synthetics_attributes=None, + synthetics_info_header=None, + is_part_of_cat=False, + trip_id="4485b89db608aece", + path_hash=None, + referring_path_hash=None, + alternate_path_hashes=[], + trace_intrinsics={}, + distributed_trace_intrinsics={}, + agent_attributes=[], + user_attributes=[], + priority=1.0, + parent_transport_duration=None, + parent_span=None, + parent_type=None, + parent_account=None, + parent_app=None, + parent_tx=None, + parent_transport_type=None, + sampled=True, + root_span_guid=None, + trace_id="4485b89db608aece", + loop_time=0.0, + partial_granularity_sampled=partial_granularity, + ) + return node + + return _transaction_node def validate_metric_payload(metrics=None, endpoints_called=None): @@ -321,14 +349,32 @@ def test_serverless_application_harvest(): @pytest.mark.parametrize( - "distributed_tracing_enabled,span_events_enabled,spans_created", - [(True, True, 1), (True, True, 15), (True, False, 1), (True, True, 0), (True, False, 0), (False, True, 0)], + "distributed_tracing_enabled,full_granularity_enabled,partial_granularity_enabled,span_events_enabled,spans_created", + [ + (True, True, False, True, 1), + (True, True, True, True, 1), + (True, True, False, True, 15), + (True, True, False, False, 1), + (True, True, False, True, 0), + (True, True, False, False, 0), + (False, True, False, True, 0), + ], ) -def test_application_harvest_with_spans(distributed_tracing_enabled, span_events_enabled, spans_created): +def test_application_harvest_with_spans( + distributed_tracing_enabled, + full_granularity_enabled, + partial_granularity_enabled, + span_events_enabled, + spans_created, +): span_endpoints_called = [] max_samples_stored = 10 - if distributed_tracing_enabled and span_events_enabled: + if ( + distributed_tracing_enabled + and span_events_enabled + and (full_granularity_enabled or partial_granularity_enabled) + ): seen = spans_created sent = min(spans_created, max_samples_stored) else: @@ -348,6 +394,8 @@ def test_application_harvest_with_spans(distributed_tracing_enabled, span_events "developer_mode": True, "license_key": "**NOT A LICENSE KEY**", "distributed_tracing.enabled": distributed_tracing_enabled, + "distributed_tracing.sampler.full_granularity.enabled": full_granularity_enabled, + "distributed_tracing.sampler.partial_granularity.enabled": partial_granularity_enabled, "span_events.enabled": span_events_enabled, # Uses the name from post-translation as this is modifying the settings object, not a config file "event_harvest_config.harvest_limits.span_event_data": max_samples_stored, @@ -366,12 +414,12 @@ def _test(): # Verify that the metric_data endpoint is the 2nd to last and # span_event_data is the 3rd to last endpoint called - assert span_endpoints_called[-2] == "metric_data" + assert span_endpoints_called[-2] == "metric_data", span_endpoints_called if span_events_enabled and spans_created > 0: - assert span_endpoints_called[-3] == "span_event_data" + assert span_endpoints_called[-3] == "span_event_data", span_endpoints_called else: - assert span_endpoints_called[-3] != "span_event_data" + assert span_endpoints_called[-3] != "span_event_data", span_endpoints_called _test() @@ -451,10 +499,11 @@ def _test(): }, ) def test_transaction_count(transaction_node): + txn_node = transaction_node() app = Application("Python Agent Test (Harvest Loop)") app.connect_to_data_collector(None) - app.record_transaction(transaction_node) + app.record_transaction(txn_node) # Harvest has not run yet assert app._transaction_count == 1 @@ -465,9 +514,47 @@ def test_transaction_count(transaction_node): assert app._transaction_count == 0 # Record a transaction - app.record_transaction(transaction_node) + app.record_transaction(txn_node) + assert app._transaction_count == 1 + + app.harvest() + + # Harvest resets the transaction count + assert app._transaction_count == 0 + + +@override_generic_settings( + settings, + { + "developer_mode": True, + "license_key": "**NOT A LICENSE KEY**", + "feature_flag": set(), + "collect_custom_events": False, + "application_logging.forwarding.enabled": False, + "distributed_tracing.sampler.full_granularity.enabled": False, + "distributed_tracing.sampler.partial_granularity.enabled": True, + "distributed_tracing.sampler.partial_granularity.type": "compact", + }, +) +def test_partial_granularity_metrics(transaction_node): + txn_node = transaction_node(True) + app = Application("Python Agent Test (Harvest Loop)") + app.connect_to_data_collector(None) + + app.record_transaction(txn_node) + + # Harvest has not run yet assert app._transaction_count == 1 + instrumented = "Supportability/DistributedTrace/PartialGranularity/compact/Span/Instrumented" + kept = "Supportability/DistributedTrace/PartialGranularity/compact/Span/Kept" + pg = "Supportability/Python/PartialGranularity/compact" + dropped_ids = "Supportability/Python/PartialGranularity/NrIds/Dropped" + assert app._stats_engine.stats_table[(instrumented, "")][0] == 203 + assert app._stats_engine.stats_table[(kept, "")][0] == 2 + assert app._stats_engine.stats_table[(pg, "")][0] == 1 + assert app._stats_engine.stats_table[(dropped_ids, "")][0] == 37 + app.harvest() # Harvest resets the transaction count @@ -478,18 +565,19 @@ def test_transaction_count(transaction_node): settings, {"developer_mode": True, "license_key": "**NOT A LICENSE KEY**", "feature_flag": set()} ) def test_adaptive_sampling(transaction_node, monkeypatch): + txn_node = transaction_node() app = Application("Python Agent Test (Harvest Loop)") # Should always return false for sampling prior to connect - assert app.compute_sampled() is False + assert app.compute_sampled(True, 0) is False app.connect_to_data_collector(None) # First harvest, first N should be sampled for _ in range(settings.sampling_target): - assert app.compute_sampled() is True + assert app.compute_sampled(True, 0) is True - assert app.compute_sampled() is False + assert app.compute_sampled(True, 0) is False # fix random.randrange to return 0 monkeypatch.setattr(random, "randrange", lambda *args, **kwargs: 0) @@ -497,14 +585,14 @@ def test_adaptive_sampling(transaction_node, monkeypatch): # Multiple resets should behave the same for _ in range(2): # Set the last_reset to longer than the period so a reset will occur. - app.adaptive_sampler.last_reset = time.time() - app.adaptive_sampler.period + app.sampler.get_sampler(True, 0).last_reset = time.time() - app.sampler.get_sampler(True, 0).period # Subsequent harvests should allow sampling of 2X the target for _ in range(2 * settings.sampling_target): - assert app.compute_sampled() is True + assert app.compute_sampled(True, 0) is True # No further samples should be saved - assert app.compute_sampled() is False + assert app.compute_sampled(True, 0) is False @override_generic_settings( @@ -522,11 +610,12 @@ def test_adaptive_sampling(transaction_node, monkeypatch): }, ) def test_reservoir_sizes(transaction_node): + txn_node = transaction_node() app = Application("Python Agent Test (Harvest Loop)") app.connect_to_data_collector(None) # Record a transaction with events - app.record_transaction(transaction_node) + app.record_transaction(txn_node) # Test that the samples have been recorded assert app._stats_engine.custom_events.num_samples == 101 @@ -534,7 +623,7 @@ def test_reservoir_sizes(transaction_node): assert app._stats_engine.log_events.num_samples == 101 # Add 1 for the root span - assert app._stats_engine.span_events.num_samples == 102 + assert app._stats_engine.span_events.num_samples == 203 @pytest.mark.parametrize( @@ -647,20 +736,20 @@ def test_serverless_mode_adaptive_sampling(time_to_next_reset, computed_count, c app = Application("Python Agent Test (Harvest Loop)") app.connect_to_data_collector(None) - app.adaptive_sampler.computed_count = 123 - app.adaptive_sampler.last_reset = time.time() - 60 + time_to_next_reset + app.sampler.get_sampler(True, 0).computed_count = 123 + app.sampler.get_sampler(True, 0).last_reset = time.time() - 60 + time_to_next_reset - assert app.compute_sampled() is True - assert app.adaptive_sampler.computed_count == computed_count - assert app.adaptive_sampler.computed_count_last == computed_count_last + assert app.compute_sampled(True, 0) is True + assert app.sampler.get_sampler(True, 0).computed_count == computed_count + assert app.sampler.get_sampler(True, 0).computed_count_last == computed_count_last -@validate_function_not_called("newrelic.core.adaptive_sampler", "AdaptiveSampler._reset") +@validate_function_not_called("newrelic.core.samplers.adaptive_sampler", "AdaptiveSampler._reset") @override_generic_settings(settings, {"developer_mode": True}) def test_compute_sampled_no_reset(): app = Application("Python Agent Test (Harvest Loop)") app.connect_to_data_collector(None) - assert app.compute_sampled() is True + assert app.compute_sampled(True, 0) is True def test_analytic_event_sampling_info(): diff --git a/tests/agent_unittests/test_infinite_trace_settings.py b/tests/agent_unittests/test_infinite_trace_settings.py index 4b47a72398..31c8e6819e 100644 --- a/tests/agent_unittests/test_infinite_trace_settings.py +++ b/tests/agent_unittests/test_infinite_trace_settings.py @@ -14,6 +14,8 @@ import pytest +from newrelic.core.config import finalize_application_settings + INI_FILE_EMPTY = b""" [newrelic] """ @@ -77,3 +79,18 @@ def test_infinite_tracing_port(ini, env, expected_port, global_settings): def test_infinite_tracing_span_queue_size(ini, env, expected_size, global_settings): settings = global_settings() assert settings.infinite_tracing.span_queue_size == expected_size + + +@pytest.mark.parametrize( + "ini,env", + ((INI_FILE_INFINITE_TRACING, {"NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_PARTIAL_GRANULARITY_ENABLED": "true"}),), +) +def test_partial_granularity_dissabled_when_infinite_tracing_enabled(ini, env, global_settings): + settings = global_settings() + assert settings.distributed_tracing.sampler.partial_granularity.enabled + assert settings.infinite_tracing.enabled + + app_settings = finalize_application_settings(settings=settings) + + assert not app_settings.distributed_tracing.sampler.partial_granularity.enabled + assert app_settings.infinite_tracing.enabled diff --git a/tests/application_celery/test_task_methods.py b/tests/application_celery/test_task_methods.py index eda1907ee1..86f90a299e 100644 --- a/tests/application_celery/test_task_methods.py +++ b/tests/application_celery/test_task_methods.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import pytest from _target_application import add, add_with_run, add_with_super, tsum from celery import chain, chord, group from testing_support.validators.validate_code_level_metrics import validate_code_level_metrics @@ -129,6 +130,7 @@ def test_celery_task_signature(): assert result == 7 +@pytest.mark.xfail(reason="This test has a race condition that causes failures in CI.") @validate_transaction_metrics( name="_target_application.add", group="Celery", diff --git a/tests/coroutines_asyncio/test_context_propagation.py b/tests/coroutines_asyncio/test_context_propagation.py index b338b6ec3e..eb5c358745 100644 --- a/tests/coroutines_asyncio/test_context_propagation.py +++ b/tests/coroutines_asyncio/test_context_propagation.py @@ -36,16 +36,31 @@ import uvloop loop_policies = (pytest.param(None, id="asyncio"), pytest.param(uvloop.EventLoopPolicy(), id="uvloop")) + uvloop_factory = (pytest.param(uvloop.new_event_loop, id="uvloop"), pytest.param(None, id="None")) except ImportError: loop_policies = (pytest.param(None, id="asyncio"),) + uvloop_factory = (pytest.param(None, id="None"),) + + +def loop_factories(): + import asyncio + + if sys.platform == "win32": + return (pytest.param(asyncio.ProactorEventLoop, id="asyncio.ProactorEventLoop"), *uvloop_factory) + else: + return (pytest.param(asyncio.SelectorEventLoop, id="asyncio.SelectorEventLoop"), *uvloop_factory) @pytest.fixture(autouse=True) def reset_event_loop(): - from asyncio import set_event_loop, set_event_loop_policy + try: + from asyncio import set_event_loop, set_event_loop_policy + + # Remove the loop policy to avoid side effects + set_event_loop_policy(None) + except ImportError: + from asyncio import set_event_loop - # Remove the loop policy to avoid side effects - set_event_loop_policy(None) set_event_loop(None) @@ -102,6 +117,7 @@ async def _test(asyncio, schedule, nr_enabled=True): return trace +@pytest.mark.skipif(sys.version_info >= (3, 16), reason="loop_policy is not available") @pytest.mark.parametrize("loop_policy", loop_policies) @pytest.mark.parametrize("schedule", ("create_task", "ensure_future")) @validate_transaction_metrics( @@ -166,10 +182,12 @@ def handle_exception(loop, context): memcache_trace("cmd"), ], ) -def test_two_transactions(event_loop, trace): +def test_two_transactions_with_global_event_loop(event_loop, trace): """ Instantiate a coroutine in one transaction and await it in another. This should not cause any errors. + This uses the global event loop policy, which has been deprecated + since Python 3.11 and is scheduled for removal in Python 3.16. """ import asyncio @@ -211,6 +229,99 @@ async def await_task(): event_loop.run_until_complete(asyncio.gather(afut, bfut)) +@pytest.mark.skipif(sys.version_info < (3, 11), reason="asyncio.Runner is not available") +@validate_transaction_metrics("await_task", background_task=True) +@validate_transaction_metrics("create_coro", background_task=True, index=-2) +@pytest.mark.parametrize("loop_factory", loop_factories()) +@pytest.mark.parametrize( + "trace", + [ + function_trace(name="simple_gen"), + external_trace(library="lib", url="http://foo.com"), + database_trace("select * from foo"), + datastore_trace("lib", "foo", "bar"), + message_trace("lib", "op", "typ", "name"), + memcache_trace("cmd"), + ], +) +def test_two_transactions_with_loop_factory(trace, loop_factory): + """ + Instantiate a coroutine in one transaction and await it in + another. This should not cause any errors. + Starting in Python 3.11, the asyncio.Runner class was added + as well as the loop_factory parameter. The loop_factory + parameter provides a replacement for loop policies (which + are scheduled for removal in Python 3.16). + """ + import asyncio + + @trace + async def task(): + pass + + @background_task(name="create_coro") + async def create_coro(): + return asyncio.create_task(task()) + + @background_task(name="await_task") + async def await_task(task_to_await): + return await task_to_await + + async def _main(): + _task = await create_coro() + return await await_task(_task) + + with asyncio.Runner(loop_factory=loop_factory) as runner: + runner.run(_main()) + + +@pytest.mark.skipif(sys.version_info < (3, 11), reason="loop_factory/asyncio.Runner is not available") +@pytest.mark.parametrize("loop_factory", loop_factories()) +@validate_transaction_metrics( + "test_context_propagation:test_context_propagation_with_loop_factory", + background_task=True, + scoped_metrics=(("Function/waiter2", 2), ("Function/waiter3", 2)), +) +@background_task() +def test_context_propagation_with_loop_factory(loop_factory): + import asyncio + + exceptions = [] + + def handle_exception(loop, context): + exceptions.append(context) + + # Call default handler for standard logging + loop.default_exception_handler(context) + + async def subtask(): + with FunctionTrace(name="waiter2", terminal=True): + pass + + await child() + + async def _task(trace): + assert current_trace() == trace + + await subtask() + + trace = current_trace() + + with asyncio.Runner(loop_factory=loop_factory) as runner: + assert trace == current_trace() + runner._loop.set_exception_handler(handle_exception) + runner.run(_task(trace)) + runner.run(_task(trace)) + + # The agent should have removed all traces from the cache since + # run_until_complete has terminated (all callbacks scheduled inside the + # task have run) + assert len(trace_cache()) == 1 # Sentinel is all that remains + + # # Assert that no exceptions have occurred + assert not exceptions, exceptions + + # Sentinel left in cache transaction exited async def sentinel_in_cache_txn_exited(asyncio, bg): event = asyncio.Event() diff --git a/tests/cross_agent/fixtures/distributed_tracing/README.md b/tests/cross_agent/fixtures/distributed_tracing/README.md index 0090d527a0..c49f614852 100644 --- a/tests/cross_agent/fixtures/distributed_tracing/README.md +++ b/tests/cross_agent/fixtures/distributed_tracing/README.md @@ -1,59 +1,3 @@ -# Distributed Tracing Cross Agent Tests - -The `distributed_tracing.json` file included here is our local copy of the -Better Better CAT CAT [tests](https://source.datanerd.us/agents/cross_agent_tests/pull/96/files). - -The tests in json format and is a list of dictionaries, each dictionary -representing one test. - -## Required fields - -Each test will include these fields: - -+ `test_name`: A string representing the test name. -+ `comment`: String; optional field. Describes the test. -+ `inbound_payloads`: List representing distributed trace payloads as described - in the spec. AcceptDistributedTracePayload should be called with each payload - in turn. -+ `account_id` String. The account_id to use for this test. -+ `transport_type` String. If the transport_type is 'HTTP', payloads will be - generated via a request to a dummy WSGI server. Otherwise, they'll be - generated manually. -+ `major_version`: Integer. The expected major version of the payload. -+ `minor_version`: Integer. The expected minor version of the payload. -+ `trusted_account_key`: String. The earliest ancestor trusted account id. -+ `force_sampled_true`: Boolean. Currently does nothing. -+ `expected_metrics`: List. Each list item itself is also a list of length two. - The first item in the list is a metric name (unscoped). The second item is - the expected number of occurrences of that metric. When a `null` is - encountered, then the metric is expected to be absent. -+ `web_transaction`: Boolean. Whether the test should be run - as a web transaction (as opposed to a background task). -+ `raises_exception`: Boolean, defaults to false. Whether the test should raise - and record an exception (thus creating error traces, error events, etc). -+ `span_events_enabled`: Boolean. Whether span events are expected to be - enabled for this test. (that is to say, whether span events are generated and - validated or not) -+ `outbound_payloads`: List. For each item in the list, an outgoing request - should be made during the distributed trace transaction. The list item itself - is a dict, with the key, value pairs that should be asserted. Note that - keys prepended with "d." are in the `data` portion of the outgoing payload. -+ `intrinsics`: Dictionary. Has a specific format, as described below. - -## "Intrinsics" field attributes - -+ `target_events`: List of strings. Each string represents an event type that - will be generated by this test. Each string will also appear as a field in - this `intrinsics` dict, with the same format as `common`. - -+ `common`: A dict representing common attributes for all generated event - types. The key/value pairs in `common` should be unioned with the values for - each specific event type. Both `common` and the specific events will each - have three subfields: `expected`, `unexpected`, and `exact`. The first two - list of fields that should and should not, respectably, be present in the - are event's attributes. `exact`, on the other hand, is a dict with key/value - pairs describing what each attribute's exact value should be. - ### Trace Context test details The Trace Context test cases in `trace_context.json` are meant to be used to verify the @@ -65,17 +9,39 @@ the agent under test. Here's what the various fields in each test case mean: | Name | Meaning | | ---- | ------- | -| `name` | A human-meaningful name for the test case. | +| `test_name` | A human-meaningful name for the test case. | | `trusted_account_key` | The account ids the agent can trust. | | `account_id` | The account id the agent would receive on connect. | | `web_transaction` | Whether the transaction that's tested is a web transaction or not. | | `raises_exception` | Whether to simulate an exception happening within the transaction or not, resulting in a transaction error event. | -| `force_sampled_true` | Whether to force a transaction to be sampled or not. | +| `distributed_tracing_enabled` | If `false`, then distributed tracing is disabled. If `true` or absent, then distributed tracing is enabled (default behavior). | +| `full_granularity_enabled` | If `false`, then full granularity tracing is disabled. If `true` or absent, then full granularity is enabled (default behavior). | +| `root` | The full granularity sampler to use for transactions at the root of a trace. | +| `remote_parent_sampled` | The full granularity sampler to use for transactions with a remote parent that was sampled. | +| `remote_parent_not_sampled` | The full granularity sampler to use for transactions with a remote parent that is not sampled. | +| `force_adaptive_sampled` | The sampling decision to force on a transaction whenever the adaptive sampler is used. This applies to all adaptive samplers used in the test, whether they are the global sampler or an individual sampler instance. | +| `full_granularity_ratio` | The ratio to use for all of the full granularity trace ID ratio samplers defined in the test. For testing purposes we are not defining different ratios for each trace ID ratio sampler instance. If that is necessary, we will need a different way to configure the ratios. | +| `partial_granularity_enabled` | If `true`, then partial granularity is enabled. If `false` or absent, then partial granularity is disabled (default behavior). | +| `partial_granularity_root` | The partial granularity sampler to use for root transactions. | +| `partial_granularity_remote_parent_sampled` | The partial granularity sampler to use for transactions with a remote parent that was sampled. | +| `partial_granularity_remote_parent_not_sampled` | The partial granularity sampler to use for transaction with a remote parent that was not sampled. | +| `partial_granularity_ratio` | The partial granularity ratio to use for all the partial granularity ratio samplers defined in the test. As with `full_granularity_ratio` we're limiting these tests to have one ratio configured for all partial granularity samplers.| +| `expected_priority_between` | The inclusive range of the expected priority value on the generated transaction event. | | `transport_type` | The transport type for the inbound request. | | `inbound_headers` | The headers you should mock coming into the agent. | -| `outbound_payloads` | The exact/expected/unexpected values for outbound headers. | +| `outbound_payloads` | The exact/expected/unexpected values for outbound `w3c` headers. | | `intrinsics` | The exact/expected/unexpected attributes for events. | | `expected_metrics` | The expected metrics and associated counts as a result of the test. | +| `span_events_enabled` | Whether span events are enabled in the agent or not. | +| `transaction_events_enabled` | Whether transaction events are enabled in the agent or not. | + +The samplers that can referenced in the `root`, `remote_parent_sampled`, and `remote_parent_not_sampled` fields are: + +- `default`: Use the adaptive sampler. +- `adaptive`: Use the adaptive sampler. +- `trace_id_ratio_based`: Use the trace ID ratio sampler. +- `always_on`: Use the always on sampler. +- `always_off`: Use the always off sampler. The `outbound_payloads` and `intrinsics` field can have nested values, for example: ```javascript @@ -119,3 +85,43 @@ have a `guid`, both have `da8bc8cc6d062849b0efcf3c169afb5a` as the `traceId`, an The `Transaction` block means anything in there should only apply to the transaction object. Same for the `Span` block. The same idea goes for the `outbound_payloads` block but will apply specifically for the outbound `traceparent` header and `tracestate` header. + +`outbound_payloads` may also target `newrelic` headers and follow same basic structure inline with trace context headers, for example: +```javascript + ... + "outbound_payloads": [ + { + "exact": { + "traceparent.version": "00", + "traceparent.trace_id": "00000000000000006e2fea0b173fdad0", + "traceparent.trace_flags": "01", + "tracestate.tenant_id": "33", + "tracestate.version": 0, + "tracestate.parent_type": 0, + "tracestate.parent_account_id": "33", + "tracestate.sampled": true, + "tracestate.priority": 1.123432, + "newrelic.v": [0, 1], + "newrelic.d.ty": "App", + "newrelic.d.ac": "33", + "newrelic.d.ap": "2827902", + "newrelic.d.tr": "6E2fEA0B173FDAD0", + "newrelic.d.sa": true, + "newrelic.d.pr": 1.1234321 + }, + "expected": [ + "traceparent.parent_id", + "tracestate.timestamp", + "tracestate.parent_application_id", + "tracestate.span_id", + "tracestate.transaction_id", + "newrelic.d.ap", + "newrelic.d.tx", + "newrelic.d.ti", + "newrelic.d.id" + ], + "unexpected": ["newrelic.d.tk"] + } + ], + ... +``` \ No newline at end of file diff --git a/tests/cross_agent/fixtures/distributed_tracing/distributed_tracing.json b/tests/cross_agent/fixtures/distributed_tracing/distributed_tracing.json index ab82f11d8b..aab8a670b7 100644 --- a/tests/cross_agent/fixtures/distributed_tracing/distributed_tracing.json +++ b/tests/cross_agent/fixtures/distributed_tracing/distributed_tracing.json @@ -64,6 +64,66 @@ ["Supportability/DistributedTrace/AcceptPayload/Success", 1] ] }, + { + "test_name": "high_priority_but_sampled_false", + "comment": "this should never happen, but is here to verify your agent only creates a span event if sampled=true, not just based off of priority", + "trusted_account_key": "33", + "account_id": "33", + "web_transaction": true, + "raises_exception": false, + "force_sampled_true": false, + "span_events_enabled": true, + "major_version": 0, + "minor_version": 1, + "transport_type": "HTTP", + "inbound_payloads": [ + { + "v": [0, 1], + "d": { + "ac": "33", + "ap": "2827902", + "id": "7d3efb1b173fecfa", + "tx": "e8b91a159289ff74", + "pr": 1.234567, + "sa": false, + "ti": 1518469636035, + "tr": "d6b4ba0c3a712ca", + "ty": "App" + } + } + ], + "intrinsics": { + "target_events": ["Transaction"], + "common":{ + "exact": { + "traceId": "d6b4ba0c3a712ca", + "priority": 1.234567, + "sampled": false + }, + "expected": ["guid"], + "unexpected": ["grandparentId", "cross_process_id", "nr.tripId", "nr.pathHash", "nr.referringPathHash", "nr.guid", "nr.referringTransactionGuid", "nr.alternatePathHashes"] + }, + "Transaction": { + "exact": { + "parent.type": "App", + "parent.app": "2827902", + "parent.account": "33", + "parent.transportType": "HTTP", + "parentId": "e8b91a159289ff74", + "parentSpanId": "7d3efb1b173fecfa" + }, + "expected": ["parent.transportDuration"] + }, + "unexpected_events": ["Span"] + }, + "expected_metrics": [ + ["DurationByCaller/App/33/2827902/HTTP/all", 1], + ["DurationByCaller/App/33/2827902/HTTP/allWeb", 1], + ["TransportDuration/App/33/2827902/HTTP/all", 1], + ["TransportDuration/App/33/2827902/HTTP/allWeb", 1], + ["Supportability/DistributedTrace/AcceptPayload/Success", 1] + ] + }, { "test_name": "multiple_accept_calls", "trusted_account_key": "33", @@ -736,78 +796,6 @@ ["Supportability/DistributedTrace/CreatePayload/Success", 2] ] }, - { - "test_name": "payload_from_trusted_partnership_account", - "trusted_account_key": "44", - "account_id": "11", - "web_transaction": true, - "raises_exception": false, - "force_sampled_true": false, - "span_events_enabled": true, - "major_version": 0, - "minor_version": 1, - "transport_type": "HTTP", - "inbound_payloads": [ - { - "v": [0, 1], - "d": { - "ac": "33", - "ap": "2827902", - "tx": "e8b91a159289ff74", - "pr": 0.123456, - "sa": false, - "ti": 1518469636035, - "tr": "d6b4ba0c3a712ca", - "tk": "44", - "ty": "App" - } - } - ], - "outbound_payloads": [ - { - "exact": { - "v": [0, 1], - "d.ac": "11", - "d.pr": 0.123456, - "d.sa": false, - "d.tr": "d6b4ba0c3a712ca", - "d.tk": "44", - "d.ty": "App" - }, - "expected": ["d.ap", "d.tx", "d.ti", "d.id"], - "unexpected": [] - } - ], - "intrinsics": { - "target_events": ["Transaction"], - "common":{ - "exact": { - "parent.type": "App", - "parent.app": "2827902", - "parent.account": "33", - "parent.transportType": "HTTP", - "traceId": "d6b4ba0c3a712ca", - "priority": 0.123456, - "sampled": false - }, - "expected": ["parent.transportDuration", "guid"], - "unexpected": ["grandparentId", "cross_process_id", "nr.tripId", "nr.pathHash", "nr.referringPathHash", "nr.guid", "nr.referringTransactionGuid", "nr.alternatePathHashes"] - }, - "Transaction": { - "exact": { - "parentId": "e8b91a159289ff74" - } - } - }, - "expected_metrics": [ - ["DurationByCaller/App/33/2827902/HTTP/all", 1], - ["DurationByCaller/App/33/2827902/HTTP/allWeb", 1], - ["TransportDuration/App/33/2827902/HTTP/all", 1], - ["TransportDuration/App/33/2827902/HTTP/allWeb", 1], - ["Supportability/DistributedTrace/AcceptPayload/Success", 1], - ["Supportability/DistributedTrace/CreatePayload/Success", 1] - ] - }, { "test_name": "payload_has_larger_minor_version", "trusted_account_key": "33", @@ -895,7 +883,7 @@ "target_events": ["Transaction"], "common":{ "expected": ["guid", "traceId", "priority", "sampled"], - "unexpected": ["grandparentId", "cross_process_id", "nr.tripId", "nr.pathHash", "nr.referringPathHash", "nr.guid", "nr.referringTransactionGuid", "nr.alternatePathHashes", "parent.transportDuration", "parent.type", "parent.app", "parent.account", "parent.transportType", "parentId"] + "unexpected": ["grandparentId", "cross_process_id", "nr.tripId", "nr.pathHash", "nr.referringPathHash", "nr.guid", "nr.referringTransactionGuid", "nr.alternatePathHashes", "parent.transportDuration", "parent.type", "parent.app", "parent.account", "parentId"] } }, "expected_metrics": [ @@ -932,7 +920,7 @@ "target_events": ["Transaction"], "common":{ "expected": ["guid", "traceId", "priority", "sampled"], - "unexpected": ["grandparentId", "cross_process_id", "nr.tripId", "nr.pathHash", "nr.referringPathHash", "nr.guid", "nr.referringTransactionGuid", "nr.alternatePathHashes", "parent.transportDuration", "parent.type", "parent.app", "parent.account", "parent.transportType", "parentId"] + "unexpected": ["grandparentId", "cross_process_id", "nr.tripId", "nr.pathHash", "nr.referringPathHash", "nr.guid", "nr.referringTransactionGuid", "nr.alternatePathHashes", "parent.transportDuration", "parent.type", "parent.app", "parent.account", "parentId"] } }, "expected_metrics": [ @@ -969,7 +957,7 @@ "target_events": ["Transaction"], "common":{ "expected": ["guid", "traceId", "priority", "sampled"], - "unexpected": ["grandparentId", "cross_process_id", "nr.tripId", "nr.pathHash", "nr.referringPathHash", "nr.guid", "nr.referringTransactionGuid", "nr.alternatePathHashes", "parent.transportDuration", "parent.type", "parent.app", "parent.account", "parent.transportType", "parentId"] + "unexpected": ["grandparentId", "cross_process_id", "nr.tripId", "nr.pathHash", "nr.referringPathHash", "nr.guid", "nr.referringTransactionGuid", "nr.alternatePathHashes", "parent.transportDuration", "parent.type", "parent.app", "parent.account", "parentId"] } }, "expected_metrics": [ @@ -992,7 +980,7 @@ "target_events": ["Transaction"], "common":{ "expected": ["guid", "traceId", "priority", "sampled"], - "unexpected": ["grandparentId", "cross_process_id", "nr.tripId", "nr.pathHash", "nr.referringPathHash", "nr.guid", "nr.referringTransactionGuid", "nr.alternatePathHashes", "parent.transportDuration", "parent.type", "parent.app", "parent.account", "parent.transportType", "parentId"] + "unexpected": ["grandparentId", "cross_process_id", "nr.tripId", "nr.pathHash", "nr.referringPathHash", "nr.guid", "nr.referringTransactionGuid", "nr.alternatePathHashes", "parent.transportDuration", "parent.type", "parent.app", "parent.account", "parentId"] } }, "expected_metrics": [ @@ -1028,7 +1016,7 @@ "target_events": ["Transaction"], "common":{ "expected": ["guid", "traceId", "priority", "sampled"], - "unexpected": ["grandparentId", "cross_process_id", "nr.tripId", "nr.pathHash", "nr.referringPathHash", "nr.guid", "nr.referringTransactionGuid", "nr.alternatePathHashes", "parent.transportDuration", "parent.type", "parent.app", "parent.account", "parent.transportType", "parentId"] + "unexpected": ["grandparentId", "cross_process_id", "nr.tripId", "nr.pathHash", "nr.referringPathHash", "nr.guid", "nr.referringTransactionGuid", "nr.alternatePathHashes", "parent.transportDuration", "parent.type", "parent.app", "parent.account", "parentId"] } }, "expected_metrics": [ @@ -1055,7 +1043,7 @@ "target_events": ["Transaction"], "common":{ "expected": ["guid", "traceId", "priority", "sampled"], - "unexpected": ["grandparentId", "cross_process_id", "nr.tripId", "nr.pathHash", "nr.referringPathHash", "nr.guid", "nr.referringTransactionGuid", "nr.alternatePathHashes", "parent.transportDuration", "parent.type", "parent.app", "parent.account", "parent.transportType", "parentId"] + "unexpected": ["grandparentId", "cross_process_id", "nr.tripId", "nr.pathHash", "nr.referringPathHash", "nr.guid", "nr.referringTransactionGuid", "nr.alternatePathHashes", "parent.transportDuration", "parent.type", "parent.app", "parent.account", "parentId"] } }, "expected_metrics": [ @@ -1091,7 +1079,7 @@ "target_events": ["Transaction"], "common":{ "expected": ["guid", "traceId", "priority", "sampled"], - "unexpected": ["grandparentId", "cross_process_id", "nr.tripId", "nr.pathHash", "nr.referringPathHash", "nr.guid", "nr.referringTransactionGuid", "nr.alternatePathHashes", "parent.transportDuration", "parent.type", "parent.app", "parent.account", "parent.transportType", "parentId"] + "unexpected": ["grandparentId", "cross_process_id", "nr.tripId", "nr.pathHash", "nr.referringPathHash", "nr.guid", "nr.referringTransactionGuid", "nr.alternatePathHashes", "parent.transportDuration", "parent.type", "parent.app", "parent.account", "parentId"] } }, "expected_metrics": [ @@ -1127,7 +1115,7 @@ "target_events": ["Transaction"], "common":{ "expected": ["guid", "traceId", "priority", "sampled"], - "unexpected": ["grandparentId", "cross_process_id", "nr.tripId", "nr.pathHash", "nr.referringPathHash", "nr.guid", "nr.referringTransactionGuid", "nr.alternatePathHashes", "parent.transportDuration", "parent.type", "parent.app", "parent.account", "parent.transportType", "parentId"] + "unexpected": ["grandparentId", "cross_process_id", "nr.tripId", "nr.pathHash", "nr.referringPathHash", "nr.guid", "nr.referringTransactionGuid", "nr.alternatePathHashes", "parent.transportDuration", "parent.type", "parent.app", "parent.account", "parentId"] } }, "expected_metrics": [ @@ -1163,7 +1151,7 @@ "target_events": ["Transaction"], "common":{ "expected": ["guid", "traceId", "priority", "sampled"], - "unexpected": ["grandparentId", "cross_process_id", "nr.tripId", "nr.pathHash", "nr.referringPathHash", "nr.guid", "nr.referringTransactionGuid", "nr.alternatePathHashes", "parent.transportDuration", "parent.type", "parent.app", "parent.account", "parent.transportType", "parentId"] + "unexpected": ["grandparentId", "cross_process_id", "nr.tripId", "nr.pathHash", "nr.referringPathHash", "nr.guid", "nr.referringTransactionGuid", "nr.alternatePathHashes", "parent.transportDuration", "parent.type", "parent.app", "parent.account", "parentId"] } }, "expected_metrics": [ @@ -1199,7 +1187,7 @@ "target_events": ["Transaction"], "common":{ "expected": ["guid", "traceId", "priority", "sampled"], - "unexpected": ["grandparentId", "cross_process_id", "nr.tripId", "nr.pathHash", "nr.referringPathHash", "nr.guid", "nr.referringTransactionGuid", "nr.alternatePathHashes", "parent.transportDuration", "parent.type", "parent.app", "parent.account", "parent.transportType", "parentId"] + "unexpected": ["grandparentId", "cross_process_id", "nr.tripId", "nr.pathHash", "nr.referringPathHash", "nr.guid", "nr.referringTransactionGuid", "nr.alternatePathHashes", "parent.transportDuration", "parent.type", "parent.app", "parent.account", "parentId"] } }, "expected_metrics": [ @@ -1235,7 +1223,7 @@ "target_events": ["Transaction"], "common":{ "expected": ["guid", "traceId", "priority", "sampled"], - "unexpected": ["grandparentId", "cross_process_id", "nr.tripId", "nr.pathHash", "nr.referringPathHash", "nr.guid", "nr.referringTransactionGuid", "nr.alternatePathHashes", "parent.transportDuration", "parent.type", "parent.app", "parent.account", "parent.transportType", "parentId"] + "unexpected": ["grandparentId", "cross_process_id", "nr.tripId", "nr.pathHash", "nr.referringPathHash", "nr.guid", "nr.referringTransactionGuid", "nr.alternatePathHashes", "parent.transportDuration", "parent.type", "parent.app", "parent.account", "parentId"] } }, "expected_metrics": [ @@ -1271,7 +1259,7 @@ "target_events": ["Transaction"], "common":{ "expected": ["guid", "traceId", "priority", "sampled"], - "unexpected": ["grandparentId", "cross_process_id", "nr.tripId", "nr.pathHash", "nr.referringPathHash", "nr.guid", "nr.referringTransactionGuid", "nr.alternatePathHashes", "parent.transportDuration", "parent.type", "parent.app", "parent.account", "parent.transportType", "parentId"] + "unexpected": ["grandparentId", "cross_process_id", "nr.tripId", "nr.pathHash", "nr.referringPathHash", "nr.guid", "nr.referringTransactionGuid", "nr.alternatePathHashes", "parent.transportDuration", "parent.type", "parent.app", "parent.account", "parentId"] } }, "expected_metrics": [ diff --git a/tests/cross_agent/fixtures/distributed_tracing/trace_context.json b/tests/cross_agent/fixtures/distributed_tracing/trace_context.json index 61f6be3d84..fb4cd686eb 100644 --- a/tests/cross_agent/fixtures/distributed_tracing/trace_context.json +++ b/tests/cross_agent/fixtures/distributed_tracing/trace_context.json @@ -1,12 +1,926 @@ [ + { + "test_name": "w3c_sampled_remote_parent_sampled_default_uses_adaptive_sampling_algo", + "comment": "W3C parent header is used to determine remote parent is sampled but W3C trace state is not present so decision goes to random adaptive sampler algo", + "trusted_account_key": "33", + "account_id": "33", + "web_transaction": true, + "raises_exception": false, + "span_events_enabled": true, + "transaction_events_enabled": true, + "transport_type": "HTTP", + "remote_parent_sampled": "adaptive", + "remote_parent_not_sampled": "adaptive", + "force_adaptive_sampled": false, + "expected_priority_between": [ 0, 1 ], + "inbound_headers": [ + { + "traceparent": "00-0af7651916cd43dd8448eb211c80319c-00f067aa0ba902b7-01", + "newrelic": "{\"v\":[0,1],\"d\":{\"ty\":\"App\",\"ac\":\"33\",\"ap\":\"2827902\",\"id\":\"7d3efb1b173fecfa\",\"tr\":\"0af7651916cd43dd8448eb211c80319c\",\"pr\":1.234567,\"sa\":true,\"ti\":1518469636035,\"tx\":\"0af7651916cd43dd\"}}" + } + ], + "intrinsics": { + "target_events": [ "Transaction" ], + "common": { + "exact": { + "traceId": "0af7651916cd43dd8448eb211c80319c", + "sampled": false + }, + "expected": [ "guid" ] + } + } + }, + { + "test_name": "w3c_sampled_remote_parent_sampled_default_uses_w3c_trace_state_remote_sampled", + "comment": "W3C parent header is used to determine remote parent is sampled and W3C trace state is used to set sampling decision", + "trusted_account_key": "33", + "account_id": "33", + "web_transaction": true, + "raises_exception": false, + "span_events_enabled": true, + "transaction_events_enabled": true, + "transport_type": "HTTP", + "remote_parent_sampled": "adaptive", + "remote_parent_not_sampled": "adaptive", + "force_adaptive_sampled": false, + "inbound_headers": [ + { + "traceparent": "00-0af7651916cd43dd8448eb211c80319c-00f067aa0ba902b7-01", + "tracestate": "33@nr=0-0-33-2827902-0af7651916cd43dd--1-1.2-1518469636035", + "newrelic": "{\"v\":[0,1],\"d\":{\"ty\":\"App\",\"ac\":\"33\",\"ap\":\"2827902\",\"id\":\"7d3efb1b173fecfa\",\"tr\":\"0af7651916cd43dd8448eb211c80319c\",\"pr\":0,\"sa\":false,\"ti\":1518469636035,\"tx\":\"0af7651916cd43dd\"}}" + } + ], + "intrinsics": { + "target_events": [ "Transaction" ], + "common": { + "exact": { + "traceId": "0af7651916cd43dd8448eb211c80319c", + "priority": 1.2, + "sampled": true + }, + "expected": [ "guid" ] + } + } + }, + { + "test_name": "newrelic_sampled_remote_parent_sampled_default_uses_newrelic_remote_sampled", + "comment": "New Relic header is used to determine remote parent is sampled and to set sampling decision", + "trusted_account_key": "33", + "account_id": "33", + "web_transaction": true, + "raises_exception": false, + "span_events_enabled": true, + "transaction_events_enabled": true, + "transport_type": "HTTP", + "remote_parent_sampled": "adaptive", + "remote_parent_not_sampled": "adaptive", + "force_adaptive_sampled": false, + "inbound_headers": [ + { + "newrelic": "{\"v\":[0,1],\"d\":{\"ty\":\"App\",\"ac\":\"33\",\"ap\":\"2827902\",\"id\":\"7d3efb1b173fecfa\",\"tr\":\"0af7651916cd43dd8448eb211c80319c\",\"pr\":1.2,\"sa\":true,\"ti\":1518469636035,\"tx\":\"0af7651916cd43dd\"}}" + } + ], + "intrinsics": { + "target_events": [ "Transaction" ], + "common": { + "exact": { + "traceId": "0af7651916cd43dd8448eb211c80319c", + "priority": 1.2, + "sampled": true + }, + "expected": [ "guid" ] + } + } + }, + { + "test_name": "no_remote_parent_sampled_decision_so_default_uses_adaptive_sampling_algo", + "comment": "No headers so root default is used and sampling decision goes to random adaptive sampling algorithm", + "trusted_account_key": "33", + "account_id": "33", + "web_transaction": true, + "raises_exception": false, + "span_events_enabled": true, + "transaction_events_enabled": true, + "transport_type": "HTTP", + "remote_parent_sampled": "adaptive", + "remote_parent_not_sampled": "adaptive", + "expected_priority_between": [ 2, 3 ], + "force_adaptive_sampled": true, + "inbound_headers": [ + {} + ], + "intrinsics": { + "target_events": [ "Transaction" ], + "common": { + "exact": { + "sampled": true + }, + "expected": [ "guid", "traceId" ] + } + } + }, + { + "test_name": "w3c_remote_parent_not_sampled_so_adaptive_uses_adaptive_sampling_algo", + "comment": "W3C parent header indicates remote parent not sampled, adaptive is used, and no W3C trace state header present so sampling decision goes to random adaptive sampling algorithm", + "trusted_account_key": "33", + "account_id": "33", + "web_transaction": true, + "raises_exception": false, + "span_events_enabled": true, + "transaction_events_enabled": true, + "transport_type": "HTTP", + "remote_parent_sampled": "adaptive", + "remote_parent_not_sampled": "adaptive", + "expected_priority_between": [ 2, 3 ], + "force_adaptive_sampled": true, + "inbound_headers": [ + { + "traceparent": "00-0af7651916cd43dd8448eb211c80319c-00f067aa0ba902b7-00" + } + ], + "intrinsics": { + "target_events": [ "Transaction" ], + "common": { + "exact": { + "traceId": "0af7651916cd43dd8448eb211c80319c", + "sampled": true + }, + "expected": [ "guid" ] + } + } + }, + { + "test_name": "w3c_remote_parent_not_sampled_so_always_off", + "comment": "W3C parent header indicates remote parent not sampled and is set to always off", + "trusted_account_key": "33", + "account_id": "33", + "web_transaction": true, + "raises_exception": false, + "span_events_enabled": true, + "transaction_events_enabled": true, + "transport_type": "HTTP", + "remote_parent_sampled": "adaptive", + "remote_parent_not_sampled": "always_off", + "force_adaptive_sampled": true, + "inbound_headers": [ + { + "traceparent": "00-0af7651916cd43dd8448eb211c80319c-00f067aa0ba902b7-00", + "newrelic": "{\"v\":[0,1],\"d\":{\"ty\":\"App\",\"ac\":\"33\",\"ap\":\"2827902\",\"id\":\"7d3efb1b173fecfa\",\"tr\":\"0af7651916cd43dd8448eb211c80319c\",\"pr\":1.2,\"sa\":true,\"ti\":1518469636035,\"tx\":\"0af7651916cd43dd\"}}" + } + ], + "intrinsics": { + "target_events": [ "Transaction" ], + "common": { + "exact": { + "traceId": "0af7651916cd43dd8448eb211c80319c", + "priority": 0.0, + "sampled": false + }, + "expected": [ "guid" ] + } + } + }, + { + "test_name": "newrelic_remote_parent_not_sampled_so_always_off", + "comment": "New Relic parent header indicates remote parent not sampled and is set to always off", + "trusted_account_key": "33", + "account_id": "33", + "web_transaction": true, + "raises_exception": false, + "span_events_enabled": true, + "transaction_events_enabled": true, + "transport_type": "HTTP", + "remote_parent_sampled": "adaptive", + "remote_parent_not_sampled": "always_off", + "force_adaptive_sampled": true, + "inbound_headers": [ + { + "newrelic": "{\"v\":[0,1],\"d\":{\"ty\":\"App\",\"ac\":\"33\",\"ap\":\"2827902\",\"id\":\"7d3efb1b173fecfa\",\"tr\":\"0af7651916cd43dd8448eb211c80319c\",\"pr\":1.2,\"sa\":false,\"ti\":1518469636035,\"tx\":\"0af7651916cd43dd\"}}" + } + ], + "intrinsics": { + "target_events": [ "Transaction" ], + "common": { + "exact": { + "traceId": "0af7651916cd43dd8448eb211c80319c", + "priority": 0.0, + "sampled": false + }, + "expected": [ "guid" ] + } + } + }, + { + "test_name": "newrelic_remote_parent_not_sampled_so_always_on", + "comment": "New Relic parent header indicates remote parent not sampled and is set to always on", + "trusted_account_key": "33", + "account_id": "33", + "web_transaction": true, + "raises_exception": false, + "span_events_enabled": true, + "transaction_events_enabled": true, + "transport_type": "HTTP", + "remote_parent_sampled": "adaptive", + "remote_parent_not_sampled": "always_on", + "force_adaptive_sampled": true, + "inbound_headers": [ + { + "newrelic": "{\"v\":[0,1],\"d\":{\"ty\":\"App\",\"ac\":\"33\",\"ap\":\"2827902\",\"id\":\"7d3efb1b173fecfa\",\"tr\":\"0af7651916cd43dd8448eb211c80319c\",\"pr\":1.2,\"sa\":false,\"ti\":1518469636035,\"tx\":\"0af7651916cd43dd\"}}" + } + ], + "intrinsics": { + "target_events": [ "Transaction" ], + "common": { + "exact": { + "traceId": "0af7651916cd43dd8448eb211c80319c", + "priority": 3.0, + "sampled": true + }, + "expected": [ "guid" ] + } + } + }, + { + "test_name": "newrelic_remote_parent_sampled_so_always_on", + "comment": "New Relic parent header indicates remote parent sampled and is set to always on", + "trusted_account_key": "33", + "account_id": "33", + "web_transaction": true, + "raises_exception": false, + "span_events_enabled": true, + "transaction_events_enabled": true, + "transport_type": "HTTP", + "remote_parent_sampled": "always_on", + "remote_parent_not_sampled": "always_off", + "force_adaptive_sampled": true, + "inbound_headers": [ + { + "newrelic": "{\"v\":[0,1],\"d\":{\"ty\":\"App\",\"ac\":\"33\",\"ap\":\"2827902\",\"id\":\"7d3efb1b173fecfa\",\"tr\":\"0af7651916cd43dd8448eb211c80319c\",\"pr\":1.2,\"sa\":true,\"ti\":1518469636035,\"tx\":\"0af7651916cd43dd\"}}" + } + ], + "intrinsics": { + "target_events": [ "Transaction" ], + "common": { + "exact": { + "traceId": "0af7651916cd43dd8448eb211c80319c", + "priority": 3.0, + "sampled": true + }, + "expected": [ "guid" ] + } + } + }, + { + "test_name": "no_headers_root_always_off", + "comment": "Traces originating from current service are never sampled", + "trusted_account_key": "33", + "account_id": "33", + "web_transaction": true, + "raises_exception": false, + "span_events_enabled": true, + "transaction_events_enabled": true, + "transport_type": "HTTP", + "root": "always_off", + "remote_parent_sampled": "adaptive", + "remote_parent_not_sampled": "adaptive", + "force_adaptive_sampled": true, + "inbound_headers": [ + {} + ], + "intrinsics": { + "target_events": [ "Transaction" ], + "common": { + "exact": { + "priority": 0.0, + "sampled": false + }, + "expected": [ "guid", "traceId" ] + } + } + }, + { + "test_name": "no_headers_root_always_on", + "comment": "Traces originating from current service are always sampled", + "trusted_account_key": "33", + "account_id": "33", + "web_transaction": true, + "raises_exception": false, + "span_events_enabled": true, + "transaction_events_enabled": true, + "transport_type": "HTTP", + "root": "always_on", + "remote_parent_sampled": "adaptive", + "remote_parent_not_sampled": "adaptive", + "force_adaptive_sampled": true, + "inbound_headers": [ + {} + ], + "intrinsics": { + "target_events": [ + "Transaction" + ], + "common": { + "exact": { + "priority": 3.0, + "sampled": true + }, + "expected": [ + "guid", + "traceId" + ] + } + } + }, + { + "test_name": "no_headers_root_uses_ratio_sampler", + "comment": "Traces originating from current service are always sampled (because ratio=1.0)", + "trusted_account_key": "33", + "account_id": "33", + "web_transaction": true, + "raises_exception": false, + "span_events_enabled": true, + "transaction_events_enabled": true, + "transport_type": "HTTP", + "root": "trace_id_ratio_based", + "full_granularity_ratio": 1.0, + "remote_parent_sampled": "adaptive", + "remote_parent_not_sampled": "adaptive", + + "expected_priority_between": [ 2, 3 ], + "force_adaptive_sampled": false, + "inbound_headers": [ + {} + ], + "intrinsics": { + "target_events": [ "Transaction" ], + "common": { + "exact": { + "sampled": true + }, + "expected": [ "guid", "traceId" ] + } + } + }, + { + "test_name": "no_headers_root_uses_adaptive_sampler_when_no_ratio", + "comment": "Traces originating from current service use adaptive sampler when no ratio is configured", + "trusted_account_key": "33", + "account_id": "33", + "web_transaction": true, + "raises_exception": false, + "span_events_enabled": true, + "transaction_events_enabled": true, + "transport_type": "HTTP", + "root": "trace_id_ratio_based", + "remote_parent_sampled": "adaptive", + "remote_parent_not_sampled": "adaptive", + "expected_priority_between": [ 2, 3 ], + "force_adaptive_sampled": true, + "inbound_headers": [ + { + "traceparent": "00-0af7651916cd43dd8448eb211c80319c-00f067aa0ba902b7-00" + } + ], + "intrinsics": { + "target_events": [ "Transaction" ], + "common": { + "exact": { + "traceId": "0af7651916cd43dd8448eb211c80319c", + "sampled": true + }, + "expected": [ "guid" ] + } + } + }, + { + "test_name": "w3c_remote_parent_sampled_uses_ratio_sampler_w3c_trace_id", + "comment": "W3C parent header determines parent is sampled and W3C trace id is passed to ratio sampler", + "trusted_account_key": "33", + "account_id": "33", + "web_transaction": true, + "raises_exception": false, + "span_events_enabled": true, + "transaction_events_enabled": true, + "transport_type": "HTTP", + "root": "always_off", + "full_granularity_ratio": 1.0, + "remote_parent_sampled": "trace_id_ratio_based", + "remote_parent_not_sampled": "always_off", + + "expected_priority_between": [ 2, 3 ], + "force_adaptive_sampled": false, + "inbound_headers": [ + { + "traceparent": "00-0af7651916cd43dd8448eb211c80319c-00f067aa0ba902b7-01" + } + ], + "intrinsics": { + "target_events": [ "Transaction" ], + "common": { + "exact": { + "traceId": "0af7651916cd43dd8448eb211c80319c", + "sampled": true + }, + "expected": [ "guid" ] + } + } + }, + { + "test_name": "w3c_remote_parent_sampled_uses_ratio_sampler_w3c_trace_id_not_sampled", + "comment": "W3C parent header determines parent is sampled and W3C trace id is passed to ratio sampler but makes a sampling decision of false because ratio is 0", + "trusted_account_key": "33", + "account_id": "33", + "web_transaction": true, + "raises_exception": false, + "span_events_enabled": true, + "transaction_events_enabled": true, + "transport_type": "HTTP", + "root": "trace_id_ratio_based", + "full_granularity_ratio": 0.00000001, + "remote_parent_sampled": "trace_id_ratio_based", + "remote_parent_not_sampled": "trace_id_ratio_based", + + "expected_priority_between": [ 0, 1 ], + "force_adaptive_sampled": false, + "inbound_headers": [ + { + "traceparent": "00-0af7651916cd43dd8448eb211c80319c-00f067aa0ba902b7-01" + } + ], + "intrinsics": { + "target_events": [ "Transaction" ], + "common": { + "exact": { + "traceId": "0af7651916cd43dd8448eb211c80319c", + "sampled": false + }, + "expected": [ "guid" ] + } + } + }, + { + "test_name": "newrelic_remote_parent_sampled_uses_ratio_sampler_newrelic_trace_id", + "comment": "New Relic parent header determines parent is sampled and New Relic trace id is passed to ratio sampler", + "trusted_account_key": "33", + "account_id": "33", + "web_transaction": true, + "raises_exception": false, + "span_events_enabled": true, + "transaction_events_enabled": true, + "transport_type": "HTTP", + "root": "always_off", + "full_granularity_ratio": 1.0, + "remote_parent_sampled": "trace_id_ratio_based", + "remote_parent_not_sampled": "always_off", + "expected_priority_between": [ 2, 3 ], + "force_adaptive_sampled": false, + "inbound_headers": [ + { + "newrelic": "{\"v\":[0,1],\"d\":{\"ty\":\"App\",\"ac\":\"33\",\"ap\":\"2827902\",\"id\":\"7d3efb1b173fecfa\",\"tr\":\"0af7651916cd43dd8448eb211c80319c\",\"pr\":0.2,\"sa\":true,\"ti\":1518469636035,\"tx\":\"0af7651916cd43dd\"}}" + } + ], + "intrinsics": { + "target_events": [ "Transaction" ], + "common": { + "exact": { + "traceId": "0af7651916cd43dd8448eb211c80319c", + "sampled": true + }, + "expected": [ "guid" ] + } + } + }, + { + "test_name": "newrelic_remote_parent_sampled_uses_ratio_sampler_newrelic_trace_id_not_sampled", + "comment": "New Relic parent header determines parent is sampled and New Relic trace id is passed to ratio sampler but makes a sampling decision of false because ratio is 0", + "trusted_account_key": "33", + "account_id": "33", + "web_transaction": true, + "raises_exception": false, + "span_events_enabled": true, + "transaction_events_enabled": true, + "transport_type": "HTTP", + "root": "trace_id_ratio_based", + "full_granularity_ratio": 0.00000001, + "remote_parent_sampled": "trace_id_ratio_based", + "remote_parent_not_sampled": "trace_id_ratio_based", + "expected_priority_between": [ 0, 1 ], + "force_adaptive_sampled": false, + "inbound_headers": [ + { + "newrelic": "{\"v\":[0,1],\"d\":{\"ty\":\"App\",\"ac\":\"33\",\"ap\":\"2827902\",\"id\":\"7d3efb1b173fecfa\",\"tr\":\"0af7651916cd43dd8448eb211c80319c\",\"pr\":0.2,\"sa\":true,\"ti\":1518469636035,\"tx\":\"0af7651916cd43dd\"}}" + } + ], + "intrinsics": { + "target_events": [ "Transaction" ], + "common": { + "exact": { + "traceId": "0af7651916cd43dd8448eb211c80319c", + "sampled": false + }, + "expected": [ "guid" ] + } + } + }, + { + "test_name": "no_headers_root_uses_partial_ratio_sampler", + "comment": "Traces originating from current service are always sampled at partial granularity (ratio=1.0)", + "trusted_account_key": "33", + "account_id": "33", + "web_transaction": true, + "raises_exception": false, + "span_events_enabled": true, + "transaction_events_enabled": true, + "transport_type": "HTTP", + "root": "trace_id_ratio_based", + "full_granularity_ratio": 0.00000001, + "remote_parent_sampled": "adaptive", + "remote_parent_not_sampled": "adaptive", + "partial_granularity_enabled": true, + "partial_granularity_root": "trace_id_ratio_based", + "partial_granularity_ratio": 1.0, + "partial_granularity_remote_parent_sampled": "adaptive", + "partial_granularity_remote_parent_not_sampled": "adaptive", + "force_adaptive_sampled": false, + "inbound_headers": [ + {} + ], + + "expected_priority_between": [ 1, 2 ], + "intrinsics": { + "target_events": [ "Transaction" ], + "common": { + "exact": { + "sampled": true + }, + "expected": [ "guid", "traceId" ] + } + } + }, + { + "test_name": "newrelic_remote_parent_sampled_uses_partial_ratio_sampler", + "comment": "New Relic parent header determines parent is sampled and New Relic trace id is passed to partial ratio sampler", + "trusted_account_key": "33", + "account_id": "33", + "web_transaction": true, + "raises_exception": false, + "span_events_enabled": true, + "transaction_events_enabled": true, + "transport_type": "HTTP", + "root": "always_off", + "remote_parent_sampled": "always_off", + "remote_parent_not_sampled": "always_off", + "partial_granularity_enabled": true, + "partial_granularity_root": "adaptive", + "partial_granularity_remote_parent_sampled": "trace_id_ratio_based", + "partial_granularity_ratio": 1.0, + "partial_granularity_remote_parent_not_sampled": "adaptive", + "force_adaptive_sampled": false, + + "expected_priority_between": [ 1, 2 ], + "inbound_headers": [ + { + "newrelic": "{\"v\":[0,1],\"d\":{\"ty\":\"App\",\"ac\":\"33\",\"ap\":\"2827902\",\"id\":\"7d3efb1b173fecfa\",\"tr\":\"0af7651916cd43dd8448eb211c80319c\",\"pr\":0.2,\"sa\":true,\"ti\":1518469636035,\"tx\":\"0af7651916cd43dd\"}}" + } + ], + "intrinsics": { + "target_events": [ "Transaction" ], + "common": { + "exact": { + "traceId": "0af7651916cd43dd8448eb211c80319c", + "sampled": true + }, + "expected": [ "guid" ] + } + } + }, + { + "test_name": "newrelic_remote_parent_sampled_uses_partial_ratio_sampler_full_ratio_not_sampled", + "comment": "New Relic parent header determines parent is sampled and New Relic trace id is passed first to full ratio and then partial ratio sampler", + "trusted_account_key": "33", + "account_id": "33", + "web_transaction": true, + "raises_exception": false, + "span_events_enabled": true, + "transaction_events_enabled": true, + "transport_type": "HTTP", + "root": "trace_id_ratio_based", + "remote_parent_sampled": "trace_id_ratio_based", + "remote_parent_not_sampled": "trace_id_ratio_based", + "full_granularity_ratio": 0.00000001, + "partial_granularity_enabled": true, + "partial_granularity_root": "trace_id_ratio_based", + "partial_granularity_remote_parent_sampled": "trace_id_ratio_based", + "partial_granularity_ratio": 1, + "partial_granularity_remote_parent_not_sampled": "trace_id_ratio_based", + "force_adaptive_sampled": false, + + "expected_priority_between": [ 1, 2 ], + "inbound_headers": [ + { + "newrelic": "{\"v\":[0,1],\"d\":{\"ty\":\"App\",\"ac\":\"33\",\"ap\":\"2827902\",\"id\":\"7d3efb1b173fecfa\",\"tr\":\"0af7651916cd43dd8448eb211c80319c\",\"pr\":0.2,\"sa\":true,\"ti\":1518469636035,\"tx\":\"0af7651916cd43dd\"}}" + } + ], + "intrinsics": { + "target_events": [ "Transaction" ], + "common": { + "exact": { + "traceId": "0af7651916cd43dd8448eb211c80319c", + "sampled": true + }, + "expected": [ "guid" ] + } + } + }, + { + "test_name": "newrelic_remote_parent_sampled_uses_partial_ratio_sampler_not_sampled", + "comment": "New Relic parent header determines parent is sampled and New Relic trace id is passed to partial ratio sampler and returns false because ratio is 0", + "trusted_account_key": "33", + "account_id": "33", + "web_transaction": true, + "raises_exception": false, + "span_events_enabled": true, + "transaction_events_enabled": true, + "transport_type": "HTTP", + "root": "trace_id_ratio_based", + "remote_parent_sampled": "trace_id_ratio_based", + "remote_parent_not_sampled": "trace_id_ratio_based", + "full_granularity_ratio": 0.00000001, + "partial_granularity_enabled": true, + "partial_granularity_root": "trace_id_ratio_based", + "partial_granularity_remote_parent_sampled": "trace_id_ratio_based", + "partial_granularity_ratio": 0.00000001, + "partial_granularity_remote_parent_not_sampled": "trace_id_ratio_based", + "force_adaptive_sampled": false, + + "expected_priority_between": [ 0, 1 ], + "inbound_headers": [ + { + "newrelic": "{\"v\":[0,1],\"d\":{\"ty\":\"App\",\"ac\":\"33\",\"ap\":\"2827902\",\"id\":\"7d3efb1b173fecfa\",\"tr\":\"0af7651916cd43dd8448eb211c80319c\",\"pr\":0.2,\"sa\":true,\"ti\":1518469636035,\"tx\":\"0af7651916cd43dd\"}}" + } + ], + "intrinsics": { + "target_events": [ "Transaction" ], + "common": { + "exact": { + "traceId": "0af7651916cd43dd8448eb211c80319c", + "sampled": false + }, + "expected": [ "guid" ] + } + } + }, + { + "test_name": "no_headers_full_traces_disabled_uses_partial_adaptive_sampler", + "comment": "Root traces are always sampled at partial granularity with the global adaptive sampler", + "trusted_account_key": "33", + "account_id": "33", + "web_transaction": true, + "raises_exception": false, + "span_events_enabled": true, + "transaction_events_enabled": true, + "transport_type": "HTTP", + "full_granularity_enabled": false, + "partial_granularity_enabled": true, + "partial_granularity_root": "adaptive", + "force_adaptive_sampled": true, + "partial_granularity_remote_parent_sampled": "always_off", + "partial_granularity_remote_parent_not_sampled": "always_off", + "inbound_headers": [ + {} + ], + + "expected_priority_between": [ 1, 2 ], + "intrinsics": { + "target_events": [ "Transaction" ], + "common": { + "exact": { + "sampled": true + }, + "expected": [ "guid", "traceId" ] + } + } + }, + { + "test_name": "w3c_remote_parent_not_sampled_uses_partial_ratio_sampler", + "comment": "Traceparent header determines parent is not sampled. Full granularity tries to run the adaptive sampler, chooses sampled=false, and W3C trace id is passed to partial ratio sampler", + "trusted_account_key": "33", + "account_id": "33", + "web_transaction": true, + "raises_exception": false, + "span_events_enabled": true, + "transaction_events_enabled": true, + "transport_type": "HTTP", + "root": "always_off", + "remote_parent_sampled": "always_off", + "remote_parent_not_sampled": "adaptive", + "force_adaptive_sampled": false, + "partial_granularity_enabled": true, + "partial_granularity_root": "always_off", + "partial_granularity_remote_parent_sampled": "always_off", + "partial_granularity_remote_parent_not_sampled": "trace_id_ratio_based", + "partial_granularity_ratio": 1.0, + "inbound_headers": [ + { + "traceparent": "00-0af7651916cd43dd8448eb211c80319c-00f067aa0ba902b7-00" + } + ], + + "expected_priority_between": [ 1, 2 ], + "intrinsics": { + "target_events": [ "Transaction" ], + "common": { + "exact": { + "traceId": "0af7651916cd43dd8448eb211c80319c", + "sampled": true + }, + "expected": [ "guid" ] + } + } + }, + { + "test_name": "w3c_remote_parent_sampled_uses_partial_always_on", + "comment": "W3C parent header determines parent is sampled and passes to partial granularity always on sampler", + "trusted_account_key": "33", + "account_id": "33", + "web_transaction": true, + "raises_exception": false, + "span_events_enabled": true, + "transaction_events_enabled": true, + "transport_type": "HTTP", + "full_granularity_enabled": false, + "partial_granularity_enabled": true, + "partial_granularity_remote_parent_sampled": "always_on", + + "force_adaptive_sampled": false, + "inbound_headers": [ + { + "traceparent": "00-0af7651916cd43dd8448eb211c80319c-00f067aa0ba902b7-01" + } + ], + "intrinsics": { + "target_events": [ "Transaction" ], + "common": { + "exact": { + "traceId": "0af7651916cd43dd8448eb211c80319c", + "sampled": true, + "priority": 2.0 + }, + "expected": [ "guid" ] + } + } + }, + { + "test_name": "no_headers_root_uses_partial_always_on", + "comment": "Traces originating from current service are always sampled at partial granularity (with priority 2.0)", + "trusted_account_key": "33", + "account_id": "33", + "web_transaction": true, + "raises_exception": false, + "span_events_enabled": true, + "transaction_events_enabled": true, + "transport_type": "HTTP", + "root": "always_off", + "partial_granularity_enabled": true, + "partial_granularity_root": "always_on", + "inbound_headers": [ + {} + ], + "intrinsics": { + "target_events": [ + "Transaction" + ], + "common": { + "exact": { + "priority": 2.0, + "sampled": true + }, + "expected": [ + "guid", + "traceId" + ] + } + } + }, + { + "test_name": "payload_missing_priority_incremented_to_full_granularity_priority", + "comment": "Remote parent was sampled and tracestate has a sampled=true flag but no priority, so a random priority between 2 and 3 should be generated without use of the sampler", + "trusted_account_key": "33", + "account_id": "33", + "web_transaction": true, + "raises_exception": false, + "span_events_enabled": true, + "transaction_events_enabled": true, + "transport_type": "HTTP", + "remote_parent_sampled": "adaptive", + "force_adaptive_sampled": false, + "inbound_headers": [ + { + "traceparent": "00-f6e9c09812b22fba2f1999b318ddfc8b-7d3efb1b173fecfa-01", + "tracestate": "33@nr=0-2-33-2827902-7d3efb1b173fecfa--1--1518469636035" + } + ], + "expected_priority_between": [2,3], + "intrinsics": { + "target_events": ["Transaction"], + "common":{ + "exact": { + "traceId": "f6e9c09812b22fba2f1999b318ddfc8b", + "sampled": true + }, + "expected": ["guid", "priority"] + } + } + }, + { + "test_name": "payload_missing_priority_incremented_to_partial_granularity_priority", + "comment": "Remote parent was sampled and tracestate has a sampled=true flag but no priority, so a random priority between 1 and 2 should be generated without use of the sampler", + "trusted_account_key": "33", + "account_id": "33", + "web_transaction": true, + "raises_exception": false, + "span_events_enabled": true, + "transaction_events_enabled": true, + "transport_type": "HTTP", + "full_granularity_enabled": false, + "partial_granularity_enabled": true, + "partial_granularity_remote_parent_sampled": "adaptive", + "force_adaptive_sampled": false, + "inbound_headers": [ + { + "traceparent": "00-f6e9c09812b22fba2f1999b318ddfc8b-7d3efb1b173fecfa-01", + "tracestate": "33@nr=0-2-33-2827902-7d3efb1b173fecfa--1--1518469636035" + } + ], + "expected_priority_between": [1, 2], + "intrinsics": { + "target_events": ["Transaction"], + "common":{ + "exact": { + "traceId": "f6e9c09812b22fba2f1999b318ddfc8b", + "sampled": true + }, + "expected": ["guid", "priority"] + } + } + }, + + { + "test_name": "no_headers_root_full_and_partial_disabled", + "comment": "Traces are assigned a random priority <1 when full and partial tracing are both disabled. The expected_priority_between min is arbitrarily small (so that it does not include 0)", + "trusted_account_key": "33", + "account_id": "33", + "web_transaction": true, + "raises_exception": false, + "span_events_enabled": true, + "transaction_events_enabled": true, + "transport_type": "HTTP", + "full_granularity_enabled": false, + "root": "always_on", + "partial_granularity_enabled": false, + "partial_granularity_root": "always_on", + "force_adaptive_sampled": true, + "inbound_headers": [ + {} + ], + "expected_priority_between": [0.000000001, 1], + "intrinsics": { + "target_events": [ "Transaction" ], + "common": { + "exact": { + "sampled": false + }, + "expected": [ "guid", "traceId" ] + } + } + }, + { + "test_name": "no_headers_distributed_tracing_disabled", + "comment": "Traces are assigned a random priority <1 when distributed tracing is disabled. The expected_priority_between min is arbitrarily small (so that it does not include 0)", + "trusted_account_key": "33", + "account_id": "33", + "web_transaction": true, + "raises_exception": false, + "span_events_enabled": true, + "transaction_events_enabled": true, + "distributed_tracing_enabled": false, + "transport_type": "HTTP", + "root": "always_on", + "force_adaptive_sampled": true, + "inbound_headers": [ + {} + ], + "expected_priority_between": [0.000000001, 1] + }, { "test_name": "accept_payload", "trusted_account_key": "33", "account_id": "33", "web_transaction": true, "raises_exception": false, - "force_sampled_true": false, + "force_adaptive_sampled": false, "span_events_enabled": true, + "transaction_events_enabled": true, "transport_type": "HTTP", "inbound_headers": [ { @@ -59,8 +973,9 @@ "account_id": "33", "web_transaction": true, "raises_exception": false, - "force_sampled_true": false, + "force_adaptive_sampled": false, "span_events_enabled": true, + "transaction_events_enabled": true, "transport_type": "HTTP", "inbound_headers": [ { @@ -75,11 +990,11 @@ "traceparent.trace_id": "37375fc353f345b5801b166e31b76136", "traceparent.trace_flags": "00", "tracestate.tenant_id": "33", - "tracestate.version": 0, - "tracestate.parent_type": 0, + "tracestate.version": "0", + "tracestate.parent_type": "0", "tracestate.parent_account_id": "33", - "tracestate.sampled": false, - "tracestate.priority": 0.123456 + "tracestate.sampled": "0", + "tracestate.priority": "0.123456" }, "expected": [ "traceparent.parent_id", @@ -120,24 +1035,108 @@ ["Supportability/TraceContext/Accept/Success", 1] ] }, + { + "test_name": "non_new_relic_parent", + "comment": [ "If a New Relic agent started a trace, and then a non-New", + "Relic tracer propagated the trace, then the traceparent span ID would", + "updated by the non-New Relic tracer, but not the span ID in the New", + "Relic tracestate entry" ], + "trusted_account_key": "33", + "account_id": "33", + "web_transaction": true, + "raises_exception": false, + "force_adaptive_sampled": false, + "span_events_enabled": true, + "transaction_events_enabled": true, + "transport_type": "HTTP", + "inbound_headers": [ + { + "traceparent": "00-7a933b0e517e8c1f6bc6a7466be6f2a0-e8b91a159289ff74-01", + "tracestate": "33@nr=0-0-33-2827902-5093db371f0ba945-3ac44d37ece29bd2-1-1.23456-1518469636035" + } + ], + "intrinsics": { + "target_events": ["Transaction", "Span"], + "common":{ + "exact": { + "traceId": "7a933b0e517e8c1f6bc6a7466be6f2a0", + "priority": 1.23456, + "sampled": true + }, + "expected": ["guid"], + "unexpected": ["grandparentId", "cross_process_id", "nr.tripId", "nr.pathHash", "nr.referringPathHash", "nr.guid", "nr.referringTransactionGuid", "nr.alternatePathHashes"] + }, + "Transaction": { + "exact": { + "parent.type": "App", + "parent.app": "2827902", + "parent.account": "33", + "parent.transportType": "HTTP", + "parentId": "3ac44d37ece29bd2", + "parentSpanId": "e8b91a159289ff74" + }, + "expected": ["parent.transportDuration"] + }, + "Span": { + "exact": { + "parentId": "e8b91a159289ff74", + "trustedParentId": "5093db371f0ba945" + }, + "expected": ["transactionId"], + "unexpected": ["parent.transportDuration", "parent.type", "parent.app", "parent.account", "parent.transportType", "tracingVendors"] + + } + }, + "expected_metrics": [ + ["DurationByCaller/App/33/2827902/HTTP/all", 1], + ["DurationByCaller/App/33/2827902/HTTP/allWeb", 1], + ["TransportDuration/App/33/2827902/HTTP/all", 1], + ["TransportDuration/App/33/2827902/HTTP/allWeb", 1], + ["Supportability/TraceContext/Accept/Success", 1] + ] + }, { "test_name": "spans_disabled_in_parent", - "comment": [ - "If the spans are disabled in a New Relic agent, it will forward its ", - "tracestate, and generate a new traceparent. ", - "The traceparent.parent_id and tracestate.parent_id will mismatch." - ], + "comment": [ "If the parent is a New Relic agent with span events disabled it SHOULD omit span", + "id from the tracestate. This verifies agents propagate Trace Context payloads when the", + "parent is a New Relic agent with span events disabled" ], "trusted_account_key": "33", "account_id": "33", "web_transaction": true, "raises_exception": false, - "force_sampled_true": false, + "force_adaptive_sampled": false, "span_events_enabled": true, + "transaction_events_enabled": true, "transport_type": "HTTP", "inbound_headers": [ { "traceparent": "00-7a933b0e517e8c1f6bc6a7466be6f2a0-e8b91a159289ff74-01", - "tracestate": "33@nr=0-0-33-2827902-5093db371f0ba945-3ac44d37ece29bd2-1-1.23456-1518469636035" + "tracestate": "33@nr=0-0-33-2827902--3ac44d37ece29bd2-1-1.23456-1518469636035" + } + ], + "outbound_payloads": [ + { + "exact": { + "traceparent.version": "00", + "traceparent.trace_id": "7a933b0e517e8c1f6bc6a7466be6f2a0", + "traceparent.trace_flags": "01", + "tracestate.tenant_id": "33", + "tracestate.version": "0", + "tracestate.parent_type": "0", + "tracestate.parent_account_id": "33", + "tracestate.sampled": "1", + "tracestate.priority": "1.23456" + }, + "expected": [ + "traceparent.parent_id", + "tracestate.timestamp", + "tracestate.parent_application_id", + "tracestate.span_id", + "tracestate.transaction_id" + ], + "notequal": { + "traceparent.parent_id": "7d3efb1b173fecfa" + } } ], "intrinsics": { @@ -149,7 +1148,9 @@ "sampled": true }, "expected": ["guid"], - "unexpected": ["grandparentId", "cross_process_id", "nr.tripId", "nr.pathHash", "nr.referringPathHash", "nr.guid", "nr.referringTransactionGuid", "nr.alternatePathHashes"] + "unexpected": ["grandparentId", "cross_process_id", "nr.tripId", "nr.pathHash", + "nr.referringPathHash", "nr.guid", "nr.referringTransactionGuid", + "nr.alternatePathHashes"] }, "Transaction": { "exact": { @@ -164,11 +1165,11 @@ }, "Span": { "exact": { - "parentId": "e8b91a159289ff74", - "trustedParentId": "5093db371f0ba945" + "parentId": "e8b91a159289ff74" }, "expected": ["transactionId"], - "unexpected": ["parent.transportDuration", "parent.type", "parent.app", "parent.account", "parent.transportType", "tracingVendors"] + "unexpected": ["parent.transportDuration", "parent.type", "parent.app", "parent.account", + "parent.transportType", "tracingVendors", "trustedParentId"] } }, @@ -186,8 +1187,9 @@ "account_id": "33", "web_transaction": true, "raises_exception": false, - "force_sampled_true": true, + "force_adaptive_sampled": true, "span_events_enabled": false, + "transaction_events_enabled": true, "transport_type": "HTTP", "inbound_headers": [ { @@ -202,11 +1204,11 @@ "traceparent.trace_id": "2c7a33d956d44531b48ec6f2e535e5c4", "traceparent.trace_flags": "01", "tracestate.tenant_id": "33", - "tracestate.version": 0, - "tracestate.parent_type": 0, + "tracestate.version": "0", + "tracestate.parent_type": "0", "tracestate.parent_account_id": "33", - "tracestate.sampled": true, - "tracestate.priority": 1.23456 + "tracestate.sampled": "1", + "tracestate.priority": "1.23456" }, "expected": [ "traceparent.parent_id", @@ -255,8 +1257,9 @@ "account_id": "33", "web_transaction": true, "raises_exception": false, - "force_sampled_true": true, + "force_adaptive_sampled": true, "span_events_enabled": false, + "transaction_events_enabled": true, "transport_type": "HTTP", "inbound_headers": [ ], @@ -301,8 +1304,7 @@ } }, "expected_metrics": [ - ["DurationByCaller/Unknown/Unknown/Unknown/HTTP/all", 1], - ["DurationByCaller/Unknown/Unknown/Unknown/HTTP/allWeb", 1] + ["Supportability/TraceContext/Create/Success", 1] ] }, { @@ -311,8 +1313,9 @@ "account_id": "33", "web_transaction": true, "raises_exception": true, - "force_sampled_true": false, + "force_adaptive_sampled": false, "span_events_enabled": true, + "transaction_events_enabled": true, "transport_type": "HTTP", "inbound_headers": [ { @@ -368,8 +1371,9 @@ "account_id": "33", "web_transaction": false, "raises_exception": false, - "force_sampled_true": true, + "force_adaptive_sampled": true, "span_events_enabled": true, + "transaction_events_enabled": true, "transport_type": "HTTP", "inbound_headers": [ { @@ -422,8 +1426,9 @@ "account_id": "33", "web_transaction": true, "raises_exception": false, - "force_sampled_true": true, + "force_adaptive_sampled": true, "span_events_enabled": true, + "transaction_events_enabled": true, "transport_type": "HTTP", "inbound_headers": [ { @@ -476,8 +1481,9 @@ "account_id": "33", "web_transaction": true, "raises_exception": false, - "force_sampled_true": false, + "force_adaptive_sampled": false, "span_events_enabled": true, + "transaction_events_enabled": true, "transport_type": "kafka", "inbound_headers": [ { @@ -529,8 +1535,9 @@ "account_id": "33", "web_transaction": true, "raises_exception": false, - "force_sampled_true": true, + "force_adaptive_sampled": true, "span_events_enabled": true, + "transaction_events_enabled": true, "transport_type": "HTTP", "inbound_headers": [ { @@ -545,11 +1552,11 @@ "traceparent.trace_id": "e22175eb1d68b6de32bf70e38458ccc3", "traceparent.trace_flags": "01", "tracestate.tenant_id": "33", - "tracestate.version": 0, - "tracestate.parent_type": 0, + "tracestate.version": "0", + "tracestate.parent_type": "0", "tracestate.parent_account_id": "33", - "tracestate.sampled": true, - "tracestate.priority": 1.23456 + "tracestate.sampled": "1", + "tracestate.priority": "1.23456" }, "expected": [ "traceparent.parent_id", @@ -606,8 +1613,9 @@ "account_id": "33", "web_transaction": true, "raises_exception": false, - "force_sampled_true": false, + "force_adaptive_sampled": false, "span_events_enabled": true, + "transaction_events_enabled": true, "transport_type": "HTTP", "inbound_headers": [ { @@ -622,11 +1630,11 @@ "traceparent.trace_id": "099ae207600a34ecdd5902aba9c8c6c3", "traceparent.trace_flags": "01", "tracestate.tenant_id": "33", - "tracestate.version": 0, - "tracestate.parent_type": 0, + "tracestate.version": "0", + "tracestate.parent_type": "0", "tracestate.parent_account_id": "33", - "tracestate.sampled": true, - "tracestate.priority": 1.23456 + "tracestate.sampled": "1", + "tracestate.priority": "1.23456" }, "expected": [ "traceparent.parent_id", @@ -642,11 +1650,11 @@ "traceparent.trace_id": "099ae207600a34ecdd5902aba9c8c6c3", "traceparent.trace_flags": "01", "tracestate.tenant_id": "33", - "tracestate.version": 0, - "tracestate.parent_type": 0, + "tracestate.version": "0", + "tracestate.parent_type": "0", "tracestate.parent_account_id": "33", - "tracestate.sampled": true, - "tracestate.priority": 1.23456 + "tracestate.sampled": "1", + "tracestate.priority": "1.23456" }, "expected": [ "traceparent.parent_id", @@ -704,7 +1712,8 @@ "web_transaction": true, "raises_exception": false, "span_events_enabled": true, - "force_sampled_true": false, + "transaction_events_enabled": true, + "force_adaptive_sampled": false, "transport_type": "HTTP", "inbound_headers": [ { @@ -719,11 +1728,11 @@ "traceparent.trace_id": "44673569f54fad422c3795b6cd4aef69", "traceparent.trace_flags": "01", "tracestate.tenant_id": "65", - "tracestate.version": 0, - "tracestate.parent_type": 0, + "tracestate.version": "0", + "tracestate.parent_type": "0", "tracestate.parent_account_id": "11", - "tracestate.sampled": true, - "tracestate.priority": 1.23456 + "tracestate.sampled": "1", + "tracestate.priority": "1.23456" }, "expected": [ "traceparent.parent_id", @@ -779,8 +1788,9 @@ "account_id": "33", "web_transaction": true, "raises_exception": false, - "force_sampled_true": true, + "force_adaptive_sampled": true, "span_events_enabled": true, + "transaction_events_enabled": true, "transport_type": "HTTP", "inbound_headers": [ { @@ -818,8 +1828,9 @@ "account_id": "33", "web_transaction": true, "raises_exception": false, - "force_sampled_true": true, + "force_adaptive_sampled": true, "span_events_enabled": true, + "transaction_events_enabled": true, "transport_type": "HTTP", "inbound_headers": [ { @@ -858,15 +1869,16 @@ "test_name": "tracestate_has_larger_version", "comment": [ "If the new relic payload's version is higher than 0, with extra new fields, all ", - "the existing fields should be read and used, and the extra future feilds should ", + "the existing fields should be read and used, and the extra future fields should ", "be ignored." ], "trusted_account_key": "33", "account_id": "33", "web_transaction": true, "raises_exception": false, - "force_sampled_true": false, + "force_adaptive_sampled": false, "span_events_enabled": true, + "transaction_events_enabled": true, "transport_type": "HTTP", "inbound_headers": [ { @@ -881,11 +1893,11 @@ "traceparent.trace_id": "ccaa36c833b26ce54bafa6c4102fd740", "traceparent.trace_flags": "01", "tracestate.tenant_id": "33", - "tracestate.version": 0, - "tracestate.parent_type": 0, + "tracestate.version": "0", + "tracestate.parent_type": "0", "tracestate.parent_account_id": "33", - "tracestate.sampled": true, - "tracestate.priority": 1.23456 + "tracestate.sampled": "1", + "tracestate.priority": "1.23456" }, "expected": [ "traceparent.parent_id", @@ -941,8 +1953,9 @@ "account_id": "33", "web_transaction": true, "raises_exception": false, - "force_sampled_true": true, + "force_adaptive_sampled": true, "span_events_enabled": true, + "transaction_events_enabled": true, "transport_type": "HTTP", "inbound_headers": [ { @@ -980,8 +1993,9 @@ "account_id": "33", "web_transaction": true, "raises_exception": false, - "force_sampled_true": false, + "force_adaptive_sampled": false, "span_events_enabled": true, + "transaction_events_enabled": true, "transport_type": "HTTP", "inbound_headers": [ { @@ -1009,8 +2023,9 @@ "account_id": "33", "web_transaction": true, "raises_exception": false, - "force_sampled_true": false, + "force_adaptive_sampled": false, "span_events_enabled": true, + "transaction_events_enabled": true, "transport_type": "HTTP", "inbound_headers": [ { @@ -1038,8 +2053,9 @@ "account_id": "33", "web_transaction": true, "raises_exception": false, - "force_sampled_true": false, + "force_adaptive_sampled": false, "span_events_enabled": true, + "transaction_events_enabled": true, "transport_type": "HTTP", "inbound_headers": [ { @@ -1067,8 +2083,9 @@ "account_id": "33", "web_transaction": true, "raises_exception": false, - "force_sampled_true": false, + "force_adaptive_sampled": false, "span_events_enabled": true, + "transaction_events_enabled": true, "transport_type": "HTTP", "inbound_headers": [ { @@ -1121,8 +2138,9 @@ "account_id": "33", "web_transaction": true, "raises_exception": false, - "force_sampled_true": false, + "force_adaptive_sampled": false, "span_events_enabled": true, + "transaction_events_enabled": true, "transport_type": "HTTP", "inbound_headers": [ { @@ -1147,8 +2165,9 @@ "account_id": "33", "web_transaction": true, "raises_exception": false, - "force_sampled_true": false, + "force_adaptive_sampled": false, "span_events_enabled": true, + "transaction_events_enabled": true, "transport_type": "HTTP", "inbound_headers": [ { @@ -1165,34 +2184,285 @@ "expected": ["guid", "traceId", "priority", "sampled", "parent.transportType"], "unexpected": ["grandparentId", "cross_process_id", "nr.tripId", "nr.pathHash", "nr.referringPathHash", "nr.guid", "nr.referringTransactionGuid", "nr.alternatePathHashes", "parent.transportDuration", "parent.type", "parent.app", "parent.account", "parentId"] } - }, + }, + "expected_metrics": [ + ["Supportability/TraceContext/TraceState/InvalidNrEntry", 1] + ] + }, + { + "test_name": "multiple_vendors_in_tracestate", + "trusted_account_key": "33", + "account_id": "33", + "web_transaction": true, + "raises_exception": false, + "force_adaptive_sampled": true, + "span_events_enabled": true, + "transaction_events_enabled": true, + "transport_type": "HTTP", + "inbound_headers": [ + { + "traceparent": "00-5f2796876f44a3c898994ce2668e2222-b4a146e3237b4df1-01", + "tracestate": "foo=1,bar=2" + } + ], + "outbound_payloads": [ + { + "exact": { + "traceparent.version": "00", + "traceparent.trace_id": "5f2796876f44a3c898994ce2668e2222", + "tracestate.tenant_id": "33", + "tracestate.version": "0", + "tracestate.parent_type": "0" + }, + "expected": [ + "traceparent.trace_flags", + "traceparent.parent_id", + "tracestate.span_id", + "tracestate.transaction_id", + "tracestate.parent_application_id", + "tracestate.timestamp", + "tracestate.sampled", + "tracestate.priority" + ], + "vendors": [ + "foo", + "bar" + ] + } + ], + "intrinsics": { + "target_events": ["Transaction", "Span"], + "common":{ + "exact": { + "sampled": true, + "traceId": "5f2796876f44a3c898994ce2668e2222" + }, + "expected": ["guid", "priority"], + "unexpected": ["grandparentId", "cross_process_id", "nr.tripId", "nr.pathHash", "nr.referringPathHash", "nr.guid", "nr.referringTransactionGuid", "nr.alternatePathHashes"] + }, + "Transaction": { + "exact": { + "parentSpanId": "b4a146e3237b4df1" + }, + "expected": ["parent.transportType"], + "unexpected": [ + "parent.transportDuration", + "parent.type", + "parent.app", + "parent.account", + "parentId" + ] + }, + "Span": { + "exact": { + "parentId": "b4a146e3237b4df1", + "tracingVendors": "foo,bar" + }, + "expected": ["transactionId"], + "unexpected": [ + "parent.transportDuration", + "parent.type", + "parent.app", + "parent.account", + "parent.transportType" + ] + } + }, + "expected_metrics": [ + ["DurationByCaller/Unknown/Unknown/Unknown/HTTP/all", 1], + ["DurationByCaller/Unknown/Unknown/Unknown/HTTP/allWeb", 1] + ] + }, + { + "test_name": "missing_tracestate", + "trusted_account_key": "33", + "account_id": "33", + "web_transaction": true, + "raises_exception": false, + "force_adaptive_sampled": true, + "span_events_enabled": true, + "transaction_events_enabled": true, + "transport_type": "HTTP", + "inbound_headers": [ + { + "traceparent": "00-5f2796876f44a3c898994ce2668e2222-b4a146e3237b4df1-01" + } + ], + "outbound_payloads": [ + { + "exact": { + "traceparent.version": "00", + "traceparent.trace_id": "5f2796876f44a3c898994ce2668e2222", + "tracestate.tenant_id": "33", + "tracestate.version": "0", + "tracestate.parent_type": "0", + "tracestate.parent_account_id": "33" + }, + "expected": [ + "traceparent.trace_flags", + "traceparent.parent_id", + "tracestate.span_id", + "tracestate.transaction_id", + "tracestate.parent_application_id", + "tracestate.timestamp", + "tracestate.sampled", + "tracestate.priority" + ] + } + ], + "intrinsics": { + "target_events": ["Transaction", "Span"], + "common":{ + "exact": { + "traceId": "5f2796876f44a3c898994ce2668e2222", + "sampled": true + }, + "expected": ["guid", "priority"], + "unexpected": ["grandparentId", "cross_process_id", "nr.tripId", "nr.pathHash", "nr.referringPathHash", "nr.guid", "nr.referringTransactionGuid", "nr.alternatePathHashes"] + }, + "Transaction": { + "exact": { + "parentSpanId": "b4a146e3237b4df1" + }, + "expected": ["parent.transportType"], + "unexpected": [ + "parent.transportDuration", + "parent.type", + "parent.app", + "parent.account", + "parentId" + ] + }, + "Span": { + "exact": { + "parentId": "b4a146e3237b4df1" + }, + "expected": ["transactionId"], + "unexpected": [ + "parent.transportDuration", + "parent.type", + "parent.app", + "parent.account", + "parent.transportType", + "tracingVendors", + "trustedParentId" + ] + } + }, + "expected_metrics": [ + ["DurationByCaller/Unknown/Unknown/Unknown/HTTP/all", 1], + ["DurationByCaller/Unknown/Unknown/Unknown/HTTP/allWeb", 1] + ] + }, + { + "test_name": "missing_traceparent", + "trusted_account_key": "33", + "account_id": "33", + "web_transaction": true, + "raises_exception": false, + "force_adaptive_sampled": true, + "span_events_enabled": true, + "transaction_events_enabled": true, + "transport_type": "HTTP", + "inbound_headers": [ + { + "tracestate": "foo=1,bar=2" + } + ], + "outbound_payloads": [ + { + "exact": { + "traceparent.version": "00", + "tracestate.tenant_id": "33", + "tracestate.version": "0", + "tracestate.parent_type": "0", + "tracestate.parent_account_id": "33" + }, + "expected": [ + "traceparent.trace_id", + "traceparent.trace_flags", + "traceparent.parent_id", + "tracestate.span_id", + "tracestate.transaction_id", + "tracestate.parent_application_id", + "tracestate.timestamp", + "tracestate.sampled", + "tracestate.priority" + ], + "vendors": [ + ] + } + ], + "expected_metrics": [ + ["Supportability/TraceContext/Create/Success", 1] + ] + }, + { + "test_name": "missing_traceparent_and_tracestate", + "trusted_account_key": "33", + "account_id": "33", + "web_transaction": true, + "raises_exception": false, + "force_adaptive_sampled": true, + "span_events_enabled": true, + "transaction_events_enabled": true, + "transport_type": "HTTP", + "inbound_headers": [ + { } + ], + "outbound_payloads": [ + { + "exact": { + "traceparent.version": "00", + "tracestate.tenant_id": "33", + "tracestate.version": "0", + "tracestate.parent_type": "0", + "tracestate.parent_account_id": "33" + }, + "expected": [ + "traceparent.trace_id", + "traceparent.trace_flags", + "traceparent.parent_id", + "tracestate.span_id", + "tracestate.transaction_id", + "tracestate.parent_application_id", + "tracestate.timestamp", + "tracestate.sampled", + "tracestate.priority" + ] + } + ], "expected_metrics": [ - ["Supportability/TraceContext/TraceState/InvalidNrEntry", 1] + ["Supportability/TraceContext/Create/Success", 1] ] }, { - "test_name": "multiple_vendors_in_tracestate", + "test_name": "multiple_new_relic_trace_state_entries", "trusted_account_key": "33", - "account_id": "33", + "account_id": "99", "web_transaction": true, "raises_exception": false, - "force_sampled_true": true, + "force_adaptive_sampled": false, "span_events_enabled": true, + "transaction_events_enabled": true, "transport_type": "HTTP", "inbound_headers": [ { - "traceparent": "00-5f2796876f44a3c898994ce2668e2222-b4a146e3237b4df1-01", - "tracestate": "foo=1,bar=2" + "traceparent": "00-87b1c9a429205b25e5b687d890d4821f-afe162ae3117a892-00", + "tracestate": "44@nr=0-0-11-30299-afe162ae3117a892-0b752e7f02c85205-0-0.123456-1518469636035,33@nr=0-0-33-2827902-7d3efb1b173fecfa-b79e301bd0ffed87-1-1.23456-1518469636025" } ], "outbound_payloads": [ { "exact": { "traceparent.version": "00", - "traceparent.trace_id": "5f2796876f44a3c898994ce2668e2222", + "traceparent.trace_id": "87b1c9a429205b25e5b687d890d4821f", "tracestate.tenant_id": "33", - "tracestate.version": 0, - "tracestate.parent_type": 0 + "tracestate.version": "0", + "tracestate.parent_type": "0", + "tracestate.parent_account_id": "99", + "tracestate.sampled": "1", + "tracestate.priority": "1.23456" }, "expected": [ "traceparent.trace_flags", @@ -1200,13 +2470,10 @@ "tracestate.span_id", "tracestate.transaction_id", "tracestate.parent_application_id", - "tracestate.timestamp", - "tracestate.sampled", - "tracestate.priority" + "tracestate.timestamp" ], "vendors": [ - "foo", - "bar" + "44@nr" ] } ], @@ -1214,68 +2481,67 @@ "target_events": ["Transaction", "Span"], "common":{ "exact": { - "sampled": true, - "traceId": "5f2796876f44a3c898994ce2668e2222" + "traceId": "87b1c9a429205b25e5b687d890d4821f", + "priority": 1.23456, + "sampled": true }, - "expected": ["guid", "priority"], + "expected": ["guid"], "unexpected": ["grandparentId", "cross_process_id", "nr.tripId", "nr.pathHash", "nr.referringPathHash", "nr.guid", "nr.referringTransactionGuid", "nr.alternatePathHashes"] }, "Transaction": { "exact": { - "parentSpanId": "b4a146e3237b4df1" + "parent.type": "App", + "parent.app": "2827902", + "parent.account": "33", + "parent.transportType": "HTTP", + "parentId": "b79e301bd0ffed87", + "parentSpanId": "afe162ae3117a892" }, - "expected": ["parent.transportType"], - "unexpected": [ - "parent.transportDuration", - "parent.type", - "parent.app", - "parent.account", - "parentId" - ] + "expected": ["parent.transportDuration"] }, "Span": { "exact": { - "parentId": "b4a146e3237b4df1", - "tracingVendors": "foo,bar" + "parentId": "afe162ae3117a892", + "trustedParentId": "7d3efb1b173fecfa", + "tracingVendors": "44@nr" }, "expected": ["transactionId"], - "unexpected": [ - "parent.transportDuration", - "parent.type", - "parent.app", - "parent.account", - "parent.transportType" - ] + "unexpected": ["parent.transportDuration", "parent.type", "parent.app", "parent.account", "parent.transportType"] } }, "expected_metrics": [ - ["DurationByCaller/Unknown/Unknown/Unknown/HTTP/all", 1], - ["DurationByCaller/Unknown/Unknown/Unknown/HTTP/allWeb", 1] + ["DurationByCaller/App/33/2827902/HTTP/all", 1], + ["DurationByCaller/App/33/2827902/HTTP/allWeb", 1], + ["Supportability/TraceContext/Accept/Success", 1] ] }, { - "test_name": "missing_tracestate", + "test_name": "priority_not_converted_to_scientific_notation", "trusted_account_key": "33", - "account_id": "33", + "account_id": "99", "web_transaction": true, "raises_exception": false, - "force_sampled_true": true, + "force_adaptive_sampled": false, "span_events_enabled": true, + "transaction_events_enabled": true, "transport_type": "HTTP", "inbound_headers": [ { - "traceparent": "00-5f2796876f44a3c898994ce2668e2222-b4a146e3237b4df1-01" + "traceparent": "00-87b1c9a429205b25e5b687d890d4821f-afe162ae3117a892-00", + "tracestate": "33@nr=0-0-11-30299-afe162ae3117a892-0b752e7f02c85205-0-0.000012-1518469636035" } ], "outbound_payloads": [ { "exact": { "traceparent.version": "00", - "traceparent.trace_id": "5f2796876f44a3c898994ce2668e2222", + "traceparent.trace_id": "87b1c9a429205b25e5b687d890d4821f", "tracestate.tenant_id": "33", - "tracestate.version": 0, - "tracestate.parent_type": 0, - "tracestate.parent_account_id": "33" + "tracestate.version": "0", + "tracestate.parent_type": "0", + "tracestate.parent_account_id": "99", + "tracestate.sampled": "0", + "tracestate.priority": "0.000012" }, "expected": [ "traceparent.trace_flags", @@ -1283,254 +2549,381 @@ "tracestate.span_id", "tracestate.transaction_id", "tracestate.parent_application_id", - "tracestate.timestamp", - "tracestate.sampled", - "tracestate.priority" - ] + "tracestate.timestamp" + ] } + ], + "intrinsics": { + "target_events": ["Transaction"], + "common":{ + "exact": { + "traceId": "87b1c9a429205b25e5b687d890d4821f", + "priority": 0.000012, + "sampled": false + }, + "expected": ["guid"], + "unexpected": ["grandparentId", "cross_process_id", "nr.tripId", "nr.pathHash", "nr.referringPathHash", "nr.guid", "nr.referringTransactionGuid", "nr.alternatePathHashes"] + }, + "Transaction": { + "exact": { + "parent.type": "App", + "parent.app": "30299", + "parent.account": "11", + "parent.transportType": "HTTP", + "parentId": "0b752e7f02c85205", + "parentSpanId": "afe162ae3117a892" + }, + "expected": ["parent.transportDuration"] + } + }, + "expected_metrics": [ + ["DurationByCaller/App/11/30299/HTTP/all", 1], + ["DurationByCaller/App/11/30299/HTTP/allWeb", 1], + ["Supportability/TraceContext/Accept/Success", 1] + ] + }, + { + "test_name": "w3c_and_newrelic_headers_present", + "comment": "outbound newrelic headers are built from w3c headers, ignoring inbound newrelic headers", + "trusted_account_key": "33", + "account_id": "33", + "web_transaction": true, + "raises_exception": false, + "force_adaptive_sampled": false, + "span_events_enabled": true, + "transaction_events_enabled": true, + "transport_type": "HTTP", + "inbound_headers": [ + { + "traceparent": "00-da8bc8cc6d062849b0efcf3c169afb5a-7d3efb1b173fecfa-01", + "tracestate": "33@nr=0-0-33-2827902-7d3efb1b173fecfa-e8b91a159289ff74-1-1.23456-1518469636035", + "newrelic": "{\"v\":[0,1],\"d\":{\"ty\":\"Mobile\",\"ac\":\"123\",\"ap\":\"51424\",\"id\":\"5f474d64b9cc9b2a\",\"tr\":\"6e2fea0b173fdad0\",\"pr\":0.1234,\"sa\":true,\"ti\":1482959525577,\"tx\":\"27856f70d3d314b7\"}}" + } + ], + "outbound_payloads": [ + { + "exact": { + "newrelic.v": [0, 1], + "newrelic.d.ty": "App", + "newrelic.d.ac": "33", + "newrelic.d.tr": "da8bc8cc6d062849b0efcf3c169afb5a", + "newrelic.d.pr": 1.23456, + "newrelic.d.sa": true + }, + "expected": [ + "newrelic.d.pr", + "newrelic.d.ap", + "newrelic.d.tx", + "newrelic.d.ti", + "newrelic.d.id"], + "unexpected": ["newrelic.d.tk"] } ], "intrinsics": { "target_events": ["Transaction", "Span"], "common":{ "exact": { - "traceId": "5f2796876f44a3c898994ce2668e2222", + "traceId": "da8bc8cc6d062849b0efcf3c169afb5a", + "priority": 1.23456, "sampled": true }, - "expected": ["guid", "priority"], + "expected": ["guid"], "unexpected": ["grandparentId", "cross_process_id", "nr.tripId", "nr.pathHash", "nr.referringPathHash", "nr.guid", "nr.referringTransactionGuid", "nr.alternatePathHashes"] }, "Transaction": { "exact": { - "parentSpanId": "b4a146e3237b4df1" + "parent.type": "App", + "parent.app": "2827902", + "parent.account": "33", + "parent.transportType": "HTTP", + "parentId": "e8b91a159289ff74", + "parentSpanId": "7d3efb1b173fecfa" }, - "expected": ["parent.transportType"], - "unexpected": [ - "parent.transportDuration", - "parent.type", - "parent.app", - "parent.account", - "parentId" - ] + "expected": ["parent.transportDuration"] }, "Span": { "exact": { - "parentId": "b4a146e3237b4df1" + "parentId": "7d3efb1b173fecfa", + "trustedParentId": "7d3efb1b173fecfa" }, "expected": ["transactionId"], - "unexpected": [ - "parent.transportDuration", - "parent.type", - "parent.app", - "parent.account", - "parent.transportType", - "tracingVendors", - "trustedParentId" - ] + "unexpected": ["parent.transportDuration", "parent.type", "parent.app", "parent.account", "parent.transportType", "tracingVendors"] } }, "expected_metrics": [ - ["DurationByCaller/Unknown/Unknown/Unknown/HTTP/all", 1], - ["DurationByCaller/Unknown/Unknown/Unknown/HTTP/allWeb", 1] + ["DurationByCaller/App/33/2827902/HTTP/all", 1], + ["DurationByCaller/App/33/2827902/HTTP/allWeb", 1], + ["TransportDuration/App/33/2827902/HTTP/all", 1], + ["TransportDuration/App/33/2827902/HTTP/allWeb", 1], + ["Supportability/TraceContext/Accept/Success", 1] ] }, { - "test_name": "missing_traceparent", + "test_name": "w3c_and_newrelic_headers_present_error_parsing_traceparent", + "comment": [ + "If the traceparent header is present on an inbound request, conforming agents MUST", + "ignore any newrelic header. If the traceparent header is invalid, a new trace MUST", + "be started. The newrelic header MUST be used _only_ when traceparent is _missing_." + ], "trusted_account_key": "33", "account_id": "33", "web_transaction": true, "raises_exception": false, - "force_sampled_true": true, + "force_adaptive_sampled": true, "span_events_enabled": true, + "transaction_events_enabled": true, "transport_type": "HTTP", "inbound_headers": [ { - "tracestate": "foo=1,bar=2" + "traceparent": "garbage", + "tracestate": "33@nr=0-0-33-2827902-7d3efb1b173fecfa-e8b91a159289ff74-1-1.23456-1518469636035", + "newrelic": "{\"v\":[0,1],\"d\":{\"ty\":\"Mobile\",\"ac\":\"33\",\"ap\":\"51424\",\"id\":\"5f474d64b9cc9b2a\",\"tr\":\"6e2fea0b173fdad0\",\"pr\":0.1234,\"sa\":true,\"ti\":1482959525577,\"tx\":\"27856f70d3d314b7\"}}" } ], "outbound_payloads": [ { "exact": { - "traceparent.version": "00", - "tracestate.tenant_id": "33", - "tracestate.version": 0, - "tracestate.parent_type": 0, - "tracestate.parent_account_id": "33" + "newrelic.v": [0, 1], + "newrelic.d.ty": "App", + "newrelic.d.ac": "33", + "newrelic.d.sa": true + }, + "notequal": { + "newrelic.d.tr": "6e2fea0b173fdad0" }, "expected": [ - "traceparent.trace_id", - "traceparent.trace_flags", - "traceparent.parent_id", - "tracestate.span_id", - "tracestate.transaction_id", - "tracestate.parent_application_id", - "tracestate.timestamp", - "tracestate.sampled", - "tracestate.priority" + "newrelic.d.pr", + "newrelic.d.ap", + "newrelic.d.tx", + "newrelic.d.ti", + "newrelic.d.id", + "newrelic.d.tr" ], - "vendors": [ - ] + "unexpected": ["newrelic.d.tk"] } ], + "intrinsics": { + "target_events": ["Span"], + "common":{ + "expected": ["guid", "traceId", "priority", "sampled"], + "unexpected": ["grandparentId", "cross_process_id", "nr.tripId", "nr.pathHash", "nr.referringPathHash", "nr.guid", "nr.referringTransactionGuid", "nr.alternatePathHashes", "parent.type", "parent.app", "parent.account", "parentId", "parentSpanId", "parent.transportDuration", "tracingVendors"] + }, + "Span": { + "expected": ["transactionId"] + } + }, "expected_metrics": [ - ["DurationByCaller/Unknown/Unknown/Unknown/HTTP/all", 1], - ["DurationByCaller/Unknown/Unknown/Unknown/HTTP/allWeb", 1] + ["Supportability/TraceContext/TraceParent/Parse/Exception", 1] ] }, { - "test_name": "missing_traceparent_and_tracestate", + "test_name": "w3c_and_newrelic_headers_present_error_parsing_tracestate", "trusted_account_key": "33", "account_id": "33", "web_transaction": true, "raises_exception": false, - "force_sampled_true": true, + "force_adaptive_sampled": true, "span_events_enabled": true, + "transaction_events_enabled": true, "transport_type": "HTTP", "inbound_headers": [ - { } + { + "traceparent": "00-da8bc8cc6d062849b0efcf3c169afb5a-7d3efb1b173fecfa-01", + "tracestate": "33@nr=garbage", + "newrelic": "{\"v\":[0,1],\"d\":{\"ty\":\"Mobile\",\"ac\":\"123\",\"ap\":\"51424\",\"id\":\"5f474d64b9cc9b2a\",\"tr\":\"6e2fea0b173fdad0\",\"pr\":0.1234,\"sa\":true,\"ti\":1482959525577,\"tx\":\"27856f70d3d314b7\"}}" + } ], "outbound_payloads": [ { "exact": { - "traceparent.version": "00", - "tracestate.tenant_id": "33", - "tracestate.version": 0, - "tracestate.parent_type": 0, - "tracestate.parent_account_id": "33" + "newrelic.v": [0, 1], + "newrelic.d.ty": "App", + "newrelic.d.ac": "33", + "newrelic.d.tr": "da8bc8cc6d062849b0efcf3c169afb5a", + "newrelic.d.sa": true }, "expected": [ - "traceparent.trace_id", - "traceparent.trace_flags", - "traceparent.parent_id", - "tracestate.span_id", - "tracestate.transaction_id", - "tracestate.parent_application_id", - "tracestate.timestamp", - "tracestate.sampled", - "tracestate.priority" - ] + "newrelic.d.pr", + "newrelic.d.ap", + "newrelic.d.tx", + "newrelic.d.ti", + "newrelic.d.id" + ], + "unexpected": ["newrelic.d.tk"] } ], + "intrinsics": { + "target_events": ["Transaction", "Span"], + "common":{ + "exact": { + "traceId": "da8bc8cc6d062849b0efcf3c169afb5a", + "sampled": true + }, + "expected": ["guid", "priority"], + "unexpected": ["grandparentId", "cross_process_id", "nr.tripId", "nr.pathHash", "nr.referringPathHash", "nr.guid", "nr.referringTransactionGuid", "nr.alternatePathHashes", "parent.type", "parent.app", "parent.account", "parent.transportDuration", "trustedParentId"] + }, + "Transaction": { + "exact": { + "parent.transportType": "HTTP", + "parentSpanId": "7d3efb1b173fecfa" + }, + "unexpected": ["parentId"] + }, + "Span": { + "exact": { + "parentId": "7d3efb1b173fecfa" + }, + "expected": ["transactionId"], + "unexpected": ["tracingVendors"] + } + }, "expected_metrics": [ ["DurationByCaller/Unknown/Unknown/Unknown/HTTP/all", 1], - ["DurationByCaller/Unknown/Unknown/Unknown/HTTP/allWeb", 1] + ["DurationByCaller/Unknown/Unknown/Unknown/HTTP/allWeb", 1], + ["Supportability/TraceContext/TraceState/InvalidNrEntry", 1] ] }, { - "test_name": "multiple_new_relic_trace_state_entries", + "test_name": "newrelic_origin_trace_id_correctly_transformed_for_w3c", + "comment": [ + "Tests correct handling of newrelic headers", + "Agents may receive a traceId in upper-case, or shorter than 32 characters.", + "In this case, the traceId MUST be left-padded with zeros AND lower-cased", + "The outbound newrelic header, if configured, should include the traceId as-received." + ], "trusted_account_key": "33", - "account_id": "99", + "account_id": "33", "web_transaction": true, "raises_exception": false, - "force_sampled_true": false, + "force_adaptive_sampled": false, "span_events_enabled": true, + "transaction_events_enabled": true, "transport_type": "HTTP", "inbound_headers": [ { - "traceparent": "00-87b1c9a429205b25e5b687d890d4821f-afe162ae3117a892-00", - "tracestate": "44@nr=0-0-11-30299-afe162ae3117a892-0b752e7f02c85205-0-0.123456-1518469636035,33@nr=0-0-33-2827902-7d3efb1b173fecfa-b79e301bd0ffed87-1-1.23456-1518469636025" + "newrelic": "{\"v\":[0,1],\"d\":{\"ty\":\"App\",\"ac\":\"33\",\"ap\":\"51424\",\"id\":\"5f474d64b9cc9b2a\",\"tr\":\"6E2fEA0B173FDAD0\",\"pr\":1.1234321,\"sa\":true,\"ti\":1482959525577,\"tx\":\"27856f70d3d314b7\"}}" } ], "outbound_payloads": [ { "exact": { "traceparent.version": "00", - "traceparent.trace_id": "87b1c9a429205b25e5b687d890d4821f", + "traceparent.trace_id": "00000000000000006e2fea0b173fdad0", + "traceparent.trace_flags": "01", "tracestate.tenant_id": "33", - "tracestate.version": 0, - "tracestate.parent_type": 0, - "tracestate.parent_account_id": "99", - "tracestate.sampled": true, - "tracestate.priority": 1.23456 + "tracestate.version": "0", + "tracestate.parent_type": "0", + "tracestate.parent_account_id": "33", + "tracestate.sampled": "1", + "tracestate.priority": "1.123432", + "newrelic.v": [0, 1], + "newrelic.d.ty": "App", + "newrelic.d.ac": "33", + "newrelic.d.tr": "6E2fEA0B173FDAD0", + "newrelic.d.sa": true }, "expected": [ - "traceparent.trace_flags", "traceparent.parent_id", + "tracestate.timestamp", + "tracestate.parent_application_id", "tracestate.span_id", "tracestate.transaction_id", - "tracestate.parent_application_id", - "tracestate.timestamp" + "newrelic.d.ap", + "newrelic.d.tx", + "newrelic.d.ti", + "newrelic.d.id", + "newrelic.d.pr" ], - "vendors": [ - "44@nr" - ] + "unexpected": ["newrelic.d.tk"] } ], "intrinsics": { "target_events": ["Transaction", "Span"], "common":{ "exact": { - "traceId": "87b1c9a429205b25e5b687d890d4821f", - "priority": 1.23456, + "traceId": "6E2fEA0B173FDAD0", "sampled": true }, - "expected": ["guid"], + "expected": ["guid", "priority"], "unexpected": ["grandparentId", "cross_process_id", "nr.tripId", "nr.pathHash", "nr.referringPathHash", "nr.guid", "nr.referringTransactionGuid", "nr.alternatePathHashes"] }, "Transaction": { "exact": { "parent.type": "App", - "parent.app": "2827902", "parent.account": "33", + "parent.app": "51424", "parent.transportType": "HTTP", - "parentId": "b79e301bd0ffed87", - "parentSpanId": "afe162ae3117a892" + "parentSpanId": "5f474d64b9cc9b2a", + "parentId": "27856f70d3d314b7" }, "expected": ["parent.transportDuration"] }, "Span": { "exact": { - "parentId": "afe162ae3117a892", - "trustedParentId": "7d3efb1b173fecfa", - "tracingVendors": "44@nr" + "parentId": "5f474d64b9cc9b2a" }, "expected": ["transactionId"], - "unexpected": ["parent.transportDuration", "parent.type", "parent.app", "parent.account", "parent.transportType"] + "unexpected": ["tracingVendors"] } }, "expected_metrics": [ - ["DurationByCaller/App/33/2827902/HTTP/all", 1], - ["DurationByCaller/App/33/2827902/HTTP/allWeb", 1], - ["Supportability/TraceContext/Accept/Success", 1] + ["DurationByCaller/App/33/51424/HTTP/all", 1], + ["DurationByCaller/App/33/51424/HTTP/allWeb", 1], + ["TransportDuration/App/33/51424/HTTP/all", 1], + ["TransportDuration/App/33/51424/HTTP/allWeb", 1], + ["Supportability/DistributedTrace/AcceptPayload/Success", 1] ] }, { - "test_name": "w3c_and_newrelc_headers_present", + "test_name": "span_events_enabled_transaction_events_disabled", "trusted_account_key": "33", "account_id": "33", "web_transaction": true, "raises_exception": false, - "force_sampled_true": false, + "force_adaptive_sampled": true, "span_events_enabled": true, + "transaction_events_enabled": false, "transport_type": "HTTP", "inbound_headers": [ { - "traceparent": "00-da8bc8cc6d062849b0efcf3c169afb5a-7d3efb1b173fecfa-01", - "tracestate": "33@nr=0-0-33-2827902-7d3efb1b173fecfa-e8b91a159289ff74-1-1.23456-1518469636035", - "newrelic": "{\"v\":[0,1],\"d\":{\"ty\":\"Mobile\",\"ac\":\"123\",\"ap\":\"51424\",\"id\":\"5f474d64b9cc9b2a\",\"tr\":\"6e2fea0b173fdad0\",\"pr\":0.1234,\"sa\":true,\"ti\":1482959525577,\"tx\":\"27856f70d3d314b7\"}}" + "traceparent": "00-e22175eb1d68b6de32bf70e38458ccc3-7d3efb1b173fecfa-01", + "tracestate": "33@nr=0-0-33-2827902-7d3efb1b173fecfa-e8b91a159289ff74-1-1.23456-1518469636035" + } + ], + "outbound_payloads": [ + { + "exact": { + "traceparent.version": "00", + "traceparent.trace_id": "e22175eb1d68b6de32bf70e38458ccc3", + "traceparent.trace_flags": "01", + "tracestate.tenant_id": "33", + "tracestate.version": "0", + "tracestate.parent_type": "0", + "tracestate.parent_account_id": "33", + "tracestate.sampled": "1", + "tracestate.priority": "1.23456" + }, + "expected": [ + "traceparent.parent_id", + "tracestate.span_id", + "tracestate.timestamp", + "tracestate.parent_application_id" + ], + "unexpected": [ + "tracestate.transaction_id" + ] } ], "intrinsics": { - "target_events": ["Transaction", "Span"], + "target_events": ["Span"], "common":{ "exact": { - "traceId": "da8bc8cc6d062849b0efcf3c169afb5a", + "traceId": "e22175eb1d68b6de32bf70e38458ccc3", "priority": 1.23456, "sampled": true }, "expected": ["guid"], "unexpected": ["grandparentId", "cross_process_id", "nr.tripId", "nr.pathHash", "nr.referringPathHash", "nr.guid", "nr.referringTransactionGuid", "nr.alternatePathHashes"] }, - "Transaction": { - "exact": { - "parent.type": "App", - "parent.app": "2827902", - "parent.account": "33", - "parent.transportType": "HTTP", - "parentId": "e8b91a159289ff74", - "parentSpanId": "7d3efb1b173fecfa" - }, - "expected": ["parent.transportDuration"] - }, "Span": { "exact": { "parentId": "7d3efb1b173fecfa", @@ -1545,159 +2938,298 @@ ["DurationByCaller/App/33/2827902/HTTP/allWeb", 1], ["TransportDuration/App/33/2827902/HTTP/all", 1], ["TransportDuration/App/33/2827902/HTTP/allWeb", 1], - ["Supportability/TraceContext/Accept/Success", 1] + ["Supportability/TraceContext/Accept/Success", 1], + ["Supportability/TraceContext/Create/Success", 1] ] }, { - "test_name": "w3c_and_newrelc_headers_present_error_parsing_traceparent", + "test_name": "span_events_disabled_transaction_events_disabled", + "comment": [ + "With both spans and transaction events disabled, there will be no ", + "events to verify intrinsics against. tracestate.span_id and ", + "tracestate.transaction_id are not expected on outbound payloads." + ], "trusted_account_key": "33", "account_id": "33", "web_transaction": true, "raises_exception": false, - "force_sampled_true": true, - "span_events_enabled": true, + "force_adaptive_sampled": true, + "span_events_enabled": false, + "transaction_events_enabled": false, "transport_type": "HTTP", "inbound_headers": [ { - "traceparent": "garbage", - "tracestate": "33@nr=0-0-33-2827902-7d3efb1b173fecfa-e8b91a159289ff74-1-1.23456-1518469636035", - "newrelic": "{\"v\":[0,1],\"d\":{\"ty\":\"Mobile\",\"ac\":\"123\",\"ap\":\"51424\",\"id\":\"5f474d64b9cc9b2a\",\"tr\":\"6e2fea0b173fdad0\",\"pr\":0.1234,\"sa\":true,\"ti\":1482959525577,\"tx\":\"27856f70d3d314b7\"}}" + "traceparent": "00-e22175eb1d68b6de32bf70e38458ccc3-7d3efb1b173fecfa-01", + "tracestate": "33@nr=0-0-33-2827902-7d3efb1b173fecfa-e8b91a159289ff74-1-1.23456-1518469636035" } ], - "intrinsics": { - "target_events": ["Transaction", "Span"], - "common":{ - "expected": ["guid", "traceId", "priority", "sampled"], - "unexpected": ["grandparentId", "cross_process_id", "nr.tripId", "nr.pathHash", "nr.referringPathHash", "nr.guid", "nr.referringTransactionGuid", "nr.alternatePathHashes", "parent.type", "parent.app", "parent.account", "parentId", "parentSpanId", "parent.transportDuration", "tracingVendors"] - }, - "Span": { - "expected": ["transactionId"] + "outbound_payloads": [ + { + "exact": { + "traceparent.version": "00", + "traceparent.trace_id": "e22175eb1d68b6de32bf70e38458ccc3", + "traceparent.trace_flags": "01", + "tracestate.tenant_id": "33", + "tracestate.version": "0", + "tracestate.parent_type": "0", + "tracestate.parent_account_id": "33", + "tracestate.sampled": "1", + "tracestate.priority": "1.23456" + }, + "expected": [ + "traceparent.parent_id", + "tracestate.timestamp", + "tracestate.parent_application_id" + ], + "unexpected": [ + "tracestate.span_id", + "tracestate.transaction_id" + ] } - }, + ], "expected_metrics": [ - ["DurationByCaller/Unknown/Unknown/Unknown/HTTP/all", 1], - ["DurationByCaller/Unknown/Unknown/Unknown/HTTP/allWeb", 1], - ["Supportability/TraceContext/TraceParent/Parse/Exception", 1] + ["DurationByCaller/App/33/2827902/HTTP/all", 1], + ["DurationByCaller/App/33/2827902/HTTP/allWeb", 1], + ["TransportDuration/App/33/2827902/HTTP/all", 1], + ["TransportDuration/App/33/2827902/HTTP/allWeb", 1], + ["Supportability/TraceContext/Accept/Success", 1], + ["Supportability/TraceContext/Create/Success", 1] ] }, { - "test_name": "w3c_and_newrelc_headers_present_error_parsing_tracestate", + "test_name": "w3c_and_newrelic_headers_present_emit_both_header_types", "trusted_account_key": "33", "account_id": "33", "web_transaction": true, "raises_exception": false, - "force_sampled_true": true, + "force_adaptive_sampled": false, "span_events_enabled": true, + "transaction_events_enabled": true, "transport_type": "HTTP", "inbound_headers": [ { "traceparent": "00-da8bc8cc6d062849b0efcf3c169afb5a-7d3efb1b173fecfa-01", - "tracestate": "33@nr=garbage", + "tracestate": "33@nr=0-0-33-2827902-7d3efb1b173fecfa-e8b91a159289ff74-1-1.23456-1518469636035", "newrelic": "{\"v\":[0,1],\"d\":{\"ty\":\"Mobile\",\"ac\":\"123\",\"ap\":\"51424\",\"id\":\"5f474d64b9cc9b2a\",\"tr\":\"6e2fea0b173fdad0\",\"pr\":0.1234,\"sa\":true,\"ti\":1482959525577,\"tx\":\"27856f70d3d314b7\"}}" } ], - "intrinsics": { - "target_events": ["Transaction", "Span"], - "common":{ - "exact": { - "traceId": "da8bc8cc6d062849b0efcf3c169afb5a", - "sampled": true - }, - "expected": ["guid", "priority"], - "unexpected": ["grandparentId", "cross_process_id", "nr.tripId", "nr.pathHash", "nr.referringPathHash", "nr.guid", "nr.referringTransactionGuid", "nr.alternatePathHashes", "parent.type", "parent.app", "parent.account", "parent.transportDuration", "trustedParentId"] - }, - "Transaction": { + "outbound_payloads": [ + { "exact": { - "parent.transportType": "HTTP", - "parentSpanId": "7d3efb1b173fecfa" + "traceparent.version": "00", + "traceparent.trace_id": "da8bc8cc6d062849b0efcf3c169afb5a", + "tracestate.tenant_id": "33", + "tracestate.version": "0", + "tracestate.parent_type": "0", + "tracestate.parent_account_id": "33", + "tracestate.sampled": "1", + "tracestate.priority": "1.23456", + "newrelic.v": [0, 1], + "newrelic.d.ty": "App", + "newrelic.d.ac": "33", + "newrelic.d.tr": "da8bc8cc6d062849b0efcf3c169afb5a", + "newrelic.d.sa": true, + "newrelic.d.pr": 1.23456 }, - "unexpected": ["parentId"] - }, - "Span": { + "expected": [ + "traceparent.trace_flags", + "traceparent.parent_id", + "tracestate.span_id", + "tracestate.transaction_id", + "tracestate.parent_application_id", + "tracestate.timestamp", + "newrelic.d.ap", + "newrelic.d.tx", + "newrelic.d.ti", + "newrelic.d.id" + ], + "unexpected": ["newrelic.d.tk"] + } + ], + "expected_metrics": [ + ["DurationByCaller/App/33/2827902/HTTP/all", 1], + ["DurationByCaller/App/33/2827902/HTTP/allWeb", 1], + ["TransportDuration/App/33/2827902/HTTP/all", 1], + ["TransportDuration/App/33/2827902/HTTP/allWeb", 1], + ["Supportability/TraceContext/Accept/Success", 1], + ["Supportability/TraceContext/Create/Success", 1], + ["Supportability/DistributedTrace/CreatePayload/Success", 1] + ] + }, + { + "test_name": "only_w3c_headers_present_emit_both_header_types", + "trusted_account_key": "33", + "account_id": "33", + "web_transaction": true, + "raises_exception": false, + "force_adaptive_sampled": false, + "span_events_enabled": true, + "transaction_events_enabled": true, + "transport_type": "HTTP", + "inbound_headers": [ + { + "traceparent": "00-da8bc8cc6d062849b0efcf3c169afb5a-7d3efb1b173fecfa-01", + "tracestate": "33@nr=0-0-33-2827902-7d3efb1b173fecfa-e8b91a159289ff74-1-1.23456-1518469636035" + } + ], + "outbound_payloads": [ + { "exact": { - "parentId": "7d3efb1b173fecfa" + "traceparent.version": "00", + "traceparent.trace_id": "da8bc8cc6d062849b0efcf3c169afb5a", + "tracestate.tenant_id": "33", + "tracestate.version": "0", + "tracestate.parent_type": "0", + "tracestate.parent_account_id": "33", + "tracestate.sampled": "1", + "tracestate.priority": "1.23456", + "newrelic.v": [0, 1], + "newrelic.d.ty": "App", + "newrelic.d.ac": "33", + "newrelic.d.tr": "da8bc8cc6d062849b0efcf3c169afb5a", + "newrelic.d.sa": true, + "newrelic.d.pr": 1.23456 }, - "expected": ["transactionId"], - "unexpected": ["tracingVendors"] + "expected": [ + "traceparent.trace_flags", + "traceparent.parent_id", + "tracestate.span_id", + "tracestate.transaction_id", + "tracestate.parent_application_id", + "tracestate.timestamp", + "newrelic.d.ap", + "newrelic.d.tx", + "newrelic.d.ti", + "newrelic.d.id" + ], + "unexpected": ["newrelic.d.tk"] } - }, + ], "expected_metrics": [ - ["DurationByCaller/Unknown/Unknown/Unknown/HTTP/all", 1], - ["DurationByCaller/Unknown/Unknown/Unknown/HTTP/allWeb", 1], - ["Supportability/TraceContext/TraceState/InvalidNrEntry", 1] + ["DurationByCaller/App/33/2827902/HTTP/all", 1], + ["DurationByCaller/App/33/2827902/HTTP/allWeb", 1], + ["TransportDuration/App/33/2827902/HTTP/all", 1], + ["TransportDuration/App/33/2827902/HTTP/allWeb", 1], + ["Supportability/TraceContext/Accept/Success", 1], + ["Supportability/TraceContext/Create/Success", 1], + ["Supportability/DistributedTrace/CreatePayload/Success", 1] ] }, { - "test_name": "trace_id_is_left_padded_and_priority_rounded", + "test_name": "only_newrelic_headers_present_emit_both_header_types", "trusted_account_key": "33", "account_id": "33", "web_transaction": true, "raises_exception": false, - "force_sampled_true": false, + "force_adaptive_sampled": false, "span_events_enabled": true, + "transaction_events_enabled": true, "transport_type": "HTTP", "inbound_headers": [ { - "newrelic": "{\"v\":[0,1],\"d\":{\"ty\":\"App\",\"ac\":\"33\",\"ap\":\"51424\",\"id\":\"5f474d64b9cc9b2a\",\"tr\":\"6e2fea0b173fdad0\",\"pr\":1.1234321,\"sa\":true,\"ti\":1482959525577,\"tx\":\"27856f70d3d314b7\"}}" + "newrelic": "{\"v\":[0,1],\"d\":{\"ty\":\"App\",\"ac\":\"33\",\"ap\":\"2827902\",\"id\":\"5f474d64b9cc9b2a\",\"tr\":\"da8bc8cc6d062849b0efcf3c169afb5a\",\"pr\":1.23456,\"sa\":true,\"ti\":1482959525577,\"tx\":\"27856f70d3d314b7\"}}" } ], "outbound_payloads": [ { "exact": { "traceparent.version": "00", - "traceparent.trace_id": "00000000000000006e2fea0b173fdad0", - "traceparent.trace_flags": "01", + "traceparent.trace_id": "da8bc8cc6d062849b0efcf3c169afb5a", "tracestate.tenant_id": "33", - "tracestate.version": 0, - "tracestate.parent_type": 0, + "tracestate.version": "0", + "tracestate.parent_type": "0", "tracestate.parent_account_id": "33", - "tracestate.sampled": true, - "tracestate.priority": 1.123432 + "tracestate.sampled": "1", + "tracestate.priority": "1.23456", + "newrelic.v": [0, 1], + "newrelic.d.ty": "App", + "newrelic.d.ac": "33", + "newrelic.d.tr": "da8bc8cc6d062849b0efcf3c169afb5a", + "newrelic.d.sa": true, + "newrelic.d.pr": 1.23456 }, "expected": [ + "traceparent.trace_flags", "traceparent.parent_id", - "tracestate.timestamp", - "tracestate.parent_application_id", "tracestate.span_id", - "tracestate.transaction_id" - ] + "tracestate.transaction_id", + "tracestate.parent_application_id", + "tracestate.timestamp", + "newrelic.d.ap", + "newrelic.d.tx", + "newrelic.d.ti", + "newrelic.d.id" + ], + "unexpected": ["newrelic.d.tk"] + } + ], + "expected_metrics": [ + ["DurationByCaller/App/33/2827902/HTTP/all", 1], + ["DurationByCaller/App/33/2827902/HTTP/allWeb", 1], + ["TransportDuration/App/33/2827902/HTTP/all", 1], + ["TransportDuration/App/33/2827902/HTTP/allWeb", 1], + ["Supportability/DistributedTrace/AcceptPayload/Success", 1], + ["Supportability/TraceContext/Create/Success", 1], + ["Supportability/DistributedTrace/CreatePayload/Success", 1] + ] + }, + { + "test_name": "inbound_payload_from_agent_in_serverless_mode", + "comment": [ + "Test a payload that originates from a serverless agent. The only", + "difference in the payload between a serverless and non-serverless agent", + "is the `appId` in the tracestate header will be 'Unknown'." + ], + "trusted_account_key": "33", + "account_id": "33", + "web_transaction": true, + "raises_exception": false, + "force_adaptive_sampled": false, + "span_events_enabled": true, + "transaction_events_enabled": true, + "transport_type": "HTTP", + "inbound_headers": [ + { + "traceparent": "00-da8bc8cc6d062849b0efcf3c169afb5a-7d3efb1b173fecfa-01", + "tracestate": "33@nr=0-0-33-Unknown-7d3efb1b173fecfa-e8b91a159289ff74-1-1.23456-1518469636035" } ], "intrinsics": { "target_events": ["Transaction", "Span"], "common":{ "exact": { - "traceId": "6e2fea0b173fdad0", + "traceId": "da8bc8cc6d062849b0efcf3c169afb5a", + "priority": 1.23456, "sampled": true }, - "expected": ["guid", "priority"], + "expected": ["guid"], "unexpected": ["grandparentId", "cross_process_id", "nr.tripId", "nr.pathHash", "nr.referringPathHash", "nr.guid", "nr.referringTransactionGuid", "nr.alternatePathHashes"] }, "Transaction": { "exact": { "parent.type": "App", + "parent.app": "Unknown", "parent.account": "33", - "parent.app": "51424", "parent.transportType": "HTTP", - "parentSpanId": "5f474d64b9cc9b2a", - "parentId": "27856f70d3d314b7" + "parentId": "e8b91a159289ff74", + "parentSpanId": "7d3efb1b173fecfa" }, "expected": ["parent.transportDuration"] }, "Span": { "exact": { - "parentId": "5f474d64b9cc9b2a" + "parentId": "7d3efb1b173fecfa", + "trustedParentId": "7d3efb1b173fecfa" }, "expected": ["transactionId"], - "unexpected": ["tracingVendors"] + "unexpected": ["parent.transportDuration", "parent.type", "parent.app", "parent.account", "parent.transportType", "tracingVendors"] } }, "expected_metrics": [ - ["DurationByCaller/App/33/51424/HTTP/all", 1], - ["DurationByCaller/App/33/51424/HTTP/allWeb", 1], - ["TransportDuration/App/33/51424/HTTP/all", 1], - ["TransportDuration/App/33/51424/HTTP/allWeb", 1], - ["Supportability/DistributedTrace/AcceptPayload/Success", 1] + ["DurationByCaller/App/33/Unknown/HTTP/all", 1], + ["DurationByCaller/App/33/Unknown/HTTP/allWeb", 1], + ["TransportDuration/App/33/Unknown/HTTP/all", 1], + ["TransportDuration/App/33/Unknown/HTTP/allWeb", 1] ] } ] diff --git a/tests/cross_agent/fixtures/samplers/README.md b/tests/cross_agent/fixtures/samplers/README.md new file mode 100644 index 0000000000..13c35ca8ed --- /dev/null +++ b/tests/cross_agent/fixtures/samplers/README.md @@ -0,0 +1,92 @@ +# Samplers + +With the introduction of Otel-style sampling algorithms and core tracing, we now have many configurable samplers +that may be working simultaneously within one application. These tests describe how the samplers should be set up +based on local config, and how they are expected to behave under traffic. + +## sampler_configuration.json + +This is a small test describing the samplers that should be created based on local config. + +### Full Test Parameters + +| Parameter | Description | +| --- | --- | +| `test_name` | The name of this test | +|`comment`| A longer description of the test | +|`config`| The local sampler config, provided as a nested JSON Object | +|`expected_samplers`|The samplers that should have been created based on the local config, provided as a nested JSON object whose keys are one or more of `full_root`, `full_remote_parent_sampled`, `full_remote_parent_not_sampled`, `partial_root`, `partial_remote_parent_sampled`, `partial_remote_parent_not_sampled`. If a sampler is not specified in `expected_samplers`, this is because the sampler is expected to have been disabled by the local config. | + +Additionally, each expected sampler will have one or more of the properties below: + +| Expected sampler property | Description | +| --- | --- | +| `type` | The type of the sampler that was created. Options are `always_on`, `always_off`, `trace_id_ratio_based`, and `adaptive`. | +| `is_global_adaptive_sampler` | Whether this sampler is the shared global instance of the adaptive sampler. If `false`, then this sampler MUST be a unique adaptive sampler instance. | +| `ratio` | The expected ratio this sampler should use, if this is a `trace_id_ratio_based` sampler. | +| `target` | The sampling target the sampler should use, if this is an `adaptive` sampler. If a test with an adaptive sampler is missing this, it is because the global adaptive sampler is in use and no global `adaptive_sampling_target` has been configured (so the target will vary depending on each team's default).| + + +## harvest_sampling_rates.json + +This test describes expected sampling rates during **one (the first), slow (60-sec) harvest** based on local config and specified traffic. + +### Test setup + +Every test case in this suite must be able to simulate a single slow harvest and the following types of transactions. Example headers are provided to help clarify the situation (you can use but you must generate random trace ids within the traceparent, otherwise ratio sampling will not work as expected). + +| Transaction type | Description | Example headers creating this scenario | +| --- | --- | --- | +| `root` | This is a root trace originating from this service | none | +| `parent_sampled_no_matching_acct_id` | The remote parent was sampled, and there was not a matching trusted account id in the headers. | trusted_account_key: 33,
{
traceparent: 00-0af7651916cd43dd8448eb211c80319c-00f067aa0ba902b7-01,
tracestate: 44@nr=0-0-44-2827902-0af7651916cd43dd--1--1518469636035
} | +| `parent_sampled_matching_acct_id_sampled_true` | The remote parent was sampled, there was a matching trusted acct id in the headers, and the tracestate sampled flag was set to true. | trusted_account_key: 33,
{
traceparent: 00-0af7651916cd43dd8448eb211c80319c-00f067aa0ba902b7-01,
tracestate: 33@nr=0-0-33-2827902-0af7651916cd43dd--1--1518469636035
} | +| `parent_not_sampled_no_matching_acct_id` | The remote parent was not sampled, and there was not a matching trusted acct id in the headers. |trusted_account_key: 33,
{
traceparent: 00-0af7651916cd43dd8448eb211c80319c-00f067aa0ba902b7-00,
tracestate: 44@nr=0-0-44-2827902-0af7651916cd43dd--1-1.2-1518469636035
}| +| `parent_not_sampled_matching_acct_id_sampled_true` | The remote parent not sampled, there was a matching trusted acct id in the headers, and the tracestate sampled flag was set to true.| trusted_account_key: 33,
{
traceparent: 00-0af7651916cd43dd8448eb211c80319c-00f067aa0ba902b7-00,
tracestate: 33@nr=0-0-33-2827902-0af7651916cd43dd--1--1518469636035
} | + +### Full Test Parameters + +| Parameter | Description | +| --- | --- | +| `test_name` | The name of this test | +|`comment` | A longer description of the test | +|`config` | The local sampler config, provided as a nested JSON Object | +|`root`| The number of transactions of this type to simulate during this harvest | +|`parent_sampled_no_matching_acct_id`|(as above)| +|`parent_sampled_matching_acct_id_sampled_true`|(as above)| +|`parent_not_sampled_no_matching_acct_id`|(as above)| +|`parent_not_sampled_matching_acct_id_sampled_true`|(as above)| +|`expected_sampled`| The total number of transactions that should have been sampled. | +|`expected_sampled_full`| The total number of transactions that should have been sampled at full granularity. | +|`expected_sampled_partial`| The total number of transactions that should have been sampled at partial granularity. | +|`expected_adaptive_sampler_decisions`| The number of new sampling decisions that the adaptive sampler had to compute. **Note**: This is an optional assertion, and would require mocking, spying or instrumentating the AdaptiveSampler to indicate it's being used to compute a sampling decision.| +|`variance`|The acceptable variance in the expected values for this test, expressed as a decimal. Eg: if `variance = 0.1` and `expected_sampled = 100`, the test passes if `90 <= actual total sampled <= 110`. This is provided for tests that include a trace id ratio based sampler (see Nondeterministic Sampler Behavior below).| + +### Explanation of traffic types + +The list of cases above might seem odd. The cases are intentionally specific to hone in on important details of how our samplers should work. +We need to verify the behavior for `root`, `remote_parent_sampled`, and `remote_parent_not_sampled` transactions for all of our samplers. + +We also need to verify additional `remote_parent` behavior for our adaptive sampler. For brevity, the explanation below is in WC3-speak (though proprietary newrelic headers also apply). +- After `remote_parent_sampled` or `remote_parent_not_sampled` has been determined from the traceparent header, +the adaptive sampler looks for a matching trusted account id off the tracestate header. The id it finds may or may not match. + - => We need to vary our cases based on whether or not a matching account id was found (eg, `parent_sampled_no_matching_acct_id` vs `parent_sampled_matching_acct_id_sampled_true`). +- Next, if a matching id was found, the adaptive sampler should not run, and instead reuse the `sampled` flag on the tracestate header. This `sampled` flag may or may not +match the `remote_parent_sampled/not_sampled` flag we pulled earlier from the traceparent. (It is also possible for the tracestate `sampled` flag to be missing, but that is out of scope for these tests). + - => We need to vary our test cases to cover scenarios where the two sampling flags match (`parent_sampled_matching_acct_id_sampled_true`) or + do not match (`parent_not_sampled_matching_acct_id_sampled_true`). + +By including this breadth of cases, we hope to ensure these tests cover common mistakes and gotchas. + +### Nondeterministic Sampler Behavior + +The always_on and always_off samplers behave deterministically. Their expected sampling totals are always exact. + +The adaptive and trace_id_ratio_based samplers are probabilistic in the wild, so their expected sampling totals not usually exact. +To account for this non-deterministic behavior in these tests, we make the following simplifications: + +- Adaptive Sampler: this sampler **does** behave deterministically in the first harvest, when it samples exactly its target. So, we use this +to our advantage, by running each test as though it is the first harvest. Please be aware, that testing harvests after the first is still important, +and **SHOULD** be done by each team in team-specific unit tests. +- Trace Id Ratio Based Sampler: this sampler is not deterministic, but it is highly faithful to its configured ratio, especially as the number of +samples increases. Any test with a trace_id_ratio_based sampler will include a **variance** parameter (described in the test parameters table above) to account for this small margin of error. + diff --git a/tests/cross_agent/fixtures/samplers/harvest_sampling_rates.json b/tests/cross_agent/fixtures/samplers/harvest_sampling_rates.json new file mode 100644 index 0000000000..c5472bf677 --- /dev/null +++ b/tests/cross_agent/fixtures/samplers/harvest_sampling_rates.json @@ -0,0 +1,216 @@ +[ + { + "test_name": "default_configuration_root_samples_global_sampling_target", + "comment": "When only root transactions are flowing, adaptive_sampling_target-many of them should be sampled", + "config": { + "sampler": { + "adaptive_sampling_target": 10 + } + }, + "root": 50, + + "expected_sampled": 10, + "expected_sampled_full": 10, + "expected_sampled_partial": 0, + "expected_adaptive_sampler_decisions": 50 + }, + { + "test_name": "multiple_transaction_types_count_towards_global_sampling_target", + "comment": "Transactions with remote parents still count towards the global target if they lack trusted acct ids.", + "config": { + "sampler": { + "adaptive_sampling_target": 10 + } + }, + "root": 50, + "parent_sampled_no_matching_acct_id": 50, + "parent_not_sampled_no_matching_acct_id": 50, + + "expected_sampled": 10, + "expected_sampled_full": 10, + "expected_sampled_partial": 0, + "expected_adaptive_sampler_decisions": 150 + }, + { + "test_name": "txns_with_remote_parents_and_matching_acct_ids_do_not_count_towards_global_sampling_target", + "comment": "Transactions with remote parents that have matching acct key ids do not run the adaptive sampler or count towards the sampling target. They reuse the sampling decision from the tracestate.", + "config": { + "sampler": { + "adaptive_sampling_target": 25 + } + }, + "root": 50, + "parent_sampled_matching_acct_id_sampled_true": 50, + "parent_not_sampled_matching_acct_id_sampled_true": 50, + + "expected_sampled": 125, + "expected_sampled_full": 125, + "expected_sampled_partial": 0, + "expected_adaptive_sampler_decisions": 50 + }, + { + "test_name": "inbound_decisions_do_not_influence_non_adaptive_samplers", + "comment": "All samplers other than adaptive type ignore the inbound sampling decision (only sampled/not sampled flag matters)", + "config": { + "sampler": { + "remote_parent_sampled": { + "trace_id_ratio_based": { + "ratio": 0.3 + } + }, + "remote_parent_not_sampled": "always_on" + } + }, + "parent_sampled_matching_acct_id_sampled_true": 5000, + "parent_sampled_no_matching_acct_id": 5000, + "parent_not_sampled_matching_acct_id_sampled_true": 60, + "parent_not_sampled_no_matching_acct_id": 40, + + "expected_sampled": 3100, + "expected_sampled_full": 3100, + "expected_sampled_partial": 0, + "variance": 0.05 + }, + { + "test_name": "trace_ratios_should_be_additive_when_layered", + "comment": "The partial granularity sampling ratio should be added to the full granularity ratio when samplers are used simultaneously.", + "config": { + "sampler": { + "root": { + "trace_id_ratio_based": { + "ratio": 0.5 + } + }, + "remote_parent_sampled": "always_off", + "remote_parent_not_sampled": "always_off", + "partial_granularity": { + "enabled": true, + "root": { + "trace_id_ratio_based": { + "ratio": 0.3 + } + }, + "remote_parent_sampled": "always_off", + "remote_parent_not_sampled": "always_off" + } + } + }, + "root": 10000, + "expected_sampled": 8000, + "expected_sampled_full": 5000, + "expected_sampled_partial": 3000, + "variance": 0.05 + }, + { + "test_name": "should_create_multiple_instances_of_adaptive_sampler", + "config": { + "sampler": { + "root": { + "adaptive": { + "sampling_target": 10 + } + }, + "remote_parent_sampled": { + "adaptive": { + "sampling_target": 10 + } + }, + "remote_parent_not_sampled": { + "adaptive": { + "sampling_target": 10 + } + }, + "partial_granularity": { + "enabled": true, + "root": { + "adaptive": { + "sampling_target": 15 + } + }, + "remote_parent_sampled": { + "adaptive": { + "sampling_target": 15 + } + }, + "remote_parent_not_sampled": { + "adaptive": { + "sampling_target": 15 + } + } + } + } + }, + "root": 100, + "parent_sampled_no_matching_acct_id": 100, + "parent_not_sampled_no_matching_acct_id": 100, + + "expected_sampled": 75, + "expected_sampled_full": 30, + "expected_sampled_partial": 45 + }, + { + "test_name": "giant_example_from_spec", + "comment": "Multiple sampler types are configured. 15000 root + 500 remote_parent_sampled + 500 remote_parent_not_sampled transactions are sampled.", + "config": { + "sampler": { + "root": { + "trace_id_ratio_based": { + "ratio": 0.1 + } + }, + "remote_parent_sampled": "always_off", + "remote_parent_not_sampled": "always_on", + "partial_granularity": { + "enabled": true, + "type": "essential", + "root": { + "trace_id_ratio_based": { + "ratio": 0.4 + } + }, + "remote_parent_sampled": { + "trace_id_ratio_based": { + "ratio": 1.0 + } + }, + "remote_parent_not_sampled": "always_off" + } + } + }, + "root": 15000, + "parent_sampled_matching_acct_id_sampled_true": 500, + "parent_not_sampled_matching_acct_id_sampled_true": 500, + + "expected_sampled": 8500, + "expected_sampled_full": 2000, + "expected_sampled_partial": 6500, + "variance": 0.05 + }, + { + "test_name": "adaptive_and_ratio_samplers_are_layered", + "comment": "30 roots are sampled by the adaptive sampler at full granularity, followed by 0.5 * (10000 remaining) = 5000 roots sampled at partial granularity.", + "config": { + "sampler": { + "root": { + "adaptive": { + "sampling_target": 30 + } + }, + "partial_granularity": { + "enabled": true, + "root": { + "trace_id_ratio_based": { + "ratio": 0.5 + } + } + } + } + }, + "root": 10030, + + "expected_sampled": 5030, + "expected_sampled_full": 30, + "expected_sampled_partial": 5000, + "variance": 0.05 + } +] diff --git a/tests/cross_agent/fixtures/samplers/sampler_configuration.json b/tests/cross_agent/fixtures/samplers/sampler_configuration.json new file mode 100644 index 0000000000..96ebac6054 --- /dev/null +++ b/tests/cross_agent/fixtures/samplers/sampler_configuration.json @@ -0,0 +1,207 @@ +[ + { + "test_name": "default_configuration", + "comment": "The default configuration uses the global adaptive sampler at full granularity.", + "config": {}, + "expected_samplers": { + "full_root": { + "type": "adaptive", + "is_global_adaptive_sampler": true + }, + "full_remote_parent_sampled": { + "type": "adaptive", + "is_global_adaptive_sampler": true + }, + "full_remote_parent_not_sampled": { + "type": "adaptive", + "is_global_adaptive_sampler": true + } + } + }, + + { + "test_name": "sampling_target_specified_uses_new_sampler_instance", + "comment": "When a valid sampling_target is specified, the global sampler instance is not used.", + "config": { + "sampler": { + "adaptive_sampling_target": 10, + "root": { + "adaptive": { + "sampling_target" : 35 + } + }, + "remote_parent_sampled": { + "adaptive": { + "sampling_target" : 10 + } + }, + "remote_parent_not_sampled" : "adaptive", + "partial_granularity": { + "enabled": true, + "root" : { + "banana" : {} + } + } + } + }, + "expected_samplers": { + "full_root": { + "type": "adaptive", + "is_global_adaptive_sampler": false, + "target": 35 + }, + "full_remote_parent_sampled": { + "type": "adaptive", + "is_global_adaptive_sampler": false, + "target": 10 + }, + "full_remote_parent_not_sampled": { + "type": "adaptive", + "is_global_adaptive_sampler": true, + "target": 10 + }, + "partial_root": { + "type": "adaptive", + "is_global_adaptive_sampler": true, + "target": 10 + }, + "partial_remote_parent_sampled": { + "type": "adaptive", + "is_global_adaptive_sampler": true, + "target": 10 + }, + "partial_remote_parent_not_sampled": { + "type": "adaptive", + "is_global_adaptive_sampler": true, + "target": 10 + } + } + }, + { + "test_name": "no_ratio_falls_back_to_global_sampler", + "comment": "When a trace_id_ratio_based sampler is configured without a valid ratio, we should fall back to the global adaptive sampler.", + "config" : { + "sampler": { + "root" : { + "trace_id_ratio_based": {} + }, + "remote_parent_sampled": { + "trace_id_ratio_based": { + } + }, + "remote_parent_not_sampled": { + "trace_id_ratio_based": { + "ratio": 0.33 + } + } + } + }, + "expected_samplers": { + "full_root": { + "type": "adaptive", + "is_global_adaptive_sampler": true + }, + "full_remote_parent_sampled": { + "type": "adaptive", + "is_global_adaptive_sampler": true + }, + "full_remote_parent_not_sampled": { + "type": "trace_id_ratio_based", + "ratio": 0.33 + } + } + }, + { + "test_name": "layering_ratio_samplers_adjusts_partial_ratio", + "comment": "The ratio of partial gran samplers should be added to the ratio of simultaneously configured full gran samplers.", + "config": { + "sampler" : { + "root": { + "trace_id_ratio_based": { + "ratio" : 0.4 + } + }, + "partial_granularity": { + "enabled": true, + "root": { + "trace_id_ratio_based" : { + "ratio" : 0.25 + } + }, + "remote_parent_sampled": { + "trace_id_ratio_based": { + "ratio": 0.1 + } + } + } + } + }, + "expected_samplers": { + "full_root": { + "type": "trace_id_ratio_based", + "ratio": 0.4 + }, + "full_remote_parent_sampled": { + "type": "adaptive", + "is_global_adaptive_sampler": true + }, + "full_remote_parent_not_sampled": { + "type": "adaptive", + "is_global_adaptive_sampler": true + }, + "partial_root": { + "type": "trace_id_ratio_based", + "ratio": 0.65 + }, + "partial_remote_parent_sampled": { + "type": "trace_id_ratio_based", + "ratio": 0.1 + }, + "partial_remote_parent_not_sampled": { + "type": "adaptive", + "is_global_adaptive_sampler": true + } + } + }, + { + "test_name": "layering_ratios_has_no_effect_if_full_disabled", + "comment": "Ratios are not additive if full granularity is disabled.", + "config": { + "sampler": { + "full_granularity": { + "enabled": false + }, + "root": { + "trace_id_ratio_based": { + "ratio": 0.5 + } + }, + "partial_granularity": { + "enabled": true, + "root": { + "trace_id_ratio_based": { + "ratio": 0.2 + } + }, + "remote_parent_sampled": "always_on" + } + } + }, + + "expected_full_granularity_enabled": false, + "expected_partial_granularity_enabled": true, + "expected_samplers": { + "partial_root": { + "type": "trace_id_ratio_based", + "ratio": 0.2 + }, + "partial_remote_parent_sampled": { + "type": "always_on" + }, + "partial_remote_parent_not_sampled": { + "type": "adaptive", + "is_global_adaptive_sampler": true + } + } + } +] diff --git a/tests/cross_agent/test_datastore_instance.py b/tests/cross_agent/test_datastore_instance.py index a35b3e65dd..9e2fd8a392 100644 --- a/tests/cross_agent/test_datastore_instance.py +++ b/tests/cross_agent/test_datastore_instance.py @@ -86,6 +86,8 @@ class FakeModule: guid=None, agent_attributes={}, user_attributes={}, + span_link_events={}, + span_event_events={}, ) empty_stats = StatsEngine() diff --git a/tests/cross_agent/test_distributed_tracing.py b/tests/cross_agent/test_distributed_tracing.py index 2d4ca1ed72..136b94ec5e 100644 --- a/tests/cross_agent/test_distributed_tracing.py +++ b/tests/cross_agent/test_distributed_tracing.py @@ -65,7 +65,7 @@ def load_tests(): def override_compute_sampled(override): - @transient_function_wrapper("newrelic.core.adaptive_sampler", "AdaptiveSampler.compute_sampled") + @transient_function_wrapper("newrelic.core.samplers.adaptive_sampler", "AdaptiveSampler.compute_sampled") def _override_compute_sampled(wrapped, instance, args, kwargs): if override: return True diff --git a/tests/cross_agent/test_distributed_tracing_trace_context.py b/tests/cross_agent/test_distributed_tracing_trace_context.py new file mode 100644 index 0000000000..fcd65a0147 --- /dev/null +++ b/tests/cross_agent/test_distributed_tracing_trace_context.py @@ -0,0 +1,355 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import base64 +import json +from pathlib import Path + +import pytest +import webtest +from testing_support.fixtures import override_application_settings, validate_attributes +from testing_support.validators.validate_error_event_attributes import validate_error_event_attributes +from testing_support.validators.validate_span_events import validate_span_events +from testing_support.validators.validate_transaction_event_attributes import validate_transaction_event_attributes +from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics + +import newrelic.agent +from newrelic.api.transaction import current_transaction +from newrelic.api.wsgi_application import wsgi_application +from newrelic.common.encoding_utils import ( + PARENT_TYPE, + DistributedTracePayload, + NrTraceState, + W3CTraceParent, + W3CTraceState, +) +from newrelic.common.object_wrapper import transient_function_wrapper + +FIXTURE = Path(__file__).parent / "fixtures" / "distributed_tracing" / "trace_context.json" + + +def load_tests(): + result = [] + with FIXTURE.open(encoding="utf-8") as fh: + tests = json.load(fh) + + for test in tests: + _id = test.pop("test_name", None) + test_desc = test.pop("comment", None) + settings = { + "distributed_tracing.sampler.full_granularity.enabled": test.pop("full_granularity_enabled", True), + "distributed_tracing.enabled": test.pop("distributed_tracing_enabled", True), + "span_events.enabled": test.pop("span_events_enabled", True), + "transaction_events.enabled": test.pop("transaction_events_enabled", True), + "distributed_tracing.sampler._root": test.pop("root", "adaptive"), + "distributed_tracing.sampler._remote_parent_sampled": test.pop("remote_parent_sampled", "adaptive"), + "distributed_tracing.sampler._remote_parent_not_sampled": test.pop("remote_parent_not_sampled", "adaptive"), + "trusted_account_key": test.pop("trusted_account_key", None), + "account_id": test.pop("account_id", None), + "distributed_tracing.sampler.partial_granularity.enabled": test.pop("partial_granularity_enabled", False), + "distributed_tracing.sampler.partial_granularity._root": test.pop("partial_granularity_root", "adaptive"), + "distributed_tracing.sampler.partial_granularity._remote_parent_sampled": test.pop( + "partial_granularity_remote_parent_sampled", "adaptive" + ), + "distributed_tracing.sampler.partial_granularity._remote_parent_not_sampled": test.pop( + "partial_granularity_remote_parent_not_sampled", "adaptive" + ), + } + full_gran_ratio = test.pop("full_granularity_ratio", None) + if full_gran_ratio is not None: + if settings["distributed_tracing.sampler._root"] == "trace_id_ratio_based": + settings["distributed_tracing.sampler.root.trace_id_ratio_based.ratio"] = full_gran_ratio + if settings["distributed_tracing.sampler._remote_parent_sampled"] == "trace_id_ratio_based": + settings["distributed_tracing.sampler.remote_parent_sampled.trace_id_ratio_based.ratio"] = ( + full_gran_ratio + ) + if settings["distributed_tracing.sampler._remote_parent_not_sampled"] == "trace_id_ratio_based": + settings["distributed_tracing.sampler.remote_parent_not_sampled.trace_id_ratio_based.ratio"] = ( + full_gran_ratio + ) + partial_gran_ratio = test.pop("partial_granularity_ratio", None) + if partial_gran_ratio is not None: + if settings["distributed_tracing.sampler.partial_granularity._root"] == "trace_id_ratio_based": + settings["distributed_tracing.sampler.partial_granularity.root.trace_id_ratio_based.ratio"] = ( + partial_gran_ratio + ) + if ( + settings["distributed_tracing.sampler.partial_granularity._remote_parent_sampled"] + == "trace_id_ratio_based" + ): + settings[ + "distributed_tracing.sampler.partial_granularity.remote_parent_sampled.trace_id_ratio_based.ratio" + ] = partial_gran_ratio + if ( + settings["distributed_tracing.sampler.partial_granularity._remote_parent_not_sampled"] + == "trace_id_ratio_based" + ): + settings[ + "distributed_tracing.sampler.partial_granularity.remote_parent_not_sampled.trace_id_ratio_based.ratio" + ] = partial_gran_ratio + + force_adaptive_sampled = test.pop("force_adaptive_sampled", None) + expected_metrics = test.pop("expected_metrics", []) + raises_exception = test.pop("raises_exception", False) + web_transaction = test.pop("web_transaction", False) + inbound_headers = (test.pop("inbound_headers", None) or [{}])[0] + expected_priority_between = test.pop("expected_priority_between", None) + intrinsics = test.pop("intrinsics", {}) + common_exact_intrinsics = intrinsics.get("common", {}).get("exact", {}) + common_expected_intrinsics = intrinsics.get("common", {}).get("expected", []) + common_unexpected_intrinsics = intrinsics.get("common", {}).get("unexpected", []) + span_exact_intrinsics = intrinsics.get("Span", {}).get("exact", {}) + span_expected_intrinsics = intrinsics.get("Span", {}).get("expected", []) + span_unexpected_intrinsics = intrinsics.get("Span", {}).get("unexpected", []) + transaction_exact_intrinsics = intrinsics.get("Transaction", {}).get("exact", {}) + transaction_expected_intrinsics = intrinsics.get("Transaction", {}).get("expected", []) + transaction_unexpected_intrinsics = intrinsics.get("Transaction", {}).get("unexpected", []) + if "Transaction" in intrinsics.get("target_events", []): + transaction_exact_intrinsics.update(common_exact_intrinsics) + transaction_expected_intrinsics.extend(common_expected_intrinsics) + transaction_unexpected_intrinsics.extend(common_unexpected_intrinsics) + check_span_events = False + if "Span" in intrinsics.get("target_events", []): + check_span_events = True + span_exact_intrinsics.update(common_exact_intrinsics) + span_expected_intrinsics.extend(common_expected_intrinsics) + span_unexpected_intrinsics.extend(common_unexpected_intrinsics) + + payload = (test.pop("outbound_payloads", None) or [{}])[0] + traceparent_key_map = {"trace_id": "tr", "parent_id": "id", "version": "v", "trace_flags": "sa"} + tracestate_key_map = { + "tenant_id": "ac", + "version": "v", + "parent_type": "ty", + "parent_account_id": "ac", + "sampled": "sa", + "priority": "pr", + "parent_id": "pi", + "timestamp": "ti", + "parent_application_id": "ap", + "span_id": "id", + "transaction_id": "tx", + } + expected_traceparent_exact = { + traceparent_key_map[key.split(".")[1]]: value + for key, value in payload.get("exact", {}).items() + if "traceparent" in key + } + expected_tracestate_exact = { + tracestate_key_map[key.split(".")[1]]: value + for key, value in payload.get("exact", {}).items() + if "tracestate" in key + } + expected_traceparent = [ + traceparent_key_map[key.split(".")[1]] for key in payload.get("expected", []) if "traceparent" in key + ] + expected_tracestate = [ + tracestate_key_map[key.split(".")[1]] for key in payload.get("expected", []) if "tracestate" in key + ] + outbound_payloads = ( + expected_traceparent_exact, + expected_tracestate_exact, + expected_traceparent, + expected_tracestate, + ) + + transport_type = test.pop("transport_type", "HTTP") + + assert not test, f"{test} has not been fully parsed" + + param = pytest.param( + settings, + force_adaptive_sampled, + transport_type, + raises_exception, + web_transaction, + inbound_headers, + expected_priority_between, + common_exact_intrinsics, + common_expected_intrinsics, + common_unexpected_intrinsics, + transaction_exact_intrinsics, + transaction_expected_intrinsics, + transaction_unexpected_intrinsics, + check_span_events, + span_exact_intrinsics, + span_expected_intrinsics, + span_unexpected_intrinsics, + outbound_payloads, + expected_metrics, + id=_id, + ) + result.append(param) + + return result + + +def override_compute_sampled(override): + @transient_function_wrapper("newrelic.core.samplers.adaptive_sampler", "AdaptiveSampler.compute_sampled") + def _override_compute_sampled(wrapped, instance, args, kwargs): + sampled = wrapped(*args, **kwargs) + if override is None: + return sampled + return override + + return _override_compute_sampled + + +@pytest.fixture +def create_transaction(): + def _create_transaction( + test_name, + web_transaction, + transport_type, + raises_exception, + inbound_headers, + outbound_payloads, + expected_priority_between, + ): + def _task(): + txn = current_transaction() + application = txn._application._agent._applications.get(txn.settings.app_name) + # Re-initialize sampler proxy after overriding settings. + application.sampler.__init__(txn.settings) + + if raises_exception: + try: + 1 / 0 # noqa: B018 + except ZeroDivisionError: + txn.notice_error() + + txn.accept_distributed_trace_headers(inbound_headers, transport_type) + + if outbound_payloads: + headers = [] + txn.insert_distributed_trace_headers(headers) + if test_name == "multiple_create_calls": + txn.insert_distributed_trace_headers(headers) + headers = dict(headers) + + expected_traceparent_exact, expected_tracestate_exact, expected_traceparent, expected_tracestate = ( + outbound_payloads + ) + + if expected_traceparent_exact or expected_traceparent: + traceparent = W3CTraceParent.decode(headers["traceparent"]) + for key, value in expected_traceparent_exact.items(): + if key == "sa": + assert traceparent.get(key, None) == bool(int(value, 2)) + continue + assert traceparent.get(key, None) == value + for key in expected_traceparent: + assert key in traceparent + + if expected_tracestate_exact or expected_tracestate: + vendors = W3CTraceState.decode(headers["tracestate"]) + trusted_account_key = txn.settings.trusted_account_key + newrelic = vendors.pop(f"{trusted_account_key}@nr", "") + tracestate = NrTraceState.decode(newrelic, trusted_account_key) + for key, value in expected_tracestate_exact.items(): + if key == "v": + assert tracestate.get(key, None) == value + continue + if key == "ty": + assert tracestate.get(key, None) == PARENT_TYPE[str(value)] + continue + if key == "sa": + assert tracestate.get(key, None) == bool(int(value)) + continue + if key == "pr": + assert f"{tracestate.get(key, None):.6f}".rstrip("0") == value + continue + assert tracestate.get(key, None) == value + for key in expected_tracestate: + assert key in tracestate + + if expected_priority_between: + assert expected_priority_between[0] < tracestate[key] < expected_priority_between[1] + + if web_transaction: + request = newrelic.agent.WebTransactionWrapper(_task, name=test_name) + else: + request = newrelic.agent.BackgroundTaskWrapper(_task, name=test_name) + + return request + + return _create_transaction + + +@pytest.mark.parametrize( + "settings,force_adaptive_sampled,transport_type,raises_exception,web_transaction,inbound_headers,expected_priority_between,exact_intrinsics,expected_intrinsics,unexpected_intrinsics,transaction_exact_intrinsics,transaction_expected_intrinsics,transaction_unexpected_intrinsics,check_span_events,span_exact_intrinsics,span_expected_intrinsics,span_unexpected_intrinsics,outbound_payloads,expected_metrics", + load_tests(), +) +def test_distributed_tracing( + settings, + force_adaptive_sampled, + transport_type, + raises_exception, + web_transaction, + inbound_headers, + expected_priority_between, + exact_intrinsics, + expected_intrinsics, + unexpected_intrinsics, + transaction_exact_intrinsics, + transaction_expected_intrinsics, + transaction_unexpected_intrinsics, + check_span_events, + span_exact_intrinsics, + span_expected_intrinsics, + span_unexpected_intrinsics, + outbound_payloads, + expected_metrics, + request, + create_transaction, +): + test_name = request.node.callspec.id + txn_event_required = {"agent": [], "user": [], "intrinsic": transaction_expected_intrinsics} + txn_event_forgone = {"agent": [], "user": [], "intrinsic": transaction_unexpected_intrinsics} + txn_event_exact = {"agent": {}, "user": {}, "intrinsic": transaction_exact_intrinsics} + + @validate_transaction_metrics(test_name, rollup_metrics=expected_metrics, background_task=not web_transaction) + @validate_transaction_event_attributes(txn_event_required, txn_event_forgone, txn_event_exact) + @override_compute_sampled(force_adaptive_sampled) + @override_application_settings(settings) + def _test(): + transaction = create_transaction( + test_name, + web_transaction, + transport_type, + raises_exception, + inbound_headers, + outbound_payloads, + expected_priority_between, + ) + + transaction() + + if raises_exception: + error_event_required = {"agent": [], "user": [], "intrinsic": expected_intrinsics} + error_event_forgone = {"agent": [], "user": [], "intrinsic": unexpected_intrinsics} + error_event_exact = {"agent": {}, "user": {}, "intrinsic": exact_intrinsics} + _test = validate_error_event_attributes(error_event_required, error_event_forgone, error_event_exact)(_test) + + if settings["span_events.enabled"]: + if check_span_events: + _test = validate_span_events( + exact_intrinsics=span_exact_intrinsics, + expected_intrinsics=span_expected_intrinsics, + unexpected_intrinsics=span_unexpected_intrinsics, + )(_test) + else: + _test = validate_span_events(count=0)(_test) + + _test() diff --git a/tests/cross_agent/test_harvest_sampling_rates.py b/tests/cross_agent/test_harvest_sampling_rates.py new file mode 100644 index 0000000000..bd09535fa5 --- /dev/null +++ b/tests/cross_agent/test_harvest_sampling_rates.py @@ -0,0 +1,261 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import random +import tempfile +import time +from pathlib import Path + +import pytest +from testing_support.fixtures import failing_endpoint, override_application_settings, override_generic_settings +from testing_support.validators.validate_function_not_called import validate_function_not_called + +from newrelic.api.application import application_instance, register_application +from newrelic.api.background_task import background_task +from newrelic.api.transaction import current_transaction +from newrelic.common.agent_http import DeveloperModeClient +from newrelic.common.object_wrapper import function_wrapper, transient_function_wrapper +from newrelic.core.application import Application +from newrelic.core.config import finalize_application_settings, global_settings +from newrelic.core.custom_event import create_custom_event +from newrelic.core.error_node import ErrorNode +from newrelic.core.external_node import ExternalNode +from newrelic.core.function_node import FunctionNode +from newrelic.core.log_event_node import LogEventNode +from newrelic.core.root_node import RootNode +from newrelic.core.stats_engine import CustomMetrics, DimensionalMetrics, SampledDataSet +from newrelic.core.transaction_node import TransactionNode +from newrelic.network.exceptions import RetryDataForRequest + +FIXTURE = Path(__file__).parent / "fixtures" / "samplers" / "harvest_sampling_rates.json" + + +def replace_section(setting): + end = setting.split(".")[-1] + if end in ("root", "remote_parent_sampled", "remote_parent_not_sampled"): + return ".".join([*setting.split(".")[:-1], f"_{end}"]) + return setting + + +def parse_to_config_paths(settings, setting, config): + if isinstance(config, dict): + for key, value in config.items(): + if isinstance(value, dict) and not value: + new_setting = replace_section(setting) + settings[new_setting] = key + continue + # If there's a case where we have root.adaptive.sampling_target we also + # need to set _root = "adaptive". This usually happens in the config or + # env var parsing so this is special handling only needed for tests. + if key in ("root", "remote_parent_sampled", "remote_parent_not_sampled"): + new_setting = f"{setting}._{key}" + v = list(value.keys())[0] if isinstance(value, dict) else value + settings[new_setting] = v + new_setting = f"{setting}.{key}" + parse_to_config_paths(settings, new_setting, value) + else: + new_setting = replace_section(setting) + settings[new_setting] = config + + +def load_tests(): + result = [] + with FIXTURE.open(encoding="utf-8") as fh: + tests = json.load(fh) + + for test in tests: + _id = test.pop("test_name", None) + test_desc = test.pop("comment", None) + + config = test.pop("config", {}) + settings = {} + parse_to_config_paths(settings, "distributed_tracing", config) + + if "distributed_tracing.sampler.adaptive_sampling_target" in settings: + settings["sampling_target"] = settings.pop("distributed_tracing.sampler.adaptive_sampling_target") + root = test.pop("root", 0) + parent_sampled_no_matching_acct_id = test.pop("parent_sampled_no_matching_acct_id", 0) + parent_not_sampled_no_matching_acct_id = test.pop("parent_not_sampled_no_matching_acct_id", 0) + parent_sampled_matching_acct_id_sampled_true = test.pop("parent_sampled_matching_acct_id_sampled_true", 0) + parent_not_sampled_matching_acct_id_sampled_true = test.pop( + "parent_not_sampled_matching_acct_id_sampled_true", 0 + ) + expected_sampled = test.pop("expected_sampled", None) + expected_sampled_full = test.pop("expected_sampled_full", None) + expected_sampled_partial = test.pop("expected_sampled_partial", None) + expected_adaptive_sampler_decisions = test.pop("expected_adaptive_sampler_decisions", None) + variance = test.pop("variance", 0) + assert not test, f"{test} has not been fully parsed." + + param = pytest.param( + settings, + root, + parent_sampled_no_matching_acct_id, + parent_not_sampled_no_matching_acct_id, + parent_sampled_matching_acct_id_sampled_true, + parent_not_sampled_matching_acct_id_sampled_true, + expected_sampled, + expected_sampled_full, + expected_sampled_partial, + expected_adaptive_sampler_decisions, + variance, + id=_id, + ) + result.append(param) + + return result + + +@pytest.mark.skip(reason="These are too time consuming in CI") +@pytest.mark.parametrize( + "settings,root,parent_sampled_no_matching_acct_id,parent_not_sampled_no_matching_acct_id,parent_sampled_matching_acct_id_sampled_true,parent_not_sampled_matching_acct_id_sampled_true,expected_sampled,expected_sampled_full,expected_sampled_partial,expected_adaptive_sampler_decisions,variance", + load_tests(), +) +def test_harvest_sampling_rates( + settings, + root, + parent_sampled_no_matching_acct_id, + parent_not_sampled_no_matching_acct_id, + parent_sampled_matching_acct_id_sampled_true, + parent_not_sampled_matching_acct_id_sampled_true, + expected_sampled, + expected_sampled_full, + expected_sampled_partial, + expected_adaptive_sampler_decisions, + variance, +): + global total + total = 0 + global partial + partial = 0 + + overide_settings = { + "trusted_account_id": "33", + "trusted_account_key": "33", + "event_harvest_config.harvest_limits.span_event_data": 10000, + } + overide_settings.update(settings) + + @override_application_settings(overide_settings) + def _test(test_adaptive=False, test_totals=False, num_tests=1): + app = application_instance("Python Agent Test (cross_agent_tests)") + application = app._agent._applications.get("Python Agent Test (cross_agent_tests)") + # Re-initialize sampler proxy after overriding settings. + application.sampler.__init__(app.settings) + # Re-initialize span event with new harvest value. + application._stats_engine.reset_span_events() + # Reset adaptive sampling decision count. + global adaptive_sampling_decisons + adaptive_sampling_decisons = 0 + + @count_adaptive_sampling_decisions() + @background_task() + def _transaction(headers): + txn = current_transaction() + + txn.accept_distributed_trace_headers(headers, "HTTP") + + for _ in range(root): + _transaction({}) + + for _ in range(parent_sampled_no_matching_acct_id): + trace_id = f"{random.getrandbits(128):032x}" + _transaction( + { + "traceparent": f"00-{trace_id}-00f067aa0ba902b7-01", + "tracestate": "22@nr=0-0-33-2827902-0af7651916cd43dd--1-1.2-1518469636035", + } + ) + + for _ in range(parent_not_sampled_no_matching_acct_id): + trace_id = f"{random.getrandbits(128):032x}" + _transaction( + { + "traceparent": f"00-{trace_id}-00f067aa0ba902b7-00", + "tracestate": "22@nr=0-0-33-2827902-0af7651916cd43dd--1-1.2-1518469636035", + } + ) + + for _ in range(parent_sampled_matching_acct_id_sampled_true): + trace_id = f"{random.getrandbits(128):032x}" + _transaction( + { + "traceparent": f"00-{trace_id}-00f067aa0ba902b7-01", + "tracestate": "33@nr=0-0-33-2827902-0af7651916cd43dd--1-1.2-1518469636035", + } + ) + + for _ in range(parent_not_sampled_matching_acct_id_sampled_true): + trace_id = f"{random.getrandbits(128):032x}" + _transaction( + { + "traceparent": f"00-{trace_id}-00f067aa0ba902b7-00", + "tracestate": "33@nr=0-0-33-2827902-0af7651916cd43dd--1-1.2-1518469636035", + } + ) + + global total + total += application._stats_engine.span_events.num_samples + global partial + partial += len([event for event in application._stats_engine.span_events.samples if event[0].get("nr.pg")]) + + if test_totals: + if expected_sampled is not None: + assert ( + expected_sampled - expected_sampled * variance + <= total / num_tests + <= expected_sampled + expected_sampled * variance + ) + if expected_sampled_partial is not None: + assert ( + expected_sampled_partial - expected_sampled_partial * variance + <= partial / num_tests + <= expected_sampled_partial + expected_sampled_partial * variance + ) + if expected_sampled_full is not None: + assert ( + expected_sampled_full - expected_sampled_full * variance + <= (total - partial) / num_tests + <= expected_sampled_full + expected_sampled_full * variance + ) + + if test_adaptive and expected_adaptive_sampler_decisions is not None: + assert ( + expected_adaptive_sampler_decisions - expected_adaptive_sampler_decisions * variance + <= adaptive_sampling_decisons + <= expected_adaptive_sampler_decisions + expected_adaptive_sampler_decisions * variance + ) + + application.harvest() + assert application._stats_engine.span_events.num_samples == 0 + + num_tests = 5 + for n in range(num_tests): + if n == 0: + _test(test_adaptive=True) + elif n == num_tests - 1: + _test(num_tests=num_tests, test_totals=True) + else: + _test() + + +def count_adaptive_sampling_decisions(): + @transient_function_wrapper("newrelic.core.samplers.adaptive_sampler", "AdaptiveSampler.compute_sampled") + def _count_adaptive_sampling_decisions(wrapped, instance, args, kwargs): + global adaptive_sampling_decisons + adaptive_sampling_decisons += 1 + return wrapped(*args, **kwargs) + + return _count_adaptive_sampling_decisions diff --git a/tests/cross_agent/test_sampler_configuration.py b/tests/cross_agent/test_sampler_configuration.py new file mode 100644 index 0000000000..d9ebcb4ea1 --- /dev/null +++ b/tests/cross_agent/test_sampler_configuration.py @@ -0,0 +1,145 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import copy +import json +import random +import time +from pathlib import Path + +import pytest +import webtest +from testing_support.fixtures import override_application_settings, validate_attributes, validate_attributes_complete +from testing_support.validators.validate_error_event_attributes import validate_error_event_attributes +from testing_support.validators.validate_function_called import validate_function_called +from testing_support.validators.validate_function_not_called import validate_function_not_called +from testing_support.validators.validate_transaction_event_attributes import validate_transaction_event_attributes +from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics +from testing_support.validators.validate_transaction_object_attributes import validate_transaction_object_attributes + +from newrelic.api.application import application_instance +from newrelic.api.function_trace import function_trace +from newrelic.common.object_names import callable_name +from newrelic.common.object_wrapper import function_wrapper, transient_function_wrapper +from newrelic.core.samplers.adaptive_sampler import AdaptiveSampler +from newrelic.core.samplers.trace_id_ratio_based_sampler import TraceIdRatioBasedSampler + +try: + from newrelic.core.infinite_tracing_pb2 import AttributeValue, Span +except: + AttributeValue = None + Span = None + +from testing_support.mock_external_http_server import MockExternalHTTPHResponseHeadersServer +from testing_support.validators.validate_span_events import check_value_equals, validate_span_events + +from newrelic.api.application import application_instance +from newrelic.api.background_task import BackgroundTask, background_task +from newrelic.api.external_trace import ExternalTrace +from newrelic.api.time_trace import current_trace +from newrelic.api.transaction import ( + accept_distributed_trace_headers, + current_span_id, + current_trace_id, + current_transaction, + insert_distributed_trace_headers, +) +from newrelic.api.web_transaction import WSGIWebTransaction +from newrelic.api.wsgi_application import wsgi_application +from newrelic.core.attribute import Attribute + +FIXTURE = Path(__file__).parent / "fixtures" / "samplers" / "sampler_configuration.json" + + +def replace_section(setting): + end = setting.split(".")[-1] + if end in ("root", "remote_parent_sampled", "remote_parent_not_sampled"): + return ".".join([*setting.split(".")[:-1], f"_{end}"]) + return setting + + +def parse_to_config_paths(settings, setting, config): + if isinstance(config, dict): + for key, value in config.items(): + if isinstance(value, dict) and not value: + new_setting = replace_section(setting) + settings[new_setting] = key + continue + # If there's a case where we have root.adaptive.sampling_target we also + # need to set _root = "adaptive". This usually happens in the config or + # env var parsing so this is special handling only needed for tests. + if key in ("root", "remote_parent_sampled", "remote_parent_not_sampled"): + new_setting = ".".join([setting, f"_{key}"]) + v = list(value.keys())[0] if isinstance(value, dict) else value + settings[new_setting] = v + new_setting = f"{setting}.{key}" + parse_to_config_paths(settings, new_setting, value) + else: + new_setting = replace_section(setting) + settings[new_setting] = config + + +def load_tests(): + result = [] + with FIXTURE.open(encoding="utf-8") as fh: + tests = json.load(fh) + + for test in tests: + _id = test.pop("test_name", None) + test_desc = test.pop("comment", None) + + config = test.pop("config", {}) + settings = {} + parse_to_config_paths(settings, "distributed_tracing", config) + expected_samplers = test.pop("expected_samplers", {}) + param = pytest.param(settings, expected_samplers, id=_id) + result.append(param) + + return result + + +SECTIONS = { + "full_root": (True, 0), + "full_remote_parent_sampled": (True, 1), + "full_remote_parent_not_sampled": (True, 2), + "partial_root": (False, 0), + "partial_remote_parent_sampled": (False, 1), + "partial_remote_parent_not_sampled": (False, 2), +} + + +@pytest.mark.parametrize("settings,expected_samplers", load_tests()) +def test_sampler_configuration(settings, expected_samplers): + @override_application_settings(settings) + @background_task() + def _test(): + txn = current_transaction() + application = txn._application._agent._applications.get(txn.settings.app_name) + # Re-initialize sampler proxy after overriding settings. + application.sampler.__init__(txn.settings) + + for sampler, attributes in expected_samplers.items(): + instance = SECTIONS[sampler] + sampler_instance = application.sampler.get_sampler(*instance) + if attributes["type"] == "adaptive": + assert isinstance(sampler_instance, AdaptiveSampler) + elif attributes["type"] == "trace_id_ratio_based": + assert isinstance(sampler_instance, TraceIdRatioBasedSampler) + if "ratio" in attributes: + assert sampler_instance.ratio == attributes["ratio"] + if attributes.get("is_global_adaptive_sampler", False): + assert sampler_instance is application.sampler._samplers["global"] + + _test() diff --git a/tests/cross_agent/test_w3c_trace_context.py b/tests/cross_agent/test_w3c_trace_context.py deleted file mode 100644 index 0c51184f28..0000000000 --- a/tests/cross_agent/test_w3c_trace_context.py +++ /dev/null @@ -1,257 +0,0 @@ -# Copyright 2010 New Relic, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -from pathlib import Path - -import pytest -import webtest -from testing_support.fixtures import override_application_settings, validate_attributes -from testing_support.validators.validate_span_events import validate_span_events -from testing_support.validators.validate_transaction_event_attributes import validate_transaction_event_attributes -from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics - -from newrelic.api.transaction import ( - accept_distributed_trace_headers, - current_transaction, - insert_distributed_trace_headers, -) -from newrelic.api.wsgi_application import wsgi_application -from newrelic.common.encoding_utils import W3CTraceState -from newrelic.common.object_wrapper import transient_function_wrapper - -FIXTURE = Path(__file__).parent / "fixtures" / "distributed_tracing" / "trace_context.json" - - -_parameters_list = ( - "test_name", - "trusted_account_key", - "account_id", - "web_transaction", - "raises_exception", - "force_sampled_true", - "span_events_enabled", - "transport_type", - "inbound_headers", - "outbound_payloads", - "intrinsics", - "expected_metrics", -) - -_parameters = ",".join(_parameters_list) - - -XFAIL_TESTS = [ - "spans_disabled_root", - "missing_traceparent", - "missing_traceparent_and_tracestate", - "w3c_and_newrelc_headers_present_error_parsing_traceparent", -] - - -def load_tests(): - result = [] - with FIXTURE.open(encoding="utf-8") as fh: - tests = json.load(fh) - - for test in tests: - values = (test.get(param, None) for param in _parameters_list) - param = pytest.param(*values, id=test.get("test_name")) - result.append(param) - - return result - - -ATTR_MAP = { - "traceparent.version": 0, - "traceparent.trace_id": 1, - "traceparent.parent_id": 2, - "traceparent.trace_flags": 3, - "tracestate.version": 0, - "tracestate.parent_type": 1, - "tracestate.parent_account_id": 2, - "tracestate.parent_application_id": 3, - "tracestate.span_id": 4, - "tracestate.transaction_id": 5, - "tracestate.sampled": 6, - "tracestate.priority": 7, - "tracestate.timestamp": 8, - "tracestate.tenant_id": None, -} - - -def validate_outbound_payload(actual, expected, trusted_account_key): - traceparent = "" - tracestate = "" - for key, value in actual: - if key == "traceparent": - traceparent = value.split("-") - elif key == "tracestate": - vendors = W3CTraceState.decode(value) - nr_entry = vendors.pop(f"{trusted_account_key}@nr", "") - tracestate = nr_entry.split("-") - exact_values = expected.get("exact", {}) - expected_attrs = expected.get("expected", []) - unexpected_attrs = expected.get("unexpected", []) - expected_vendors = expected.get("vendors", []) - for key, value in exact_values.items(): - header = traceparent if key.startswith("traceparent.") else tracestate - attr = ATTR_MAP[key] - if attr is not None: - if isinstance(value, bool): - assert header[attr] == str(int(value)) - elif isinstance(value, int): - assert int(header[attr]) == value - else: - assert header[attr] == str(value) - - for key in expected_attrs: - header = traceparent if key.startswith("traceparent.") else tracestate - attr = ATTR_MAP[key] - if attr is not None: - assert header[attr], key - - for key in unexpected_attrs: - header = traceparent if key.startswith("traceparent.") else tracestate - attr = ATTR_MAP[key] - if attr is not None: - assert not header[attr], key - - for vendor in expected_vendors: - assert vendor in vendors - - -@wsgi_application() -def target_wsgi_application(environ, start_response): - transaction = current_transaction() - - if not environ[".web_transaction"]: - transaction.background_task = True - - if environ[".raises_exception"]: - try: - raise ValueError("oops") - except: - transaction.notice_error() - - if ".inbound_headers" in environ: - accept_distributed_trace_headers(environ[".inbound_headers"], transport_type=environ[".transport_type"]) - - payloads = [] - for _ in range(environ[".outbound_calls"]): - payloads.append([]) - insert_distributed_trace_headers(payloads[-1]) - - start_response("200 OK", [("Content-Type", "application/json")]) - return [json.dumps(payloads).encode("utf-8")] - - -test_application = webtest.TestApp(target_wsgi_application) - - -def override_compute_sampled(override): - @transient_function_wrapper("newrelic.core.adaptive_sampler", "AdaptiveSampler.compute_sampled") - def _override_compute_sampled(wrapped, instance, args, kwargs): - if override: - return True - return wrapped(*args, **kwargs) - - return _override_compute_sampled - - -@pytest.mark.parametrize(_parameters, load_tests()) -def test_trace_context( - test_name, - trusted_account_key, - account_id, - web_transaction, - raises_exception, - force_sampled_true, - span_events_enabled, - transport_type, - inbound_headers, - outbound_payloads, - intrinsics, - expected_metrics, -): - if test_name in XFAIL_TESTS: - pytest.xfail("Waiting on cross agent tests update.") - # Prepare assertions - if not intrinsics: - intrinsics = {} - - common = intrinsics.get("common", {}) - common_required = common.get("expected", []) - common_forgone = common.get("unexpected", []) - common_exact = common.get("exact", {}) - - txn_intrinsics = intrinsics.get("Transaction", {}) - txn_event_required = {"agent": [], "user": [], "intrinsic": txn_intrinsics.get("expected", [])} - txn_event_required["intrinsic"].extend(common_required) - txn_event_forgone = {"agent": [], "user": [], "intrinsic": txn_intrinsics.get("unexpected", [])} - txn_event_forgone["intrinsic"].extend(common_forgone) - txn_event_exact = {"agent": {}, "user": {}, "intrinsic": txn_intrinsics.get("exact", {})} - txn_event_exact["intrinsic"].update(common_exact) - - override_settings = { - "distributed_tracing.enabled": True, - "span_events.enabled": span_events_enabled, - "account_id": account_id, - "trusted_account_key": trusted_account_key, - } - - extra_environ = { - ".web_transaction": web_transaction, - ".raises_exception": raises_exception, - ".transport_type": transport_type, - ".outbound_calls": (outbound_payloads and len(outbound_payloads)) or 0, - } - - inbound_headers = (inbound_headers and inbound_headers[0]) or None - if transport_type != "HTTP": - extra_environ[".inbound_headers"] = inbound_headers - inbound_headers = None - - @validate_transaction_metrics( - test_name, group="Uri", rollup_metrics=expected_metrics, background_task=not web_transaction - ) - @validate_transaction_event_attributes(txn_event_required, txn_event_forgone, txn_event_exact) - @validate_attributes("intrinsic", common_required, common_forgone) - @override_application_settings(override_settings) - @override_compute_sampled(force_sampled_true) - def _test(): - return test_application.get(f"/{test_name}", headers=inbound_headers, extra_environ=extra_environ) - - if "Span" in intrinsics: - span_intrinsics = intrinsics.get("Span") - span_expected = span_intrinsics.get("expected", []) - span_expected.extend(common_required) - span_unexpected = span_intrinsics.get("unexpected", []) - span_unexpected.extend(common_forgone) - span_exact = span_intrinsics.get("exact", {}) - span_exact.update(common_exact) - - _test = validate_span_events( - exact_intrinsics=span_exact, expected_intrinsics=span_expected, unexpected_intrinsics=span_unexpected - )(_test) - elif not span_events_enabled: - _test = validate_span_events(count=0)(_test) - - response = _test() - assert response.status == "200 OK" - payloads = response.json - if outbound_payloads: - assert len(payloads) == len(outbound_payloads) - for actual, expected in zip(payloads, outbound_payloads): - validate_outbound_payload(actual, expected, trusted_account_key) diff --git a/tests/datastore_oracledb/test_async_connection.py b/tests/datastore_oracledb/test_async_connection.py index 60b2d088a4..4ee81a2397 100644 --- a/tests/datastore_oracledb/test_async_connection.py +++ b/tests/datastore_oracledb/test_async_connection.py @@ -60,7 +60,8 @@ async def execute_db_calls_with_cursor(cursor): END; """ ) - await cursor.callproc(PROCEDURE_NAME, [cursor.var(str)]) # Must specify a container for the OUT parameter + # Must specify a container for the OUT parameter + await cursor.callproc(name=PROCEDURE_NAME, parameters=[cursor.var(str)]) _test_execute_scoped_metrics = [ diff --git a/tests/datastore_oracledb/test_connection.py b/tests/datastore_oracledb/test_connection.py index f8789e78ff..b99b0ef366 100644 --- a/tests/datastore_oracledb/test_connection.py +++ b/tests/datastore_oracledb/test_connection.py @@ -54,7 +54,8 @@ def execute_db_calls_with_cursor(cursor): END; """ ) - cursor.callproc(PROCEDURE_NAME, [cursor.var(str)]) # Must specify a container for the OUT parameter + # Must specify a container for the OUT parameter + cursor.callproc(name=PROCEDURE_NAME, parameters=[cursor.var(str)]) _test_execute_scoped_metrics = [ diff --git a/tests/datastore_redis/test_uninstrumented_methods.py b/tests/datastore_redis/test_uninstrumented_methods.py index 87a6a4ac0d..34f20b3301 100644 --- a/tests/datastore_redis/test_uninstrumented_methods.py +++ b/tests/datastore_redis/test_uninstrumented_methods.py @@ -26,6 +26,7 @@ "MODULE_CALLBACKS", "MODULE_VERSION", "NAME", + "RESPONSE_CALLBACKS", "add_edge", "add_node", "append_bucket_size", @@ -72,6 +73,7 @@ "load_document", "load_external_module", "lock", + "maint_notifications_config", "name", "nodes", "parse_response", @@ -80,7 +82,6 @@ "register_script", "relationship_types", "response_callbacks", - "RESPONSE_CALLBACKS", "sentinel", "set_file", "set_path", diff --git a/tests/datastore_rediscluster/test_uninstrumented_rediscluster_methods.py b/tests/datastore_rediscluster/test_uninstrumented_rediscluster_methods.py index 3f2a258355..658c5d6519 100644 --- a/tests/datastore_rediscluster/test_uninstrumented_rediscluster_methods.py +++ b/tests/datastore_rediscluster/test_uninstrumented_rediscluster_methods.py @@ -67,6 +67,7 @@ "load_document", "load_external_module", "lock", + "maint_notifications_config", "name", "nodes", "parse_response", @@ -118,10 +119,14 @@ "get_node", "get_node_from_key", "get_nodes", + "get_nodes_from_slot", "get_primaries", "get_random_node", + "get_random_primary_node", + "get_random_primary_or_all_nodes", "get_redis_connection", "get_replicas", + "get_special_nodes", "keyslot", "mget_nonatomic", "monitor", diff --git a/tests/external_aiobotocore/test_bedrock_chat_completion_converse.py b/tests/external_aiobotocore/test_bedrock_chat_completion_converse.py index da9c5818e7..516787c2e5 100644 --- a/tests/external_aiobotocore/test_bedrock_chat_completion_converse.py +++ b/tests/external_aiobotocore/test_bedrock_chat_completion_converse.py @@ -15,9 +15,16 @@ import botocore.exceptions import pytest from conftest import BOTOCORE_VERSION +from external_botocore._test_bedrock_chat_completion_converse import ( + chat_completion_expected_events, + chat_completion_expected_streaming_events, + chat_completion_invalid_access_key_error_events, + chat_completion_invalid_model_error_events, +) from testing_support.fixtures import override_llm_token_callback_settings, reset_core_stats_engine, validate_attributes from testing_support.ml_testing_utils import ( - add_token_count_to_events, + add_token_count_streaming_events, + add_token_counts_to_chat_events, disabled_ai_monitoring_record_content_settings, disabled_ai_monitoring_settings, events_sans_content, @@ -36,113 +43,65 @@ from newrelic.api.transaction import add_custom_attribute from newrelic.common.object_names import callable_name -chat_completion_expected_events = [ - ( - {"type": "LlmChatCompletionSummary"}, - { - "id": None, # UUID that varies with each run - "llm.conversation_id": "my-awesome-id", - "llm.foo": "bar", - "span_id": None, - "trace_id": "trace-id", - "request_id": "c20d345e-6878-4778-b674-6b187bae8ecf", - "duration": None, # Response time varies each test run - "request.model": "anthropic.claude-3-sonnet-20240229-v1:0", - "response.model": "anthropic.claude-3-sonnet-20240229-v1:0", - "request.temperature": 0.7, - "request.max_tokens": 100, - "response.choices.finish_reason": "max_tokens", - "vendor": "bedrock", - "ingest_source": "Python", - "response.number_of_messages": 3, - }, - ), - ( - {"type": "LlmChatCompletionMessage"}, - { - "id": None, # UUID that varies with each run - "llm.conversation_id": "my-awesome-id", - "llm.foo": "bar", - "request_id": "c20d345e-6878-4778-b674-6b187bae8ecf", - "span_id": None, - "trace_id": "trace-id", - "content": "You are a scientist.", - "role": "system", - "completion_id": None, - "sequence": 0, - "response.model": "anthropic.claude-3-sonnet-20240229-v1:0", - "vendor": "bedrock", - "ingest_source": "Python", - }, - ), - ( - {"type": "LlmChatCompletionMessage"}, - { - "id": None, # UUID that varies with each run - "llm.conversation_id": "my-awesome-id", - "llm.foo": "bar", - "request_id": "c20d345e-6878-4778-b674-6b187bae8ecf", - "span_id": None, - "trace_id": "trace-id", - "content": "What is 212 degrees Fahrenheit converted to Celsius?", - "role": "user", - "completion_id": None, - "sequence": 1, - "response.model": "anthropic.claude-3-sonnet-20240229-v1:0", - "vendor": "bedrock", - "ingest_source": "Python", - }, - ), - ( - {"type": "LlmChatCompletionMessage"}, - { - "id": None, # UUID that varies with each run - "llm.conversation_id": "my-awesome-id", - "llm.foo": "bar", - "request_id": "c20d345e-6878-4778-b674-6b187bae8ecf", - "span_id": None, - "trace_id": "trace-id", - "content": "To convert 212°F to Celsius, we can use the formula:\n\nC = (F - 32) × 5/9\n\nWhere:\nC is the temperature in Celsius\nF is the temperature in Fahrenheit\n\nPlugging in 212°F, we get:\n\nC = (212 - 32) × 5/9\nC = 180 × 5/9\nC = 100\n\nTherefore, 212°", # noqa: RUF001 - "role": "assistant", - "completion_id": None, - "sequence": 2, - "response.model": "anthropic.claude-3-sonnet-20240229-v1:0", - "vendor": "bedrock", - "ingest_source": "Python", - "is_response": True, - }, - ), -] + +@pytest.fixture(scope="session", params=[False, True], ids=["ResponseStandard", "ResponseStreaming"]) +def response_streaming(request): + return request.param + + +@pytest.fixture(scope="session") +def expected_metric(response_streaming): + return ("Llm/completion/Bedrock/converse" + ("_stream" if response_streaming else ""), 1) + + +@pytest.fixture(scope="session") +def expected_events(response_streaming): + return chat_completion_expected_streaming_events if response_streaming else chat_completion_expected_events @pytest.fixture(scope="module") -def exercise_model(loop, bedrock_converse_server): +def exercise_model(loop, bedrock_converse_server, response_streaming): def _exercise_model(message): async def coro(): inference_config = {"temperature": 0.7, "maxTokens": 100} - response = await bedrock_converse_server.converse( + _response = await bedrock_converse_server.converse( + modelId="anthropic.claude-3-sonnet-20240229-v1:0", + messages=message, + system=[{"text": "You are a scientist."}], + inferenceConfig=inference_config, + ) + + return loop.run_until_complete(coro()) + + def _exercise_model_streaming(message): + async def coro(): + inference_config = {"temperature": 0.7, "maxTokens": 100} + + response = await bedrock_converse_server.converse_stream( modelId="anthropic.claude-3-sonnet-20240229-v1:0", messages=message, system=[{"text": "You are a scientist."}], inferenceConfig=inference_config, ) - assert response + _responses = [r async for r in response["stream"]] # Consume the response stream return loop.run_until_complete(coro()) - return _exercise_model + return _exercise_model_streaming if response_streaming else _exercise_model @reset_core_stats_engine() -def test_bedrock_chat_completion_in_txn_with_llm_metadata(set_trace_info, exercise_model): - @validate_custom_events(events_with_context_attrs(chat_completion_expected_events)) - # One summary event, one user message, and one response message from the assistant +def test_bedrock_chat_completion_in_txn_with_llm_metadata( + set_trace_info, exercise_model, expected_metric, expected_events +): + @validate_custom_events(events_with_context_attrs(expected_events)) + # One summary event, one system message, one user message, and one response message from the assistant @validate_custom_event_count(count=4) @validate_transaction_metrics( name="test_bedrock_chat_completion_in_txn_with_llm_metadata", - scoped_metrics=[("Llm/completion/Bedrock/converse", 1)], - rollup_metrics=[("Llm/completion/Bedrock/converse", 1)], + scoped_metrics=[expected_metric], + rollup_metrics=[expected_metric], custom_metrics=[(f"Supportability/Python/ML/Bedrock/{BOTOCORE_VERSION}", 1)], background_task=True, ) @@ -162,14 +121,14 @@ def _test(): @disabled_ai_monitoring_record_content_settings @reset_core_stats_engine() -def test_bedrock_chat_completion_no_content(set_trace_info, exercise_model): - @validate_custom_events(events_sans_content(chat_completion_expected_events)) +def test_bedrock_chat_completion_no_content(set_trace_info, exercise_model, expected_metric, expected_events): + @validate_custom_events(events_sans_content(expected_events)) # One summary event, one user message, and one response message from the assistant @validate_custom_event_count(count=4) @validate_transaction_metrics( name="test_bedrock_chat_completion_no_content", - scoped_metrics=[("Llm/completion/Bedrock/converse", 1)], - rollup_metrics=[("Llm/completion/Bedrock/converse", 1)], + scoped_metrics=[expected_metric], + rollup_metrics=[expected_metric], custom_metrics=[(f"Supportability/Python/ML/Bedrock/{BOTOCORE_VERSION}", 1)], background_task=True, ) @@ -188,14 +147,18 @@ def _test(): @reset_core_stats_engine() @override_llm_token_callback_settings(llm_token_count_callback) -def test_bedrock_chat_completion_with_token_count(set_trace_info, exercise_model): - @validate_custom_events(add_token_count_to_events(chat_completion_expected_events)) +def test_bedrock_chat_completion_with_token_count(set_trace_info, exercise_model, expected_metric, expected_events): + expected_events = add_token_counts_to_chat_events(expected_events) + if response_streaming: + expected_events = add_token_count_streaming_events(expected_events) + + @validate_custom_events(expected_events) # One summary event, one user message, and one response message from the assistant @validate_custom_event_count(count=4) @validate_transaction_metrics( name="test_bedrock_chat_completion_with_token_count", - scoped_metrics=[("Llm/completion/Bedrock/converse", 1)], - rollup_metrics=[("Llm/completion/Bedrock/converse", 1)], + scoped_metrics=[expected_metric], + rollup_metrics=[expected_metric], custom_metrics=[(f"Supportability/Python/ML/Bedrock/{BOTOCORE_VERSION}", 1)], background_task=True, ) @@ -213,13 +176,13 @@ def _test(): @reset_core_stats_engine() -def test_bedrock_chat_completion_no_llm_metadata(set_trace_info, exercise_model): - @validate_custom_events(events_sans_llm_metadata(chat_completion_expected_events)) +def test_bedrock_chat_completion_no_llm_metadata(set_trace_info, exercise_model, expected_metric, expected_events): + @validate_custom_events(events_sans_llm_metadata(expected_events)) @validate_custom_event_count(count=4) @validate_transaction_metrics( name="test_bedrock_chat_completion_in_txn_no_llm_metadata", - scoped_metrics=[("Llm/completion/Bedrock/converse", 1)], - rollup_metrics=[("Llm/completion/Bedrock/converse", 1)], + scoped_metrics=[expected_metric], + rollup_metrics=[expected_metric], custom_metrics=[(f"Supportability/Python/ML/Bedrock/{BOTOCORE_VERSION}", 1)], background_task=True, ) @@ -250,54 +213,37 @@ def test_bedrock_chat_completion_disabled_ai_monitoring_settings(set_trace_info, exercise_model(message) -chat_completion_invalid_access_key_error_events = [ - ( - {"type": "LlmChatCompletionSummary"}, - { - "id": None, # UUID that varies with each run - "llm.conversation_id": "my-awesome-id", - "llm.foo": "bar", - "span_id": None, - "trace_id": "trace-id", - "request_id": "e1206e19-2318-4a9d-be98-017c73f06118", - "duration": None, # Response time varies each test run - "request.model": "anthropic.claude-3-sonnet-20240229-v1:0", - "response.model": "anthropic.claude-3-sonnet-20240229-v1:0", - "request.temperature": 0.7, - "request.max_tokens": 100, - "vendor": "bedrock", - "ingest_source": "Python", - "response.number_of_messages": 1, - "error": True, - }, - ), - ( - {"type": "LlmChatCompletionMessage"}, - { - "id": None, # UUID that varies with each run - "llm.conversation_id": "my-awesome-id", - "llm.foo": "bar", - "request_id": "e1206e19-2318-4a9d-be98-017c73f06118", - "span_id": None, - "trace_id": "trace-id", - "content": "Invalid Token", - "role": "user", - "completion_id": None, - "sequence": 0, - "response.model": "anthropic.claude-3-sonnet-20240229-v1:0", - "vendor": "bedrock", - "ingest_source": "Python", - }, - ), -] - _client_error = botocore.exceptions.ClientError _client_error_name = callable_name(_client_error) +@pytest.fixture +def exercise_converse_incorrect_access_key(loop, bedrock_converse_server, response_streaming, monkeypatch): + def _exercise_converse_incorrect_access_key(): + async def _coro(): + monkeypatch.setattr( + bedrock_converse_server._request_signer._credentials, "access_key", "INVALID-ACCESS-KEY" + ) + + message = [{"role": "user", "content": [{"text": "Invalid Token"}]}] + request = ( + bedrock_converse_server.converse_stream if response_streaming else bedrock_converse_server.converse + ) + with pytest.raises(_client_error): + await request( + modelId="anthropic.claude-3-sonnet-20240229-v1:0", + messages=message, + inferenceConfig={"temperature": 0.7, "maxTokens": 100}, + ) + + loop.run_until_complete(_coro()) + + return _exercise_converse_incorrect_access_key + + @reset_core_stats_engine() def test_bedrock_chat_completion_error_incorrect_access_key( - loop, monkeypatch, bedrock_converse_server, exercise_model, set_trace_info + exercise_converse_incorrect_access_key, set_trace_info, expected_metric ): """ A request is made to the server with invalid credentials. botocore will reach out to the server and receive an @@ -320,8 +266,8 @@ def test_bedrock_chat_completion_error_incorrect_access_key( ) @validate_transaction_metrics( name="test_bedrock_chat_completion", - scoped_metrics=[("Llm/completion/Bedrock/converse", 1)], - rollup_metrics=[("Llm/completion/Bedrock/converse", 1)], + scoped_metrics=[expected_metric], + rollup_metrics=[expected_metric], custom_metrics=[(f"Supportability/Python/ML/Bedrock/{BOTOCORE_VERSION}", 1)], background_task=True, ) @@ -332,72 +278,36 @@ def _test(): add_custom_attribute("llm.foo", "bar") add_custom_attribute("non_llm_attr", "python-agent") - converse_incorrect_access_key(loop, bedrock_converse_server, monkeypatch) + exercise_converse_incorrect_access_key() _test() -def converse_incorrect_access_key(loop, bedrock_converse_server, monkeypatch): - async def _coro(): - monkeypatch.setattr(bedrock_converse_server._request_signer._credentials, "access_key", "INVALID-ACCESS-KEY") +@pytest.fixture +def exercise_converse_invalid_model(loop, bedrock_converse_server, response_streaming, monkeypatch): + def _exercise_converse_invalid_model(): + async def _coro(): + monkeypatch.setattr( + bedrock_converse_server._request_signer._credentials, "access_key", "INVALID-ACCESS-KEY" + ) - with pytest.raises(_client_error): - message = [{"role": "user", "content": [{"text": "Invalid Token"}]}] - response = await bedrock_converse_server.converse( - modelId="anthropic.claude-3-sonnet-20240229-v1:0", - messages=message, - inferenceConfig={"temperature": 0.7, "maxTokens": 100}, + message = [{"role": "user", "content": [{"text": "Model does not exist."}]}] + request = ( + bedrock_converse_server.converse_stream if response_streaming else bedrock_converse_server.converse ) - assert response - - loop.run_until_complete(_coro()) - - -chat_completion_invalid_model_error_events = [ - ( - {"type": "LlmChatCompletionSummary"}, - { - "id": None, # UUID that varies with each run - "llm.conversation_id": "my-awesome-id", - "llm.foo": "bar", - "request_id": "f4908827-3db9-4742-9103-2bbc34578b03", - "span_id": None, - "trace_id": "trace-id", - "duration": None, # Response time varies each test run - "request.model": "does-not-exist", - "response.model": "does-not-exist", - "request.temperature": 0.7, - "request.max_tokens": 100, - "response.number_of_messages": 1, - "vendor": "bedrock", - "ingest_source": "Python", - "error": True, - }, - ), - ( - {"type": "LlmChatCompletionMessage"}, - { - "id": None, - "llm.conversation_id": "my-awesome-id", - "llm.foo": "bar", - "span_id": None, - "trace_id": "trace-id", - "request_id": "f4908827-3db9-4742-9103-2bbc34578b03", - "content": "Model does not exist.", - "role": "user", - "completion_id": None, - "response.model": "does-not-exist", - "sequence": 0, - "vendor": "bedrock", - "ingest_source": "Python", - }, - ), -] + with pytest.raises(_client_error): + await request( + modelId="does-not-exist", messages=message, inferenceConfig={"temperature": 0.7, "maxTokens": 100} + ) + + loop.run_until_complete(_coro()) + + return _exercise_converse_invalid_model @reset_core_stats_engine() -def test_bedrock_chat_completion_error_invalid_model(loop, bedrock_converse_server, set_trace_info): - @validate_custom_events(chat_completion_invalid_model_error_events) +def test_bedrock_chat_completion_error_invalid_model(exercise_converse_invalid_model, set_trace_info, expected_metric): + @validate_custom_events(events_with_context_attrs(chat_completion_invalid_model_error_events)) @validate_error_trace_attributes( "botocore.errorfactory:ValidationException", exact_attrs={ @@ -412,8 +322,8 @@ def test_bedrock_chat_completion_error_invalid_model(loop, bedrock_converse_serv ) @validate_transaction_metrics( name="test_bedrock_chat_completion_error_invalid_model", - scoped_metrics=[("Llm/completion/Bedrock/converse", 1)], - rollup_metrics=[("Llm/completion/Bedrock/converse", 1)], + scoped_metrics=[expected_metric], + rollup_metrics=[expected_metric], custom_metrics=[(f"Supportability/Python/ML/Bedrock/{BOTOCORE_VERSION}", 1)], background_task=True, ) @@ -424,28 +334,17 @@ def _test(): add_custom_attribute("llm.foo", "bar") add_custom_attribute("non_llm_attr", "python-agent") - converse_invalid_model(loop, bedrock_converse_server) + with WithLlmCustomAttributes({"context": "attr"}): + exercise_converse_invalid_model() _test() -def converse_invalid_model(loop, bedrock_converse_server): - async def _coro(): - with pytest.raises(_client_error): - message = [{"role": "user", "content": [{"text": "Model does not exist."}]}] - - response = await bedrock_converse_server.converse( - modelId="does-not-exist", messages=message, inferenceConfig={"temperature": 0.7, "maxTokens": 100} - ) - - assert response - - loop.run_until_complete(_coro()) - - @reset_core_stats_engine() @disabled_ai_monitoring_record_content_settings -def test_bedrock_chat_completion_error_invalid_model_no_content(loop, bedrock_converse_server, set_trace_info): +def test_bedrock_chat_completion_error_invalid_model_no_content( + exercise_converse_invalid_model, set_trace_info, expected_metric +): @validate_custom_events(events_sans_content(chat_completion_invalid_model_error_events)) @validate_error_trace_attributes( "botocore.errorfactory:ValidationException", @@ -461,8 +360,8 @@ def test_bedrock_chat_completion_error_invalid_model_no_content(loop, bedrock_co ) @validate_transaction_metrics( name="test_bedrock_chat_completion_error_invalid_model_no_content", - scoped_metrics=[("Llm/completion/Bedrock/converse", 1)], - rollup_metrics=[("Llm/completion/Bedrock/converse", 1)], + scoped_metrics=[expected_metric], + rollup_metrics=[expected_metric], custom_metrics=[(f"Supportability/Python/ML/Bedrock/{BOTOCORE_VERSION}", 1)], background_task=True, ) @@ -473,49 +372,6 @@ def _test(): add_custom_attribute("llm.foo", "bar") add_custom_attribute("non_llm_attr", "python-agent") - converse_invalid_model(loop, bedrock_converse_server) - - _test() - - -@reset_core_stats_engine() -@override_llm_token_callback_settings(llm_token_count_callback) -def test_bedrock_chat_completion_error_incorrect_access_key_with_token_count( - monkeypatch, bedrock_converse_server, loop, set_trace_info -): - """ - A request is made to the server with invalid credentials. botocore will reach out to the server and receive an - UnrecognizedClientException as a response. Information from the request will be parsed and reported in customer - events. The error response can also be parsed, and will be included as attributes on the recorded exception. - """ - - @validate_custom_events(add_token_count_to_events(chat_completion_invalid_access_key_error_events)) - @validate_error_trace_attributes( - _client_error_name, - exact_attrs={ - "agent": {}, - "intrinsic": {}, - "user": { - "http.statusCode": 403, - "error.message": "The security token included in the request is invalid.", - "error.code": "UnrecognizedClientException", - }, - }, - ) - @validate_transaction_metrics( - name="test_bedrock_chat_completion_incorrect_access_key_with_token_count", - scoped_metrics=[("Llm/completion/Bedrock/converse", 1)], - rollup_metrics=[("Llm/completion/Bedrock/converse", 1)], - custom_metrics=[(f"Supportability/Python/ML/Bedrock/{BOTOCORE_VERSION}", 1)], - background_task=True, - ) - @background_task(name="test_bedrock_chat_completion_incorrect_access_key_with_token_count") - def _test(): - set_trace_info() - add_custom_attribute("llm.conversation_id", "my-awesome-id") - add_custom_attribute("llm.foo", "bar") - add_custom_attribute("non_llm_attr", "python-agent") - - converse_incorrect_access_key(loop, bedrock_converse_server, monkeypatch) + exercise_converse_invalid_model() _test() diff --git a/tests/external_aiobotocore/test_bedrock_chat_completion_invoke_model.py b/tests/external_aiobotocore/test_bedrock_chat_completion_invoke_model.py index e02cc5b543..40c21c35ee 100644 --- a/tests/external_aiobotocore/test_bedrock_chat_completion_invoke_model.py +++ b/tests/external_aiobotocore/test_bedrock_chat_completion_invoke_model.py @@ -14,13 +14,13 @@ import json import os from io import BytesIO +from pprint import pformat -import botocore.errorfactory import botocore.eventstream import botocore.exceptions import pytest from conftest import BOTOCORE_VERSION -from external_botocore._test_bedrock_chat_completion import ( +from external_botocore._test_bedrock_chat_completion_invoke_model import ( chat_completion_expected_events, chat_completion_expected_malformed_request_body_events, chat_completion_expected_malformed_response_body_events, @@ -34,7 +34,8 @@ ) from testing_support.fixtures import override_llm_token_callback_settings, reset_core_stats_engine, validate_attributes from testing_support.ml_testing_utils import ( - add_token_count_to_events, + add_token_count_streaming_events, + add_token_counts_to_chat_events, disabled_ai_monitoring_record_content_settings, disabled_ai_monitoring_settings, disabled_ai_monitoring_streaming_settings, @@ -207,7 +208,7 @@ def _test(): @reset_core_stats_engine() @override_llm_token_callback_settings(llm_token_count_callback) def test_bedrock_chat_completion_with_token_count(set_trace_info, exercise_model, expected_events, expected_metrics): - @validate_custom_events(add_token_count_to_events(expected_events)) + @validate_custom_events(add_token_counts_to_chat_events(add_token_count_streaming_events(expected_events))) # One summary event, one user message, and one response message from the assistant @validate_custom_event_count(count=3) @validate_transaction_metrics( @@ -456,51 +457,6 @@ def _test(): _test() -@reset_core_stats_engine() -@override_llm_token_callback_settings(llm_token_count_callback) -def test_bedrock_chat_completion_error_incorrect_access_key_with_token( - monkeypatch, - bedrock_server, - exercise_model, - set_trace_info, - expected_invalid_access_key_error_events, - expected_metrics, -): - @validate_custom_events(add_token_count_to_events(expected_invalid_access_key_error_events)) - @validate_error_trace_attributes( - _client_error_name, - exact_attrs={ - "agent": {}, - "intrinsic": {}, - "user": { - "http.statusCode": 403, - "error.message": "The security token included in the request is invalid.", - "error.code": "UnrecognizedClientException", - }, - }, - ) - @validate_transaction_metrics( - name="test_bedrock_chat_completion", - scoped_metrics=expected_metrics, - rollup_metrics=expected_metrics, - custom_metrics=[(f"Supportability/Python/ML/Bedrock/{BOTOCORE_VERSION}", 1)], - background_task=True, - ) - @background_task(name="test_bedrock_chat_completion") - def _test(): - monkeypatch.setattr(bedrock_server._request_signer._credentials, "access_key", "INVALID-ACCESS-KEY") - - with pytest.raises(_client_error): # not sure where this exception actually comes from - set_trace_info() - add_custom_attribute("llm.conversation_id", "my-awesome-id") - add_custom_attribute("llm.foo", "bar") - add_custom_attribute("non_llm_attr", "python-agent") - - exercise_model(prompt="Invalid Token", temperature=0.7, max_tokens=100) - - _test() - - def invoke_model_malformed_request_body(loop, bedrock_server, response_streaming): async def _coro(): with pytest.raises(_client_error): @@ -799,58 +755,6 @@ async def _test(): loop.run_until_complete(_test()) -@reset_core_stats_engine() -@override_llm_token_callback_settings(llm_token_count_callback) -@validate_custom_events(add_token_count_to_events(chat_completion_expected_streaming_error_events)) -@validate_custom_event_count(count=2) -@validate_error_trace_attributes( - _event_stream_error_name, - exact_attrs={ - "agent": {}, - "intrinsic": {}, - "user": { - "error.message": "Malformed input request, please reformat your input and try again.", - "error.code": "ValidationException", - }, - }, - forgone_params={"agent": (), "intrinsic": (), "user": ("http.statusCode")}, -) -@validate_transaction_metrics( - name="test_bedrock_chat_completion", - scoped_metrics=[("Llm/completion/Bedrock/invoke_model_with_response_stream", 1)], - rollup_metrics=[("Llm/completion/Bedrock/invoke_model_with_response_stream", 1)], - custom_metrics=[(f"Supportability/Python/ML/Bedrock/{BOTOCORE_VERSION}", 1)], - background_task=True, -) -@background_task(name="test_bedrock_chat_completion") -def test_bedrock_chat_completion_error_streaming_exception_with_token_count(loop, bedrock_server, set_trace_info): - """ - Duplicate of test_bedrock_chat_completion_error_streaming_exception, but with token callback being set. - - See the original test for a description of the error case. - """ - - async def _test(): - with pytest.raises(_event_stream_error): - model = "amazon.titan-text-express-v1" - body = (chat_completion_payload_templates[model] % ("Streaming Exception", 0.7, 100)).encode("utf-8") - - set_trace_info() - add_custom_attribute("llm.conversation_id", "my-awesome-id") - add_custom_attribute("llm.foo", "bar") - add_custom_attribute("non_llm_attr", "python-agent") - - response = await bedrock_server.invoke_model_with_response_stream( - body=body, modelId=model, accept="application/json", contentType="application/json" - ) - - body = response.get("body") - async for resp in body: - assert resp - - loop.run_until_complete(_test()) - - def test_bedrock_chat_completion_functions_marked_as_wrapped_for_sdk_compatibility(bedrock_server): assert bedrock_server._nr_wrapped @@ -858,7 +762,12 @@ def test_bedrock_chat_completion_functions_marked_as_wrapped_for_sdk_compatibili def test_chat_models_instrumented(loop): import aiobotocore - SUPPORTED_MODELS = [model for model, _, _, _ in MODEL_EXTRACTORS if "embed" not in model] + def _is_supported_model(model): + supported_models = [model for model, _, _, _ in MODEL_EXTRACTORS if "embed" not in model] + for supported_model in supported_models: + if supported_model in model: + return True + return False _id = os.environ.get("AWS_ACCESS_KEY_ID") key = os.environ.get("AWS_SECRET_ACCESS_KEY") @@ -871,12 +780,8 @@ def test_chat_models_instrumented(loop): try: response = loop.run_until_complete(client.list_foundation_models(byOutputModality="TEXT")) models = [model["modelId"] for model in response["modelSummaries"]] - not_supported = [] - for model in models: - is_supported = any(model.startswith(supported_model) for supported_model in SUPPORTED_MODELS) - if not is_supported: - not_supported.append(model) + not_supported = [model for model in models if not _is_supported_model(model)] - assert not not_supported, f"The following unsupported models were found: {not_supported}" + assert not not_supported, f"The following unsupported models were found: {pformat(not_supported)}" finally: loop.run_until_complete(client.__aexit__(None, None, None)) diff --git a/tests/external_aiobotocore/test_bedrock_embeddings.py b/tests/external_aiobotocore/test_bedrock_embeddings.py index 96b930feb5..1f9359934b 100644 --- a/tests/external_aiobotocore/test_bedrock_embeddings.py +++ b/tests/external_aiobotocore/test_bedrock_embeddings.py @@ -14,6 +14,7 @@ import json import os from io import BytesIO +from pprint import pformat import botocore.exceptions import pytest @@ -27,7 +28,7 @@ ) from testing_support.fixtures import override_llm_token_callback_settings, reset_core_stats_engine, validate_attributes from testing_support.ml_testing_utils import ( - add_token_count_to_events, + add_token_count_to_embedding_events, disabled_ai_monitoring_record_content_settings, disabled_ai_monitoring_settings, events_sans_content, @@ -164,7 +165,7 @@ def _test(): @reset_core_stats_engine() @override_llm_token_callback_settings(llm_token_count_callback) def test_bedrock_embedding_with_token_count(set_trace_info, exercise_model, expected_events): - @validate_custom_events(add_token_count_to_events(expected_events)) + @validate_custom_events(add_token_count_to_embedding_events(expected_events)) @validate_custom_event_count(count=1) @validate_transaction_metrics( name="test_bedrock_embedding", @@ -289,45 +290,6 @@ def _test(): _test() -@reset_core_stats_engine() -@override_llm_token_callback_settings(llm_token_count_callback) -def test_bedrock_embedding_error_incorrect_access_key_with_token_count( - monkeypatch, bedrock_server, exercise_model, set_trace_info, expected_invalid_access_key_error_events -): - @validate_custom_events(add_token_count_to_events(expected_invalid_access_key_error_events)) - @validate_error_trace_attributes( - _client_error_name, - exact_attrs={ - "agent": {}, - "intrinsic": {}, - "user": { - "http.statusCode": 403, - "error.message": "The security token included in the request is invalid.", - "error.code": "UnrecognizedClientException", - }, - }, - ) - @validate_transaction_metrics( - name="test_bedrock_embedding", - scoped_metrics=[("Llm/embedding/Bedrock/invoke_model", 1)], - rollup_metrics=[("Llm/embedding/Bedrock/invoke_model", 1)], - background_task=True, - ) - @background_task(name="test_bedrock_embedding") - def _test(): - monkeypatch.setattr(bedrock_server._request_signer._credentials, "access_key", "INVALID-ACCESS-KEY") - - with pytest.raises(_client_error): # not sure where this exception actually comes from - set_trace_info() - add_custom_attribute("llm.conversation_id", "my-awesome-id") - add_custom_attribute("llm.foo", "bar") - add_custom_attribute("non_llm_attr", "python-agent") - - exercise_model(prompt="Invalid Token") - - _test() - - @reset_core_stats_engine() @validate_custom_events(embedding_expected_malformed_request_body_events) @validate_custom_event_count(count=1) @@ -414,7 +376,12 @@ async def _test(): def test_embedding_models_instrumented(loop): import aiobotocore - SUPPORTED_MODELS = [model for model, _, _, _ in MODEL_EXTRACTORS if "embed" in model] + def _is_supported_model(model): + supported_models = [model for model, _, _, _ in MODEL_EXTRACTORS if "embed" in model] + for supported_model in supported_models: + if supported_model in model: + return True + return False _id = os.environ.get("AWS_ACCESS_KEY_ID") key = os.environ.get("AWS_SECRET_ACCESS_KEY") @@ -427,12 +394,8 @@ def test_embedding_models_instrumented(loop): try: response = client.list_foundation_models(byOutputModality="EMBEDDING") models = [model["modelId"] for model in response["modelSummaries"]] - not_supported = [] - for model in models: - is_supported = any(model.startswith(supported_model) for supported_model in SUPPORTED_MODELS) - if not is_supported: - not_supported.append(model) + not_supported = [model for model in models if not _is_supported_model(model)] - assert not not_supported, f"The following unsupported models were found: {not_supported}" + assert not not_supported, f"The following unsupported models were found: {pformat(not_supported)}" finally: loop.run_until_complete(client.__aexit__(None, None, None)) diff --git a/tests/external_botocore/_mock_external_bedrock_server_converse.py b/tests/external_botocore/_mock_external_bedrock_server_converse.py index aef6d52856..bb34315fc0 100644 --- a/tests/external_botocore/_mock_external_bedrock_server_converse.py +++ b/tests/external_botocore/_mock_external_bedrock_server_converse.py @@ -16,6 +16,105 @@ from testing_support.mock_external_http_server import MockExternalHTTPServer +STREAMED_RESPONSES = { + "What is 212 degrees Fahrenheit converted to Celsius?": [ + { + "Content-Type": "application/vnd.amazon.eventstream", + "x-amzn-RequestId": "f070b880-e0fb-4537-8093-796671c39239", + }, + 200, + [ + "000000b2000000528a40b4c50b3a6576656e742d7479706507000c6d65737361676553746172740d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b2270223a226162636465666768696a6b6c6d6e6f707172737475767778797a4142434445464748494a4b4c4d4e4f505152535455565758595a30222c22726f6c65223a22617373697374616e74227d40ff8268000000ae000000575f3a3ac90b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a22546f227d2c2270223a226162636465666768696a6b6c6d6e6f70717273227d57b47eb0", + "000000b800000057b09a58eb0b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a2220636f6e76657274227d2c2270223a226162636465666768696a6b6c6d6e6f7071727374757677227d7f921878", + "000000c600000057f67806450b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a222046616872656e227d2c2270223a226162636465666768696a6b6c6d6e6f707172737475767778797a4142434445464748494a4b4c227d725b3c0b", + "000000a800000057d07acf690b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a2268656974227d2c2270223a226162636465666768696a6b227d926527fe", + "000000b400000057756ab5ea0b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a2220746f227d2c2270223a226162636465666768696a6b6c6d6e6f707172737475767778227d47f66bd8", + "000000a400000057158a22680b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a222043656c73697573227d2c2270223a22616263227dc03a975f", + "000000c8000000574948b8240b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a222c227d2c2270223a226162636465666768696a6b6c6d6e6f707172737475767778797a4142434445464748494a4b4c4d4e4f5051525354227db2e3dafb", + "000000ad00000057189a40190b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a2220796f75227d2c2270223a226162636465666768696a6b6c6d6e6f70227d76c0e56b", + "000000c500000057b1d87c950b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a2220757365227d2c2270223a226162636465666768696a6b6c6d6e6f707172737475767778797a4142434445464748494a4b4c4d4e227de3731476", + "000000cb000000570ee8c2f40b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a2220746865227d2c2270223a226162636465666768696a6b6c6d6e6f707172737475767778797a4142434445464748494a4b4c4d4e4f5051525354227dd4810232", + "000000d3000000575e781eb70b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a2220666f726d756c61227d2c2270223a226162636465666768696a6b6c6d6e6f707172737475767778797a4142434445464748494a4b4c4d4e4f505152535455565758227df6672f41", + "000000d00000005719d864670b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a223a227d2c2270223a226162636465666768696a6b6c6d6e6f707172737475767778797a4142434445464748494a4b4c4d4e4f505152535455565758595a3031227dbd8afb45", + "000000b6000000570faae68a0b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a225c6e5c6e43227d2c2270223a226162636465666768696a6b6c6d6e6f707172737475767778227d088d049f", + "000000a700000057522a58b80b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a22203d227d2c2270223a226162636465666768696a6b6c227d88e54236", + "000000b70000005732cacf3a0b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a222028227d2c2270223a226162636465666768696a6b6c6d6e6f707172737475767778797a4142227de6ec1ebe", + "000000b400000057756ab5ea0b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a2246227d2c2270223a226162636465666768696a6b6c6d6e6f707172737475767778797a227d02007761", + "000000c900000057742891940b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a22202d227d2c2270223a226162636465666768696a6b6c6d6e6f707172737475767778797a4142434445464748494a4b4c4d4e4f5051525354227d3b3f080c", + "000000ab0000005797dab5b90b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a2220227d2c2270223a226162636465666768696a6b6c6d6e6f7071227d5638cc83", + "0000009d00000057b9bbf89f0b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a223332227d2c2270223a226162227dc02cb212", + "000000bc00000057451afe2b0b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a2229227d2c2270223a226162636465666768696a6b6c6d6e6f707172737475767778797a4142434445464748227da0e9aee9", + "000000c700000057cb182ff50b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a22202a227d2c2270223a226162636465666768696a6b6c6d6e6f707172737475767778797a4142434445464748494a4b4c4d4e4f505152227d0e3821bb", + "000000b70000005732cacf3a0b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a2220227d2c2270223a226162636465666768696a6b6c6d6e6f707172737475767778797a414243227d1daf3cc5", + "000000b400000057756ab5ea0b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a2235227d2c2270223a226162636465666768696a6b6c6d6e6f707172737475767778797a227dada5d973", + "000000d10000005724b84dd70b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a222f227d2c2270223a226162636465666768696a6b6c6d6e6f707172737475767778797a4142434445464748494a4b4c4d4e4f505152535455565758595a303132227db97b8201", + "000000bc00000057451afe2b0b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a2239227d2c2270223a226162636465666768696a6b6c6d6e6f707172737475767778797a4142434445464748227d99250da7", + "000000ad00000057189a40190b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a225c6e5c6e5768657265227d2c2270223a226162636465666768696a6b227d5f2ed4ef", + "0000009f00000057c37babff0b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a223a227d2c2270223a226162636465227d85a07294", + "000000a900000057ed1ae6d90b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a225c6e43227d2c2270223a226162636465666768696a6b6c6d227d50fa22de", + "000000ce00000057c6084d840b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a22206973227d2c2270223a226162636465666768696a6b6c6d6e6f707172737475767778797a4142434445464748494a4b4c4d4e4f505152535455565758227dfe3dc5ac", + "000000c8000000574948b8240b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a2220746865227d2c2270223a226162636465666768696a6b6c6d6e6f707172737475767778797a4142434445464748494a4b4c4d4e4f5051227d3f77fbbc", + "000000c1000000574458da550b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a222074656d7065726174757265227d2c2270223a226162636465666768696a6b6c6d6e6f707172737475767778797a4142227d402a7229", + "000000d200000057631837070b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a2220696e227d2c2270223a226162636465666768696a6b6c6d6e6f707172737475767778797a4142434445464748494a4b4c4d4e4f505152535455565758595a3031227df5f66d94", + "000000d90000005714c806160b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a222043656c73697573227d2c2270223a226162636465666768696a6b6c6d6e6f707172737475767778797a4142434445464748494a4b4c4d4e4f505152535455565758595a30313233227d3daccf94", + "000000b500000057480a9c5a0b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a225c6e46227d2c2270223a226162636465666768696a6b6c6d6e6f70717273747576777879227d5042c3ff", + "000000cf00000057fb6864340b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a22206973227d2c2270223a226162636465666768696a6b6c6d6e6f707172737475767778797a4142434445464748494a4b4c4d4e4f50515253545556575859227da79da7ad", + "000000bd00000057787ad79b0b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a2220746865227d2c2270223a226162636465666768696a6b6c6d6e6f707172737475767778797a414243444546227dbd3a0aec", + "000000b70000005732cacf3a0b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a222074656d7065726174757265227d2c2270223a226162636465666768696a6b6c6d6e6f707172227d1560b810", + "000000bf0000005702ba84fb0b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a2220696e227d2c2270223a226162636465666768696a6b6c6d6e6f707172737475767778797a414243444546474849227d40f78c16", + "000000ce00000057c6084d840b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a222046616872656e227d2c2270223a226162636465666768696a6b6c6d6e6f707172737475767778797a4142434445464748494a4b4c4d4e4f5051525354227d47b98626", + "000000a2000000579acad7c80b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a2268656974227d2c2270223a226162636465227d54cc33be", + "000000da0000005753687cc60b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a225c6e5c6e506c7567227d2c2270223a226162636465666768696a6b6c6d6e6f707172737475767778797a4142434445464748494a4b4c4d4e4f505152535455565758595a3031323334227d9eb4ac9a", + "000000bc00000057451afe2b0b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a2267696e67227d2c2270223a226162636465666768696a6b6c6d6e6f707172737475767778797a4142434445227d3a11d9ac000000c500000057b1d87c950b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a2220696e227d2c2270223a226162636465666768696a6b6c6d6e6f707172737475767778797a4142434445464748494a4b4c4d4e4f227d391bdff3", + "0000009e00000057fe1b824f0b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a2220227d2c2270223a2261626364227da292de09", + "000000b70000005732cacf3a0b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a22323132227d2c2270223a226162636465666768696a6b6c6d6e6f707172737475767778797a41227dbfd117db", + "000000c20000005703f8a0850b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a22c2b0227d2c2270223a226162636465666768696a6b6c6d6e6f707172737475767778797a4142434445464748494a4b4c4d227d1166f202", + "000000a100000057dd6aad180b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a2246227d2c2270223a2261626364656667227dcba24fa6", + "000000b300000057c74a69fa0b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a2220666f72227d2c2270223a226162636465666768696a6b6c6d6e6f70717273747576227dd306dee6", + "000000c700000057cb182ff50b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a222046227d2c2270223a226162636465666768696a6b6c6d6e6f707172737475767778797a4142434445464748494a4b4c4d4e4f505152227d3bdbedf1", + "000000c600000057f67806450b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a223a227d2c2270223a226162636465666768696a6b6c6d6e6f707172737475767778797a4142434445464748494a4b4c4d4e4f505152227d71d79c49", + "000000ae000000575f3a3ac90b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a225c6e5c6e43227d2c2270223a226162636465666768696a6b6c6d6e6f70227d2d8a1cce", + "000000bf0000005702ba84fb0b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a22203d227d2c2270223a226162636465666768696a6b6c6d6e6f707172737475767778797a4142434445464748494a227de81a06eb", + "000000b6000000570faae68a0b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a222028227d2c2270223a226162636465666768696a6b6c6d6e6f707172737475767778797a41227dea662b27", + "000000d500000057d138eb170b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a22323132227d2c2270223a226162636465666768696a6b6c6d6e6f707172737475767778797a4142434445464748494a4b4c4d4e4f505152535455565758595a3031323334227da7888b21", + "000000d700000057abf8b8770b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a2220227d2c2270223a226162636465666768696a6b6c6d6e6f707172737475767778797a4142434445464748494a4b4c4d4e4f505152535455565758595a303132333435363738227d63107603", + "000000c0000000577938f3e50b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a222d227d2c2270223a226162636465666768696a6b6c6d6e6f707172737475767778797a4142434445464748494a4b4c227d9e32b6f5", + "000000c600000057f67806450b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a2220227d2c2270223a226162636465666768696a6b6c6d6e6f707172737475767778797a4142434445464748494a4b4c4d4e4f505152227db3145f6b", + "0000009f00000057c37babff0b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a223332227d2c2270223a2261626364227d277c3f97", + "000000a300000057a7aafe780b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a2229227d2c2270223a22616263646566676869227dd05f85ca", + "000000bc00000057451afe2b0b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a22202a227d2c2270223a226162636465666768696a6b6c6d6e6f707172737475767778797a41424344454647227db0dfade1", + "000000aa00000057aaba9c090b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a2220227d2c2270223a226162636465666768696a6b6c6d6e6f70227da476449e", + "000000ac0000005725fa69a90b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a2235227d2c2270223a226162636465666768696a6b6c6d6e6f707172227deedc54f0", + "000000ca000000573388eb440b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a222f227d2c2270223a226162636465666768696a6b6c6d6e6f707172737475767778797a4142434445464748494a4b4c4d4e4f50515253545556227d7abef087", + "000000d00000005719d864670b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a2239227d2c2270223a226162636465666768696a6b6c6d6e6f707172737475767778797a4142434445464748494a4b4c4d4e4f505152535455565758595a3031227de7c50a2e", + "0000009f00000057c37babff0b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a225c6e43227d2c2270223a22616263227df88e9dc2", + "000000ac0000005725fa69a90b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a22203d227d2c2270223a226162636465666768696a6b6c6d6e6f7071227d6f5c7d17", + "000000bd00000057787ad79b0b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a2220227d2c2270223a226162636465666768696a6b6c6d6e6f707172737475767778797a414243444546474849227d1c650877", + "000000a400000057158a22680b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a22313830227d2c2270223a226162636465666768227dba33e936", + "000000bb00000057f73a223b0b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a2220227d2c2270223a226162636465666768696a6b6c6d6e6f707172737475767778797a41424344454647227df14100ef", + "000000a400000057158a22680b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a222a227d2c2270223a226162636465666768696a227da79b0693", + "000000c700000057cb182ff50b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a2220227d2c2270223a226162636465666768696a6b6c6d6e6f707172737475767778797a4142434445464748494a4b4c4d4e4f50515253227de52ff51e", + "000000aa00000057aaba9c090b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a2235227d2c2270223a226162636465666768696a6b6c6d6e6f70227df5cf9fcf", + "000000b9000000578dfa715b0b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a222f227d2c2270223a226162636465666768696a6b6c6d6e6f707172737475767778797a4142434445227dc22fcb78", + "0000009d00000057b9bbf89f0b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a2239227d2c2270223a22616263227db33d112d", + "000000b9000000578dfa715b0b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a225c6e43227d2c2270223a226162636465666768696a6b6c6d6e6f707172737475767778797a414243227d6e135792", + "000000c20000005703f8a0850b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a22203d227d2c2270223a226162636465666768696a6b6c6d6e6f707172737475767778797a4142434445464748494a4b4c4d227d242e22f6", + "000000a000000057e00a84a80b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a2220227d2c2270223a22616263646566227d64c7e90b", + "000000a800000057d07acf690b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a22313030227d2c2270223a226162636465666768696a6b6c227dee65d4c5", + "000000e200000057c2398f810b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a225c6e5c6e5468657265666f7265227d2c2270223a226162636465666768696a6b6c6d6e6f707172737475767778797a4142434445464748494a4b4c4d4e4f505152535455565758595a3031323334353637227d43ae3a9e", + "000000c600000057f67806450b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a222c227d2c2270223a226162636465666768696a6b6c6d6e6f707172737475767778797a4142434445464748494a4b4c4d4e4f505152227df0760dea", + "000000a50000005728ea0bd80b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a2220227d2c2270223a226162636465666768696a6b227db714fc15", + "000000ab0000005797dab5b90b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a22323132227d2c2270223a226162636465666768696a6b6c6d6e6f227de9fc19df", + "000000be000000573fdaad4b0b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a2220227d2c2270223a226162636465666768696a6b6c6d6e6f707172737475767778797a4142434445464748494a227dd7107790", + "000000c600000057f67806450b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a2264656772656573227d2c2270223a226162636465666768696a6b6c6d6e6f707172737475767778797a4142434445464748494a4b4c227d15374080", + "000000dd00000057e148a0d60b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a222046616872656e227d2c2270223a226162636465666768696a6b6c6d6e6f707172737475767778797a4142434445464748494a4b4c4d4e4f505152535455565758595a303132333435363738227d8993e5c9", + "000000a800000056a77dffff0b3a6576656e742d74797065070010636f6e74656e74426c6f636b53746f700d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2270223a226162636465666768696a6b6c6d6e6f707172737475767778797a4142434445464748494a227d1c361897", + "000000bd00000051911972ae0b3a6576656e742d7479706507000b6d65737361676553746f700d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b2270223a226162636465666768696a6b6c6d6e6f707172737475767778797a4142434445464748494a4b4c4d4e4f505152535455565758595a303132333435222c2273746f70526561736f6e223a226d61785f746f6b656e73227d2963d7e1", + "000000f00000004ebc72e3a30b3a6576656e742d747970650700086d657461646174610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226d657472696373223a7b226c6174656e63794d73223a323134397d2c2270223a226162636465666768696a6b6c6d6e6f707172737475767778222c227573616765223a7b22696e707574546f6b656e73223a32362c226f7574707574546f6b656e73223a3130302c22736572766572546f6f6c5573616765223a7b7d2c22746f74616c546f6b656e73223a3132367d7dd415e186", + ], + ] +} + RESPONSES = { "What is 212 degrees Fahrenheit converted to Celsius?": [ {"Content-Type": "application/json", "x-amzn-RequestId": "c20d345e-6878-4778-b674-6b187bae8ecf"}, @@ -65,6 +164,7 @@ def simple_get(self): except Exception: content = body + stream = self.path.endswith("converse-stream") prompt = extract_shortened_prompt_converse(content) if not prompt: self.send_response(500) @@ -73,17 +173,29 @@ def simple_get(self): return headers, status_code, response = ({}, 0, "") - - for k, v in RESPONSES.items(): - if prompt.startswith(k): - headers, status_code, response = v - break + if stream: + for k, v in STREAMED_RESPONSES.items(): + if prompt.startswith(k): + headers, status_code, response = v + break + if not response: + for k, v in RESPONSES.items(): + # Only look for error responses returned immediately instead of in a stream + if prompt.startswith(k) and v[1] >= 400: + headers, status_code, response = v + stream = False # Response will not be streamed + break + else: + for k, v in RESPONSES.items(): + if prompt.startswith(k): + headers, status_code, response = v + break if not response: # If no matches found self.send_response(500) self.end_headers() - self.wfile.write(f"Unknown Prompt:\n{prompt}".encode()) + self.wfile.write(f"Unknown Prompt ({'Streaming' if stream else 'Non-Streaming'}):\n{prompt}".encode()) return # Send response code @@ -94,10 +206,19 @@ def simple_get(self): self.send_header(k, v) self.end_headers() - # Send response body - response_body = json.dumps(response).encode("utf-8") + if stream: + # Send response body + for resp in response: + self.wfile.write(bytes.fromhex(resp)) + else: + # Send response body + response_body = json.dumps(response).encode("utf-8") + + if "Malformed Body" in prompt: + # Remove end of response to make invalid JSON + response_body = response_body[:-4] - self.wfile.write(response_body) + self.wfile.write(response_body) return diff --git a/tests/external_botocore/_mock_external_bedrock_server_invoke_model.py b/tests/external_botocore/_mock_external_bedrock_server_invoke_model.py index 6dd1fbaac0..09b3937ce2 100644 --- a/tests/external_botocore/_mock_external_bedrock_server_invoke_model.py +++ b/tests/external_botocore/_mock_external_bedrock_server_invoke_model.py @@ -6772,7 +6772,7 @@ def simple_get(self): # If no matches found self.send_response(500) self.end_headers() - self.wfile.write(f"Unknown Prompt:\n{prompt}".encode()) + self.wfile.write(f"Unknown Prompt ({'Streaming' if stream else 'Non-Streaming'}):\n{prompt}".encode()) return if stream: diff --git a/tests/external_botocore/_test_bedrock_chat_completion_converse.py b/tests/external_botocore/_test_bedrock_chat_completion_converse.py new file mode 100644 index 0000000000..50c67a2260 --- /dev/null +++ b/tests/external_botocore/_test_bedrock_chat_completion_converse.py @@ -0,0 +1,269 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Ignore unicode characters in this file from LLM responses +# ruff: noqa: RUF001 + +chat_completion_expected_events = [ + ( + {"type": "LlmChatCompletionSummary"}, + { + "id": None, # UUID that varies with each run + "timestamp": None, + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "span_id": None, + "trace_id": "trace-id", + "request_id": "c20d345e-6878-4778-b674-6b187bae8ecf", + "duration": None, # Response time varies each test run + "request.model": "anthropic.claude-3-sonnet-20240229-v1:0", + "response.model": "anthropic.claude-3-sonnet-20240229-v1:0", + "response.usage.prompt_tokens": 26, + "response.usage.completion_tokens": 100, + "response.usage.total_tokens": 126, + "request.temperature": 0.7, + "request.max_tokens": 100, + "response.choices.finish_reason": "max_tokens", + "vendor": "bedrock", + "ingest_source": "Python", + "response.number_of_messages": 3, + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, # UUID that varies with each run + "timestamp": None, + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "request_id": "c20d345e-6878-4778-b674-6b187bae8ecf", + "span_id": None, + "trace_id": "trace-id", + "content": "You are a scientist.", + "role": "system", + "completion_id": None, + "sequence": 0, + "token_count": 0, + "response.model": "anthropic.claude-3-sonnet-20240229-v1:0", + "vendor": "bedrock", + "ingest_source": "Python", + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, # UUID that varies with each run + "timestamp": None, + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "request_id": "c20d345e-6878-4778-b674-6b187bae8ecf", + "span_id": None, + "trace_id": "trace-id", + "content": "What is 212 degrees Fahrenheit converted to Celsius?", + "role": "user", + "completion_id": None, + "sequence": 1, + "token_count": 0, + "response.model": "anthropic.claude-3-sonnet-20240229-v1:0", + "vendor": "bedrock", + "ingest_source": "Python", + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "request_id": "c20d345e-6878-4778-b674-6b187bae8ecf", + "span_id": None, + "trace_id": "trace-id", + "content": "To convert 212°F to Celsius, we can use the formula:\n\nC = (F - 32) × 5/9\n\nWhere:\nC is the temperature in Celsius\nF is the temperature in Fahrenheit\n\nPlugging in 212°F, we get:\n\nC = (212 - 32) × 5/9\nC = 180 × 5/9\nC = 100\n\nTherefore, 212°", + "role": "assistant", + "completion_id": None, + "sequence": 2, + "token_count": 0, + "response.model": "anthropic.claude-3-sonnet-20240229-v1:0", + "vendor": "bedrock", + "ingest_source": "Python", + "is_response": True, + }, + ), +] + +chat_completion_expected_streaming_events = [ + ( + {"type": "LlmChatCompletionSummary"}, + { + "id": None, # UUID that varies with each run + "timestamp": None, + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "span_id": None, + "trace_id": "trace-id", + "request_id": "f070b880-e0fb-4537-8093-796671c39239", + "duration": None, # Response time varies each test run + "request.model": "anthropic.claude-3-sonnet-20240229-v1:0", + "response.model": "anthropic.claude-3-sonnet-20240229-v1:0", + "request.temperature": 0.7, + "request.max_tokens": 100, + "response.choices.finish_reason": "max_tokens", + "vendor": "bedrock", + "ingest_source": "Python", + "response.number_of_messages": 3, + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, # UUID that varies with each run + "timestamp": None, + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "request_id": "f070b880-e0fb-4537-8093-796671c39239", + "span_id": None, + "trace_id": "trace-id", + "content": "You are a scientist.", + "role": "system", + "completion_id": None, + "sequence": 0, + "response.model": "anthropic.claude-3-sonnet-20240229-v1:0", + "vendor": "bedrock", + "ingest_source": "Python", + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, # UUID that varies with each run + "timestamp": None, + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "request_id": "f070b880-e0fb-4537-8093-796671c39239", + "span_id": None, + "trace_id": "trace-id", + "content": "What is 212 degrees Fahrenheit converted to Celsius?", + "role": "user", + "completion_id": None, + "sequence": 1, + "response.model": "anthropic.claude-3-sonnet-20240229-v1:0", + "vendor": "bedrock", + "ingest_source": "Python", + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "request_id": "f070b880-e0fb-4537-8093-796671c39239", + "span_id": None, + "trace_id": "trace-id", + "content": "To convert Fahrenheit to Celsius, you use the formula:\n\nC = (F - 32) * 5/9\n\nWhere:\nC is the temperature in Celsius\nF is the temperature in Fahrenheit\n\nPlugging in 212°F for F:\n\nC = (212 - 32) * 5/9\nC = 180 * 5/9\nC = 100\n\nTherefore, 212 degrees Fahren", + "role": "assistant", + "completion_id": None, + "sequence": 2, + "response.model": "anthropic.claude-3-sonnet-20240229-v1:0", + "vendor": "bedrock", + "ingest_source": "Python", + "is_response": True, + }, + ), +] + +chat_completion_invalid_access_key_error_events = [ + ( + {"type": "LlmChatCompletionSummary"}, + { + "id": None, # UUID that varies with each run + "timestamp": None, + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "span_id": None, + "trace_id": "trace-id", + "request_id": "e1206e19-2318-4a9d-be98-017c73f06118", + "duration": None, # Response time varies each test run + "request.model": "anthropic.claude-3-sonnet-20240229-v1:0", + "response.model": "anthropic.claude-3-sonnet-20240229-v1:0", + "request.temperature": 0.7, + "request.max_tokens": 100, + "vendor": "bedrock", + "ingest_source": "Python", + "response.number_of_messages": 1, + "error": True, + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, # UUID that varies with each run + "timestamp": None, + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "request_id": "e1206e19-2318-4a9d-be98-017c73f06118", + "span_id": None, + "trace_id": "trace-id", + "content": "Invalid Token", + "role": "user", + "completion_id": None, + "sequence": 0, + "response.model": "anthropic.claude-3-sonnet-20240229-v1:0", + "vendor": "bedrock", + "ingest_source": "Python", + }, + ), +] +chat_completion_invalid_model_error_events = [ + ( + {"type": "LlmChatCompletionSummary"}, + { + "id": None, # UUID that varies with each run + "timestamp": None, + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "request_id": "f4908827-3db9-4742-9103-2bbc34578b03", + "span_id": None, + "trace_id": "trace-id", + "duration": None, # Response time varies each test run + "request.model": "does-not-exist", + "response.model": "does-not-exist", + "request.temperature": 0.7, + "request.max_tokens": 100, + "response.number_of_messages": 1, + "vendor": "bedrock", + "ingest_source": "Python", + "error": True, + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, + "timestamp": None, + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "span_id": None, + "trace_id": "trace-id", + "request_id": "f4908827-3db9-4742-9103-2bbc34578b03", + "content": "Model does not exist.", + "role": "user", + "completion_id": None, + "response.model": "does-not-exist", + "sequence": 0, + "vendor": "bedrock", + "ingest_source": "Python", + }, + ), +] diff --git a/tests/external_botocore/_test_bedrock_chat_completion.py b/tests/external_botocore/_test_bedrock_chat_completion_invoke_model.py similarity index 95% rename from tests/external_botocore/_test_bedrock_chat_completion.py rename to tests/external_botocore/_test_bedrock_chat_completion_invoke_model.py index fd970b0603..382b5375f1 100644 --- a/tests/external_botocore/_test_bedrock_chat_completion.py +++ b/tests/external_botocore/_test_bedrock_chat_completion_invoke_model.py @@ -31,6 +31,7 @@ {"type": "LlmChatCompletionSummary"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "span_id": None, @@ -51,6 +52,7 @@ {"type": "LlmChatCompletionMessage"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "request_id": "48c7ee13-7790-461f-959f-04b0a4cf91c8", @@ -90,6 +92,7 @@ {"type": "LlmChatCompletionSummary"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "span_id": None, @@ -98,6 +101,9 @@ "duration": None, # Response time varies each test run "request.model": "amazon.titan-text-express-v1", "response.model": "amazon.titan-text-express-v1", + "response.usage.completion_tokens": 32, + "response.usage.total_tokens": 44, + "response.usage.prompt_tokens": 12, "request.temperature": 0.7, "request.max_tokens": 100, "response.choices.finish_reason": "FINISH", @@ -110,6 +116,7 @@ {"type": "LlmChatCompletionMessage"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "request_id": "81508a1c-33a8-4294-8743-f0c629af2f49", @@ -119,6 +126,7 @@ "role": "user", "completion_id": None, "sequence": 0, + "token_count": 0, "response.model": "amazon.titan-text-express-v1", "vendor": "bedrock", "ingest_source": "Python", @@ -137,6 +145,7 @@ "role": "assistant", "completion_id": None, "sequence": 1, + "token_count": 0, "response.model": "amazon.titan-text-express-v1", "vendor": "bedrock", "ingest_source": "Python", @@ -149,6 +158,7 @@ {"type": "LlmChatCompletionSummary"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "span_id": None, @@ -170,6 +180,7 @@ {"type": "LlmChatCompletionMessage"}, { "id": "1234-0", + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "request_id": "228ee63f-4eca-4b7d-b679-bc920de63525", @@ -209,6 +220,7 @@ {"type": "LlmChatCompletionSummary"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "span_id": None, @@ -229,6 +241,7 @@ {"type": "LlmChatCompletionMessage"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "request_id": "6a886158-b39f-46ce-b214-97458ab76f2f", @@ -268,6 +281,7 @@ {"type": "LlmChatCompletionSummary"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "span_id": None, @@ -276,6 +290,9 @@ "duration": None, # Response time varies each test run "request.model": "anthropic.claude-3-sonnet-20240229-v1:0", "response.model": "anthropic.claude-3-sonnet-20240229-v1:0", + "response.usage.completion_tokens": 31, + "response.usage.prompt_tokens": 21, + "response.usage.total_tokens": 52, "request.temperature": 0.7, "request.max_tokens": 100, "response.choices.finish_reason": "end_turn", @@ -288,6 +305,7 @@ {"type": "LlmChatCompletionMessage"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "request_id": "ab38295d-df9c-4141-8173-38221651bf46", @@ -297,6 +315,7 @@ "role": "user", "completion_id": None, "sequence": 0, + "token_count": 0, "response.model": "anthropic.claude-3-sonnet-20240229-v1:0", "vendor": "bedrock", "ingest_source": "Python", @@ -315,6 +334,7 @@ "role": "assistant", "completion_id": None, "sequence": 1, + "token_count": 0, "response.model": "anthropic.claude-3-sonnet-20240229-v1:0", "vendor": "bedrock", "ingest_source": "Python", @@ -327,6 +347,7 @@ {"type": "LlmChatCompletionSummary"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "span_id": None, @@ -348,6 +369,7 @@ {"type": "LlmChatCompletionMessage"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "request_id": "12912a17-aa13-45f3-914c-cc82166f3601", @@ -387,6 +409,7 @@ {"type": "LlmChatCompletionSummary"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "span_id": None, @@ -395,6 +418,9 @@ "duration": None, # Response time varies each test run "request.model": "meta.llama2-13b-chat-v1", "response.model": "meta.llama2-13b-chat-v1", + "response.usage.prompt_tokens": 17, + "response.usage.completion_tokens": 69, + "response.usage.total_tokens": 86, "request.temperature": 0.7, "request.max_tokens": 100, "response.choices.finish_reason": "stop", @@ -407,6 +433,7 @@ {"type": "LlmChatCompletionMessage"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "request_id": "a168214d-742d-4244-bd7f-62214ffa07df", @@ -416,6 +443,7 @@ "role": "user", "completion_id": None, "sequence": 0, + "token_count": 0, "response.model": "meta.llama2-13b-chat-v1", "vendor": "bedrock", "ingest_source": "Python", @@ -434,6 +462,7 @@ "role": "assistant", "completion_id": None, "sequence": 1, + "token_count": 0, "response.model": "meta.llama2-13b-chat-v1", "vendor": "bedrock", "ingest_source": "Python", @@ -448,6 +477,7 @@ {"type": "LlmChatCompletionSummary"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "span_id": None, @@ -468,6 +498,7 @@ {"type": "LlmChatCompletionMessage"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "request_id": "48c7ee13-7790-461f-959f-04b0a4cf91c8", @@ -507,6 +538,7 @@ {"type": "LlmChatCompletionSummary"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "span_id": None, @@ -525,6 +557,7 @@ {"type": "LlmChatCompletionMessage"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "request_id": "884db5c9-18ab-4f27-8892-33656176a2e6", @@ -564,6 +597,7 @@ {"type": "LlmChatCompletionSummary"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "span_id": None, @@ -581,6 +615,7 @@ {"type": "LlmChatCompletionMessage"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "request_id": "1a72a1f6-310f-469c-af1d-2c59eb600089", @@ -620,6 +655,7 @@ {"type": "LlmChatCompletionSummary"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "span_id": None, @@ -676,6 +712,7 @@ {"type": "LlmChatCompletionSummary"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "span_id": None, @@ -694,6 +731,7 @@ {"type": "LlmChatCompletionMessage"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "request_id": "cce6b34c-812c-4f97-8885-515829aa9639", @@ -735,6 +773,7 @@ {"type": "LlmChatCompletionSummary"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "span_id": None, @@ -755,6 +794,7 @@ {"type": "LlmChatCompletionMessage"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "request_id": "48c7ee13-7790-461f-959f-04b0a4cf91c8", @@ -794,6 +834,7 @@ {"type": "LlmChatCompletionSummary"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "span_id": None, @@ -812,6 +853,7 @@ {"type": "LlmChatCompletionMessage"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "request_id": "884db5c9-18ab-4f27-8892-33656176a2e6", @@ -851,6 +893,7 @@ {"type": "LlmChatCompletionSummary"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "span_id": None, @@ -869,6 +912,7 @@ {"type": "LlmChatCompletionMessage"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "request_id": "1a72a1f6-310f-469c-af1d-2c59eb600089", @@ -908,6 +952,7 @@ {"type": "LlmChatCompletionSummary"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "span_id": None, @@ -926,6 +971,7 @@ {"type": "LlmChatCompletionMessage"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "request_id": "96c7306d-2d60-4629-83e9-dbd6befb0e4e", @@ -965,6 +1011,7 @@ {"type": "LlmChatCompletionSummary"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "span_id": None, @@ -983,6 +1030,7 @@ {"type": "LlmChatCompletionMessage"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "request_id": "cce6b34c-812c-4f97-8885-515829aa9639", @@ -1025,6 +1073,7 @@ {"type": "LlmChatCompletionSummary"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "span_id": None, @@ -1045,6 +1094,7 @@ {"type": "LlmChatCompletionMessage"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "request_id": "48c7ee13-7790-461f-959f-04b0a4cf91c8", @@ -1084,6 +1134,7 @@ {"type": "LlmChatCompletionSummary"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "span_id": None, @@ -1092,6 +1143,9 @@ "duration": None, # Response time varies each test run "request.model": "amazon.titan-text-express-v1", "response.model": "amazon.titan-text-express-v1", + "response.usage.completion_tokens": 35, + "response.usage.total_tokens": 47, + "response.usage.prompt_tokens": 12, "request.temperature": 0.7, "request.max_tokens": 100, "response.choices.finish_reason": "FINISH", @@ -1104,6 +1158,7 @@ {"type": "LlmChatCompletionMessage"}, { "id": None, # UUID that varies with each run + "timestamp": None, "request_id": "b427270f-371a-458d-81b6-a05aafb2704c", "span_id": None, "trace_id": "trace-id", @@ -1113,6 +1168,7 @@ "role": "user", "completion_id": None, "sequence": 0, + "token_count": 0, "response.model": "amazon.titan-text-express-v1", "vendor": "bedrock", "ingest_source": "Python", @@ -1131,6 +1187,7 @@ "role": "assistant", "completion_id": None, "sequence": 1, + "token_count": 0, "response.model": "amazon.titan-text-express-v1", "vendor": "bedrock", "ingest_source": "Python", @@ -1143,6 +1200,7 @@ {"type": "LlmChatCompletionSummary"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "span_id": None, @@ -1151,6 +1209,9 @@ "duration": None, # Response time varies each test run "request.model": "anthropic.claude-instant-v1", "response.model": "anthropic.claude-instant-v1", + "response.usage.completion_tokens": 99, + "response.usage.prompt_tokens": 19, + "response.usage.total_tokens": 118, "request.temperature": 0.7, "request.max_tokens": 100, "response.choices.finish_reason": "stop_sequence", @@ -1163,6 +1224,7 @@ {"type": "LlmChatCompletionMessage"}, { "id": None, # UUID that varies with each run + "timestamp": None, "request_id": "a645548f-0b3a-47ce-a675-f51e6e9037de", "span_id": None, "trace_id": "trace-id", @@ -1172,6 +1234,7 @@ "role": "user", "completion_id": None, "sequence": 0, + "token_count": 0, "response.model": "anthropic.claude-instant-v1", "vendor": "bedrock", "ingest_source": "Python", @@ -1190,6 +1253,7 @@ "role": "assistant", "completion_id": None, "sequence": 1, + "token_count": 0, "response.model": "anthropic.claude-instant-v1", "vendor": "bedrock", "ingest_source": "Python", @@ -1202,6 +1266,7 @@ {"type": "LlmChatCompletionSummary"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "span_id": None, @@ -1221,6 +1286,7 @@ {"type": "LlmChatCompletionMessage"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "request_id": "1efe6197-80f9-43a6-89a5-bb536c1b822f", @@ -1260,6 +1326,7 @@ {"type": "LlmChatCompletionSummary"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "span_id": None, @@ -1269,6 +1336,9 @@ "duration": None, # Response time varies each test run "request.model": "cohere.command-text-v14", "response.model": "cohere.command-text-v14", + "response.usage.completion_tokens": 91, + "response.usage.total_tokens": 100, + "response.usage.prompt_tokens": 9, "request.temperature": 0.7, "request.max_tokens": 100, "response.choices.finish_reason": "COMPLETE", @@ -1281,6 +1351,7 @@ {"type": "LlmChatCompletionMessage"}, { "id": None, # UUID that varies with each run + "timestamp": None, "request_id": "4f8ab6c5-42d1-4e35-9573-30f9f41f821e", "span_id": None, "trace_id": "trace-id", @@ -1290,6 +1361,7 @@ "role": "user", "completion_id": None, "sequence": 0, + "token_count": 0, "response.model": "cohere.command-text-v14", "vendor": "bedrock", "ingest_source": "Python", @@ -1308,6 +1380,7 @@ "role": "assistant", "completion_id": None, "sequence": 1, + "token_count": 0, "response.model": "cohere.command-text-v14", "vendor": "bedrock", "ingest_source": "Python", @@ -1320,6 +1393,7 @@ {"type": "LlmChatCompletionSummary"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "span_id": None, @@ -1328,6 +1402,9 @@ "duration": None, # Response time varies each test run "request.model": "meta.llama2-13b-chat-v1", "response.model": "meta.llama2-13b-chat-v1", + "response.usage.prompt_tokens": 17, + "response.usage.completion_tokens": 100, + "response.usage.total_tokens": 117, "request.temperature": 0.7, "request.max_tokens": 100, "response.choices.finish_reason": "length", @@ -1340,6 +1417,7 @@ {"type": "LlmChatCompletionMessage"}, { "id": None, # UUID that varies with each run + "timestamp": None, "request_id": "6dd99878-0919-4f92-850c-48f50f923b76", "span_id": None, "trace_id": "trace-id", @@ -1349,6 +1427,7 @@ "role": "user", "completion_id": None, "sequence": 0, + "token_count": 0, "response.model": "meta.llama2-13b-chat-v1", "vendor": "bedrock", "ingest_source": "Python", @@ -1367,6 +1446,7 @@ "role": "assistant", "completion_id": None, "sequence": 1, + "token_count": 0, "response.model": "meta.llama2-13b-chat-v1", "vendor": "bedrock", "ingest_source": "Python", @@ -1381,6 +1461,7 @@ {"type": "LlmChatCompletionSummary"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "request_id": "f4908827-3db9-4742-9103-2bbc34578b03", @@ -1402,6 +1483,7 @@ {"type": "LlmChatCompletionSummary"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "span_id": None, @@ -1422,6 +1504,7 @@ {"type": "LlmChatCompletionMessage"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "request_id": "48c7ee13-7790-461f-959f-04b0a4cf91c8", @@ -1442,6 +1525,7 @@ {"type": "LlmChatCompletionSummary"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "span_id": None, @@ -1462,6 +1546,7 @@ {"type": "LlmChatCompletionMessage"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "request_id": "15b39c8b-8e85-42c9-9623-06720301bda3", @@ -1482,6 +1567,7 @@ {"type": "LlmChatCompletionSummary"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "span_id": None, @@ -1502,6 +1588,7 @@ {"type": "LlmChatCompletionMessage"}, { "id": None, + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "request_id": "9021791d-3797-493d-9277-e33aa6f6d544", @@ -1522,6 +1609,7 @@ {"type": "LlmChatCompletionSummary"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "span_id": None, @@ -1542,6 +1630,7 @@ {"type": "LlmChatCompletionMessage"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "request_id": "37396f55-b721-4bae-9461-4c369f5a080d", @@ -1562,6 +1651,7 @@ {"type": "LlmChatCompletionSummary"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "span_id": None, @@ -1582,6 +1672,7 @@ {"type": "LlmChatCompletionMessage"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "request_id": "282ba076-576f-46aa-a2e6-680392132e87", @@ -1602,6 +1693,7 @@ {"type": "LlmChatCompletionSummary"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "span_id": None, @@ -1622,6 +1714,7 @@ {"type": "LlmChatCompletionMessage"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "request_id": "22476490-a0d6-42db-b5ea-32d0b8a7f751", @@ -1642,6 +1735,7 @@ {"type": "LlmChatCompletionSummary"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "span_id": None, @@ -1662,6 +1756,7 @@ {"type": "LlmChatCompletionMessage"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "request_id": "22476490-a0d6-42db-b5ea-32d0b8a7f751", @@ -1685,6 +1780,7 @@ {"type": "LlmChatCompletionSummary"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "span_id": None, @@ -1705,6 +1801,7 @@ {"type": "LlmChatCompletionSummary"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "span_id": None, @@ -1724,6 +1821,7 @@ {"type": "LlmChatCompletionMessage"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "request_id": "81508a1c-33a8-4294-8743-f0c629af2f49", @@ -1745,6 +1843,7 @@ {"type": "LlmChatCompletionSummary"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "span_id": None, @@ -1764,6 +1863,7 @@ {"type": "LlmChatCompletionMessage"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "request_id": "a5a8cebb-fd33-4437-8168-5667fbdfc1fb", @@ -1785,6 +1885,7 @@ {"type": "LlmChatCompletionSummary"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "span_id": None, @@ -1804,6 +1905,7 @@ {"type": "LlmChatCompletionMessage"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "request_id": "a5a8cebb-fd33-4437-8168-5667fbdfc1fb", @@ -1826,6 +1928,7 @@ {"type": "LlmChatCompletionSummary"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "span_id": None, @@ -1845,6 +1948,7 @@ {"type": "LlmChatCompletionMessage"}, { "id": None, # UUID that varies with each run + "timestamp": None, "llm.conversation_id": "my-awesome-id", "llm.foo": "bar", "span_id": None, diff --git a/tests/external_botocore/_test_bedrock_embeddings.py b/tests/external_botocore/_test_bedrock_embeddings.py index f5c227b9c3..af544af001 100644 --- a/tests/external_botocore/_test_bedrock_embeddings.py +++ b/tests/external_botocore/_test_bedrock_embeddings.py @@ -33,6 +33,7 @@ "response.model": "amazon.titan-embed-text-v1", "request.model": "amazon.titan-embed-text-v1", "request_id": "11233989-07e8-4ecb-9ba6-79601ba6d8cc", + "response.usage.total_tokens": 6, "vendor": "bedrock", "ingest_source": "Python", }, @@ -52,6 +53,7 @@ "response.model": "amazon.titan-embed-g1-text-02", "request.model": "amazon.titan-embed-g1-text-02", "request_id": "b10ac895-eae3-4f07-b926-10b2866c55ed", + "response.usage.total_tokens": 6, "vendor": "bedrock", "ingest_source": "Python", }, diff --git a/tests/external_botocore/test_bedrock_chat_completion_converse.py b/tests/external_botocore/test_bedrock_chat_completion_converse.py new file mode 100644 index 0000000000..b613b6c3a8 --- /dev/null +++ b/tests/external_botocore/test_bedrock_chat_completion_converse.py @@ -0,0 +1,357 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import botocore.exceptions +import pytest +from _test_bedrock_chat_completion_converse import ( + chat_completion_expected_events, + chat_completion_expected_streaming_events, + chat_completion_invalid_access_key_error_events, + chat_completion_invalid_model_error_events, +) +from conftest import BOTOCORE_VERSION +from testing_support.fixtures import override_llm_token_callback_settings, reset_core_stats_engine, validate_attributes +from testing_support.ml_testing_utils import ( + add_token_count_streaming_events, + add_token_counts_to_chat_events, + disabled_ai_monitoring_record_content_settings, + disabled_ai_monitoring_settings, + events_sans_content, + events_sans_llm_metadata, + events_with_context_attrs, + llm_token_count_callback, + set_trace_info, +) +from testing_support.validators.validate_custom_event import validate_custom_event_count +from testing_support.validators.validate_custom_events import validate_custom_events +from testing_support.validators.validate_error_trace_attributes import validate_error_trace_attributes +from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics + +from newrelic.api.background_task import background_task +from newrelic.api.llm_custom_attributes import WithLlmCustomAttributes +from newrelic.api.transaction import add_custom_attribute +from newrelic.common.object_names import callable_name + + +@pytest.fixture(scope="session", params=[False, True], ids=["ResponseStandard", "ResponseStreaming"]) +def response_streaming(request): + return request.param + + +@pytest.fixture(scope="session") +def expected_metric(response_streaming): + return ("Llm/completion/Bedrock/converse" + ("_stream" if response_streaming else ""), 1) + + +@pytest.fixture(scope="session") +def expected_events(response_streaming): + return chat_completion_expected_streaming_events if response_streaming else chat_completion_expected_events + + +@pytest.fixture(scope="module") +def exercise_model(bedrock_converse_server, response_streaming): + def _exercise_model(message): + inference_config = {"temperature": 0.7, "maxTokens": 100} + + _response = bedrock_converse_server.converse( + modelId="anthropic.claude-3-sonnet-20240229-v1:0", + messages=message, + system=[{"text": "You are a scientist."}], + inferenceConfig=inference_config, + ) + + def _exercise_model_streaming(message): + inference_config = {"temperature": 0.7, "maxTokens": 100} + + response = bedrock_converse_server.converse_stream( + modelId="anthropic.claude-3-sonnet-20240229-v1:0", + messages=message, + system=[{"text": "You are a scientist."}], + inferenceConfig=inference_config, + ) + _responses = list(response["stream"]) # Consume the response stream + + return _exercise_model_streaming if response_streaming else _exercise_model + + +@reset_core_stats_engine() +def test_bedrock_chat_completion_in_txn_with_llm_metadata( + set_trace_info, exercise_model, expected_metric, expected_events +): + @validate_custom_events(events_with_context_attrs(expected_events)) + # One summary event, one system message, one user message, and one response message from the assistant + @validate_custom_event_count(count=4) + @validate_transaction_metrics( + name="test_bedrock_chat_completion_in_txn_with_llm_metadata", + scoped_metrics=[expected_metric], + rollup_metrics=[expected_metric], + custom_metrics=[(f"Supportability/Python/ML/Bedrock/{BOTOCORE_VERSION}", 1)], + background_task=True, + ) + @validate_attributes("agent", ["llm"]) + @background_task(name="test_bedrock_chat_completion_in_txn_with_llm_metadata") + def _test(): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + add_custom_attribute("llm.foo", "bar") + add_custom_attribute("non_llm_attr", "python-agent") + with WithLlmCustomAttributes({"context": "attr"}): + message = [{"role": "user", "content": [{"text": "What is 212 degrees Fahrenheit converted to Celsius?"}]}] + exercise_model(message) + + _test() + + +@disabled_ai_monitoring_record_content_settings +@reset_core_stats_engine() +def test_bedrock_chat_completion_no_content(set_trace_info, exercise_model, expected_metric, expected_events): + @validate_custom_events(events_sans_content(expected_events)) + # One summary event, one user message, and one response message from the assistant + @validate_custom_event_count(count=4) + @validate_transaction_metrics( + name="test_bedrock_chat_completion_no_content", + scoped_metrics=[expected_metric], + rollup_metrics=[expected_metric], + custom_metrics=[(f"Supportability/Python/ML/Bedrock/{BOTOCORE_VERSION}", 1)], + background_task=True, + ) + @validate_attributes("agent", ["llm"]) + @background_task(name="test_bedrock_chat_completion_no_content") + def _test(): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + add_custom_attribute("llm.foo", "bar") + add_custom_attribute("non_llm_attr", "python-agent") + message = [{"role": "user", "content": [{"text": "What is 212 degrees Fahrenheit converted to Celsius?"}]}] + exercise_model(message) + + _test() + + +@reset_core_stats_engine() +@override_llm_token_callback_settings(llm_token_count_callback) +def test_bedrock_chat_completion_with_token_count( + set_trace_info, exercise_model, expected_metric, expected_events, response_streaming +): + expected_events = add_token_counts_to_chat_events(expected_events) + if response_streaming: + expected_events = add_token_count_streaming_events(expected_events) + + @validate_custom_events(expected_events) + # One summary event, one user message, and one response message from the assistant + @validate_custom_event_count(count=4) + @validate_transaction_metrics( + name="test_bedrock_chat_completion_with_token_count", + scoped_metrics=[expected_metric], + rollup_metrics=[expected_metric], + custom_metrics=[(f"Supportability/Python/ML/Bedrock/{BOTOCORE_VERSION}", 1)], + background_task=True, + ) + @validate_attributes("agent", ["llm"]) + @background_task(name="test_bedrock_chat_completion_with_token_count") + def _test(): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + add_custom_attribute("llm.foo", "bar") + add_custom_attribute("non_llm_attr", "python-agent") + message = [{"role": "user", "content": [{"text": "What is 212 degrees Fahrenheit converted to Celsius?"}]}] + exercise_model(message) + + _test() + + +@reset_core_stats_engine() +def test_bedrock_chat_completion_no_llm_metadata(set_trace_info, exercise_model, expected_metric, expected_events): + @validate_custom_events(events_sans_llm_metadata(expected_events)) + @validate_custom_event_count(count=4) + @validate_transaction_metrics( + name="test_bedrock_chat_completion_in_txn_no_llm_metadata", + scoped_metrics=[expected_metric], + rollup_metrics=[expected_metric], + custom_metrics=[(f"Supportability/Python/ML/Bedrock/{BOTOCORE_VERSION}", 1)], + background_task=True, + ) + @background_task(name="test_bedrock_chat_completion_in_txn_no_llm_metadata") + def _test(): + set_trace_info() + message = [{"role": "user", "content": [{"text": "What is 212 degrees Fahrenheit converted to Celsius?"}]}] + exercise_model(message) + + _test() + + +@reset_core_stats_engine() +@validate_custom_event_count(count=0) +def test_bedrock_chat_completion_outside_txn(exercise_model): + add_custom_attribute("llm.conversation_id", "my-awesome-id") + message = [{"role": "user", "content": [{"text": "What is 212 degrees Fahrenheit converted to Celsius?"}]}] + exercise_model(message) + + +@disabled_ai_monitoring_settings +@reset_core_stats_engine() +@validate_custom_event_count(count=0) +@background_task(name="test_bedrock_chat_completion_disabled_ai_monitoring_settings") +def test_bedrock_chat_completion_disabled_ai_monitoring_settings(set_trace_info, exercise_model): + set_trace_info() + message = [{"role": "user", "content": [{"text": "What is 212 degrees Fahrenheit converted to Celsius?"}]}] + exercise_model(message) + + +_client_error = botocore.exceptions.ClientError +_client_error_name = callable_name(_client_error) + + +@pytest.fixture +def exercise_converse_incorrect_access_key(bedrock_converse_server, response_streaming, monkeypatch): + def _exercise_converse_incorrect_access_key(): + monkeypatch.setattr(bedrock_converse_server._request_signer._credentials, "access_key", "INVALID-ACCESS-KEY") + + message = [{"role": "user", "content": [{"text": "Invalid Token"}]}] + request = bedrock_converse_server.converse_stream if response_streaming else bedrock_converse_server.converse + with pytest.raises(_client_error): + request( + modelId="anthropic.claude-3-sonnet-20240229-v1:0", + messages=message, + inferenceConfig={"temperature": 0.7, "maxTokens": 100}, + ) + + return _exercise_converse_incorrect_access_key + + +@reset_core_stats_engine() +def test_bedrock_chat_completion_error_incorrect_access_key( + exercise_converse_incorrect_access_key, set_trace_info, expected_metric +): + """ + A request is made to the server with invalid credentials. botocore will reach out to the server and receive an + UnrecognizedClientException as a response. Information from the request will be parsed and reported in customer + events. The error response can also be parsed, and will be included as attributes on the recorded exception. + """ + + @validate_custom_events(chat_completion_invalid_access_key_error_events) + @validate_error_trace_attributes( + _client_error_name, + exact_attrs={ + "agent": {}, + "intrinsic": {}, + "user": { + "http.statusCode": 403, + "error.message": "The security token included in the request is invalid.", + "error.code": "UnrecognizedClientException", + }, + }, + ) + @validate_transaction_metrics( + name="test_bedrock_chat_completion", + scoped_metrics=[expected_metric], + rollup_metrics=[expected_metric], + custom_metrics=[(f"Supportability/Python/ML/Bedrock/{BOTOCORE_VERSION}", 1)], + background_task=True, + ) + @background_task(name="test_bedrock_chat_completion") + def _test(): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + add_custom_attribute("llm.foo", "bar") + add_custom_attribute("non_llm_attr", "python-agent") + + exercise_converse_incorrect_access_key() + + _test() + + +@pytest.fixture +def exercise_converse_invalid_model(bedrock_converse_server, response_streaming, monkeypatch): + def _exercise_converse_invalid_model(): + monkeypatch.setattr(bedrock_converse_server._request_signer._credentials, "access_key", "INVALID-ACCESS-KEY") + + message = [{"role": "user", "content": [{"text": "Model does not exist."}]}] + request = bedrock_converse_server.converse_stream if response_streaming else bedrock_converse_server.converse + with pytest.raises(_client_error): + request(modelId="does-not-exist", messages=message, inferenceConfig={"temperature": 0.7, "maxTokens": 100}) + + return _exercise_converse_invalid_model + + +@reset_core_stats_engine() +def test_bedrock_chat_completion_error_invalid_model(exercise_converse_invalid_model, set_trace_info, expected_metric): + @validate_custom_events(events_with_context_attrs(chat_completion_invalid_model_error_events)) + @validate_error_trace_attributes( + "botocore.errorfactory:ValidationException", + exact_attrs={ + "agent": {}, + "intrinsic": {}, + "user": { + "http.statusCode": 400, + "error.message": "The provided model identifier is invalid.", + "error.code": "ValidationException", + }, + }, + ) + @validate_transaction_metrics( + name="test_bedrock_chat_completion_error_invalid_model", + scoped_metrics=[expected_metric], + rollup_metrics=[expected_metric], + custom_metrics=[(f"Supportability/Python/ML/Bedrock/{BOTOCORE_VERSION}", 1)], + background_task=True, + ) + @background_task(name="test_bedrock_chat_completion_error_invalid_model") + def _test(): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + add_custom_attribute("llm.foo", "bar") + add_custom_attribute("non_llm_attr", "python-agent") + + with WithLlmCustomAttributes({"context": "attr"}): + exercise_converse_invalid_model() + + _test() + + +@reset_core_stats_engine() +@disabled_ai_monitoring_record_content_settings +def test_bedrock_chat_completion_error_invalid_model_no_content( + exercise_converse_invalid_model, set_trace_info, expected_metric +): + @validate_custom_events(events_sans_content(chat_completion_invalid_model_error_events)) + @validate_error_trace_attributes( + "botocore.errorfactory:ValidationException", + exact_attrs={ + "agent": {}, + "intrinsic": {}, + "user": { + "http.statusCode": 400, + "error.message": "The provided model identifier is invalid.", + "error.code": "ValidationException", + }, + }, + ) + @validate_transaction_metrics( + name="test_bedrock_chat_completion_error_invalid_model_no_content", + scoped_metrics=[expected_metric], + rollup_metrics=[expected_metric], + custom_metrics=[(f"Supportability/Python/ML/Bedrock/{BOTOCORE_VERSION}", 1)], + background_task=True, + ) + @background_task(name="test_bedrock_chat_completion_error_invalid_model_no_content") + def _test(): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + add_custom_attribute("llm.foo", "bar") + add_custom_attribute("non_llm_attr", "python-agent") + + exercise_converse_invalid_model() + + _test() diff --git a/tests/external_botocore/test_bedrock_chat_completion_invoke_model.py b/tests/external_botocore/test_bedrock_chat_completion_invoke_model.py index 4422685b9f..ac72e458fb 100644 --- a/tests/external_botocore/test_bedrock_chat_completion_invoke_model.py +++ b/tests/external_botocore/test_bedrock_chat_completion_invoke_model.py @@ -11,16 +11,17 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + import json import os from io import BytesIO +from pprint import pformat import boto3 -import botocore.errorfactory import botocore.eventstream import botocore.exceptions import pytest -from _test_bedrock_chat_completion import ( +from _test_bedrock_chat_completion_invoke_model import ( chat_completion_expected_events, chat_completion_expected_malformed_request_body_events, chat_completion_expected_malformed_response_body_events, @@ -35,7 +36,8 @@ from conftest import BOTOCORE_VERSION from testing_support.fixtures import override_llm_token_callback_settings, reset_core_stats_engine, validate_attributes from testing_support.ml_testing_utils import ( - add_token_count_to_events, + add_token_count_streaming_events, + add_token_counts_to_chat_events, disabled_ai_monitoring_record_content_settings, disabled_ai_monitoring_settings, disabled_ai_monitoring_streaming_settings, @@ -129,6 +131,14 @@ def expected_events(model_id, response_streaming): return chat_completion_expected_events[model_id] +@pytest.fixture(scope="module") +def expected_events(model_id, response_streaming): + if response_streaming: + return chat_completion_streaming_expected_events[model_id] + else: + return chat_completion_expected_events[model_id] + + @pytest.fixture(scope="module") def expected_metrics(response_streaming): if response_streaming: @@ -200,7 +210,7 @@ def _test(): @reset_core_stats_engine() @override_llm_token_callback_settings(llm_token_count_callback) def test_bedrock_chat_completion_with_token_count(set_trace_info, exercise_model, expected_events, expected_metrics): - @validate_custom_events(add_token_count_to_events(expected_events)) + @validate_custom_events(add_token_counts_to_chat_events(add_token_count_streaming_events(expected_events))) # One summary event, one user message, and one response message from the assistant @validate_custom_event_count(count=3) @validate_transaction_metrics( @@ -438,49 +448,50 @@ def _test(): _test() -@reset_core_stats_engine() -@override_llm_token_callback_settings(llm_token_count_callback) -def test_bedrock_chat_completion_error_incorrect_access_key_with_token( - monkeypatch, - bedrock_server, - exercise_model, - set_trace_info, - expected_invalid_access_key_error_events, - expected_metrics, -): - @validate_custom_events(add_token_count_to_events(expected_invalid_access_key_error_events)) - @validate_error_trace_attributes( - _client_error_name, - exact_attrs={ - "agent": {}, - "intrinsic": {}, - "user": { - "http.statusCode": 403, - "error.message": "The security token included in the request is invalid.", - "error.code": "UnrecognizedClientException", - }, - }, - ) - @validate_transaction_metrics( - name="test_bedrock_chat_completion", - scoped_metrics=expected_metrics, - rollup_metrics=expected_metrics, - custom_metrics=[(f"Supportability/Python/ML/Bedrock/{BOTOCORE_VERSION}", 1)], - background_task=True, - ) - @background_task(name="test_bedrock_chat_completion") - def _test(): - monkeypatch.setattr(bedrock_server._request_signer._credentials, "access_key", "INVALID-ACCESS-KEY") - - with pytest.raises(_client_error): # not sure where this exception actually comes from - set_trace_info() - add_custom_attribute("llm.conversation_id", "my-awesome-id") - add_custom_attribute("llm.foo", "bar") - add_custom_attribute("non_llm_attr", "python-agent") - - exercise_model(prompt="Invalid Token", temperature=0.7, max_tokens=100) - - _test() +# +# @reset_core_stats_engine() +# @override_llm_token_callback_settings(llm_token_count_callback) +# def test_bedrock_chat_completion_error_incorrect_access_key_with_token( +# monkeypatch, +# bedrock_server, +# exercise_model, +# set_trace_info, +# expected_invalid_access_key_error_events, +# expected_metrics, +# ): +# @validate_custom_events(add_token_count_to_events(expected_invalid_access_key_error_events)) +# @validate_error_trace_attributes( +# _client_error_name, +# exact_attrs={ +# "agent": {}, +# "intrinsic": {}, +# "user": { +# "http.statusCode": 403, +# "error.message": "The security token included in the request is invalid.", +# "error.code": "UnrecognizedClientException", +# }, +# }, +# ) +# @validate_transaction_metrics( +# name="test_bedrock_chat_completion", +# scoped_metrics=expected_metrics, +# rollup_metrics=expected_metrics, +# custom_metrics=[(f"Supportability/Python/ML/Bedrock/{BOTOCORE_VERSION}", 1)], +# background_task=True, +# ) +# @background_task(name="test_bedrock_chat_completion") +# def _test(): +# monkeypatch.setattr(bedrock_server._request_signer._credentials, "access_key", "INVALID-ACCESS-KEY") +# +# with pytest.raises(_client_error): # not sure where this exception actually comes from +# set_trace_info() +# add_custom_attribute("llm.conversation_id", "my-awesome-id") +# add_custom_attribute("llm.foo", "bar") +# add_custom_attribute("non_llm_attr", "python-agent") +# +# exercise_model(prompt="Invalid Token", temperature=0.7, max_tokens=100) +# +# _test() @reset_core_stats_engine() @@ -762,61 +773,17 @@ def _test(): _test() -@reset_core_stats_engine() -@override_llm_token_callback_settings(llm_token_count_callback) -def test_bedrock_chat_completion_error_streaming_exception_with_token_count(bedrock_server, set_trace_info): - """ - Duplicate of test_bedrock_chat_completion_error_streaming_exception, but with token callback being set. - - See the original test for a description of the error case. - """ - - @validate_custom_events(add_token_count_to_events(chat_completion_expected_streaming_error_events)) - @validate_custom_event_count(count=2) - @validate_error_trace_attributes( - _event_stream_error_name, - exact_attrs={ - "agent": {}, - "intrinsic": {}, - "user": { - "error.message": "Malformed input request, please reformat your input and try again.", - "error.code": "ValidationException", - }, - }, - forgone_params={"agent": (), "intrinsic": (), "user": ("http.statusCode")}, - ) - @validate_transaction_metrics( - name="test_bedrock_chat_completion", - scoped_metrics=[("Llm/completion/Bedrock/invoke_model_with_response_stream", 1)], - rollup_metrics=[("Llm/completion/Bedrock/invoke_model_with_response_stream", 1)], - custom_metrics=[(f"Supportability/Python/ML/Bedrock/{BOTOCORE_VERSION}", 1)], - background_task=True, - ) - @background_task(name="test_bedrock_chat_completion") - def _test(): - with pytest.raises(_event_stream_error): - model = "amazon.titan-text-express-v1" - body = (chat_completion_payload_templates[model] % ("Streaming Exception", 0.7, 100)).encode("utf-8") - - set_trace_info() - add_custom_attribute("llm.conversation_id", "my-awesome-id") - add_custom_attribute("llm.foo", "bar") - add_custom_attribute("non_llm_attr", "python-agent") - - response = bedrock_server.invoke_model_with_response_stream( - body=body, modelId=model, accept="application/json", contentType="application/json" - ) - list(response["body"]) # Iterate - - _test() - - def test_bedrock_chat_completion_functions_marked_as_wrapped_for_sdk_compatibility(bedrock_server): assert bedrock_server._nr_wrapped def test_chat_models_instrumented(): - SUPPORTED_MODELS = [model for model, _, _, _ in MODEL_EXTRACTORS if "embed" not in model] + def _is_supported_model(model): + supported_models = [model for model, _, _, _ in MODEL_EXTRACTORS if "embed" not in model] + for supported_model in supported_models: + if supported_model in model: + return True + return False _id = os.environ.get("AWS_ACCESS_KEY_ID") key = os.environ.get("AWS_SECRET_ACCESS_KEY") @@ -826,10 +793,6 @@ def test_chat_models_instrumented(): client = boto3.client("bedrock", "us-east-1") response = client.list_foundation_models(byOutputModality="TEXT") models = [model["modelId"] for model in response["modelSummaries"]] - not_supported = [] - for model in models: - is_supported = any(model.startswith(supported_model) for supported_model in SUPPORTED_MODELS) - if not is_supported: - not_supported.append(model) + not_supported = [model for model in models if not _is_supported_model(model)] - assert not not_supported, f"The following unsupported models were found: {not_supported}" + assert not not_supported, f"The following unsupported models were found: {pformat(not_supported)}" diff --git a/tests/external_botocore/test_bedrock_chat_completion_via_langchain.py b/tests/external_botocore/test_bedrock_chat_completion_via_langchain.py index 82537cd10a..b25516cd5b 100644 --- a/tests/external_botocore/test_bedrock_chat_completion_via_langchain.py +++ b/tests/external_botocore/test_bedrock_chat_completion_via_langchain.py @@ -13,7 +13,7 @@ # limitations under the License. import pytest -from _test_bedrock_chat_completion import ( +from _test_bedrock_chat_completion_invoke_model import ( chat_completion_langchain_expected_events, chat_completion_langchain_expected_streaming_events, ) diff --git a/tests/external_botocore/test_bedrock_embeddings.py b/tests/external_botocore/test_bedrock_embeddings.py index 417e24b2d9..f28308354a 100644 --- a/tests/external_botocore/test_bedrock_embeddings.py +++ b/tests/external_botocore/test_bedrock_embeddings.py @@ -14,6 +14,7 @@ import json import os from io import BytesIO +from pprint import pformat import boto3 import botocore.exceptions @@ -28,7 +29,7 @@ from conftest import BOTOCORE_VERSION from testing_support.fixtures import override_llm_token_callback_settings, reset_core_stats_engine, validate_attributes from testing_support.ml_testing_utils import ( - add_token_count_to_events, + add_token_count_to_embedding_events, disabled_ai_monitoring_record_content_settings, disabled_ai_monitoring_settings, events_sans_content, @@ -161,7 +162,7 @@ def _test(): @reset_core_stats_engine() @override_llm_token_callback_settings(llm_token_count_callback) def test_bedrock_embedding_with_token_count(set_trace_info, exercise_model, expected_events): - @validate_custom_events(add_token_count_to_events(expected_events)) + @validate_custom_events(add_token_count_to_embedding_events(expected_events)) @validate_custom_event_count(count=1) @validate_transaction_metrics( name="test_bedrock_embedding", @@ -286,45 +287,6 @@ def _test(): _test() -@reset_core_stats_engine() -@override_llm_token_callback_settings(llm_token_count_callback) -def test_bedrock_embedding_error_incorrect_access_key_with_token_count( - monkeypatch, bedrock_server, exercise_model, set_trace_info, expected_invalid_access_key_error_events -): - @validate_custom_events(add_token_count_to_events(expected_invalid_access_key_error_events)) - @validate_error_trace_attributes( - _client_error_name, - exact_attrs={ - "agent": {}, - "intrinsic": {}, - "user": { - "http.statusCode": 403, - "error.message": "The security token included in the request is invalid.", - "error.code": "UnrecognizedClientException", - }, - }, - ) - @validate_transaction_metrics( - name="test_bedrock_embedding", - scoped_metrics=[("Llm/embedding/Bedrock/invoke_model", 1)], - rollup_metrics=[("Llm/embedding/Bedrock/invoke_model", 1)], - background_task=True, - ) - @background_task(name="test_bedrock_embedding") - def _test(): - monkeypatch.setattr(bedrock_server._request_signer._credentials, "access_key", "INVALID-ACCESS-KEY") - - with pytest.raises(_client_error): # not sure where this exception actually comes from - set_trace_info() - add_custom_attribute("llm.conversation_id", "my-awesome-id") - add_custom_attribute("llm.foo", "bar") - add_custom_attribute("non_llm_attr", "python-agent") - - exercise_model(prompt="Invalid Token") - - _test() - - @reset_core_stats_engine() def test_bedrock_embedding_error_malformed_request_body(bedrock_server, set_trace_info): """ @@ -409,7 +371,12 @@ def _test(): def test_embedding_models_instrumented(): - SUPPORTED_MODELS = [model for model, _, _, _ in MODEL_EXTRACTORS if "embed" in model] + def _is_supported_model(model): + supported_models = [model for model, _, _, _ in MODEL_EXTRACTORS if "embed" in model] + for supported_model in supported_models: + if supported_model in model: + return True + return False _id = os.environ.get("AWS_ACCESS_KEY_ID") key = os.environ.get("AWS_SECRET_ACCESS_KEY") @@ -419,10 +386,6 @@ def test_embedding_models_instrumented(): client = boto3.client("bedrock", "us-east-1") response = client.list_foundation_models(byOutputModality="EMBEDDING") models = [model["modelId"] for model in response["modelSummaries"]] - not_supported = [] - for model in models: - is_supported = any(model.startswith(supported_model) for supported_model in SUPPORTED_MODELS) - if not is_supported: - not_supported.append(model) + not_supported = [model for model in models if not _is_supported_model(model)] - assert not not_supported, f"The following unsupported models were found: {not_supported}" + assert not not_supported, f"The following unsupported models were found: {pformat(not_supported)}" diff --git a/tests/external_botocore/test_chat_completion_converse.py b/tests/external_botocore/test_chat_completion_converse.py deleted file mode 100644 index 96ead41dd7..0000000000 --- a/tests/external_botocore/test_chat_completion_converse.py +++ /dev/null @@ -1,524 +0,0 @@ -# Copyright 2010 New Relic, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import botocore.exceptions -import pytest -from conftest import BOTOCORE_VERSION -from testing_support.fixtures import override_llm_token_callback_settings, reset_core_stats_engine, validate_attributes -from testing_support.ml_testing_utils import ( - add_token_count_to_events, - disabled_ai_monitoring_record_content_settings, - disabled_ai_monitoring_settings, - events_sans_content, - events_sans_llm_metadata, - events_with_context_attrs, - llm_token_count_callback, - set_trace_info, -) -from testing_support.validators.validate_custom_event import validate_custom_event_count -from testing_support.validators.validate_custom_events import validate_custom_events -from testing_support.validators.validate_error_trace_attributes import validate_error_trace_attributes -from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics - -from newrelic.api.background_task import background_task -from newrelic.api.llm_custom_attributes import WithLlmCustomAttributes -from newrelic.api.transaction import add_custom_attribute -from newrelic.common.object_names import callable_name - -chat_completion_expected_events = [ - ( - {"type": "LlmChatCompletionSummary"}, - { - "id": None, # UUID that varies with each run - "llm.conversation_id": "my-awesome-id", - "llm.foo": "bar", - "span_id": None, - "trace_id": "trace-id", - "request_id": "c20d345e-6878-4778-b674-6b187bae8ecf", - "duration": None, # Response time varies each test run - "request.model": "anthropic.claude-3-sonnet-20240229-v1:0", - "response.model": "anthropic.claude-3-sonnet-20240229-v1:0", - "request.temperature": 0.7, - "request.max_tokens": 100, - "response.choices.finish_reason": "max_tokens", - "vendor": "bedrock", - "ingest_source": "Python", - "response.number_of_messages": 3, - }, - ), - ( - {"type": "LlmChatCompletionMessage"}, - { - "id": None, # UUID that varies with each run - "llm.conversation_id": "my-awesome-id", - "llm.foo": "bar", - "request_id": "c20d345e-6878-4778-b674-6b187bae8ecf", - "span_id": None, - "trace_id": "trace-id", - "content": "You are a scientist.", - "role": "system", - "completion_id": None, - "sequence": 0, - "response.model": "anthropic.claude-3-sonnet-20240229-v1:0", - "vendor": "bedrock", - "ingest_source": "Python", - }, - ), - ( - {"type": "LlmChatCompletionMessage"}, - { - "id": None, # UUID that varies with each run - "llm.conversation_id": "my-awesome-id", - "llm.foo": "bar", - "request_id": "c20d345e-6878-4778-b674-6b187bae8ecf", - "span_id": None, - "trace_id": "trace-id", - "content": "What is 212 degrees Fahrenheit converted to Celsius?", - "role": "user", - "completion_id": None, - "sequence": 1, - "response.model": "anthropic.claude-3-sonnet-20240229-v1:0", - "vendor": "bedrock", - "ingest_source": "Python", - }, - ), - ( - {"type": "LlmChatCompletionMessage"}, - { - "id": None, # UUID that varies with each run - "llm.conversation_id": "my-awesome-id", - "llm.foo": "bar", - "request_id": "c20d345e-6878-4778-b674-6b187bae8ecf", - "span_id": None, - "trace_id": "trace-id", - "content": "To convert 212°F to Celsius, we can use the formula:\n\nC = (F - 32) × 5/9\n\nWhere:\nC is the temperature in Celsius\nF is the temperature in Fahrenheit\n\nPlugging in 212°F, we get:\n\nC = (212 - 32) × 5/9\nC = 180 × 5/9\nC = 100\n\nTherefore, 212°", # noqa: RUF001 - "role": "assistant", - "completion_id": None, - "sequence": 2, - "response.model": "anthropic.claude-3-sonnet-20240229-v1:0", - "vendor": "bedrock", - "ingest_source": "Python", - "is_response": True, - }, - ), -] - - -@pytest.fixture(scope="module") -def exercise_model(bedrock_converse_server): - def _exercise_model(message): - inference_config = {"temperature": 0.7, "maxTokens": 100} - - response = bedrock_converse_server.converse( - modelId="anthropic.claude-3-sonnet-20240229-v1:0", - messages=message, - system=[{"text": "You are a scientist."}], - inferenceConfig=inference_config, - ) - - return _exercise_model - - -@reset_core_stats_engine() -def test_bedrock_chat_completion_in_txn_with_llm_metadata(set_trace_info, exercise_model): - @validate_custom_events(events_with_context_attrs(chat_completion_expected_events)) - # One summary event, one user message, and one response message from the assistant - @validate_custom_event_count(count=4) - @validate_transaction_metrics( - name="test_bedrock_chat_completion_in_txn_with_llm_metadata", - scoped_metrics=[("Llm/completion/Bedrock/converse", 1)], - rollup_metrics=[("Llm/completion/Bedrock/converse", 1)], - custom_metrics=[(f"Supportability/Python/ML/Bedrock/{BOTOCORE_VERSION}", 1)], - background_task=True, - ) - @validate_attributes("agent", ["llm"]) - @background_task(name="test_bedrock_chat_completion_in_txn_with_llm_metadata") - def _test(): - set_trace_info() - add_custom_attribute("llm.conversation_id", "my-awesome-id") - add_custom_attribute("llm.foo", "bar") - add_custom_attribute("non_llm_attr", "python-agent") - with WithLlmCustomAttributes({"context": "attr"}): - message = [{"role": "user", "content": [{"text": "What is 212 degrees Fahrenheit converted to Celsius?"}]}] - exercise_model(message) - - _test() - - -@disabled_ai_monitoring_record_content_settings -@reset_core_stats_engine() -def test_bedrock_chat_completion_no_content(set_trace_info, exercise_model): - @validate_custom_events(events_sans_content(chat_completion_expected_events)) - # One summary event, one user message, and one response message from the assistant - @validate_custom_event_count(count=4) - @validate_transaction_metrics( - name="test_bedrock_chat_completion_no_content", - scoped_metrics=[("Llm/completion/Bedrock/converse", 1)], - rollup_metrics=[("Llm/completion/Bedrock/converse", 1)], - custom_metrics=[(f"Supportability/Python/ML/Bedrock/{BOTOCORE_VERSION}", 1)], - background_task=True, - ) - @validate_attributes("agent", ["llm"]) - @background_task(name="test_bedrock_chat_completion_no_content") - def _test(): - set_trace_info() - add_custom_attribute("llm.conversation_id", "my-awesome-id") - add_custom_attribute("llm.foo", "bar") - add_custom_attribute("non_llm_attr", "python-agent") - message = [{"role": "user", "content": [{"text": "What is 212 degrees Fahrenheit converted to Celsius?"}]}] - exercise_model(message) - - _test() - - -@reset_core_stats_engine() -@override_llm_token_callback_settings(llm_token_count_callback) -def test_bedrock_chat_completion_with_token_count(set_trace_info, exercise_model): - @validate_custom_events(add_token_count_to_events(chat_completion_expected_events)) - # One summary event, one user message, and one response message from the assistant - @validate_custom_event_count(count=4) - @validate_transaction_metrics( - name="test_bedrock_chat_completion_with_token_count", - scoped_metrics=[("Llm/completion/Bedrock/converse", 1)], - rollup_metrics=[("Llm/completion/Bedrock/converse", 1)], - custom_metrics=[(f"Supportability/Python/ML/Bedrock/{BOTOCORE_VERSION}", 1)], - background_task=True, - ) - @validate_attributes("agent", ["llm"]) - @background_task(name="test_bedrock_chat_completion_with_token_count") - def _test(): - set_trace_info() - add_custom_attribute("llm.conversation_id", "my-awesome-id") - add_custom_attribute("llm.foo", "bar") - add_custom_attribute("non_llm_attr", "python-agent") - message = [{"role": "user", "content": [{"text": "What is 212 degrees Fahrenheit converted to Celsius?"}]}] - exercise_model(message) - - _test() - - -@reset_core_stats_engine() -def test_bedrock_chat_completion_no_llm_metadata(set_trace_info, exercise_model): - @validate_custom_events(events_sans_llm_metadata(chat_completion_expected_events)) - @validate_custom_event_count(count=4) - @validate_transaction_metrics( - name="test_bedrock_chat_completion_in_txn_no_llm_metadata", - scoped_metrics=[("Llm/completion/Bedrock/converse", 1)], - rollup_metrics=[("Llm/completion/Bedrock/converse", 1)], - custom_metrics=[(f"Supportability/Python/ML/Bedrock/{BOTOCORE_VERSION}", 1)], - background_task=True, - ) - @background_task(name="test_bedrock_chat_completion_in_txn_no_llm_metadata") - def _test(): - set_trace_info() - message = [{"role": "user", "content": [{"text": "What is 212 degrees Fahrenheit converted to Celsius?"}]}] - exercise_model(message) - - _test() - - -@reset_core_stats_engine() -@validate_custom_event_count(count=0) -def test_bedrock_chat_completion_outside_txn(exercise_model): - add_custom_attribute("llm.conversation_id", "my-awesome-id") - message = [{"role": "user", "content": [{"text": "What is 212 degrees Fahrenheit converted to Celsius?"}]}] - exercise_model(message) - - -@disabled_ai_monitoring_settings -@reset_core_stats_engine() -@validate_custom_event_count(count=0) -@background_task(name="test_bedrock_chat_completion_disabled_ai_monitoring_settings") -def test_bedrock_chat_completion_disabled_ai_monitoring_settings(set_trace_info, exercise_model): - set_trace_info() - message = [{"role": "user", "content": [{"text": "What is 212 degrees Fahrenheit converted to Celsius?"}]}] - exercise_model(message) - - -chat_completion_invalid_access_key_error_events = [ - ( - {"type": "LlmChatCompletionSummary"}, - { - "id": None, # UUID that varies with each run - "llm.conversation_id": "my-awesome-id", - "llm.foo": "bar", - "span_id": None, - "trace_id": "trace-id", - "request_id": "e1206e19-2318-4a9d-be98-017c73f06118", - "duration": None, # Response time varies each test run - "request.model": "anthropic.claude-3-sonnet-20240229-v1:0", - "response.model": "anthropic.claude-3-sonnet-20240229-v1:0", - "request.temperature": 0.7, - "request.max_tokens": 100, - "vendor": "bedrock", - "ingest_source": "Python", - "response.number_of_messages": 1, - "error": True, - }, - ), - ( - {"type": "LlmChatCompletionMessage"}, - { - "id": None, # UUID that varies with each run - "llm.conversation_id": "my-awesome-id", - "llm.foo": "bar", - "request_id": "e1206e19-2318-4a9d-be98-017c73f06118", - "span_id": None, - "trace_id": "trace-id", - "content": "Invalid Token", - "role": "user", - "completion_id": None, - "sequence": 0, - "response.model": "anthropic.claude-3-sonnet-20240229-v1:0", - "vendor": "bedrock", - "ingest_source": "Python", - }, - ), -] - -_client_error = botocore.exceptions.ClientError -_client_error_name = callable_name(_client_error) - - -@reset_core_stats_engine() -def test_bedrock_chat_completion_error_incorrect_access_key( - monkeypatch, bedrock_converse_server, exercise_model, set_trace_info -): - """ - A request is made to the server with invalid credentials. botocore will reach out to the server and receive an - UnrecognizedClientException as a response. Information from the request will be parsed and reported in customer - events. The error response can also be parsed, and will be included as attributes on the recorded exception. - """ - - @validate_custom_events(chat_completion_invalid_access_key_error_events) - @validate_error_trace_attributes( - _client_error_name, - exact_attrs={ - "agent": {}, - "intrinsic": {}, - "user": { - "http.statusCode": 403, - "error.message": "The security token included in the request is invalid.", - "error.code": "UnrecognizedClientException", - }, - }, - ) - @validate_transaction_metrics( - name="test_bedrock_chat_completion", - scoped_metrics=[("Llm/completion/Bedrock/converse", 1)], - rollup_metrics=[("Llm/completion/Bedrock/converse", 1)], - custom_metrics=[(f"Supportability/Python/ML/Bedrock/{BOTOCORE_VERSION}", 1)], - background_task=True, - ) - @background_task(name="test_bedrock_chat_completion") - def _test(): - monkeypatch.setattr(bedrock_converse_server._request_signer._credentials, "access_key", "INVALID-ACCESS-KEY") - - with pytest.raises(_client_error): - set_trace_info() - add_custom_attribute("llm.conversation_id", "my-awesome-id") - add_custom_attribute("llm.foo", "bar") - add_custom_attribute("non_llm_attr", "python-agent") - - message = [{"role": "user", "content": [{"text": "Invalid Token"}]}] - - response = bedrock_converse_server.converse( - modelId="anthropic.claude-3-sonnet-20240229-v1:0", - messages=message, - inferenceConfig={"temperature": 0.7, "maxTokens": 100}, - ) - - assert response - - _test() - - -chat_completion_invalid_model_error_events = [ - ( - {"type": "LlmChatCompletionSummary"}, - { - "id": None, # UUID that varies with each run - "llm.conversation_id": "my-awesome-id", - "llm.foo": "bar", - "request_id": "f4908827-3db9-4742-9103-2bbc34578b03", - "span_id": None, - "trace_id": "trace-id", - "duration": None, # Response time varies each test run - "request.model": "does-not-exist", - "response.model": "does-not-exist", - "request.temperature": 0.7, - "request.max_tokens": 100, - "response.number_of_messages": 1, - "vendor": "bedrock", - "ingest_source": "Python", - "error": True, - }, - ), - ( - {"type": "LlmChatCompletionMessage"}, - { - "id": None, - "llm.conversation_id": "my-awesome-id", - "llm.foo": "bar", - "span_id": None, - "trace_id": "trace-id", - "request_id": "f4908827-3db9-4742-9103-2bbc34578b03", - "content": "Model does not exist.", - "role": "user", - "completion_id": None, - "response.model": "does-not-exist", - "sequence": 0, - "vendor": "bedrock", - "ingest_source": "Python", - }, - ), -] - - -@reset_core_stats_engine() -def test_bedrock_chat_completion_error_invalid_model(bedrock_converse_server, set_trace_info): - @validate_custom_events(events_with_context_attrs(chat_completion_invalid_model_error_events)) - @validate_error_trace_attributes( - "botocore.errorfactory:ValidationException", - exact_attrs={ - "agent": {}, - "intrinsic": {}, - "user": { - "http.statusCode": 400, - "error.message": "The provided model identifier is invalid.", - "error.code": "ValidationException", - }, - }, - ) - @validate_transaction_metrics( - name="test_bedrock_chat_completion_error_invalid_model", - scoped_metrics=[("Llm/completion/Bedrock/converse", 1)], - rollup_metrics=[("Llm/completion/Bedrock/converse", 1)], - custom_metrics=[(f"Supportability/Python/ML/Bedrock/{BOTOCORE_VERSION}", 1)], - background_task=True, - ) - @background_task(name="test_bedrock_chat_completion_error_invalid_model") - def _test(): - set_trace_info() - add_custom_attribute("llm.conversation_id", "my-awesome-id") - add_custom_attribute("llm.foo", "bar") - add_custom_attribute("non_llm_attr", "python-agent") - - with pytest.raises(_client_error): - with WithLlmCustomAttributes({"context": "attr"}): - message = [{"role": "user", "content": [{"text": "Model does not exist."}]}] - - response = bedrock_converse_server.converse( - modelId="does-not-exist", messages=message, inferenceConfig={"temperature": 0.7, "maxTokens": 100} - ) - - assert response - - _test() - - -@reset_core_stats_engine() -@disabled_ai_monitoring_record_content_settings -def test_bedrock_chat_completion_error_invalid_model_no_content(bedrock_converse_server, set_trace_info): - @validate_custom_events(events_sans_content(chat_completion_invalid_model_error_events)) - @validate_error_trace_attributes( - "botocore.errorfactory:ValidationException", - exact_attrs={ - "agent": {}, - "intrinsic": {}, - "user": { - "http.statusCode": 400, - "error.message": "The provided model identifier is invalid.", - "error.code": "ValidationException", - }, - }, - ) - @validate_transaction_metrics( - name="test_bedrock_chat_completion_error_invalid_model_no_content", - scoped_metrics=[("Llm/completion/Bedrock/converse", 1)], - rollup_metrics=[("Llm/completion/Bedrock/converse", 1)], - custom_metrics=[(f"Supportability/Python/ML/Bedrock/{BOTOCORE_VERSION}", 1)], - background_task=True, - ) - @background_task(name="test_bedrock_chat_completion_error_invalid_model_no_content") - def _test(): - set_trace_info() - add_custom_attribute("llm.conversation_id", "my-awesome-id") - add_custom_attribute("llm.foo", "bar") - add_custom_attribute("non_llm_attr", "python-agent") - - with pytest.raises(_client_error): - message = [{"role": "user", "content": [{"text": "Model does not exist."}]}] - - response = bedrock_converse_server.converse( - modelId="does-not-exist", messages=message, inferenceConfig={"temperature": 0.7, "maxTokens": 100} - ) - - assert response - - _test() - - -@reset_core_stats_engine() -@override_llm_token_callback_settings(llm_token_count_callback) -def test_bedrock_chat_completion_error_incorrect_access_key_with_token_count( - monkeypatch, bedrock_converse_server, exercise_model, set_trace_info -): - """ - A request is made to the server with invalid credentials. botocore will reach out to the server and receive an - UnrecognizedClientException as a response. Information from the request will be parsed and reported in customer - events. The error response can also be parsed, and will be included as attributes on the recorded exception. - """ - - @validate_custom_events(add_token_count_to_events(chat_completion_invalid_access_key_error_events)) - @validate_error_trace_attributes( - _client_error_name, - exact_attrs={ - "agent": {}, - "intrinsic": {}, - "user": { - "http.statusCode": 403, - "error.message": "The security token included in the request is invalid.", - "error.code": "UnrecognizedClientException", - }, - }, - ) - @validate_transaction_metrics( - name="test_bedrock_chat_completion_incorrect_access_key_with_token_count", - scoped_metrics=[("Llm/completion/Bedrock/converse", 1)], - rollup_metrics=[("Llm/completion/Bedrock/converse", 1)], - custom_metrics=[(f"Supportability/Python/ML/Bedrock/{BOTOCORE_VERSION}", 1)], - background_task=True, - ) - @background_task(name="test_bedrock_chat_completion_incorrect_access_key_with_token_count") - def _test(): - monkeypatch.setattr(bedrock_converse_server._request_signer._credentials, "access_key", "INVALID-ACCESS-KEY") - - with pytest.raises(_client_error): - set_trace_info() - add_custom_attribute("llm.conversation_id", "my-awesome-id") - add_custom_attribute("llm.foo", "bar") - add_custom_attribute("non_llm_attr", "python-agent") - - message = [{"role": "user", "content": [{"text": "Invalid Token"}]}] - - response = bedrock_converse_server.converse( - modelId="anthropic.claude-3-sonnet-20240229-v1:0", - messages=message, - inferenceConfig={"temperature": 0.7, "maxTokens": 100}, - ) - - assert response - - _test() diff --git a/tests/framework_azurefunctions/test_utilization.py b/tests/framework_azurefunctions/test_utilization.py new file mode 100644 index 0000000000..92349fb907 --- /dev/null +++ b/tests/framework_azurefunctions/test_utilization.py @@ -0,0 +1,40 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from newrelic.common.utilization import AzureFunctionUtilization + + +def test_utilization(monkeypatch): + monkeypatch.setenv("REGION_NAME", "eastus2") + monkeypatch.setenv( + "WEBSITE_OWNER_NAME", "0b0d165f-aaaf-4a3b-b929-5f60588d95a3+testing-python-EastUS2webspace-Linux" + ) + monkeypatch.setenv("WEBSITE_SITE_NAME", "test-func-linux") + + result = AzureFunctionUtilization.fetch() + assert result, "Failed to parse utilization for Azure Functions." + + faas_app_name, cloud_region = result + expected_faas_app_name = "/subscriptions/0b0d165f-aaaf-4a3b-b929-5f60588d95a3/resourceGroups/testing-python/providers/Microsoft.Web/sites/test-func-linux" + assert faas_app_name == expected_faas_app_name + assert cloud_region == "eastus2" + + +def test_utilization_bad_website_owner_name(monkeypatch): + monkeypatch.setenv("REGION_NAME", "eastus2") + monkeypatch.setenv("WEBSITE_OWNER_NAME", "ERROR") + monkeypatch.setenv("WEBSITE_SITE_NAME", "test-func-linux") + + result = AzureFunctionUtilization.fetch() + assert result is None, f"Expected failure but got result instead. {result}" diff --git a/tests/framework_grpc/test_distributed_tracing.py b/tests/framework_grpc/test_distributed_tracing.py index 457894516d..817b8e1aef 100644 --- a/tests/framework_grpc/test_distributed_tracing.py +++ b/tests/framework_grpc/test_distributed_tracing.py @@ -138,6 +138,11 @@ def _test(): decoded["d"].pop("tk", None) w3c_data.pop("tk") + # Round priority of newrelic header to 6 decimal places so it match tracestate. + decoded["d"]["pr"] = f"{decoded['d']['pr']:.6f}" + w3c_data["pr"] = f"{w3c_data['pr']:.6f}" + del w3c_data["v"] # Remove the version before comparing. + assert decoded["d"] == w3c_data _test() diff --git a/tests/hybridagent_aiopg/conftest.py b/tests/hybridagent_aiopg/conftest.py new file mode 100644 index 0000000000..4bdaa61d2d --- /dev/null +++ b/tests/hybridagent_aiopg/conftest.py @@ -0,0 +1,41 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This test suite tests the following scenarios: +1. Hybrid Agent with a framework that is instrumented by OpenTelemetry (aiopg) +but not New Relic. +2. Despite New Relic not having instrumentation support for aiopg, there are +framework dependencies that are present in this library (psycopg2) that New +Relic does instrument, so this ensures that these hooks are disabled so that +there is no conflict with OpenTelemetry instrumentation. +3. `opentelemetry.traces.enabled` setting is toggled on and off to ensure +that traces are created or not created as expected. +""" + +from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture + +_default_settings = { + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, + "opentelemetry.enabled": True, +} + +collector_agent_registration = collector_agent_registration_fixture( + app_name="Python Agent Test (Hybrid Agent)", default_settings=_default_settings +) diff --git a/tests/hybridagent_aiopg/test_database.py b/tests/hybridagent_aiopg/test_database.py new file mode 100644 index 0000000000..dd305dbdb6 --- /dev/null +++ b/tests/hybridagent_aiopg/test_database.py @@ -0,0 +1,197 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio + +import aiopg +import pytest +from testing_support.db_settings import postgresql_settings +from testing_support.fixtures import override_application_settings +from testing_support.util import instance_hostname +from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics + +from newrelic.api.background_task import background_task + +DB_SETTINGS = postgresql_settings()[0] + +# Metrics +_base_scoped_metrics = ( + (f"Datastore/statement/postgresql/{DB_SETTINGS['table_name']}/insert", 1), + ("Datastore/operation/postgresql/create", 1), +) + +_base_rollup_metrics = ( + ("Datastore/all", 2), + ("Datastore/allOther", 2), + ("Datastore/postgresql/all", 2), + ("Datastore/postgresql/allOther", 2), + (f"Datastore/statement/postgresql/{DB_SETTINGS['table_name']}/insert", 1), + ("Datastore/operation/postgresql/insert", 1), + ("Datastore/operation/postgresql/create", 1), +) + +_disable_scoped_metrics = list(_base_scoped_metrics) +_disable_rollup_metrics = list(_base_rollup_metrics) + +_enable_scoped_metrics = list(_base_scoped_metrics) +_enable_rollup_metrics = list(_base_rollup_metrics) + +_host = instance_hostname(DB_SETTINGS["host"]) +_port = DB_SETTINGS["port"] + +_instance_metric_name = f"Datastore/instance/postgresql/{_host}/{_port}" + +_enable_rollup_metrics.append((_instance_metric_name, 2)) + +_disable_rollup_metrics.append((_instance_metric_name, None)) + + +# Query +async def _execute(cursor): + await cursor.execute(f"CREATE TABLE IF NOT EXISTS {DB_SETTINGS['table_name']} (testField INTEGER)") + await cursor.execute(f"INSERT INTO {DB_SETTINGS['table_name']} (testField) VALUES (123)") + + cursor.close() + + +async def _connect_db(): + dsn = f"dbname={DB_SETTINGS['name']} user={DB_SETTINGS['user']} password={DB_SETTINGS['password']} host={DB_SETTINGS['host']} port={DB_SETTINGS['port']}" + connection = await aiopg.connect(dsn=dsn) + + try: + cursor = await connection.cursor() + await _execute(cursor) + finally: + connection.close() + + +async def _create_pool_db(): + dsn = f"dbname={DB_SETTINGS['name']} user={DB_SETTINGS['user']} password={DB_SETTINGS['password']} host={DB_SETTINGS['host']} port={DB_SETTINGS['port']}" + pool = await aiopg.create_pool(dsn=dsn) + + try: + connection = await pool.acquire() + cursor = await connection.cursor() + await _execute(cursor) + finally: + connection.close() + + +# Tests +@pytest.mark.parametrize( + "db_instance_reporting,opentelemetry_traces_enabled", [(True, True), (True, False), (False, True), (False, False)] +) +def test_connect(db_instance_reporting, opentelemetry_traces_enabled): + kwargs = {} + if opentelemetry_traces_enabled: + kwargs = { + "scoped_metrics": _enable_scoped_metrics if db_instance_reporting else _disable_scoped_metrics, + "rollup_metrics": _enable_rollup_metrics if db_instance_reporting else _disable_rollup_metrics, + } + + @override_application_settings( + { + "datastore_tracer.instance_reporting.enabled": db_instance_reporting, + "opentelemetry.traces.enabled": opentelemetry_traces_enabled, + } + ) + @validate_transaction_metrics("test_database:test_connect.._test", background_task=True, **kwargs) + @background_task() + def _test(): + async def _inner_test(): + await _connect_db() + + asyncio.run(_inner_test()) + + _test() + + +@pytest.mark.parametrize("db_instance_reporting", (True, False)) +def test_connect_disable_opentelemetry_traces(db_instance_reporting): + with pytest.raises(AssertionError): + # This will expectedly fail when the metrics are not recorded + @override_application_settings( + { + "datastore_tracer.instance_reporting.enabled": db_instance_reporting, + "opentelemetry.traces.enabled": False, + } + ) + @validate_transaction_metrics( + "test_database:test_connect.._test", + scoped_metrics=_enable_scoped_metrics if db_instance_reporting else _disable_scoped_metrics, + rollup_metrics=_enable_rollup_metrics if db_instance_reporting else _disable_rollup_metrics, + background_task=True, + ) + @background_task() + def _test(): + async def _inner_test(): + await _connect_db() + + asyncio.run(_inner_test()) + + _test() + + +@pytest.mark.parametrize( + "db_instance_reporting,opentelemetry_traces_enabled", [(True, True), (True, False), (False, True), (False, False)] +) +def test_create_pool(db_instance_reporting, opentelemetry_traces_enabled): + kwargs = {} + if opentelemetry_traces_enabled: + kwargs = { + "scoped_metrics": _enable_scoped_metrics if db_instance_reporting else _disable_scoped_metrics, + "rollup_metrics": _enable_rollup_metrics if db_instance_reporting else _disable_rollup_metrics, + } + + @override_application_settings( + { + "datastore_tracer.instance_reporting.enabled": db_instance_reporting, + "opentelemetry.traces.enabled": opentelemetry_traces_enabled, + } + ) + @validate_transaction_metrics("test_database:test_create_pool.._test", background_task=True, **kwargs) + @background_task() + def _test(): + async def _inner_test(): + await _create_pool_db() + + asyncio.run(_inner_test()) + + _test() + + +@pytest.mark.parametrize("db_instance_reporting", (True, False)) +def test_create_pool_disable_opentelemetry_traces(db_instance_reporting): + with pytest.raises(AssertionError): + # This will expectedly fail when the metrics are not recorded + @override_application_settings( + { + "datastore_tracer.instance_reporting.enabled": db_instance_reporting, + "opentelemetry.traces.enabled": False, + } + ) + @validate_transaction_metrics( + "test_database:test_create_pool.._test", + scoped_metrics=_enable_scoped_metrics if db_instance_reporting else _disable_scoped_metrics, + rollup_metrics=_enable_rollup_metrics if db_instance_reporting else _disable_rollup_metrics, + background_task=True, + ) + @background_task() + def _test(): + async def _inner_test(): + await _create_pool_db() + + asyncio.run(_inner_test()) + + _test() diff --git a/tests/hybridagent_ariadne/__init__.py b/tests/hybridagent_ariadne/__init__.py new file mode 100644 index 0000000000..8030baccf7 --- /dev/null +++ b/tests/hybridagent_ariadne/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/hybridagent_ariadne/_target_application.py b/tests/hybridagent_ariadne/_target_application.py new file mode 100644 index 0000000000..efd4639683 --- /dev/null +++ b/tests/hybridagent_ariadne/_target_application.py @@ -0,0 +1,126 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import json + +from ariadne.contrib.tracing.opentelemetry import opentelemetry_extension + +from hybridagent_ariadne._target_schema_async import target_asgi_application as target_asgi_application_async +from hybridagent_ariadne._target_schema_async import target_schema as target_schema_async +from hybridagent_ariadne._target_schema_sync import ariadne_version_tuple +from hybridagent_ariadne._target_schema_sync import target_asgi_application as target_asgi_application_sync +from hybridagent_ariadne._target_schema_sync import target_schema as target_schema_sync +from hybridagent_ariadne._target_schema_sync import target_wsgi_application as target_wsgi_application_sync + + +def check_response(query, success, response): + if isinstance(query, str) and "error" not in query: + assert success + assert "errors" not in response, response + assert response.get("data", None), response + else: + assert "errors" in response, response + + +def run_sync(schema): + def _run_sync(query, middleware=None): + from ariadne import graphql_sync + + success, response = graphql_sync( + schema, {"query": query}, middleware=middleware, extensions=[opentelemetry_extension()] + ) + check_response(query, success, response) + + return response.get("data", {}) + + return _run_sync + + +def run_async(schema): + import asyncio + + loop = asyncio.new_event_loop() + + def _run_async(query, middleware=None): + from ariadne import graphql + + success, response = loop.run_until_complete( + graphql(schema, {"query": query}, middleware=middleware, extensions=[opentelemetry_extension()]) + ) + check_response(query, success, response) + + return response.get("data", {}) + + return _run_async + + +def run_wsgi(app): + def _run_asgi(query, middleware=None): + if not isinstance(query, str) or "error" in query: + expect_errors = True + else: + expect_errors = False + + app.app.middleware = middleware + + response = app.post( + "/", json.dumps({"query": query}), headers={"Content-Type": "application/json"}, expect_errors=expect_errors + ) + + body = json.loads(response.body.decode("utf-8")) + if expect_errors: + assert body["errors"] + else: + assert "errors" not in body or not body["errors"] + + return body.get("data", {}) + + return _run_asgi + + +def run_asgi(app): + def _run_asgi(query, middleware=None): + if ariadne_version_tuple < (0, 16): + app.asgi_application.middleware = middleware + + # In ariadne v0.16.0, the middleware attribute was removed from the GraphQL class in favor of the http_handler + elif ariadne_version_tuple >= (0, 16): + app.asgi_application.http_handler.middleware = middleware + + response = app.make_request( + "POST", "/", body=json.dumps({"query": query}), headers={"Content-Type": "application/json"} + ) + body = json.loads(response.body.decode("utf-8")) + + if not isinstance(query, str) or "error" in query: + try: + assert response.status != 200 + except AssertionError: + assert body["errors"] + else: + assert response.status == 200 + assert "errors" not in body or not body["errors"] + + return body.get("data", {}) + + return _run_asgi + + +target_application = { + "sync-sync": run_sync(target_schema_sync), + "async-sync": run_async(target_schema_sync), + "async-async": run_async(target_schema_async), + "wsgi-sync": run_wsgi(target_wsgi_application_sync), + "asgi-sync": run_asgi(target_asgi_application_sync), + "asgi-async": run_asgi(target_asgi_application_async), +} diff --git a/tests/hybridagent_ariadne/_target_schema_async.py b/tests/hybridagent_ariadne/_target_schema_async.py new file mode 100644 index 0000000000..6f35d5c527 --- /dev/null +++ b/tests/hybridagent_ariadne/_target_schema_async.py @@ -0,0 +1,91 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pathlib import Path + +from ariadne import MutationType, QueryType, UnionType, load_schema_from_path, make_executable_schema +from ariadne.asgi import GraphQL as GraphQLASGI +from ariadne.asgi.handlers import GraphQLHTTPHandler +from ariadne.contrib.tracing.opentelemetry import OpenTelemetryExtension +from hybridagent_graphql._target_schema_sync import books, libraries, magazines +from testing_support.asgi_testing import AsgiTest + +schema_file = Path(__file__).parent / "schema.graphql" +type_defs = load_schema_from_path(schema_file) + +storage = [] + +mutation = MutationType() + + +@mutation.field("storage_add") +async def resolve_storage_add(self, info, string): + storage.append(string) + return string + + +item = UnionType("Item") + + +@item.type_resolver +async def resolve_type(obj, *args): + if "isbn" in obj: + return "Book" + elif "issue" in obj: + return "Magazine" + + return None + + +query = QueryType() + + +@query.field("library") +async def resolve_library(self, info, index): + return libraries[index] + + +@query.field("storage") +async def resolve_storage(self, info): + return [storage.pop()] + + +@query.field("search") +async def resolve_search(self, info, contains): + search_books = [b for b in books if contains in b["name"]] + search_magazines = [m for m in magazines if contains in m["name"]] + return search_books + search_magazines + + +@query.field("hello") +@query.field("error_middleware") +async def resolve_hello(self, info): + return "Hello!" + + +@query.field("echo") +async def resolve_echo(self, info, echo): + return echo + + +@query.field("error_non_null") +@query.field("error") +async def resolve_error(self, info): + raise RuntimeError("Runtime Error!") + + +target_schema = make_executable_schema(type_defs, query, mutation, item) +target_asgi_application = AsgiTest( + GraphQLASGI(target_schema, debug=True, http_handler=GraphQLHTTPHandler(extensions=[OpenTelemetryExtension])) +) diff --git a/tests/hybridagent_ariadne/_target_schema_sync.py b/tests/hybridagent_ariadne/_target_schema_sync.py new file mode 100644 index 0000000000..d950441dcb --- /dev/null +++ b/tests/hybridagent_ariadne/_target_schema_sync.py @@ -0,0 +1,103 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pathlib import Path + +import webtest +from ariadne import MutationType, QueryType, UnionType, load_schema_from_path, make_executable_schema +from ariadne.asgi.handlers import GraphQLHTTPHandler +from ariadne.contrib.tracing.opentelemetry import OpenTelemetryExtension, opentelemetry_extension +from ariadne.wsgi import GraphQL as GraphQLWSGI +from hybridagent_graphql._target_schema_sync import books, libraries, magazines +from testing_support.asgi_testing import AsgiTest + +from hybridagent_ariadne.test_application import ARIADNE_VERSION + +ariadne_version_tuple = tuple(map(int, ARIADNE_VERSION.split("."))) + +if ariadne_version_tuple < (0, 16): + from ariadne.asgi import GraphQL as GraphQLASGI +elif ariadne_version_tuple >= (0, 16): + from ariadne.asgi.graphql import GraphQL as GraphQLASGI + + +schema_file = Path(__file__).parent / "schema.graphql" +type_defs = load_schema_from_path(schema_file) + +storage = [] + +mutation = MutationType() + + +@mutation.field("storage_add") +def resolve_storage_add(self, info, string): + storage.append(string) + return string + + +item = UnionType("Item") + + +@item.type_resolver +def resolve_type(obj, *args): + if "isbn" in obj: + return "Book" + elif "issue" in obj: + return "Magazine" + + return None + + +query = QueryType() + + +@query.field("library") +def resolve_library(self, info, index): + return libraries[index] + + +@query.field("storage") +def resolve_storage(self, info): + return [storage.pop()] + + +@query.field("search") +def resolve_search(self, info, contains): + search_books = [b for b in books if contains in b["name"]] + search_magazines = [m for m in magazines if contains in m["name"]] + return search_books + search_magazines + + +@query.field("hello") +@query.field("error_middleware") +def resolve_hello(self, info): + return "Hello!" + + +@query.field("echo") +def resolve_echo(self, info, echo): + return echo + + +@query.field("error_non_null") +@query.field("error") +def resolve_error(self, info): + raise RuntimeError("Runtime Error!") + + +target_schema = make_executable_schema(type_defs, query, mutation, item) +target_asgi_application = AsgiTest( + GraphQLASGI(target_schema, http_handler=GraphQLHTTPHandler(extensions=[OpenTelemetryExtension])) +) +target_wsgi_application = webtest.TestApp(GraphQLWSGI(target_schema, extensions=[opentelemetry_extension()])) diff --git a/tests/hybridagent_ariadne/conftest.py b/tests/hybridagent_ariadne/conftest.py new file mode 100644 index 0000000000..82539fda1a --- /dev/null +++ b/tests/hybridagent_ariadne/conftest.py @@ -0,0 +1,30 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture + +_default_settings = { + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, + "opentelemetry.enabled": True, + "opentelemetry.traces.enabled": True, +} + +collector_agent_registration = collector_agent_registration_fixture( + app_name="Python Agent Test (Hybrid Agent, Ariadne)", default_settings=_default_settings +) diff --git a/tests/hybridagent_ariadne/schema.graphql b/tests/hybridagent_ariadne/schema.graphql new file mode 100644 index 0000000000..d7f5797096 --- /dev/null +++ b/tests/hybridagent_ariadne/schema.graphql @@ -0,0 +1,62 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +schema { + query: Query + mutation: Mutation +} + +type Author { + first_name: String + last_name: String +} + +type Book { + id: Int + name: String + isbn: String + author: Author + branch: String +} + +union Item = Book | Magazine + +type Library { + id: Int + branch: String + magazine: [Magazine] + book: [Book] +} + +type Magazine { + id: Int + name: String + issue: Int + branch: String +} + +type Mutation { + storage_add(string: String!): String +} + +type Query { + storage: [String] + library(index: Int!): Library + hello: String + search(contains: String!): [Item] + echo(echo: String!): String + error: String + error_non_null: String! + error_middleware: String +} diff --git a/tests/hybridagent_ariadne/test_application.py b/tests/hybridagent_ariadne/test_application.py new file mode 100644 index 0000000000..122ec57833 --- /dev/null +++ b/tests/hybridagent_ariadne/test_application.py @@ -0,0 +1,36 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pytest +from hybridagent_graphql.test_application import * # noqa: F403 + +from newrelic.common.package_version_utils import get_package_version + +ARIADNE_VERSION = get_package_version("ariadne") +ariadne_version_tuple = tuple(map(int, ARIADNE_VERSION.split("."))) + + +@pytest.fixture( + scope="session", params=["sync-sync", "async-sync", "async-async", "wsgi-sync", "asgi-sync", "asgi-async"] +) +def target_application(request): + from ._target_application import target_application + + target_application = target_application[request.param] + + param = request.param.split("-") + is_wsgi_or_asgi = param[0] if (param[0] in {"wsgi", "asgi"}) else False + schema_type = param[1] + + assert ARIADNE_VERSION is not None + return "Ariadne", target_application, is_wsgi_or_asgi, schema_type diff --git a/tests/hybridagent_dynamodb/_test_botocore_dynamodb_opentelemetry.py b/tests/hybridagent_dynamodb/_test_botocore_dynamodb_opentelemetry.py new file mode 100644 index 0000000000..f2c1bcd52a --- /dev/null +++ b/tests/hybridagent_dynamodb/_test_botocore_dynamodb_opentelemetry.py @@ -0,0 +1,157 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import uuid + +import botocore.session +import pytest +from moto import mock_aws +from testing_support.fixtures import dt_enabled, override_application_settings +from testing_support.validators.validate_span_events import validate_span_events +from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics +from testing_support.validators.validate_tt_segment_params import validate_tt_segment_params + +from newrelic.api.background_task import background_task +from newrelic.common.package_version_utils import get_package_version_tuple + +""" +Disable this test suite for now. +""" + +MOTO_VERSION = get_package_version_tuple("moto") +AWS_ACCESS_KEY_ID = "AAAAAAAAAAAACCESSKEY" +AWS_SECRET_ACCESS_KEY = "AAAAAASECRETKEY" +AWS_REGION = "us-east-1" + +TEST_TABLE = f"python-agent-test-{uuid.uuid4()}" + + +_dynamodb_scoped_metrics = [ + (f"Datastore/statement/dynamodb/{TEST_TABLE}/CreateTable", 1), + (f"Datastore/statement/dynamodb/{TEST_TABLE}/PutItem", 1), + (f"Datastore/statement/dynamodb/{TEST_TABLE}/GetItem", 1), + (f"Datastore/statement/dynamodb/{TEST_TABLE}/UpdateItem", 1), + (f"Datastore/statement/dynamodb/{TEST_TABLE}/Query", 1), + (f"Datastore/statement/dynamodb/{TEST_TABLE}/Scan", 1), + (f"Datastore/statement/dynamodb/{TEST_TABLE}/DeleteItem", 1), + (f"Datastore/statement/dynamodb/{TEST_TABLE}/DeleteTable", 1), +] + +_dynamodb_rollup_metrics = [ + ("Datastore/all", 8), + ("Datastore/allOther", 8), + ("Datastore/dynamodb/all", 8), + ("Datastore/dynamodb/allOther", 8), +] + + +@pytest.mark.parametrize("account_id", (None, 12345678901)) +def test_dynamodb(account_id): + expected_aws_agent_attrs = {} + if account_id: + expected_aws_agent_attrs = { + "cloud.resource_id": f"arn:aws:dynamodb:{AWS_REGION}:{account_id:012d}:table/{TEST_TABLE}", + "db.system": "dynamodb", + } + + @override_application_settings({"cloud.aws.account_id": account_id}) + @dt_enabled + @validate_span_events(expected_agents=("aws.requestId",), count=8) + @validate_span_events(exact_agents={"aws.operation": "PutItem"}, count=1) + @validate_span_events(exact_agents={"aws.operation": "GetItem"}, count=1) + @validate_span_events(exact_agents={"aws.operation": "DeleteItem"}, count=1) + @validate_span_events(exact_agents={"aws.operation": "CreateTable"}, count=1) + @validate_span_events(exact_agents={"aws.operation": "DeleteTable"}, count=1) + @validate_span_events(exact_agents={"aws.operation": "Query"}, count=1) + @validate_span_events(exact_agents={"aws.operation": "Scan"}, count=1) + @validate_tt_segment_params(present_params=("aws.requestId",)) + @validate_transaction_metrics( + "test_botocore_dynamodb:test_dynamodb.._test", + scoped_metrics=_dynamodb_scoped_metrics, + rollup_metrics=_dynamodb_rollup_metrics, + background_task=True, + ) + @background_task() + @mock_aws + def _test(): + session = botocore.session.get_session() + client = session.create_client( + "dynamodb", + region_name=AWS_REGION, + aws_access_key_id=AWS_ACCESS_KEY_ID, + aws_secret_access_key=AWS_SECRET_ACCESS_KEY, + ) + + # Create table + resp = client.create_table( + TableName=TEST_TABLE, + AttributeDefinitions=[ + {"AttributeName": "Id", "AttributeType": "N"}, + {"AttributeName": "Foo", "AttributeType": "S"}, + ], + KeySchema=[{"AttributeName": "Id", "KeyType": "HASH"}, {"AttributeName": "Foo", "KeyType": "RANGE"}], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + assert resp["TableDescription"]["TableName"] == TEST_TABLE + + # Put item + resp = client.put_item( + TableName=TEST_TABLE, + Item={"Id": {"N": "101"}, "Foo": {"S": "hello_world"}, "SomeValue": {"S": "some_random_attribute"}}, + ) + + # Get item + resp = client.get_item(TableName=TEST_TABLE, Key={"Id": {"N": "101"}, "Foo": {"S": "hello_world"}}) + assert resp["Item"]["SomeValue"]["S"] == "some_random_attribute" + + # Update item + resp = client.update_item( + TableName=TEST_TABLE, + Key={"Id": {"N": "101"}, "Foo": {"S": "hello_world"}}, + AttributeUpdates={"Foo2": {"Value": {"S": "hello_world2"}, "Action": "PUT"}}, + ReturnValues="ALL_NEW", + ) + assert resp["Attributes"]["Foo2"] + + # Query for item + resp = client.query( + TableName=TEST_TABLE, + Select="ALL_ATTRIBUTES", + KeyConditionExpression="#Id = :v_id", + ExpressionAttributeNames={"#Id": "Id"}, + ExpressionAttributeValues={":v_id": {"N": "101"}}, + ) + assert len(resp["Items"]) == 1 + assert resp["Items"][0]["SomeValue"]["S"] == "some_random_attribute" + + # Scan + resp = client.scan(TableName=TEST_TABLE) + assert len(resp["Items"]) == 1 + + # Delete item + resp = client.delete_item(TableName=TEST_TABLE, Key={"Id": {"N": "101"}, "Foo": {"S": "hello_world"}}) + + # Delete table + resp = client.delete_table(TableName=TEST_TABLE) + assert resp["TableDescription"]["TableName"] == TEST_TABLE + + if account_id: + + @validate_span_events(exact_agents=expected_aws_agent_attrs, count=8) + def _test_apply_validator(): + _test() + + _test_apply_validator() + else: + _test() diff --git a/tests/hybridagent_dynamodb/conftest.py b/tests/hybridagent_dynamodb/conftest.py new file mode 100644 index 0000000000..dc6fd6dd90 --- /dev/null +++ b/tests/hybridagent_dynamodb/conftest.py @@ -0,0 +1,37 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture + +from newrelic.common.package_version_utils import get_package_version + +BOTOCORE_VERSION = get_package_version("botocore") + +_default_settings = { + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slowdowns. + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, + "custom_insights_events.max_attribute_value": 4096, + "ai_monitoring.enabled": True, + "opentelemetry.enabled": True, + "opentelemetry.traces.enabled": True, +} +collector_agent_registration = collector_agent_registration_fixture( + app_name="Python Agent Test (Hybrid Agent, botocore)", + default_settings=_default_settings, + linked_applications=["Python Agent Test (external_botocore)"], +) diff --git a/tests/hybridagent_dynamodb/test_botocore_dynamodb.py b/tests/hybridagent_dynamodb/test_botocore_dynamodb.py new file mode 100644 index 0000000000..00dca7445a --- /dev/null +++ b/tests/hybridagent_dynamodb/test_botocore_dynamodb.py @@ -0,0 +1,172 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import uuid + +import botocore.session +import pytest +from moto import mock_aws +from testing_support.fixtures import dt_enabled, override_application_settings +from testing_support.validators.validate_span_events import validate_span_events +from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics +from testing_support.validators.validate_tt_segment_params import validate_tt_segment_params + +from newrelic.api.background_task import background_task +from newrelic.common.package_version_utils import get_package_version_tuple + +MOTO_VERSION = get_package_version_tuple("moto") +AWS_ACCESS_KEY_ID = "AAAAAAAAAAAACCESSKEY" +AWS_SECRET_ACCESS_KEY = "AAAAAASECRETKEY" +AWS_REGION = "us-east-1" + +TEST_TABLE = f"python-agent-test-{uuid.uuid4()}" + +""" +This is taken directly from New Relic's external_botocore tests to ensure that +the hybrid agent setup is working as expected. In this case, we are verifying +that DynamoDB operations default to New Relic's instrumentation even when +using the Hybrid Agent because we have temporarily disabled that instrumentation. +""" + + +_dynamodb_scoped_metrics = [ + (f"Datastore/statement/DynamoDB/{TEST_TABLE}/create_table", 1), + (f"Datastore/statement/DynamoDB/{TEST_TABLE}/put_item", 1), + (f"Datastore/statement/DynamoDB/{TEST_TABLE}/get_item", 1), + (f"Datastore/statement/DynamoDB/{TEST_TABLE}/update_item", 1), + (f"Datastore/statement/DynamoDB/{TEST_TABLE}/query", 1), + (f"Datastore/statement/DynamoDB/{TEST_TABLE}/scan", 1), + (f"Datastore/statement/DynamoDB/{TEST_TABLE}/delete_item", 1), + (f"Datastore/statement/DynamoDB/{TEST_TABLE}/delete_table", 1), +] + +_dynamodb_rollup_metrics = [ + ("Datastore/all", 8), + ("Datastore/allOther", 8), + ("Datastore/DynamoDB/all", 8), + ("Datastore/DynamoDB/allOther", 8), +] + + +@pytest.mark.parametrize("account_id", (None, 12345678901)) +def test_dynamodb(account_id): + expected_aws_agent_attrs = {} + if account_id: + expected_aws_agent_attrs = { + "cloud.resource_id": f"arn:aws:dynamodb:{AWS_REGION}:{account_id:012d}:table/{TEST_TABLE}", + "db.system": "DynamoDB", + } + + @override_application_settings({"cloud.aws.account_id": account_id}) + @dt_enabled + @validate_span_events(expected_agents=("aws.requestId",), count=8) + @validate_span_events(exact_agents={"aws.operation": "PutItem"}, count=1) + @validate_span_events(exact_agents={"aws.operation": "GetItem"}, count=1) + @validate_span_events(exact_agents={"aws.operation": "DeleteItem"}, count=1) + @validate_span_events(exact_agents={"aws.operation": "CreateTable"}, count=1) + @validate_span_events(exact_agents={"aws.operation": "DeleteTable"}, count=1) + @validate_span_events(exact_agents={"aws.operation": "Query"}, count=1) + @validate_span_events(exact_agents={"aws.operation": "Scan"}, count=1) + @validate_tt_segment_params(present_params=("aws.requestId",)) + @validate_transaction_metrics( + "test_botocore_dynamodb:test_dynamodb.._test", + scoped_metrics=_dynamodb_scoped_metrics, + rollup_metrics=_dynamodb_rollup_metrics, + background_task=True, + ) + @background_task() + @mock_aws + def _test(): + session = botocore.session.get_session() + client = session.create_client( + "dynamodb", + region_name=AWS_REGION, + aws_access_key_id=AWS_ACCESS_KEY_ID, + aws_secret_access_key=AWS_SECRET_ACCESS_KEY, + ) + + # Create table + resp = client.create_table( + TableName=TEST_TABLE, + AttributeDefinitions=[ + {"AttributeName": "Id", "AttributeType": "N"}, + {"AttributeName": "Foo", "AttributeType": "S"}, + ], + KeySchema=[{"AttributeName": "Id", "KeyType": "HASH"}, {"AttributeName": "Foo", "KeyType": "RANGE"}], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + assert resp["TableDescription"]["TableName"] == TEST_TABLE + # moto response is ACTIVE, AWS response is CREATING + # assert resp['TableDescription']['TableStatus'] == 'ACTIVE' + + # # AWS needs time to create the table + # import time + # time.sleep(15) + + # Put item + resp = client.put_item( + TableName=TEST_TABLE, + Item={"Id": {"N": "101"}, "Foo": {"S": "hello_world"}, "SomeValue": {"S": "some_random_attribute"}}, + ) + # No checking response, due to inconsistent return values. + # moto returns resp['Attributes']. AWS returns resp['ResponseMetadata'] + + # Get item + resp = client.get_item(TableName=TEST_TABLE, Key={"Id": {"N": "101"}, "Foo": {"S": "hello_world"}}) + assert resp["Item"]["SomeValue"]["S"] == "some_random_attribute" + + # Update item + resp = client.update_item( + TableName=TEST_TABLE, + Key={"Id": {"N": "101"}, "Foo": {"S": "hello_world"}}, + AttributeUpdates={"Foo2": {"Value": {"S": "hello_world2"}, "Action": "PUT"}}, + ReturnValues="ALL_NEW", + ) + assert resp["Attributes"]["Foo2"] + + # Query for item + resp = client.query( + TableName=TEST_TABLE, + Select="ALL_ATTRIBUTES", + KeyConditionExpression="#Id = :v_id", + ExpressionAttributeNames={"#Id": "Id"}, + ExpressionAttributeValues={":v_id": {"N": "101"}}, + ) + assert len(resp["Items"]) == 1 + assert resp["Items"][0]["SomeValue"]["S"] == "some_random_attribute" + + # Scan + resp = client.scan(TableName=TEST_TABLE) + assert len(resp["Items"]) == 1 + + # Delete item + resp = client.delete_item(TableName=TEST_TABLE, Key={"Id": {"N": "101"}, "Foo": {"S": "hello_world"}}) + # No checking response, due to inconsistent return values. + # moto returns resp['Attributes']. AWS returns resp['ResponseMetadata'] + + # Delete table + resp = client.delete_table(TableName=TEST_TABLE) + assert resp["TableDescription"]["TableName"] == TEST_TABLE + # moto response is ACTIVE, AWS response is DELETING + # assert resp['TableDescription']['TableStatus'] == 'DELETING' + + if account_id: + + @validate_span_events(exact_agents=expected_aws_agent_attrs, count=8) + def _test_apply_validator(): + _test() + + _test_apply_validator() + else: + _test() diff --git a/tests/hybridagent_fastapi/_target_opentelemetry_application.py b/tests/hybridagent_fastapi/_target_opentelemetry_application.py new file mode 100644 index 0000000000..06ec691e22 --- /dev/null +++ b/tests/hybridagent_fastapi/_target_opentelemetry_application.py @@ -0,0 +1,35 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from fastapi import FastAPI +from testing_support.asgi_testing import AsgiTest + +from newrelic.api.transaction import current_transaction + +app = FastAPI() + + +@app.get("/sync") +def sync(): + assert current_transaction() is not None + return {} + + +@app.get("/async") +async def non_sync(): + assert current_transaction() is not None + return {} + + +target_application = AsgiTest(app) diff --git a/tests/hybridagent_fastapi/conftest.py b/tests/hybridagent_fastapi/conftest.py new file mode 100644 index 0000000000..68f31129d6 --- /dev/null +++ b/tests/hybridagent_fastapi/conftest.py @@ -0,0 +1,40 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture +from testing_support.fixtures import newrelic_caplog as caplog + +_default_settings = { + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, + "debug.log_autorum_middleware": True, + "opentelemetry.enabled": True, + "opentelemetry.traces.enabled": True, +} + +collector_agent_registration = collector_agent_registration_fixture( + app_name="Python Agent Test (Hybrid Agent, FastAPI)", default_settings=_default_settings +) + + +@pytest.fixture(scope="session") +def app(): + import _target_opentelemetry_application + + return _target_opentelemetry_application.target_application diff --git a/tests/hybridagent_fastapi/test_otel_application.py b/tests/hybridagent_fastapi/test_otel_application.py new file mode 100644 index 0000000000..35311dc795 --- /dev/null +++ b/tests/hybridagent_fastapi/test_otel_application.py @@ -0,0 +1,93 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +import pytest +from testing_support.fixtures import dt_enabled +from testing_support.validators.validate_span_events import validate_span_events +from testing_support.validators.validate_transaction_event_attributes import validate_transaction_event_attributes +from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics + +_exact_intrinsics = {"type": "Span"} +_exact_root_intrinsics = _exact_intrinsics.copy().update({"nr.entryPoint": True}) +_expected_intrinsics = [ + "traceId", + "transactionId", + "sampled", + "priority", + "timestamp", + "duration", + "name", + "category", + "guid", +] +_expected_root_intrinsics = [*_expected_intrinsics, "transaction.name"] +_expected_child_intrinsics = [*_expected_intrinsics, "parentId"] +_unexpected_root_intrinsics = ["parentId"] +_unexpected_child_intrinsics = ["nr.entryPoint", "transaction.name"] + +_test_application_rollup_metrics = [ + ("Supportability/DistributedTrace/CreatePayload/Success", 2), + ("Supportability/TraceContext/Create/Success", 2), + ("HttpDispatcher", 1), + ("WebTransaction", 1), + ("WebTransactionTotalTime", 1), +] + + +@pytest.mark.parametrize("endpoint", ("/sync", "/async")) +def test_application(caplog, app, endpoint): + caplog.set_level(logging.ERROR) + transaction_name = f"GET {endpoint}" + + @dt_enabled + @validate_span_events( + exact_intrinsics=_exact_root_intrinsics, + expected_intrinsics=_expected_root_intrinsics, + unexpected_intrinsics=_unexpected_root_intrinsics, + ) + @validate_transaction_event_attributes( + exact_attrs={ + "agent": { + "response.headers.contentType": "application/json", + "request.method": "GET", + "request.uri": endpoint, + "response.headers.contentLength": 2, + "response.status": "200", + }, + "intrinsic": {"name": f"WebTransaction/Uri/{transaction_name}"}, + "user": {}, + } + ) + @validate_span_events( + count=2, # "asgi.event.type": "http.response.start" and "http.response.body" + exact_intrinsics=_exact_intrinsics, + expected_intrinsics=_expected_child_intrinsics, + unexpected_intrinsics=_unexpected_child_intrinsics, + ) + @validate_transaction_metrics( + transaction_name, + group="Uri", + scoped_metrics=[(f"Function/{transaction_name} http send", 2)], + rollup_metrics=[(f"Function/{transaction_name} http send", 2), *_test_application_rollup_metrics], + ) + def _test(): + response = app.get(endpoint) + assert response.status == 200 + + # Catch context propagation error messages + assert not caplog.records + + _test() diff --git a/tests/hybridagent_flask/_test_application.py b/tests/hybridagent_flask/_test_application.py new file mode 100644 index 0000000000..2043f29b13 --- /dev/null +++ b/tests/hybridagent_flask/_test_application.py @@ -0,0 +1,70 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import webtest +from flask import Flask, abort, render_template, render_template_string +from werkzeug.exceptions import NotFound +from werkzeug.routing import Rule + +application = Flask(__name__) + + +@application.route("/index") +def index_page(): + return "INDEX RESPONSE" + + +application.url_map.add(Rule("/endpoint", endpoint="endpoint")) + + +@application.endpoint("endpoint") +def endpoint_page(): + return "ENDPOINT RESPONSE" + + +@application.route("/error") +def error_page(): + raise RuntimeError("RUNTIME ERROR") + + +@application.route("/abort_404") +def abort_404_page(): + abort(404) + + +@application.route("/exception_404") +def exception_404_page(): + raise NotFound + + +@application.route("/template_string") +def template_string(): + return render_template_string("

INDEX RESPONSE

") + + +@application.route("/template_not_found") +def template_not_found(): + return render_template("not_found") + + +@application.route("/html_insertion") +def html_insertion(): + return ( + "Some header" + "

My First Heading

My first paragraph.

" + "" + ) + + +_test_application = webtest.TestApp(application) diff --git a/tests/hybridagent_flask/_test_application_async.py b/tests/hybridagent_flask/_test_application_async.py new file mode 100644 index 0000000000..fefc2c05f3 --- /dev/null +++ b/tests/hybridagent_flask/_test_application_async.py @@ -0,0 +1,27 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import webtest +from _test_application import application +from conftest import async_handler_support + +# Async handlers only supported in Flask >2.0.0 +if async_handler_support: + + @application.route("/async") + async def async_page(): + return "ASYNC RESPONSE" + + +_test_application = webtest.TestApp(application) diff --git a/tests/hybridagent_flask/conftest.py b/tests/hybridagent_flask/conftest.py new file mode 100644 index 0000000000..093adb44c9 --- /dev/null +++ b/tests/hybridagent_flask/conftest.py @@ -0,0 +1,45 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import platform + +import pytest +from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture + +from newrelic.common.package_version_utils import get_package_version_tuple + +FLASK_VERSION = get_package_version_tuple("flask") + +is_flask_v2 = FLASK_VERSION[0] >= 2 +is_not_flask_v2_3 = FLASK_VERSION < (2, 3, 0) +is_pypy = platform.python_implementation() == "PyPy" +async_handler_support = is_flask_v2 and not is_pypy +skip_if_not_async_handler_support = pytest.mark.skipif( + not async_handler_support, reason="Requires async handler support. (Flask >=v2.0.0, CPython)" +) + +_default_settings = { + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, + "opentelemetry.enabled": True, + "opentelemetry.traces.enabled": True, +} + +collector_agent_registration = collector_agent_registration_fixture( + app_name="Python Agent Test (Hybrid Agent, Flask)", default_settings=_default_settings +) diff --git a/tests/hybridagent_flask/test_application.py b/tests/hybridagent_flask/test_application.py new file mode 100644 index 0000000000..d9f64ecd48 --- /dev/null +++ b/tests/hybridagent_flask/test_application.py @@ -0,0 +1,266 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from conftest import async_handler_support, skip_if_not_async_handler_support +from testing_support.fixtures import dt_enabled +from testing_support.validators.validate_error_event_attributes import validate_error_event_attributes +from testing_support.validators.validate_span_events import validate_span_events +from testing_support.validators.validate_transaction_errors import validate_transaction_errors +from testing_support.validators.validate_transaction_event_attributes import validate_transaction_event_attributes +from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics + +try: + # The __version__ attribute was only added in 0.7.0. + # Flask team does not use semantic versioning during development. + from flask import __version__ as flask_version + + flask_version = tuple([int(v) for v in flask_version.split(".")]) + is_gt_flask060 = True + is_dev_version = False +except ValueError: + is_gt_flask060 = True + is_dev_version = True +except ImportError: + is_gt_flask060 = False + is_dev_version = False + +requires_endpoint_decorator = pytest.mark.skipif(not is_gt_flask060, reason="The endpoint decorator is not supported.") + + +def target_application(): + # We need to delay Flask application creation because of ordering + # issues whereby the agent needs to be initialised before Flask is + # imported and the routes configured. Normally pytest only runs the + # global fixture which will initialise the agent after each test + # file is imported, which is too late. + + if not async_handler_support: + from _test_application import _test_application + else: + from _test_application_async import _test_application + return _test_application + + +_exact_intrinsics = {"type": "Span"} +_exact_root_intrinsics = _exact_intrinsics.copy().update({"nr.entryPoint": True}) +_expected_intrinsics = [ + "traceId", + "transactionId", + "sampled", + "priority", + "timestamp", + "duration", + "name", + "category", + "guid", +] +_expected_root_intrinsics = [*_expected_intrinsics.copy(), "transaction.name"] +_expected_child_intrinsics = [*_expected_intrinsics.copy(), "parentId"] +_unexpected_root_intrinsics = ["parentId"] +_unexpected_child_intrinsics = ["nr.entryPoint", "transaction.name"] + +_test_application_rollup_metrics = [ + ("Supportability/DistributedTrace/CreatePayload/Success", 1), + ("Supportability/TraceContext/Create/Success", 1), + ("Python/WSGI/Input/Bytes", 1), + ("Python/WSGI/Input/Time", 1), + ("Python/WSGI/Input/Calls/read", 1), + ("Python/WSGI/Input/Calls/readline", 1), + ("Python/WSGI/Input/Calls/readlines", 1), + ("Python/WSGI/Output/Bytes", 1), + ("Python/WSGI/Output/Time", 1), + ("Python/WSGI/Output/Calls/yield", 1), + ("Python/WSGI/Output/Calls/write", 1), + ("HttpDispatcher", 1), + ("WebTransaction", 1), + ("WebTransactionTotalTime", 1), +] + + +@dt_enabled +@validate_transaction_errors(errors=[]) +@validate_transaction_event_attributes( + required_params={"agent": ["request.headers.host", "response.headers.contentType"], "intrinsic": [], "user": []}, + exact_attrs={ + "agent": { + "request.method": "GET", + "request.uri": "/index", + "response.headers.contentLength": 14, + "response.status": "200", + }, + "intrinsic": {"name": "WebTransaction/Uri/index"}, + "user": {}, + }, +) +@validate_transaction_metrics("index", group="Uri", rollup_metrics=_test_application_rollup_metrics) +@validate_span_events( + exact_intrinsics=_exact_root_intrinsics, + expected_intrinsics=_expected_root_intrinsics, + unexpected_intrinsics=_unexpected_root_intrinsics, + exact_agents={"otel.scope.name": "flask", "otel.library.name": "flask"}, + expected_agents=["otel.scope.version", "otel.library.version"], + exact_users={"library_name": "flask", "schema_url": "https://opentelemetry.io/schemas/1.11.0"}, + expected_users=["library_version"], +) +def test_opentelemetry_application_index(): + application = target_application() + response = application.get("/index") + response.mustcontain("INDEX RESPONSE") + + +@skip_if_not_async_handler_support +@dt_enabled +@validate_transaction_errors(errors=[]) +@validate_transaction_metrics("async", group="Uri", rollup_metrics=_test_application_rollup_metrics) +@validate_span_events( + exact_intrinsics=_exact_root_intrinsics, + expected_intrinsics=_expected_root_intrinsics, + unexpected_intrinsics=_unexpected_root_intrinsics, +) +def test_opentelemetry_application_async(): + application = target_application() + response = application.get("/async") + response.mustcontain("ASYNC RESPONSE") + + +@dt_enabled +@validate_transaction_errors(errors=[]) +@validate_transaction_metrics("endpoint", group="Uri", rollup_metrics=_test_application_rollup_metrics) +@validate_span_events( + exact_intrinsics=_exact_root_intrinsics, + expected_intrinsics=_expected_root_intrinsics, + unexpected_intrinsics=_unexpected_root_intrinsics, +) +def test_opentelemetry_application_endpoint(): + application = target_application() + response = application.get("/endpoint") + response.mustcontain("ENDPOINT RESPONSE") + + +@dt_enabled +@validate_transaction_errors(errors=["builtins:RuntimeError"]) +@validate_error_event_attributes( + exact_attrs={ + "agent": {}, + "intrinsic": { + "error.message": "RUNTIME ERROR", + "error.class": "builtins:RuntimeError", + "error.expected": False, + }, + "user": {"exception.escaped": False}, + } +) +@validate_transaction_metrics("error", group="Uri", rollup_metrics=_test_application_rollup_metrics) +@validate_span_events( + exact_intrinsics=_exact_root_intrinsics, + expected_intrinsics=_expected_root_intrinsics, + unexpected_intrinsics=_unexpected_root_intrinsics, +) +def test_opentelemetry_application_error(): + application = target_application() + application.get("/error", status=500, expect_errors=True) + + +@dt_enabled +@validate_transaction_errors(errors=[]) +@validate_transaction_metrics("abort_404", group="Uri", rollup_metrics=_test_application_rollup_metrics) +@validate_span_events( + exact_intrinsics=_exact_root_intrinsics, + expected_intrinsics=_expected_root_intrinsics, + unexpected_intrinsics=_unexpected_root_intrinsics, +) +def test_opentelemetry_application_abort_404(): + application = target_application() + application.get("/abort_404", status=404) + + +@dt_enabled +@validate_transaction_errors(errors=[]) +@validate_transaction_metrics("exception_404", group="Uri", rollup_metrics=_test_application_rollup_metrics) +@validate_span_events( + exact_intrinsics=_exact_root_intrinsics, + expected_intrinsics=_expected_root_intrinsics, + unexpected_intrinsics=_unexpected_root_intrinsics, +) +def test_application_exception_404(): + application = target_application() + application.get("/exception_404", status=404) + + +@dt_enabled +@validate_transaction_errors(errors=[]) +@validate_transaction_metrics("missing", group="Uri", rollup_metrics=_test_application_rollup_metrics) +@validate_span_events( + exact_intrinsics=_exact_root_intrinsics, + expected_intrinsics=_expected_root_intrinsics, + unexpected_intrinsics=_unexpected_root_intrinsics, +) +def test_application_not_found(): + application = target_application() + application.get("/missing", status=404) + + +_test_application_render_template_string_scoped_metrics = [ + ("Template/Compile/