From 19cf43089c055a21a06f02c17864d9e236a6cdec Mon Sep 17 00:00:00 2001 From: Bas van Gijzel Date: Wed, 5 Aug 2026 12:50:28 +0000 Subject: [PATCH 1/2] ci(coverage): report solx-llvm fork coverage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A second llvm-cov show/report/export over the same profdata, scoped via positional source paths to the fork dirs the LLVM team cares about (EVM backend, MLIR Sol/Yul dialects and conversions). Produces its own HTML artifact and a flagged (llvm-fork) Codecov upload, with one Codecov component per area. Also fixes collection along the way: the integration tester never set LLVM_PROFILE_FILE, so the thousands of spawned solx subprocesses — the main source of solx-llvm C++ counters — all clobbered a single default.profraw. It now uses the same %4m online-merge pool as the unit tests. And the job gets a timeout-minutes backstop, sized for a cold instrumented LLVM build. --- .github/workflows/coverage.yaml | 67 +++++++++++++++++++++++++++++++++ codecov.yml | 26 +++++++++++++ 2 files changed, 93 insertions(+) diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml index 63ba05abc..e367df892 100644 --- a/.github/workflows/coverage.yaml +++ b/.github/workflows/coverage.yaml @@ -37,6 +37,10 @@ jobs: # summary comment step. if: ${{ github.event_name == 'push' || !github.event.pull_request.head.repo.fork }} runs-on: ubuntu-24.04 + # Backstop against hung compiles (the tester drives a pipeline still in + # development); sized for a cold instrumented LLVM build when a solx-llvm + # bump lands before cache-warmup has run for that SHA. + timeout-minutes: 180 permissions: contents: read pull-requests: write @@ -56,7 +60,9 @@ jobs: env: PROFDATA_FILE: solx.profdata LCOV_FILE: codecov.lcov + LCOV_FILE_LLVM: codecov-llvm-fork.lcov OUTPUT_HTML_DIR: COVERAGE + OUTPUT_HTML_DIR_LLVM: COVERAGE-LLVM # ^/rustc/ = std sources monomorphized into our binaries; the remapped # path exists on no runner, so llvm-cov show errors trying to render it. COVERAGE_IGNORE_REGEX: '/(solx-llvm|target-llvm|solx-solidity|\.cargo|cargo/registry)/|^/rustc/' @@ -110,6 +116,11 @@ jobs: - name: Run integration tests env: SOLX_BINARY: ${{ steps.build-solx.outputs.binary-path }} + # %Nm without %p: subprocesses merge online into a pool of 4 files + # instead of clobbering a single default.profraw. The pool is what + # preserves the solx-llvm C++ counters, which only the spawned solx + # compile subprocesses exercise. + LLVM_PROFILE_FILE: 'integration-%4m.profraw' run: | TMP="${PROFDATA_FILE}.tmp" TARGET_DIR="./target/x86_64-unknown-linux-gnu/release" @@ -176,6 +187,37 @@ jobs: -instr-profile=${PROFDATA_FILE} \ "${OBJECTS[@]}" > ./llvm/${LCOV_FILE} + # solx-llvm fork coverage for the LLVM team: same profdata, restricted + # via positional source paths to the fork-specific dirs instead of the + # ignore regex that keeps them out of the solx report above. + - name: Generate LLVM fork coverage reports + run: | + # First binary is positional: with -object only, llvm-cov consumes + # the first positional SOURCE path as the covered binary. + OBJECTS=( + "${GITHUB_WORKSPACE}/solx-instrumented" + -object "${GITHUB_WORKSPACE}/solx-tester-instrumented" + -object "${GITHUB_WORKSPACE}/solx-dev-instrumented" + ) + FORK_PATHS=( + "${GITHUB_WORKSPACE}/solx-llvm/llvm/lib/Target/EVM" + "${GITHUB_WORKSPACE}/solx-llvm/mlir/lib/Dialect/Sol" + "${GITHUB_WORKSPACE}/solx-llvm/mlir/lib/Dialect/Yul" + "${GITHUB_WORKSPACE}/solx-llvm/mlir/lib/Conversion/SolToYul" + "${GITHUB_WORKSPACE}/solx-llvm/mlir/lib/Conversion/YulToStandard" + ) + # llvm-cov only warns on a nonexistent source path, silently dropping + # the area from every report; fail instead. + for path in "${FORK_PATHS[@]}"; do + [ -d "${path}" ] || { echo "::error::fork source dir missing: ${path}"; exit 1; } + done + llvm-cov show --show-directory-coverage \ + --format=html --output-dir="${OUTPUT_HTML_DIR_LLVM}" \ + -instr-profile="${PROFDATA_FILE}" "${OBJECTS[@]}" "${FORK_PATHS[@]}" + llvm-cov export --format=lcov \ + -instr-profile="${PROFDATA_FILE}" \ + "${OBJECTS[@]}" "${FORK_PATHS[@]}" > "./llvm/${LCOV_FILE_LLVM}" + - name: Coverage summary id: coverage-summary run: | @@ -258,6 +300,12 @@ jobs: name: 'Coverage integration tests HTML' path: ${{ env.OUTPUT_HTML_DIR }} + - name: Upload LLVM fork coverage artifacts + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: 'Coverage LLVM fork HTML' + path: ${{ env.OUTPUT_HTML_DIR_LLVM }} + - name: Upload coverage to Codecov uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 if: (success() || failure()) @@ -271,3 +319,22 @@ jobs: slug: ${{ github.repository }} # Surface upload failures instead of silently going green. fail_ci_if_error: true + + # Separate flagged upload so the fork coverage can be filtered apart + # from the solx crates in the Codecov UI (see component_management in + # codecov.yml for the per-area split). + - name: Upload LLVM fork coverage to Codecov + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 + if: (success() || failure()) + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: 'llvm/codecov-llvm-fork.lcov' + flags: llvm-fork + # Enumerate the network with `git ls-files --recurse-submodules` so + # solx-llvm paths survive server-side path fixing; without this the + # fork files are silently dropped at ingest (not in the git tree). + recurse_submodules: true + disable_search: true + plugins: noop + slug: ${{ github.repository }} + fail_ci_if_error: true diff --git a/codecov.yml b/codecov.yml index 14005c2c9..4799d1b62 100644 --- a/codecov.yml +++ b/codecov.yml @@ -9,3 +9,29 @@ coverage: default: false # disable the patch status that measures patch changes comment: false + +# solx-llvm fork coverage (uploaded under the `llvm-fork` flag) split by the +# areas the LLVM team cares about. Paths are as they appear in the lcov from +# coverage.yaml, i.e. relative to the solx checkout with the submodule prefix. +component_management: + individual_components: + - component_id: llvm-evm-backend + name: LLVM EVM backend + paths: + - solx-llvm/llvm/lib/Target/EVM/** + - component_id: mlir-sol-dialect + name: MLIR Sol dialect + paths: + - solx-llvm/mlir/lib/Dialect/Sol/** + - component_id: mlir-yul-dialect + name: MLIR Yul dialect + paths: + - solx-llvm/mlir/lib/Dialect/Yul/** + - component_id: mlir-sol-to-yul + name: MLIR SolToYul + paths: + - solx-llvm/mlir/lib/Conversion/SolToYul/** + - component_id: mlir-yul-to-standard + name: MLIR YulToStandard + paths: + - solx-llvm/mlir/lib/Conversion/YulToStandard/** From f20ad9c0ef6cd9572f9f078daa376d04488c490e Mon Sep 17 00:00:00 2001 From: Bas van Gijzel Date: Wed, 5 Aug 2026 12:51:13 +0000 Subject: [PATCH 2/2] ci(coverage): replace the custom PR comment with the Codecov app comment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The hand-rolled awk per-crate table (and its red/green/yellow emoji thresholds) duplicated what Codecov already renders, minus PR-delta context. Enable the Codecov app comment instead — same style as era-compiler-llvm and EDR — with flags and components in the layout so the solx/llvm-fork split and the per-area fork numbers stay visible on the PR. The comment step was also the only reason the job needed pull-requests: write. --- .github/workflows/coverage.yaml | 81 +-------------------------------- codecov.yml | 6 ++- 2 files changed, 7 insertions(+), 80 deletions(-) diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml index e367df892..bc60fff1a 100644 --- a/.github/workflows/coverage.yaml +++ b/.github/workflows/coverage.yaml @@ -33,8 +33,8 @@ jobs: - uses: ./.github/actions/cooldown-check coverage: - # Fork PRs get a read-only GITHUB_TOKEN, which would fail the PR coverage - # summary comment step. + # Fork PRs get no repo secrets, so the Codecov uploads (which run with + # fail_ci_if_error) would fail without CODECOV_TOKEN. if: ${{ github.event_name == 'push' || !github.event.pull_request.head.repo.fork }} runs-on: ubuntu-24.04 # Backstop against hung compiles (the tester drives a pipeline still in @@ -43,7 +43,6 @@ jobs: timeout-minutes: 180 permissions: contents: read - pull-requests: write packages: read container: image: ghcr.io/nomicfoundation/solx-ci-runner@sha256:cd5a37f2630fdf1898ddb2ca11f8c2ecc4d572c60011fe923b27b1625f92ba15 @@ -218,82 +217,6 @@ jobs: -instr-profile="${PROFDATA_FILE}" \ "${OBJECTS[@]}" "${FORK_PATHS[@]}" > "./llvm/${LCOV_FILE_LLVM}" - - name: Coverage summary - id: coverage-summary - run: | - OBJECTS=( - -object "${GITHUB_WORKSPACE}/solx-instrumented" - -object "${GITHUB_WORKSPACE}/solx-tester-instrumented" - -object "${GITHUB_WORKSPACE}/solx-dev-instrumented" - ) - llvm-cov report -instr-profile=${PROFDATA_FILE} "${OBJECTS[@]}" \ - --ignore-filename-regex="${COVERAGE_IGNORE_REGEX}" \ - > coverage-report.txt - - # Parse llvm-cov report into per-crate markdown table - awk ' - /^---/ || /^Filename/ || /^$/ { next } - /^TOTAL/ { - total_lines = $(NF-5); total_missed = $(NF-4) - total_funcs = $(NF-8); total_missed_funcs = $(NF-7) - next - } - NF > 3 { - file = $1 - n = split(file, parts, "/") - crate = "" - for (i = 1; i <= n-1; i++) { - if (parts[i] ~ /^solx/ && parts[i+1] == "src") { - crate = parts[i]; break - } - } - if (crate == "") next - lines[crate] += $(NF-5) - missed[crate] += $(NF-4) - funcs[crate] += $(NF-8) - missed_f[crate] += $(NF-7) - } - END { - for (c in lines) { - lp = (lines[c] > 0) ? (lines[c] - missed[c]) / lines[c] * 100 : 0 - fp = (funcs[c] > 0) ? (funcs[c] - missed_f[c]) / funcs[c] * 100 : 0 - le = (lp >= 80) ? "🟢" : (lp >= 50) ? "🟡" : "🔴" - fe = (fp >= 80) ? "🟢" : (fp >= 50) ? "🟡" : "🔴" - printf "| %s | %s %.1f%% | %s %.1f%% |\n", c, le, lp, fe, fp - } - lp = (total_lines > 0) ? (total_lines - total_missed) / total_lines * 100 : 0 - fp = (total_funcs > 0) ? (total_funcs - total_missed_funcs) / total_funcs * 100 : 0 - le = (lp >= 80) ? "🟢" : (lp >= 50) ? "🟡" : "🔴" - fe = (fp >= 80) ? "🟢" : (fp >= 50) ? "🟡" : "🔴" - printf "ZTOTAL| **Total** | **%s %.1f%%** | **%s %.1f%%** |\n", le, lp, fe, fp - }' coverage-report.txt > coverage-rows.txt - - { - echo "| Crate | Line Coverage | Function Coverage |" - echo "|:------|:------------:|:-----------------:|" - grep -v '^ZTOTAL' coverage-rows.txt | sort - sed -n 's/^ZTOTAL//p' coverage-rows.txt - } > coverage-summary.md - - cat coverage-summary.md >> "${GITHUB_STEP_SUMMARY}" - { - echo "summary<> "${GITHUB_OUTPUT}" - - - name: Post PR coverage summary - if: github.event_name == 'pull_request' - uses: mshick/add-pr-comment@ec328af66588ab8f77cdeb2c264f14aba45bbf59 # v3.12 - with: - message-id: 'coverage-summary' - message: | - ### Coverage Summary - - ${{ steps.coverage-summary.outputs.summary }} - - [Codecov Report](https://app.codecov.io/gh/${{ github.repository }}/pull/${{ github.event.pull_request.number }}) | [HTML Report](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}#artifacts) | [Workflow Run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) - - name: Upload coverage artifacts uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: diff --git a/codecov.yml b/codecov.yml index 4799d1b62..183ef5eb3 100644 --- a/codecov.yml +++ b/codecov.yml @@ -8,7 +8,11 @@ coverage: patch: default: false # disable the patch status that measures patch changes -comment: false +# The Codecov app's PR comment. `components` adds the per-area solx-llvm +# fork table (see component_management below); `flags` splits solx vs +# llvm-fork. +comment: + layout: "header, diff, flags, components, files" # solx-llvm fork coverage (uploaded under the `llvm-fork` flag) split by the # areas the LLVM team cares about. Paths are as they appear in the lcov from