diff --git a/.archgate/adrs/ARCH-005-testing-standards.md b/.archgate/adrs/ARCH-005-testing-standards.md index e3d038d3..dc86f6ac 100644 --- a/.archgate/adrs/ARCH-005-testing-standards.md +++ b/.archgate/adrs/ARCH-005-testing-standards.md @@ -22,7 +22,7 @@ Bun's built-in test runner (`bun test`) provides a Jest-compatible API (`describ ## Decision -Use Bun's built-in test runner (`bun test`) for all tests. Test files go in `tests/` mirroring the `src/` directory structure. Fixtures go in `tests/fixtures/`. Target 80% code coverage. +Use Bun's built-in test runner (`bun test`) for all tests. Test files go in `tests/` mirroring the `src/` directory structure. Fixtures go in `tests/fixtures/`. Target 90% code coverage, enforced in CI. **Key conventions:** @@ -30,7 +30,7 @@ Use Bun's built-in test runner (`bun test`) for all tests. Test files go in `tes 2. **Fixtures in `tests/fixtures/`** — Sample ADR files and mock codebases live in a shared fixtures directory. Fixtures are reusable across test suites. 3. **Temp directories for filesystem tests** — Tests that write files use `mkdtemp` for isolation. Temp directories are cleaned up in `afterEach` or `afterAll`. 4. **Test file naming** — Test files use the `.test.ts` suffix: `.test.ts`. -5. **Coverage target: 80%** — Not enforced in CI yet, but serves as a guideline. Critical paths (engine, formats) should have higher coverage. +5. **Coverage target: 90%** — Enforced in CI. PRs that drop total line coverage below 90% are blocked by the `Validate Code` gate check. ## Do's and Don'ts @@ -190,7 +190,7 @@ globalThis.fetch = (() => - **Bun test runner API changes** — Although Bun is past 1.0, some newer APIs may still evolve between minor versions. Test runner behavior or API may change. - **Mitigation:** The project pins a specific Bun version via `.prototools`. Test runner API changes are caught during controlled Bun upgrades with full test suite validation. - **Coverage reporting gaps** — `bun test --coverage` may not report accurate coverage for all code paths, especially for dynamically imported modules. - - **Mitigation:** Coverage is a guideline (80% target), not a hard gate. Critical modules (engine, formats) are tested thoroughly regardless of coverage numbers. + - **Mitigation:** The 90% threshold is enforced on total line coverage, not per-file. Individual modules with dynamically-loaded code paths may have lower per-file coverage as long as the aggregate stays above 90%. Critical modules (engine, formats) are tested thoroughly regardless of aggregate numbers. - **Third-party SDK event loop retention** — External SDK instances that hold internal resource references may keep Bun's event loop alive on Linux after all tests complete, causing `bun test` to hang indefinitely. This does not surface on macOS (event loop drains normally there), making it a Linux-CI-only failure that is hard to reproduce locally. - **Mitigation:** Always manage external resource lifecycle in `beforeEach`/`afterEach` and call the cleanup method (`close()`, `destroy()`, `disconnect()`) in `afterEach`. Add `timeout-minutes` to CI jobs as a safety net — the `code-pull-request.yml` job is set to 10 minutes to cap any future regressions. @@ -200,6 +200,7 @@ globalThis.fetch = (() => - **Archgate rule** `ARCH-005/test-mirrors-src`: Scans all source files in `src/` and verifies a corresponding `.test.ts` file exists in `tests/`. Severity: `error`. - **CI pipeline**: `bun test --timeout 60000` runs on every pull request. Test failures and per-test timeouts block merge. All workflow jobs have `timeout-minutes` set to prevent indefinite hangs. +- **Coverage threshold**: The `Coverage Report` job enforces a 90% minimum line coverage. If total coverage drops below 90%, the job fails and the `Validate Code` gate blocks the PR. ### Manual Enforcement diff --git a/.github/actions/coverage-report/action.yml b/.github/actions/coverage-report/action.yml new file mode 100644 index 00000000..cad38163 --- /dev/null +++ b/.github/actions/coverage-report/action.yml @@ -0,0 +1,168 @@ +name: Coverage Report +description: Merge multi-platform coverage data, generate reports, post PR comment, and compute threshold status + +inputs: + min-coverage: + description: Minimum line coverage percentage required + required: false + default: "90" + github-token: + description: GitHub token for posting PR comments + required: true + linux-coverage-path: + description: Path to Linux lcov.info file + required: false + default: "coverage-linux/lcov.info" + windows-coverage-path: + description: Path to Windows lcov.info file (may not exist) + required: false + default: "coverage-windows/lcov.info" + src-filter: + description: lcov extract pattern to filter coverage to source files + required: false + default: "src/*" + event-name: + description: github.event_name value + required: true + pr-number: + description: github.event.pull_request.number value (empty for non-PR events) + required: false + default: "" + is-fork: + description: github.event.pull_request.head.repo.fork value + required: false + default: "false" + run-url: + description: Full URL to the workflow run for artifact links + required: true + +outputs: + coverage: + description: Total line coverage percentage (e.g. 92.3) + value: ${{ steps.report.outputs.coverage }} + threshold-met: + description: Whether coverage meets the minimum threshold (true/false) + value: ${{ steps.report.outputs.threshold-met }} + html-report-path: + description: Path to the generated HTML coverage report directory + value: ${{ steps.report.outputs.html-report-path }} + +runs: + using: composite + steps: + - name: Install lcov + shell: bash + run: sudo apt-get install -y -qq lcov > /dev/null 2>&1 + + - name: Merge coverage and generate report + id: report + shell: bash + env: + MIN_COVERAGE: ${{ inputs.min-coverage }} + LINUX_COVERAGE: ${{ inputs.linux-coverage-path }} + WINDOWS_COVERAGE: ${{ inputs.windows-coverage-path }} + SRC_FILTER: ${{ inputs.src-filter }} + EVENT_NAME: ${{ inputs.event-name }} + PR_NUMBER: ${{ inputs.pr-number }} + IS_FORK: ${{ inputs.is-fork }} + RUN_URL: ${{ inputs.run-url }} + GH_TOKEN: ${{ inputs.github-token }} + run: | + # Start with Linux coverage + cp "$LINUX_COVERAGE" merged.info + + # Add Windows coverage if available (normalize backslash paths first) + if [ -f "$WINDOWS_COVERAGE" ]; then + sed -i 's|\\|/|g' "$WINDOWS_COVERAGE" + lcov --add-tracefile "$LINUX_COVERAGE" \ + --add-tracefile "$WINDOWS_COVERAGE" \ + --output-file merged.info --quiet + fi + + # Filter to source files only (excludes test temp dirs) + lcov --extract merged.info "$SRC_FILTER" --output-file coverage.info --quiet + + # Generate HTML report + genhtml coverage.info --output-directory coverage-html --quiet + + # Parse summary stats + LINES_FOUND=$(awk -F: '/^LF:/{s+=$2} END{print s+0}' coverage.info) + LINES_HIT=$(awk -F: '/^LH:/{s+=$2} END{print s+0}' coverage.info) + if [ "$LINES_FOUND" -gt 0 ]; then + COVERAGE=$(awk "BEGIN{printf \"%.1f\", ($LINES_HIT/$LINES_FOUND)*100}") + else + COVERAGE="0.0" + fi + + # Determine coverage threshold status + THRESHOLD_MET=$(awk "BEGIN{print ($COVERAGE >= $MIN_COVERAGE) ? 1 : 0}") + if [ "$THRESHOLD_MET" = "1" ]; then + THRESHOLD_LABEL="met" + THRESHOLD_MET_OUTPUT="true" + else + THRESHOLD_LABEL="**not met**" + THRESHOLD_MET_OUTPUT="false" + fi + + # Per-directory coverage table + DIR_TABLE=$(awk -F: ' + /^SF:/ { + file = $2 + n = split(file, p, "/") + dir = (n >= 3) ? p[1] "/" p[2] : (n == 2 ? p[1] : file) + } + /^LF:/ { lf[dir] += $2 } + /^LH:/ { lh[dir] += $2 } + END { + for (d in lf) { + pct = (lf[d] > 0) ? (lh[d] / lf[d]) * 100 : 0 + printf "| `%s/` | %.1f%% | %d / %d |\n", d, pct, lh[d], lf[d] + } + } + ' coverage.info | sort) + + # Check if Windows coverage was merged + if [ -f "$WINDOWS_COVERAGE" ]; then + PLATFORMS="Linux + Windows" + else + PLATFORMS="Linux only" + fi + + # Build report markdown + { + echo "## Code Coverage" + echo "" + echo "| Metric | Value |" + echo "|--------|-------|" + echo "| **Lines** | **${COVERAGE}%** (${LINES_HIT} / ${LINES_FOUND}) |" + echo "| **Threshold** | ${MIN_COVERAGE}% minimum — ${THRESHOLD_LABEL} |" + echo "| **Platforms** | ${PLATFORMS} |" + echo "" + echo "Full HTML report available in [workflow artifacts](${RUN_URL}#artifacts)." + echo "" + echo "
" + echo "Per-directory breakdown" + echo "" + echo "| Directory | Coverage | Lines |" + echo "|-----------|----------|-------|" + echo "${DIR_TABLE}" + echo "" + echo "
" + } > /tmp/coverage-report.md + + # Write to GitHub Actions job summary + cat /tmp/coverage-report.md >> "$GITHUB_STEP_SUMMARY" + + # Post/update PR comment (skipped for fork PRs and non-PR events) + if [ "$EVENT_NAME" = "pull_request" ] && [ "$IS_FORK" != "true" ]; then + BODY=" + $(cat /tmp/coverage-report.md)" + + gh pr comment "${PR_NUMBER}" --body "${BODY}" --edit-last 2>/dev/null || \ + gh pr comment "${PR_NUMBER}" --body "${BODY}" + fi + + # Write outputs + echo "coverage=$COVERAGE" >> "$GITHUB_OUTPUT" + echo "threshold-met=$THRESHOLD_MET_OUTPUT" >> "$GITHUB_OUTPUT" + echo "html-report-path=coverage-html" >> "$GITHUB_OUTPUT" diff --git a/.github/actions/setup-bun-project/action.yml b/.github/actions/setup-bun-project/action.yml new file mode 100644 index 00000000..c3b4b79b --- /dev/null +++ b/.github/actions/setup-bun-project/action.yml @@ -0,0 +1,24 @@ +name: Setup Bun Project +description: Install toolchain via moonrepo and project dependencies via bun + +inputs: + cache: + description: Enable moonrepo toolchain caching + required: false + default: "false" + cache-base: + description: Base branch for moonrepo toolchain cache + required: false + default: "" + +runs: + using: composite + steps: + - uses: moonrepo/setup-toolchain@261c62cb5b0f580c7be7c8cd0f023a2e96756095 # v0 + with: + auto-install: true + cache: ${{ inputs.cache }} + cache-base: ${{ inputs.cache-base }} + - name: Install dependencies + shell: bash + run: bun install --frozen-lockfile diff --git a/.github/workflows/code-pull-request.yml b/.github/workflows/code-pull-request.yml index b027476e..7afe86bf 100644 --- a/.github/workflows/code-pull-request.yml +++ b/.github/workflows/code-pull-request.yml @@ -33,19 +33,16 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: fetch-depth: 0 - - uses: moonrepo/setup-toolchain@261c62cb5b0f580c7be7c8cd0f023a2e96756095 # v0 - with: - auto-install: true - cache: true - cache-base: main - name: Restore Bun Package Cache id: restore-bun-cache uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 with: path: /home/runner/.bun/install/cache key: bun-packages-${{ runner.os }}-v1-${{ hashFiles('bun.lock') }} - - name: Install dependencies - run: bun install --frozen-lockfile + - uses: ./.github/actions/setup-bun-project + with: + cache: "true" + cache-base: main - name: Validate commit messages if: github.event_name == 'pull_request' env: @@ -105,120 +102,52 @@ jobs: path: coverage-windows continue-on-error: true - name: Merge coverage and generate report - env: - GH_TOKEN: ${{ github.token }} - run: | - sudo apt-get install -y -qq lcov > /dev/null 2>&1 - - # Start with Linux coverage - cp coverage-linux/lcov.info merged.info - - # Add Windows coverage if available (normalize backslash paths first) - if [ -f coverage-windows/lcov.info ]; then - sed -i 's|\\|/|g' coverage-windows/lcov.info - lcov --add-tracefile coverage-linux/lcov.info \ - --add-tracefile coverage-windows/lcov.info \ - --output-file merged.info --quiet - fi - - # Filter to src/ only (excludes test temp dirs) - lcov --extract merged.info 'src/*' --output-file coverage.info --quiet - - # Generate HTML report - genhtml coverage.info --output-directory coverage-html --quiet - - # Parse summary stats - LINES_FOUND=$(awk -F: '/^LF:/{s+=$2} END{print s+0}' coverage.info) - LINES_HIT=$(awk -F: '/^LH:/{s+=$2} END{print s+0}' coverage.info) - if [ "$LINES_FOUND" -gt 0 ]; then - COVERAGE=$(awk "BEGIN{printf \"%.1f\", ($LINES_HIT/$LINES_FOUND)*100}") - else - COVERAGE="0.0" - fi - - # Per-directory coverage table - DIR_TABLE=$(awk -F: ' - /^SF:/ { - file = $2 - n = split(file, p, "/") - dir = (n >= 3) ? p[1] "/" p[2] : (n == 2 ? p[1] : file) - } - /^LF:/ { lf[dir] += $2 } - /^LH:/ { lh[dir] += $2 } - END { - for (d in lf) { - pct = (lf[d] > 0) ? (lh[d] / lf[d]) * 100 : 0 - printf "| `%s/` | %.1f%% | %d / %d |\n", d, pct, lh[d], lf[d] - } - } - ' coverage.info | sort) - - # Check if Windows coverage was merged - if [ -f coverage-windows/lcov.info ]; then - PLATFORMS="Linux + Windows" - else - PLATFORMS="Linux only" - fi - - RUN_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" - - # Build report markdown - { - echo "## Code Coverage" - echo "" - echo "| Metric | Value |" - echo "|--------|-------|" - echo "| **Lines** | **${COVERAGE}%** (${LINES_HIT} / ${LINES_FOUND}) |" - echo "| **Platforms** | ${PLATFORMS} |" - echo "" - echo "Full HTML report available in [workflow artifacts](${RUN_URL}#artifacts)." - echo "" - echo "
" - echo "Per-directory breakdown" - echo "" - echo "| Directory | Coverage | Lines |" - echo "|-----------|----------|-------|" - echo "${DIR_TABLE}" - echo "" - echo "
" - } > /tmp/coverage-report.md - - # Write to GitHub Actions job summary - cat /tmp/coverage-report.md >> "$GITHUB_STEP_SUMMARY" - - # Post/update PR comment (skipped for fork PRs and non-PR events) - if [ "${{ github.event_name }}" = "pull_request" ] && \ - [ "${{ github.event.pull_request.head.repo.fork }}" != "true" ]; then - PR_NUM="${{ github.event.pull_request.number }}" - BODY=" - $(cat /tmp/coverage-report.md)" - - gh pr comment "${PR_NUM}" --body "${BODY}" --edit-last 2>/dev/null || \ - gh pr comment "${PR_NUM}" --body "${BODY}" - fi + id: coverage-report + uses: ./.github/actions/coverage-report + with: + min-coverage: "90" + github-token: ${{ github.token }} + event-name: ${{ github.event_name }} + pr-number: ${{ github.event.pull_request.number }} + is-fork: ${{ github.event.pull_request.head.repo.fork }} + run-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - name: Upload coverage report + if: always() && steps.coverage-report.outcome == 'success' uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: name: coverage-report - path: coverage-html + path: ${{ steps.coverage-report.outputs.html-report-path }} retention-days: 30 + - name: Enforce coverage threshold + if: always() && steps.coverage-report.outcome == 'success' + run: | + COVERAGE="${{ steps.coverage-report.outputs.coverage }}" + MIN_COVERAGE=90 + BELOW=$(awk "BEGIN{print ($COVERAGE < $MIN_COVERAGE) ? 1 : 0}") + if [ "$BELOW" = "1" ]; then + echo "::error::Code coverage is ${COVERAGE}%, which is below the minimum threshold of ${MIN_COVERAGE}%." + exit 1 + fi + echo "Coverage ${COVERAGE}% meets the minimum threshold of ${MIN_COVERAGE}%." # Gate job — single required status check for branch protection. status: name: Validate Code runs-on: ubuntu-latest if: always() - needs: [validate, smoke-windows, smoke-linux] + needs: [validate, smoke-windows, smoke-linux, coverage] steps: - name: Check job results run: | if [[ "${{ needs.validate.result }}" != "success" ]] || \ [[ "${{ needs.smoke-windows.result }}" != "success" ]] || \ - [[ "${{ needs.smoke-linux.result }}" != "success" ]]; then + [[ "${{ needs.smoke-linux.result }}" != "success" ]] || \ + [[ "${{ needs.coverage.result }}" != "success" ]]; then echo "::error::One or more jobs failed:" echo " validate: ${{ needs.validate.result }}" echo " smoke-windows: ${{ needs.smoke-windows.result }}" echo " smoke-linux: ${{ needs.smoke-linux.result }}" + echo " coverage: ${{ needs.coverage.result }}" exit 1 fi echo "All checks passed." diff --git a/.github/workflows/smoke-test-linux.yml b/.github/workflows/smoke-test-linux.yml index 84910562..561e24b0 100644 --- a/.github/workflows/smoke-test-linux.yml +++ b/.github/workflows/smoke-test-linux.yml @@ -20,12 +20,7 @@ jobs: with: fetch-depth: 0 - - uses: moonrepo/setup-toolchain@261c62cb5b0f580c7be7c8cd0f023a2e96756095 # v0 - with: - auto-install: true - - - name: Install dependencies - run: bun install --frozen-lockfile + - uses: ./.github/actions/setup-bun-project - name: Build Linux binary run: bun build src/cli.ts --compile --bytecode --outfile dist/archgate-smoke-test diff --git a/.github/workflows/smoke-test-windows.yml b/.github/workflows/smoke-test-windows.yml index f3b7e17d..c55d2f68 100644 --- a/.github/workflows/smoke-test-windows.yml +++ b/.github/workflows/smoke-test-windows.yml @@ -20,12 +20,7 @@ jobs: with: fetch-depth: 0 - - uses: moonrepo/setup-toolchain@261c62cb5b0f580c7be7c8cd0f023a2e96756095 # v0 - with: - auto-install: true - - - name: Install dependencies - run: bun install --frozen-lockfile + - uses: ./.github/actions/setup-bun-project - name: Lint run: bun run lint