Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
d941b66
chore: ignore .worktrees/ directory
iliassjabali Apr 12, 2026
9e3e1d7
docs: add mutation testing design spec
iliassjabali Apr 12, 2026
6740326
docs: add mutation testing implementation plan
iliassjabali Apr 12, 2026
89c50ba
chore: add stryker mutation testing dev deps
iliassjabali Apr 12, 2026
c09f010
chore: add explicit vitest configs to adapter-claude, mcp-server, cli
iliassjabali Apr 12, 2026
54cf8f4
chore(cli): add stryker-only vitest config that excludes cli.test.ts
iliassjabali Apr 12, 2026
bc4dadd
chore: add root stryker config with shared defaults
iliassjabali Apr 12, 2026
fffbb99
chore: add per-package mutation scripts to root package.json
iliassjabali Apr 12, 2026
8c96bca
chore: ignore .stryker-tmp and reports/mutation directories
iliassjabali Apr 12, 2026
d699f53
fix: convert stryker config to mjs with env-var overrides
iliassjabali Apr 12, 2026
19e5af5
fix: add vitest dir option to stryker config for correct test discovery
iliassjabali Apr 12, 2026
dc73429
chore: set mutation thresholds from pilot scores (actual - 5)
iliassjabali Apr 12, 2026
70ba662
docs: record pilot mutation scores in design spec
iliassjabali Apr 12, 2026
5b70cc5
ci: add mutation testing workflow with per-package matrix
iliassjabali Apr 12, 2026
6973315
docs: add mutation testing user guide
iliassjabali Apr 12, 2026
2eb648c
docs: move mutation testing docs into guides/ and decisions/
iliassjabali Apr 12, 2026
41f85b1
fix(ci): specify pnpm version 10 in mutation workflow
iliassjabali Apr 12, 2026
270801f
docs: simplify mutation testing to a single guide, remove decision docs
iliassjabali Apr 12, 2026
db3ea42
refactor: replace 6 mutation scripts with single wrapper + threshold map
iliassjabali Apr 12, 2026
d90a354
ci: add per-package coverage report with PR comment
iliassjabali Apr 12, 2026
7ff1b1a
fix(ci): tolerate test failures in coverage step, skip report if no o…
iliassjabali Apr 12, 2026
b0e2a85
ci: unify coverage + mutation into single Quality workflow with PR co…
iliassjabali Apr 12, 2026
ba4fb43
fix(ci): extract mutation score from JSON report instead of parsing A…
iliassjabali Apr 12, 2026
a839a88
feat: add survivor annotations, score trend deltas, and auto-ratchet
iliassjabali Apr 12, 2026
6d3ecb2
fix: address code review findings
iliassjabali Apr 12, 2026
2f6c11f
fix: address Copilot review comments
iliassjabali Apr 12, 2026
cfc9a46
Merge branch 'main' into chore/mutation-testing
iliassjabali Apr 12, 2026
708019e
Merge branch 'main' into chore/mutation-testing
iliassjabali Apr 12, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
114 changes: 114 additions & 0 deletions .github/workflows/auto-ratchet.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
name: Auto-ratchet thresholds
on:
push:
branches: [main]
paths:
- 'packages/*/src/**'
- 'stryker.config.mjs'

permissions:
contents: write
pull-requests: write

jobs:
ratchet:
name: Check & ratchet
runs-on: ubuntu-latest
timeout-minutes: 120
steps:
- name: Checkout
uses: actions/checkout@v4

- name: Setup pnpm
uses: pnpm/action-setup@v4
with:
version: 10

- name: Setup Node
uses: actions/setup-node@v4
with:
node-version: '20'
cache: 'pnpm'

- name: Install dependencies
run: pnpm install --frozen-lockfile

- name: Build all packages
run: pnpm -r build

- name: Run mutation for all packages
run: pnpm mutation || true

- name: Update baseline scores (merge, don't overwrite)
run: |
node -e "
import { PACKAGES } from './stryker.config.mjs';
import { readFileSync, writeFileSync } from 'fs';
let existing = {};
try { existing = JSON.parse(readFileSync('quality-baseline.json','utf8')); } catch {}
const baseline = { ...existing };
for (const pkg of PACKAGES) {
const prev = existing[pkg] || {};
let coverage = prev.coverage ?? null;
let mutation = prev.mutation ?? null;
try {
coverage = JSON.parse(readFileSync('packages/' + pkg + '/coverage/coverage-summary.json','utf8')).total.lines.pct;
} catch {}
try {
const r = JSON.parse(readFileSync('reports/mutation/' + pkg + '.json','utf8'));
const t = Object.values(r.files).reduce((a,f)=>{a.k+=f.mutants.filter(m=>m.status==='Killed').length;a.t+=f.mutants.length;return a},{k:0,t:0});
mutation = parseFloat((t.t?t.k/t.t*100:0).toFixed(2));
} catch {}
baseline[pkg] = { coverage, mutation };
}
writeFileSync('quality-baseline.json', JSON.stringify(baseline, null, 2) + '\n');
" --input-type=module

- name: Try auto-ratchet
id: ratchet
run: |
if node scripts/auto-ratchet.mjs; then
echo "changed=true" >> "$GITHUB_OUTPUT"
else
echo "changed=false" >> "$GITHUB_OUTPUT"
fi

- name: Open PR with updated baseline and thresholds
if: steps.ratchet.outputs.changed == 'true'
run: |
git config user.name "github-actions[bot]"
git config user.email "github-actions[bot]@users.noreply.github.com"
branch="auto-ratchet/$(date +%Y%m%d)-$(git rev-parse --short HEAD)"
git checkout -b "$branch"
git add stryker.config.mjs quality-baseline.json
git commit -m "chore: auto-ratchet mutation thresholds and update baseline"
git push origin "$branch"
gh pr create \
--base main \
--head "$branch" \
--title "chore: auto-ratchet mutation thresholds" \
--body "Mutation scores have improved beyond their thresholds by more than 5 points. This PR bumps the thresholds to lock in the gains and updates the quality baseline."
env:
GH_TOKEN: ${{ github.token }}

- name: Open PR with updated baseline only
if: steps.ratchet.outputs.changed != 'true'
run: |
if git diff --quiet quality-baseline.json; then
echo "No baseline changes"
exit 0
fi
git config user.name "github-actions[bot]"
git config user.email "github-actions[bot]@users.noreply.github.com"
branch="quality-baseline/$(date +%Y%m%d)-$(git rev-parse --short HEAD)"
git checkout -b "$branch"
git add quality-baseline.json
git commit -m "chore: update quality baseline scores"
git push origin "$branch"
gh pr create \
--base main \
--head "$branch" \
--title "chore: update quality baseline scores" \
--body "Post-merge quality baseline update. No threshold changes."
env:
GH_TOKEN: ${{ github.token }}
2 changes: 1 addition & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ on:
branches: [main]
workflow_dispatch:

# Minimal permissions CI only needs to read code
# Minimal permissions -- CI only needs to read code
permissions:
contents: read

Expand Down
205 changes: 205 additions & 0 deletions .github/workflows/mutation.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,205 @@
name: Quality
on:
pull_request:
types: [opened, synchronize, reopened]

permissions:
contents: read
pull-requests: write
issues: write

jobs:
quality:
name: quality / ${{ matrix.package }}
runs-on: ubuntu-latest
timeout-minutes: 90
strategy:
fail-fast: false
matrix:
package: [adapter-claude, sdk, mcp-server, cli, sidecar]
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 1

- name: Setup pnpm
uses: pnpm/action-setup@v4
with:
version: 10

- name: Setup Node
uses: actions/setup-node@v4
with:
node-version: '20'
cache: 'pnpm'

- name: Install dependencies
run: pnpm install --frozen-lockfile

- name: Build all packages
run: pnpm -r build

- name: Run tests with coverage
continue-on-error: true
working-directory: packages/${{ matrix.package }}
run: npx vitest run --coverage --coverage.reporter=json-summary --coverage.reportsDirectory=coverage

- name: Restore Stryker incremental cache
uses: actions/cache@v4
with:
path: .stryker-tmp/${{ matrix.package }}-incremental.json
key: stryker-${{ matrix.package }}-${{ hashFiles(format('packages/{0}/src/**', matrix.package), 'stryker.config.mjs') }}
restore-keys: |
stryker-${{ matrix.package }}-

- name: Run Stryker
id: stryker-run
run: |
if pnpm mutation ${{ matrix.package }}; then
echo "failed=false" >> "$GITHUB_OUTPUT"
else
echo "failed=true" >> "$GITHUB_OUTPUT"
fi

- name: Annotate surviving mutants on PR diff
if: always()
run: node scripts/annotate-survivors.mjs reports/mutation/${{ matrix.package }}.json || true

- name: Extract mutation score
if: always()
id: stryker
run: |
file="reports/mutation/${{ matrix.package }}.json"
if [ -f "$file" ]; then
score=$(node -e "const r=JSON.parse(require('fs').readFileSync('$file','utf8'));const t=Object.values(r.files).reduce((a,f)=>{a.k+=f.mutants.filter(m=>m.status==='Killed').length;a.t+=f.mutants.length;return a},{k:0,t:0});console.log((t.t?t.k/t.t*100:0).toFixed(2))")
echo "score=${score}" >> "$GITHUB_OUTPUT"
else
echo "score=N/A" >> "$GITHUB_OUTPUT"
fi

- name: Extract coverage percentage
if: always()
id: coverage
run: |
file="packages/${{ matrix.package }}/coverage/coverage-summary.json"
if [ -f "$file" ]; then
pct=$(node -e "const s=JSON.parse(require('fs').readFileSync('$file','utf8'));console.log(s.total.lines.pct)")
echo "lines=${pct}" >> "$GITHUB_OUTPUT"
else
echo "lines=N/A" >> "$GITHUB_OUTPUT"
fi

- name: Write score artifact
if: always()
run: |
mkdir -p .quality-scores
echo '{"package":"${{ matrix.package }}","coverage":"${{ steps.coverage.outputs.lines }}","mutation":"${{ steps.stryker.outputs.score }}"}' > .quality-scores/${{ matrix.package }}.json

- name: Upload scores
if: always()
uses: actions/upload-artifact@v4
with:
name: quality-score-${{ matrix.package }}
path: .quality-scores/${{ matrix.package }}.json

- name: Upload Stryker HTML report
if: always()
uses: actions/upload-artifact@v4
with:
name: mutation-report-${{ matrix.package }}
path: reports/mutation/${{ matrix.package }}.html
retention-days: 14

- name: Enforce mutation threshold
if: steps.stryker-run.outputs.failed == 'true'
run: |
echo "Mutation score below threshold for ${{ matrix.package }}"
exit 1

comment:
name: PR Comment
needs: quality
if: always() && github.event.pull_request.head.repo.full_name == github.repository
runs-on: ubuntu-latest
steps:
- name: Checkout (for baseline file)
uses: actions/checkout@v4
with:
ref: main
sparse-checkout: quality-baseline.json
sparse-checkout-cone-mode: false

- name: Download all score artifacts
uses: actions/download-artifact@v4
with:
pattern: quality-score-*
merge-multiple: true
path: scores

- name: Build and post comment
uses: actions/github-script@v7
with:
script: |
const fs = require('fs');

// Load baseline from main (may not exist yet)
let baseline = {};
try { baseline = JSON.parse(fs.readFileSync('quality-baseline.json', 'utf8')); } catch {}

const files = fs.readdirSync('scores').filter(f => f.endsWith('.json'));
const rows = files
.map(f => JSON.parse(fs.readFileSync(`scores/${f}`, 'utf8')))
.sort((a, b) => a.package.localeCompare(b.package))
.map(r => {
const base = baseline[r.package] || {};
const fmtDelta = (cur, prev) => {
if (cur === 'N/A' || prev == null) return '';
const d = (parseFloat(cur) - prev).toFixed(1);
if (d > 0) return ` (+${d})`;
if (d < 0) return ` (${d})`;
return '';
};
const cov = r.coverage === 'N/A' ? 'N/A' : r.coverage + '%' + fmtDelta(r.coverage, base.coverage);
const mut = r.mutation === 'N/A' ? 'N/A' : r.mutation + '%' + fmtDelta(r.mutation, base.mutation);
return `| ${r.package} | ${cov} | ${mut} |`;
})
.join('\n');

const body = [
'## Quality Report',
'',
'| Package | Line Coverage | Mutation Score |',
'|---------|-------------|----------------|',
rows,
'',
'> Coverage = % of lines your tests touch. Mutation = % of code changes your tests catch.',
'> Deltas shown vs `main` baseline. Mutation reports available as workflow artifacts.',
].join('\n');

const marker = '<!-- quality-report -->';
const fullBody = marker + '\n' + body;

const { data: comments } = await github.rest.issues.listComments({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
});

const existing = comments.find(c => c.body?.includes(marker));

if (existing) {
await github.rest.issues.updateComment({
owner: context.repo.owner,
repo: context.repo.repo,
comment_id: existing.id,
body: fullBody,
});
} else {
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
body: fullBody,
});
}
4 changes: 4 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,10 @@ packages/sdk-python/.venv/
# ── Worktrees ─────────────────────────────────────────────────────────────────
.worktrees/

# ── Stryker (mutation testing) ────────────────────────────────────────────────
.stryker-tmp/
reports/mutation/

# ── Misc ──────────────────────────────────────────────────────────────────────
*.ots
commit.txt
Loading
Loading