diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 13d6872..854806e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,6 +5,9 @@ on: branches: [main, master] pull_request: +permissions: + contents: read + # Everything below runs fully offline after dependency install: # no LLM, no API key, no external service. jobs: @@ -64,3 +67,4 @@ jobs: - name: Claude Code plugin bundle is reproducible if: matrix.os == 'ubuntu-latest' run: git diff --exit-code integrations/claude-code-plugin/specbridge/dist + diff --git a/.github/workflows/skill-verification.yml b/.github/workflows/skill-verification.yml new file mode 100644 index 0000000..22ee4cd --- /dev/null +++ b/.github/workflows/skill-verification.yml @@ -0,0 +1,113 @@ +# Verifies the 11 specbridge-* agent skills against each build of the CLI +# using the Agent Skill Verifier +# (https://github.com/HelloThisWorld/agent-skill-verification-template): +# +# 1. the skill fixture workspace is rebuilt from this build's CLI output +# and diffed against the committed grounding facts (snapshots, +# WORKSPACE.md, spec files) — a CLI output change that would break the +# skills' citations fails here; +# 2. every skill contract + evaluation case file is statically validated +# with a pinned, checksum-verified agent-skill-verifier release. +# +# Full model-driven eval runs (verify --adapter llm) stay a local/manual +# step because they need a live model endpoint; everything here is +# deterministic and offline apart from the two pinned GitHub downloads. +# +# This is a standalone workflow (not a job in ci.yml) so the README badge +# reflects exactly the skill-test outcome. +name: skill test + +on: + push: + branches: [main, master] + pull_request: + +permissions: + contents: read + +jobs: + skill-verification: + name: agent skill verification + runs-on: ubuntu-latest + env: + # Pinned verifier release. Update the version and the asset SHA-256 + # (from the release's SHA256SUMS.txt) together. + ASV_VERSION: 0.1.0 + ASV_SHA256: 2307aa1ee6a8698fe5d7dfc12ef9413ce086e5f3fe3748501abd0d10cfe68084 + # Ref of the verification template holding the specbridge skill + # contracts, evaluation cases, and fixture builder. + TEMPLATE_REF: v0.1.0 + steps: + - uses: actions/checkout@v4 + + - uses: pnpm/action-setup@v4 + # Version comes from the "packageManager" field in package.json. + + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: pnpm + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Build the SpecBridge CLI + run: pnpm build + + - name: Checkout the verification template (pinned ref) + uses: actions/checkout@v4 + with: + repository: HelloThisWorld/agent-skill-verification-template + ref: ${{ env.TEMPLATE_REF }} + path: verification-template + + - name: Download the pinned agent-skill-verifier release and verify its checksum + run: | + curl -fsSLO "https://github.com/HelloThisWorld/agent-skill-verification-template/releases/download/v${ASV_VERSION}/agent-skill-verifier-v${ASV_VERSION}-linux-x64.tar.gz" + echo "${ASV_SHA256} agent-skill-verifier-v${ASV_VERSION}-linux-x64.tar.gz" | sha256sum -c - + mkdir -p .asv + tar -xzf "agent-skill-verifier-v${ASV_VERSION}-linux-x64.tar.gz" -C .asv + .asv/agent-skill-verifier --version + + - name: Rebuild the skill fixture from this build's CLI (grounding drift check) + env: + SPECBRIDGE_REPO: ${{ github.workspace }} + run: | + node verification-template/scripts/build-specbridge-fixture.mjs + # runner-list contains machine-dependent capability probes, so it is + # compared on its environment-independent fields by the dedicated + # script instead of byte equality. + git -C verification-template show HEAD:fixtures/specbridge-workspace/snapshots/runner-list.json > /tmp/runner-list-committed.json + node scripts/check-skill-runner-snapshot.mjs \ + /tmp/runner-list-committed.json \ + verification-template/fixtures/specbridge-workspace/snapshots/runner-list.json + # Timestamps, install-record ids, and local archive paths differ per + # rebuild by design; every other grounded fact (snapshots, + # WORKSPACE.md, specs, manifests, hashes) must be byte-identical. + if ! git -C verification-template diff --exit-code -- \ + fixtures/specbridge-workspace \ + ':(exclude)fixtures/specbridge-workspace/.specbridge/extensions/grants.json' \ + ':(exclude)fixtures/specbridge-workspace/.specbridge/extensions/records.jsonl' \ + ':(exclude)fixtures/specbridge-workspace/.specbridge/extensions/state.json' \ + ':(exclude)fixtures/specbridge-workspace/snapshots/runner-list.json'; then + echo "::error::SpecBridge CLI output drifted from the committed skill-verification fixture. If the change is intentional, regenerate the fixture in agent-skill-verification-template (scripts/build-specbridge-fixture.mjs) and pin a new TEMPLATE_REF." + exit 1 + fi + + - name: Validate every specbridge skill with agent-skill-verifier + working-directory: verification-template + run: | + fail=0 + for dir in skills/specbridge-*/; do + name="$(basename "$dir")" + for cases in "testcases/${name}.json" "testcases/${name}-negative.json"; do + [ -f "$cases" ] || continue + if "$GITHUB_WORKSPACE/.asv/agent-skill-verifier" validate --skill "$dir" --cases "$cases" --quiet; then + echo "ok: ${name} / $(basename "$cases")" + else + echo "::error::skill validation failed: ${name} / $(basename "$cases")" + fail=1 + fi + done + done + exit "$fail" diff --git a/README.md b/README.md index 47013e1..a05ecd6 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # SpecBridge [![CI](https://github.com/HelloThisWorld/specbridge/actions/workflows/ci.yml/badge.svg)](https://github.com/HelloThisWorld/specbridge/actions/workflows/ci.yml) +[![skill test](https://github.com/HelloThisWorld/specbridge/actions/workflows/skill-verification.yml/badge.svg)](https://github.com/HelloThisWorld/agent-skill-verification-template) [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE) An open, model-agnostic spec runtime for existing Kiro projects. diff --git a/scripts/check-skill-runner-snapshot.mjs b/scripts/check-skill-runner-snapshot.mjs new file mode 100644 index 0000000..7d142f5 --- /dev/null +++ b/scripts/check-skill-runner-snapshot.mjs @@ -0,0 +1,44 @@ +// Compare the environment-independent fields of a SpecBridge `runner list` +// snapshot against a freshly generated one: the profile set, each profile's +// implementation, and its enablement. Capability probes (capabilities, +// version, authentication) legitimately differ per machine — the committed +// skill-verification snapshot was generated on a workstation with runner +// CLIs installed, while CI runners have none — so those fields are ignored. +// +// Used by the skill-verification CI job together with the byte-level drift +// check that covers every other fixture file. +// +// usage: node scripts/check-skill-runner-snapshot.mjs +import { readFileSync } from "node:fs"; + +const [committedPath, rebuiltPath] = process.argv.slice(2); +if (!committedPath || !rebuiltPath) { + console.error("usage: node scripts/check-skill-runner-snapshot.mjs "); + process.exit(2); +} + +const normalize = (path) => { + const raw = JSON.parse(readFileSync(path, "utf8")); + return (raw.profiles ?? []) + .map((profile) => ({ + profile: profile.profile, + implementation: profile.implementation, + enabled: profile.enabled, + })) + .sort((a, b) => a.profile.localeCompare(b.profile)); +}; + +const committed = JSON.stringify(normalize(committedPath), null, 2); +const rebuilt = JSON.stringify(normalize(rebuiltPath), null, 2); + +if (committed !== rebuilt) { + console.error("runner-list drift in environment-independent fields (profile/implementation/enabled):"); + console.error("--- committed (verification template)"); + console.error(committed); + console.error("+++ rebuilt (this build)"); + console.error(rebuilt); + process.exit(1); +} +console.log( + `runner-list: ${normalize(rebuiltPath).length} profiles match the committed snapshot on all environment-independent fields.`, +);