From fd033cdf256f17ab728e8046d0d3765530e446b4 Mon Sep 17 00:00:00 2001 From: HelloThisWorld Date: Fri, 17 Jul 2026 17:52:24 +0800 Subject: [PATCH 1/3] ci: verify agent skills against each build with Agent Skill Verifier MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a skill-verification job that, after building the CLI: 1. rebuilds the skill fixture workspace from this build's CLI output and diffs it against the grounding facts committed in agent-skill-verification-template@v0.1.0 (snapshots, WORKSPACE.md, specs, manifests) — CLI output drift that would break the specbridge-* skills' citations now fails CI with a remediation hint; 2. statically validates all 11 specbridge-* skill contracts and their evaluation cases with a pinned, checksum-verified agent-skill-verifier v0.1.0 release binary. Deterministic and offline apart from two pinned GitHub downloads; full model-driven eval runs remain a local/manual step. Also sets explicit read-only workflow permissions. --- .github/workflows/ci.yml | 95 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 13d6872..89dfb7a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,6 +5,9 @@ on: branches: [main, master] pull_request: +permissions: + contents: read + # Everything below runs fully offline after dependency install: # no LLM, no API key, no external service. jobs: @@ -64,3 +67,95 @@ jobs: - name: Claude Code plugin bundle is reproducible if: matrix.os == 'ubuntu-latest' run: git diff --exit-code integrations/claude-code-plugin/specbridge/dist + + # Verifies the 11 specbridge-* agent skills against THIS build of the CLI + # using the Agent Skill Verifier + # (https://github.com/HelloThisWorld/agent-skill-verification-template): + # + # 1. the skill fixture workspace is rebuilt from this build's CLI output + # and diffed against the committed grounding facts (snapshots, + # WORKSPACE.md, spec files) — a CLI output change that would break the + # skills' citations fails here; + # 2. every skill contract + evaluation case file is statically validated + # with a pinned, checksum-verified agent-skill-verifier release. + # + # Full model-driven eval runs (verify --adapter llm) stay a local/manual + # step because they need a live model endpoint; everything here is + # deterministic and offline apart from the two pinned GitHub downloads. + skill-verification: + name: agent skill verification + runs-on: ubuntu-latest + env: + # Pinned verifier release. Update the version and the asset SHA-256 + # (from the release's SHA256SUMS.txt) together. + ASV_VERSION: 0.1.0 + ASV_SHA256: 2307aa1ee6a8698fe5d7dfc12ef9413ce086e5f3fe3748501abd0d10cfe68084 + # Ref of the verification template holding the specbridge skill + # contracts, evaluation cases, and fixture builder. + TEMPLATE_REF: v0.1.0 + steps: + - uses: actions/checkout@v4 + + - uses: pnpm/action-setup@v4 + # Version comes from the "packageManager" field in package.json. + + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: pnpm + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Build the SpecBridge CLI + run: pnpm build + + - name: Checkout the verification template (pinned ref) + uses: actions/checkout@v4 + with: + repository: HelloThisWorld/agent-skill-verification-template + ref: ${{ env.TEMPLATE_REF }} + path: verification-template + + - name: Download the pinned agent-skill-verifier release and verify its checksum + run: | + curl -fsSLO "https://github.com/HelloThisWorld/agent-skill-verification-template/releases/download/v${ASV_VERSION}/agent-skill-verifier-v${ASV_VERSION}-linux-x64.tar.gz" + echo "${ASV_SHA256} agent-skill-verifier-v${ASV_VERSION}-linux-x64.tar.gz" | sha256sum -c - + mkdir -p .asv + tar -xzf "agent-skill-verifier-v${ASV_VERSION}-linux-x64.tar.gz" -C .asv + .asv/agent-skill-verifier --version + + - name: Rebuild the skill fixture from this build's CLI (grounding drift check) + env: + SPECBRIDGE_REPO: ${{ github.workspace }} + run: | + node verification-template/scripts/build-specbridge-fixture.mjs + # Timestamps, install-record ids, and local archive paths differ per + # rebuild by design; every grounded fact (snapshots, WORKSPACE.md, + # specs, manifests, hashes) must be byte-identical. + if ! git -C verification-template diff --exit-code -- \ + fixtures/specbridge-workspace \ + ':(exclude)fixtures/specbridge-workspace/.specbridge/extensions/grants.json' \ + ':(exclude)fixtures/specbridge-workspace/.specbridge/extensions/records.jsonl' \ + ':(exclude)fixtures/specbridge-workspace/.specbridge/extensions/state.json'; then + echo "::error::SpecBridge CLI output drifted from the committed skill-verification fixture. If the change is intentional, regenerate the fixture in agent-skill-verification-template (scripts/build-specbridge-fixture.mjs) and pin a new TEMPLATE_REF." + exit 1 + fi + + - name: Validate every specbridge skill with agent-skill-verifier + working-directory: verification-template + run: | + fail=0 + for dir in skills/specbridge-*/; do + name="$(basename "$dir")" + for cases in "testcases/${name}.json" "testcases/${name}-negative.json"; do + [ -f "$cases" ] || continue + if "$GITHUB_WORKSPACE/.asv/agent-skill-verifier" validate --skill "$dir" --cases "$cases" --quiet; then + echo "ok: ${name} / $(basename "$cases")" + else + echo "::error::skill validation failed: ${name} / $(basename "$cases")" + fail=1 + fi + done + done + exit "$fail" From 4ae5d4386bcb28e77519d70ef445112f887060b3 Mon Sep 17 00:00:00 2001 From: HelloThisWorld Date: Fri, 17 Jul 2026 18:03:22 +0800 Subject: [PATCH 2/3] ci: compare runner-list snapshot on environment-independent fields only The committed skill-verification snapshot embeds machine-dependent capability probes (runner CLIs installed, versions, authentication) from the workstation that generated it; CI runners have none of those installed, so byte-equality failed. runner-list is now excluded from the byte-level drift diff and checked by a dedicated script that compares the CLI-determined fields (profile set, implementation, enablement) and ignores the environment probes. A genuine CLI regression in the runner catalog still fails; a machine difference no longer does. --- .github/workflows/ci.yml | 14 ++++++-- scripts/check-skill-runner-snapshot.mjs | 44 +++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 3 deletions(-) create mode 100644 scripts/check-skill-runner-snapshot.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 89dfb7a..4cbd47b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -130,14 +130,22 @@ jobs: SPECBRIDGE_REPO: ${{ github.workspace }} run: | node verification-template/scripts/build-specbridge-fixture.mjs + # runner-list contains machine-dependent capability probes, so it is + # compared on its environment-independent fields by the dedicated + # script instead of byte equality. + git -C verification-template show HEAD:fixtures/specbridge-workspace/snapshots/runner-list.json > /tmp/runner-list-committed.json + node scripts/check-skill-runner-snapshot.mjs \ + /tmp/runner-list-committed.json \ + verification-template/fixtures/specbridge-workspace/snapshots/runner-list.json # Timestamps, install-record ids, and local archive paths differ per - # rebuild by design; every grounded fact (snapshots, WORKSPACE.md, - # specs, manifests, hashes) must be byte-identical. + # rebuild by design; every other grounded fact (snapshots, + # WORKSPACE.md, specs, manifests, hashes) must be byte-identical. if ! git -C verification-template diff --exit-code -- \ fixtures/specbridge-workspace \ ':(exclude)fixtures/specbridge-workspace/.specbridge/extensions/grants.json' \ ':(exclude)fixtures/specbridge-workspace/.specbridge/extensions/records.jsonl' \ - ':(exclude)fixtures/specbridge-workspace/.specbridge/extensions/state.json'; then + ':(exclude)fixtures/specbridge-workspace/.specbridge/extensions/state.json' \ + ':(exclude)fixtures/specbridge-workspace/snapshots/runner-list.json'; then echo "::error::SpecBridge CLI output drifted from the committed skill-verification fixture. If the change is intentional, regenerate the fixture in agent-skill-verification-template (scripts/build-specbridge-fixture.mjs) and pin a new TEMPLATE_REF." exit 1 fi diff --git a/scripts/check-skill-runner-snapshot.mjs b/scripts/check-skill-runner-snapshot.mjs new file mode 100644 index 0000000..7d142f5 --- /dev/null +++ b/scripts/check-skill-runner-snapshot.mjs @@ -0,0 +1,44 @@ +// Compare the environment-independent fields of a SpecBridge `runner list` +// snapshot against a freshly generated one: the profile set, each profile's +// implementation, and its enablement. Capability probes (capabilities, +// version, authentication) legitimately differ per machine — the committed +// skill-verification snapshot was generated on a workstation with runner +// CLIs installed, while CI runners have none — so those fields are ignored. +// +// Used by the skill-verification CI job together with the byte-level drift +// check that covers every other fixture file. +// +// usage: node scripts/check-skill-runner-snapshot.mjs +import { readFileSync } from "node:fs"; + +const [committedPath, rebuiltPath] = process.argv.slice(2); +if (!committedPath || !rebuiltPath) { + console.error("usage: node scripts/check-skill-runner-snapshot.mjs "); + process.exit(2); +} + +const normalize = (path) => { + const raw = JSON.parse(readFileSync(path, "utf8")); + return (raw.profiles ?? []) + .map((profile) => ({ + profile: profile.profile, + implementation: profile.implementation, + enabled: profile.enabled, + })) + .sort((a, b) => a.profile.localeCompare(b.profile)); +}; + +const committed = JSON.stringify(normalize(committedPath), null, 2); +const rebuilt = JSON.stringify(normalize(rebuiltPath), null, 2); + +if (committed !== rebuilt) { + console.error("runner-list drift in environment-independent fields (profile/implementation/enabled):"); + console.error("--- committed (verification template)"); + console.error(committed); + console.error("+++ rebuilt (this build)"); + console.error(rebuilt); + process.exit(1); +} +console.log( + `runner-list: ${normalize(rebuiltPath).length} profiles match the committed snapshot on all environment-independent fields.`, +); From 2e17631ddf2b0189423ac4b8066c55c2006647ba Mon Sep 17 00:00:00 2001 From: HelloThisWorld Date: Fri, 17 Jul 2026 23:32:53 +0800 Subject: [PATCH 3/3] ci: standalone skill-test workflow with README status badge Moves the skill-verification job into its own workflow (name: 'skill test') so the GitHub status badge reflects exactly the skill-test outcome, and adds that badge to the README linking to the agent-skill-verification-template repository. Job content is unchanged. --- .github/workflows/ci.yml | 99 -------------------- .github/workflows/skill-verification.yml | 113 +++++++++++++++++++++++ README.md | 1 + 3 files changed, 114 insertions(+), 99 deletions(-) create mode 100644 .github/workflows/skill-verification.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4cbd47b..854806e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,102 +68,3 @@ jobs: if: matrix.os == 'ubuntu-latest' run: git diff --exit-code integrations/claude-code-plugin/specbridge/dist - # Verifies the 11 specbridge-* agent skills against THIS build of the CLI - # using the Agent Skill Verifier - # (https://github.com/HelloThisWorld/agent-skill-verification-template): - # - # 1. the skill fixture workspace is rebuilt from this build's CLI output - # and diffed against the committed grounding facts (snapshots, - # WORKSPACE.md, spec files) — a CLI output change that would break the - # skills' citations fails here; - # 2. every skill contract + evaluation case file is statically validated - # with a pinned, checksum-verified agent-skill-verifier release. - # - # Full model-driven eval runs (verify --adapter llm) stay a local/manual - # step because they need a live model endpoint; everything here is - # deterministic and offline apart from the two pinned GitHub downloads. - skill-verification: - name: agent skill verification - runs-on: ubuntu-latest - env: - # Pinned verifier release. Update the version and the asset SHA-256 - # (from the release's SHA256SUMS.txt) together. - ASV_VERSION: 0.1.0 - ASV_SHA256: 2307aa1ee6a8698fe5d7dfc12ef9413ce086e5f3fe3748501abd0d10cfe68084 - # Ref of the verification template holding the specbridge skill - # contracts, evaluation cases, and fixture builder. - TEMPLATE_REF: v0.1.0 - steps: - - uses: actions/checkout@v4 - - - uses: pnpm/action-setup@v4 - # Version comes from the "packageManager" field in package.json. - - - uses: actions/setup-node@v4 - with: - node-version: 22 - cache: pnpm - - - name: Install dependencies - run: pnpm install --frozen-lockfile - - - name: Build the SpecBridge CLI - run: pnpm build - - - name: Checkout the verification template (pinned ref) - uses: actions/checkout@v4 - with: - repository: HelloThisWorld/agent-skill-verification-template - ref: ${{ env.TEMPLATE_REF }} - path: verification-template - - - name: Download the pinned agent-skill-verifier release and verify its checksum - run: | - curl -fsSLO "https://github.com/HelloThisWorld/agent-skill-verification-template/releases/download/v${ASV_VERSION}/agent-skill-verifier-v${ASV_VERSION}-linux-x64.tar.gz" - echo "${ASV_SHA256} agent-skill-verifier-v${ASV_VERSION}-linux-x64.tar.gz" | sha256sum -c - - mkdir -p .asv - tar -xzf "agent-skill-verifier-v${ASV_VERSION}-linux-x64.tar.gz" -C .asv - .asv/agent-skill-verifier --version - - - name: Rebuild the skill fixture from this build's CLI (grounding drift check) - env: - SPECBRIDGE_REPO: ${{ github.workspace }} - run: | - node verification-template/scripts/build-specbridge-fixture.mjs - # runner-list contains machine-dependent capability probes, so it is - # compared on its environment-independent fields by the dedicated - # script instead of byte equality. - git -C verification-template show HEAD:fixtures/specbridge-workspace/snapshots/runner-list.json > /tmp/runner-list-committed.json - node scripts/check-skill-runner-snapshot.mjs \ - /tmp/runner-list-committed.json \ - verification-template/fixtures/specbridge-workspace/snapshots/runner-list.json - # Timestamps, install-record ids, and local archive paths differ per - # rebuild by design; every other grounded fact (snapshots, - # WORKSPACE.md, specs, manifests, hashes) must be byte-identical. - if ! git -C verification-template diff --exit-code -- \ - fixtures/specbridge-workspace \ - ':(exclude)fixtures/specbridge-workspace/.specbridge/extensions/grants.json' \ - ':(exclude)fixtures/specbridge-workspace/.specbridge/extensions/records.jsonl' \ - ':(exclude)fixtures/specbridge-workspace/.specbridge/extensions/state.json' \ - ':(exclude)fixtures/specbridge-workspace/snapshots/runner-list.json'; then - echo "::error::SpecBridge CLI output drifted from the committed skill-verification fixture. If the change is intentional, regenerate the fixture in agent-skill-verification-template (scripts/build-specbridge-fixture.mjs) and pin a new TEMPLATE_REF." - exit 1 - fi - - - name: Validate every specbridge skill with agent-skill-verifier - working-directory: verification-template - run: | - fail=0 - for dir in skills/specbridge-*/; do - name="$(basename "$dir")" - for cases in "testcases/${name}.json" "testcases/${name}-negative.json"; do - [ -f "$cases" ] || continue - if "$GITHUB_WORKSPACE/.asv/agent-skill-verifier" validate --skill "$dir" --cases "$cases" --quiet; then - echo "ok: ${name} / $(basename "$cases")" - else - echo "::error::skill validation failed: ${name} / $(basename "$cases")" - fail=1 - fi - done - done - exit "$fail" diff --git a/.github/workflows/skill-verification.yml b/.github/workflows/skill-verification.yml new file mode 100644 index 0000000..22ee4cd --- /dev/null +++ b/.github/workflows/skill-verification.yml @@ -0,0 +1,113 @@ +# Verifies the 11 specbridge-* agent skills against each build of the CLI +# using the Agent Skill Verifier +# (https://github.com/HelloThisWorld/agent-skill-verification-template): +# +# 1. the skill fixture workspace is rebuilt from this build's CLI output +# and diffed against the committed grounding facts (snapshots, +# WORKSPACE.md, spec files) — a CLI output change that would break the +# skills' citations fails here; +# 2. every skill contract + evaluation case file is statically validated +# with a pinned, checksum-verified agent-skill-verifier release. +# +# Full model-driven eval runs (verify --adapter llm) stay a local/manual +# step because they need a live model endpoint; everything here is +# deterministic and offline apart from the two pinned GitHub downloads. +# +# This is a standalone workflow (not a job in ci.yml) so the README badge +# reflects exactly the skill-test outcome. +name: skill test + +on: + push: + branches: [main, master] + pull_request: + +permissions: + contents: read + +jobs: + skill-verification: + name: agent skill verification + runs-on: ubuntu-latest + env: + # Pinned verifier release. Update the version and the asset SHA-256 + # (from the release's SHA256SUMS.txt) together. + ASV_VERSION: 0.1.0 + ASV_SHA256: 2307aa1ee6a8698fe5d7dfc12ef9413ce086e5f3fe3748501abd0d10cfe68084 + # Ref of the verification template holding the specbridge skill + # contracts, evaluation cases, and fixture builder. + TEMPLATE_REF: v0.1.0 + steps: + - uses: actions/checkout@v4 + + - uses: pnpm/action-setup@v4 + # Version comes from the "packageManager" field in package.json. + + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: pnpm + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Build the SpecBridge CLI + run: pnpm build + + - name: Checkout the verification template (pinned ref) + uses: actions/checkout@v4 + with: + repository: HelloThisWorld/agent-skill-verification-template + ref: ${{ env.TEMPLATE_REF }} + path: verification-template + + - name: Download the pinned agent-skill-verifier release and verify its checksum + run: | + curl -fsSLO "https://github.com/HelloThisWorld/agent-skill-verification-template/releases/download/v${ASV_VERSION}/agent-skill-verifier-v${ASV_VERSION}-linux-x64.tar.gz" + echo "${ASV_SHA256} agent-skill-verifier-v${ASV_VERSION}-linux-x64.tar.gz" | sha256sum -c - + mkdir -p .asv + tar -xzf "agent-skill-verifier-v${ASV_VERSION}-linux-x64.tar.gz" -C .asv + .asv/agent-skill-verifier --version + + - name: Rebuild the skill fixture from this build's CLI (grounding drift check) + env: + SPECBRIDGE_REPO: ${{ github.workspace }} + run: | + node verification-template/scripts/build-specbridge-fixture.mjs + # runner-list contains machine-dependent capability probes, so it is + # compared on its environment-independent fields by the dedicated + # script instead of byte equality. + git -C verification-template show HEAD:fixtures/specbridge-workspace/snapshots/runner-list.json > /tmp/runner-list-committed.json + node scripts/check-skill-runner-snapshot.mjs \ + /tmp/runner-list-committed.json \ + verification-template/fixtures/specbridge-workspace/snapshots/runner-list.json + # Timestamps, install-record ids, and local archive paths differ per + # rebuild by design; every other grounded fact (snapshots, + # WORKSPACE.md, specs, manifests, hashes) must be byte-identical. + if ! git -C verification-template diff --exit-code -- \ + fixtures/specbridge-workspace \ + ':(exclude)fixtures/specbridge-workspace/.specbridge/extensions/grants.json' \ + ':(exclude)fixtures/specbridge-workspace/.specbridge/extensions/records.jsonl' \ + ':(exclude)fixtures/specbridge-workspace/.specbridge/extensions/state.json' \ + ':(exclude)fixtures/specbridge-workspace/snapshots/runner-list.json'; then + echo "::error::SpecBridge CLI output drifted from the committed skill-verification fixture. If the change is intentional, regenerate the fixture in agent-skill-verification-template (scripts/build-specbridge-fixture.mjs) and pin a new TEMPLATE_REF." + exit 1 + fi + + - name: Validate every specbridge skill with agent-skill-verifier + working-directory: verification-template + run: | + fail=0 + for dir in skills/specbridge-*/; do + name="$(basename "$dir")" + for cases in "testcases/${name}.json" "testcases/${name}-negative.json"; do + [ -f "$cases" ] || continue + if "$GITHUB_WORKSPACE/.asv/agent-skill-verifier" validate --skill "$dir" --cases "$cases" --quiet; then + echo "ok: ${name} / $(basename "$cases")" + else + echo "::error::skill validation failed: ${name} / $(basename "$cases")" + fail=1 + fi + done + done + exit "$fail" diff --git a/README.md b/README.md index 47013e1..a05ecd6 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # SpecBridge [![CI](https://github.com/HelloThisWorld/specbridge/actions/workflows/ci.yml/badge.svg)](https://github.com/HelloThisWorld/specbridge/actions/workflows/ci.yml) +[![skill test](https://github.com/HelloThisWorld/specbridge/actions/workflows/skill-verification.yml/badge.svg)](https://github.com/HelloThisWorld/agent-skill-verification-template) [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE) An open, model-agnostic spec runtime for existing Kiro projects.