CI: verify agent skills against each build with Agent Skill Verifier

.github/workflows/ci.yml

@@ -5,6 +5,9 @@ on:
     branches: [main, master]
   pull_request:
 
+permissions:
+  contents: read
+
 # Everything below runs fully offline after dependency install:
 # no LLM, no API key, no external service.
 jobs:
@@ -64,3 +67,4 @@ jobs:
       - name: Claude Code plugin bundle is reproducible
         if: matrix.os == 'ubuntu-latest'
         run: git diff --exit-code integrations/claude-code-plugin/specbridge/dist
+

.github/workflows/skill-verification.yml

@@ -0,0 +1,113 @@
+# Verifies the 11 specbridge-* agent skills against each build of the CLI
+# using the Agent Skill Verifier
+# (https://github.com/HelloThisWorld/agent-skill-verification-template):
+#
+#   1. the skill fixture workspace is rebuilt from this build's CLI output
+#      and diffed against the committed grounding facts (snapshots,
+#      WORKSPACE.md, spec files) — a CLI output change that would break the
+#      skills' citations fails here;
+#   2. every skill contract + evaluation case file is statically validated
+#      with a pinned, checksum-verified agent-skill-verifier release.
+#
+# Full model-driven eval runs (verify --adapter llm) stay a local/manual
+# step because they need a live model endpoint; everything here is
+# deterministic and offline apart from the two pinned GitHub downloads.
+#
+# This is a standalone workflow (not a job in ci.yml) so the README badge
+# reflects exactly the skill-test outcome.
+name: skill test
+
+on:
+  push:
+    branches: [main, master]
+  pull_request:
+
+permissions:
+  contents: read
+
+jobs:
+  skill-verification:
+    name: agent skill verification
+    runs-on: ubuntu-latest
+    env:
+      # Pinned verifier release. Update the version and the asset SHA-256
+      # (from the release's SHA256SUMS.txt) together.
+      ASV_VERSION: 0.1.0
+      ASV_SHA256: 2307aa1ee6a8698fe5d7dfc12ef9413ce086e5f3fe3748501abd0d10cfe68084
+      # Ref of the verification template holding the specbridge skill
+      # contracts, evaluation cases, and fixture builder.
+      TEMPLATE_REF: v0.1.0
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: pnpm/action-setup@v4
+        # Version comes from the "packageManager" field in package.json.
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 22
+          cache: pnpm
+
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile
+
+      - name: Build the SpecBridge CLI
+        run: pnpm build
+
+      - name: Checkout the verification template (pinned ref)
+        uses: actions/checkout@v4
+        with:
+          repository: HelloThisWorld/agent-skill-verification-template
+          ref: ${{ env.TEMPLATE_REF }}
+          path: verification-template
+
+      - name: Download the pinned agent-skill-verifier release and verify its checksum
+        run: |
+          curl -fsSLO "https://github.com/HelloThisWorld/agent-skill-verification-template/releases/download/v${ASV_VERSION}/agent-skill-verifier-v${ASV_VERSION}-linux-x64.tar.gz"
+          echo "${ASV_SHA256}  agent-skill-verifier-v${ASV_VERSION}-linux-x64.tar.gz" | sha256sum -c -
+          mkdir -p .asv
+          tar -xzf "agent-skill-verifier-v${ASV_VERSION}-linux-x64.tar.gz" -C .asv
+          .asv/agent-skill-verifier --version
+
+      - name: Rebuild the skill fixture from this build's CLI (grounding drift check)
+        env:
+          SPECBRIDGE_REPO: ${{ github.workspace }}
+        run: |
+          node verification-template/scripts/build-specbridge-fixture.mjs
+          # runner-list contains machine-dependent capability probes, so it is
+          # compared on its environment-independent fields by the dedicated
+          # script instead of byte equality.
+          git -C verification-template show HEAD:fixtures/specbridge-workspace/snapshots/runner-list.json > /tmp/runner-list-committed.json
+          node scripts/check-skill-runner-snapshot.mjs \
+            /tmp/runner-list-committed.json \
+            verification-template/fixtures/specbridge-workspace/snapshots/runner-list.json
+          # Timestamps, install-record ids, and local archive paths differ per
+          # rebuild by design; every other grounded fact (snapshots,
+          # WORKSPACE.md, specs, manifests, hashes) must be byte-identical.
+          if ! git -C verification-template diff --exit-code -- \
+              fixtures/specbridge-workspace \
+              ':(exclude)fixtures/specbridge-workspace/.specbridge/extensions/grants.json' \
+              ':(exclude)fixtures/specbridge-workspace/.specbridge/extensions/records.jsonl' \
+              ':(exclude)fixtures/specbridge-workspace/.specbridge/extensions/state.json' \
+              ':(exclude)fixtures/specbridge-workspace/snapshots/runner-list.json'; then
+            echo "::error::SpecBridge CLI output drifted from the committed skill-verification fixture. If the change is intentional, regenerate the fixture in agent-skill-verification-template (scripts/build-specbridge-fixture.mjs) and pin a new TEMPLATE_REF."
+            exit 1
+          fi
+
+      - name: Validate every specbridge skill with agent-skill-verifier
+        working-directory: verification-template
+        run: |
+          fail=0
+          for dir in skills/specbridge-*/; do
+            name="$(basename "$dir")"
+            for cases in "testcases/${name}.json" "testcases/${name}-negative.json"; do
+              [ -f "$cases" ] || continue
+              if "$GITHUB_WORKSPACE/.asv/agent-skill-verifier" validate --skill "$dir" --cases "$cases" --quiet; then
+                echo "ok:   ${name} / $(basename "$cases")"
+              else
+                echo "::error::skill validation failed: ${name} / $(basename "$cases")"
+                fail=1
+              fi
+            done
+          done
+          exit "$fail"

README.md

@@ -1,6 +1,7 @@
 # SpecBridge
 
 [![CI](https://github.com/HelloThisWorld/specbridge/actions/workflows/ci.yml/badge.svg)](https://github.com/HelloThisWorld/specbridge/actions/workflows/ci.yml)
+[![skill test](https://github.com/HelloThisWorld/specbridge/actions/workflows/skill-verification.yml/badge.svg)](https://github.com/HelloThisWorld/agent-skill-verification-template)
 [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE)
 
 An open, model-agnostic spec runtime for existing Kiro projects.

scripts/check-skill-runner-snapshot.mjs

@@ -0,0 +1,44 @@
+// Compare the environment-independent fields of a SpecBridge `runner list`
+// snapshot against a freshly generated one: the profile set, each profile's
+// implementation, and its enablement. Capability probes (capabilities,
+// version, authentication) legitimately differ per machine — the committed
+// skill-verification snapshot was generated on a workstation with runner
+// CLIs installed, while CI runners have none — so those fields are ignored.
+//
+// Used by the skill-verification CI job together with the byte-level drift
+// check that covers every other fixture file.
+//
+// usage: node scripts/check-skill-runner-snapshot.mjs <committed.json> <rebuilt.json>
+import { readFileSync } from "node:fs";
+
+const [committedPath, rebuiltPath] = process.argv.slice(2);
+if (!committedPath || !rebuiltPath) {
+  console.error("usage: node scripts/check-skill-runner-snapshot.mjs <committed.json> <rebuilt.json>");
+  process.exit(2);
+}
+
+const normalize = (path) => {
+  const raw = JSON.parse(readFileSync(path, "utf8"));
+  return (raw.profiles ?? [])
+    .map((profile) => ({
+      profile: profile.profile,
+      implementation: profile.implementation,
+      enabled: profile.enabled,
+    }))
+    .sort((a, b) => a.profile.localeCompare(b.profile));
+};
+
+const committed = JSON.stringify(normalize(committedPath), null, 2);
+const rebuilt = JSON.stringify(normalize(rebuiltPath), null, 2);
+
+if (committed !== rebuilt) {
+  console.error("runner-list drift in environment-independent fields (profile/implementation/enabled):");
+  console.error("--- committed (verification template)");
+  console.error(committed);
+  console.error("+++ rebuilt (this build)");
+  console.error(rebuilt);
+  process.exit(1);
+}
+console.log(
+  `runner-list: ${normalize(rebuiltPath).length} profiles match the committed snapshot on all environment-independent fields.`,
+);