diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 812c3eeb..d8caca8d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -63,6 +63,8 @@ jobs: run: pnpm hygiene:check - name: Run docs-staleness check run: pnpm docs:staleness + - name: Run skills-conformance check + run: pnpm skills:conformance - name: Run duplication check run: pnpm dup:check - name: Install Playwright browsers diff --git a/package.json b/package.json index 0709ab22..827e95be 100644 --- a/package.json +++ b/package.json @@ -25,8 +25,9 @@ "a11y:report:html": "turbo a11y:report:html", "hygiene:check": "node scripts/code-hygiene-check.js", "docs:staleness": "pnpm --filter @pair/website docs:staleness", + "skills:conformance": "pnpm --filter @pair/knowledge-hub skills:conformance", "dup:check": "jscpd apps packages", - "quality-gate": "turbo ts:check test lint && turbo prettier:fix mdlint:fix && ./tools/markdownlint-config/bin/markdownlint-fix.sh '*.md' && pnpm hygiene:check && pnpm docs:staleness && pnpm dup:check", + "quality-gate": "turbo ts:check test lint && turbo prettier:fix mdlint:fix && ./tools/markdownlint-config/bin/markdownlint-fix.sh '*.md' && pnpm hygiene:check && pnpm docs:staleness && pnpm skills:conformance && pnpm dup:check", "e2e": "pnpm --filter @pair/website e2e", "smoke-tests": "./scripts/smoke-tests/run-all.sh --cleanup" }, diff --git a/packages/knowledge-hub/eval/README.md b/packages/knowledge-hub/eval/README.md new file mode 100644 index 00000000..63760a55 --- /dev/null +++ b/packages/knowledge-hub/eval/README.md @@ -0,0 +1,72 @@ +# Skill Trigger Eval + +Trigger/effectiveness eval harness for the skill corpus (story #313, T7). It measures whether the +frontmatter `description` of each skill in `packages/knowledge-hub/dataset/.skills/**` routes an +LLM executor to the right skill — the experiment behind principle 9 ("Evaluation") of the +authoring standard (`apps/website/content/docs/contributing/writing-skills.mdx`): a description +rewrite is a hypothesis; this eval is the experiment. + +## Contents + +- `trigger-prompts.json` — should-trigger / should-not-trigger prompt sets, one entry per skill + family. Every skill has at least one should-trigger prompt; near-miss prompts name the sibling + skill they are a decoy for (`near_miss_for`) and the correct destination (`expected`, `none` + when no skill applies). +- `results/` — committed evidence, one file per run, named `YYYY-MM-DD-