From 54f15f09baa58e2d964634f9fad27967de22477a Mon Sep 17 00:00:00 2001 From: Marco Walz Date: Mon, 23 Mar 2026 14:24:24 +0100 Subject: [PATCH 1/3] feat: replace custom validator with skill-validator CLI - Replace scripts/validate-skills.js with skill-validator check for structure, links, content analysis, and contamination detection - Add scripts/check-project.js for icskills-specific checks (required metadata fields title/category, evaluation file existence) - CI posts a sticky PR comment with the full validation report and emits inline annotations on changed lines - Add LLM quality scoring step to contributor workflow using claude-cli - Add .score_cache to .gitignore - Bump Node.js to 22 (LTS) across CI and deploy workflows - Update CONTRIBUTING.md, CLAUDE.md, and README.md Closes #114 --- .claude/CLAUDE.md | 14 ++- .github/workflows/_checks.yml | 67 +++++++++++++- .github/workflows/deploy-ic.yml | 2 +- .gitignore | 1 + CONTRIBUTING.md | 32 +++++-- README.md | 2 +- package.json | 2 +- scripts/check-project.js | 72 +++++++++++++++ scripts/validate-skills.js | 156 -------------------------------- 9 files changed, 176 insertions(+), 172 deletions(-) create mode 100644 scripts/check-project.js delete mode 100644 scripts/validate-skills.js diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md index d509677..7bc00e2 100644 --- a/.claude/CLAUDE.md +++ b/.claude/CLAUDE.md @@ -26,7 +26,7 @@ No rigid structure — organize content to best serve agents. The validator warn ```bash npm install # Install dependencies -npm run validate # Validate all skills (frontmatter, sections) +npm run validate # Runs skill-validator check + project-specific checks npm run dev # Validate + start Astro dev server npm run build # Validate + Astro production build ``` @@ -37,7 +37,15 @@ After modifying any SKILL.md file, ALWAYS run: ```bash npm run validate # Fix all errors before committing. Warnings are acceptable. ``` -Validate runs in CI and blocks deployment on errors. +Validate uses [skill-validator](https://github.com/agent-ecosystem/skill-validator) for structure, links, content analysis, and contamination checks. It runs in CI and blocks deployment on errors. + +## LLM Quality Scoring + +Before submitting a PR, run LLM scoring locally to check skill quality: +```bash +skill-validator score evaluate --provider claude-cli skills/ +``` +Uses the locally authenticated `claude` CLI — no API key needed. Low novelty scores indicate the skill may restate common knowledge. Results are cached in `.score_cache/` (gitignored). ## Evaluations @@ -77,7 +85,7 @@ skills/*/SKILL.md # Skill source files (the content) skills/skill.schema.json # JSON Schema for frontmatter skills/_template/ # Skeleton for new skills scripts/lib/parse-skill.js # Shared parsing utilities -scripts/validate-skills.js # Structural validation (CI) +scripts/check-project.js # Project-specific checks: metadata, evals (CI) src/ # Astro site source data/skills.ts # Build-time skill loader data/constants.ts # Static data (frameworks list) diff --git a/.github/workflows/_checks.yml b/.github/workflows/_checks.yml index 4b8c46b..cebadab 100644 --- a/.github/workflows/_checks.yml +++ b/.github/workflows/_checks.yml @@ -8,11 +8,74 @@ jobs: runs-on: ubuntu-latest permissions: contents: read + pull-requests: write steps: - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 with: - node-version: 20 + node-version: 22 cache: npm + + - name: Install skill-validator + env: + SKILL_VALIDATOR_VERSION: "1.4.0" + run: | + curl -sL "https://github.com/agent-ecosystem/skill-validator/releases/download/v${SKILL_VALIDATOR_VERSION}/skill-validator_${SKILL_VALIDATOR_VERSION}_linux_amd64.tar.gz" | tar xz -C /usr/local/bin skill-validator + - run: npm ci - - run: node scripts/validate-skills.js + + - name: Run skill-validator checks + id: skill-validator + run: | + set +e + report=$(skill-validator check skills/ -o markdown 2>/dev/null) + exit_code=$? + echo "$report" >> "$GITHUB_STEP_SUMMARY" + delimiter=$(openssl rand -hex 8) + { + echo "report<<${delimiter}" + echo "$report" + echo "${delimiter}" + } >> "$GITHUB_OUTPUT" + echo "exit_code=$exit_code" >> "$GITHUB_OUTPUT" + # Re-run with annotations sent to stdout/stderr for GitHub to process + skill-validator check skills/ --emit-annotations > /dev/null + exit 0 + + - name: Run project-specific checks + id: check-project + run: | + set +e + output=$(node scripts/check-project.js 2>&1) + exit_code=$? + echo "$output" + delimiter=$(openssl rand -hex 8) + { + echo "report<<${delimiter}" + echo "$output" + echo "${delimiter}" + } >> "$GITHUB_OUTPUT" + echo "exit_code=$exit_code" >> "$GITHUB_OUTPUT" + + - name: Post PR comment + if: github.event_name == 'pull_request' && always() + uses: marocchino/sticky-pull-request-comment@v2 + with: + header: skill-validator + message: | + ## Skill Validation Report + + ${{ steps.skill-validator.outputs.report }} + + ### Project Checks + + ``` + ${{ steps.check-project.outputs.report }} + ``` + + - name: Fail on validation errors + if: steps.skill-validator.outputs.exit_code == '1' || steps.check-project.outputs.exit_code == '1' + run: | + echo "::error::Validation found errors" + exit 1 diff --git a/.github/workflows/deploy-ic.yml b/.github/workflows/deploy-ic.yml index c10d935..e280434 100644 --- a/.github/workflows/deploy-ic.yml +++ b/.github/workflows/deploy-ic.yml @@ -26,7 +26,7 @@ jobs: - uses: actions/setup-node@v4 with: - node-version: 20 + node-version: 22 cache: npm - run: npm ci diff --git a/.gitignore b/.gitignore index fe007d5..d6be60b 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,5 @@ public/llms.txt lighthouse-* .eval-tmp evaluations/results/ +.score_cache _drafts diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f66f663..8230db3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -14,8 +14,12 @@ If you're not sure whether something is wrong, open an issue. We'd rather invest ## Setup ```bash -node -v # Requires Node.js >= 20 +node -v # Requires Node.js >= 22 npm ci # Install dependencies + +# Install skill-validator (required for npm run validate) +brew tap agent-ecosystem/homebrew-tap && brew install skill-validator +# or: go install github.com/agent-ecosystem/skill-validator/cmd/skill-validator@latest ``` --- @@ -121,12 +125,22 @@ Use whatever headings fit your skill. A security skill might use `## Security Pi ### 3. Validate ```bash -npm run validate # Check frontmatter and sections +npm run validate # Runs skill-validator + evals file check +``` + +This runs automatically in CI and blocks deployment on errors. Under the hood it runs [`skill-validator check`](https://github.com/agent-ecosystem/skill-validator) (structure, links, content analysis, contamination detection) plus a project-specific check for evaluation files. + +### 4. Run LLM quality scoring (recommended) + +Before submitting a PR, run LLM scoring locally to check your skill's quality: + +```bash +skill-validator score evaluate --provider claude-cli skills/ ``` -This runs automatically in CI and blocks deployment on errors. +This uses the locally authenticated `claude` CLI — no API key needed. Low novelty scores indicate the skill may restate common knowledge rather than providing genuinely new information. See the [skill-validator docs](https://github.com/agent-ecosystem/skill-validator#score-evaluate) for interpreting scores. -### 4. Add evaluation cases +### 5. Add evaluation cases Create `evaluations/.json` with test cases that verify the skill works. The eval file has two sections: @@ -150,16 +164,17 @@ This sends each prompt to Claude with and without the skill, then has a judge sc Including a summary of eval results in your PR description is recommended but not required — running evals needs `claude` CLI access and costs API credits. -### 5. That's it — the website auto-discovers skills +### 6. That's it — the website auto-discovers skills The website is automatically generated from the SKILL.md frontmatter at build time. You do **not** need to edit any source file. Astro reads all `skills/*/SKILL.md` files, parses their frontmatter, and generates the site pages, `llms.txt`, discovery endpoints, and other files. Stats (skill count, categories) all update automatically. -### 6. Submit a PR +### 7. Submit a PR - One skill per PR - Include a brief description of what the skill covers and why it's needed +- Include LLM scoring output in your PR description if you ran it locally (see step 4) - Make sure the SKILL.md is tested — code examples should compile and deploy - **All PRs require approval from a repo admin before merge.** No skill additions or updates go live without review. @@ -169,7 +184,8 @@ Stats (skill count, categories) all update automatically. 1. Edit the `SKILL.md` content 2. Run `npm run validate` -3. Submit a PR with a summary of what changed +3. Optionally run LLM scoring (see step 4 above) +4. Submit a PR with a summary of what changed The website auto-generates from SKILL.md frontmatter — no need to edit any source files. @@ -191,4 +207,4 @@ Use an existing category when possible. The validator warns on unknown categorie Current categories: **DeFi**, **Tokens**, **Auth**, **Architecture**, **Integration**, **Governance**, **Frontend**, **Security**, **Infrastructure**, **Wallet** -To add a new category: update `KNOWN_CATEGORIES` in `scripts/validate-skills.js`, the description in `skills/skill.schema.json`, and the icon in `src/components/Icons.tsx`. +To add a new category: update the enum in `skills/skill.schema.json` and the icon in `src/components/Icons.tsx`. diff --git a/README.md b/README.md index 65b5711..a6a0457 100644 --- a/README.md +++ b/README.md @@ -84,7 +84,7 @@ See [CONTRIBUTING.md](CONTRIBUTING.md) for how to add or update skills. - **Site**: [Astro](https://astro.build/) — static site generator, zero JS by default. Interactive islands with [Preact](https://preactjs.com/) (~18kb gzipped total) - **Hosting**: IC asset canister at [`skills.internetcomputer.org`](https://skills.internetcomputer.org) - **Skills**: Plain markdown files in `skills/*/SKILL.md` -- **Validation**: Structural linter for frontmatter and code blocks (`npm run validate`) +- **Validation**: [`skill-validator`](https://github.com/agent-ecosystem/skill-validator) for structure, links, content analysis, and contamination checks (`npm run validate`) - **Evaluation**: Per-skill eval cases with LLM-as-judge scoring (`node scripts/evaluate-skills.js `) - **Schema**: JSON Schema for frontmatter at `skills/skill.schema.json` - **SEO**: Per-skill meta tags, JSON-LD (TechArticle), sitemap, canonical URLs diff --git a/package.json b/package.json index 6a49e76..e43df9c 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ "license": "Apache-2.0", "type": "module", "scripts": { - "validate": "node scripts/validate-skills.js", + "validate": "skill-validator check skills/ && node scripts/check-project.js", "dev": "npm run validate && astro dev", "build": "npm run validate && astro build", "preview": "astro preview" diff --git a/scripts/check-project.js b/scripts/check-project.js new file mode 100644 index 0000000..584735f --- /dev/null +++ b/scripts/check-project.js @@ -0,0 +1,72 @@ +#!/usr/bin/env node +// Project-specific checks that complement skill-validator's generic validation. +// Covers icskills requirements not part of the Agent Skills spec: +// - Required metadata fields (title, category) used by the Astro site +// - Evaluation file existence + +import { existsSync } from "fs"; +import { join } from "path"; +import { listSkillDirs, readSkill, SKILLS_DIR } from "./lib/parse-skill.js"; + +const KNOWN_CATEGORIES = [ + "DeFi", + "Tokens", + "Auth", + "Architecture", + "Integration", + "Governance", + "Frontend", + "Security", + "Infrastructure", + "Wallet", +]; + +const evalsDir = join(SKILLS_DIR, "..", "evaluations"); +const dirs = listSkillDirs(); +const errors = []; +const warnings = []; + +for (const dir of dirs) { + const skill = readSkill(dir); + if (!skill) continue; + const label = `${dir}/SKILL.md`; + + // Required metadata fields (used by the Astro site) + if (!skill.meta.title) { + errors.push(`${label}: missing required frontmatter field: title`); + } + if (!skill.meta.category) { + errors.push(`${label}: missing required frontmatter field: category`); + } + + // Category typo detection + if (skill.meta.category && !KNOWN_CATEGORIES.includes(skill.meta.category)) { + warnings.push( + `${label}: unknown category "${skill.meta.category}" — known categories: ${KNOWN_CATEGORIES.join(", ")}` + ); + } + + // Evaluation file existence + if (!existsSync(join(evalsDir, `${dir}.json`))) { + warnings.push( + `${label}: missing evaluations/${dir}.json — see CONTRIBUTING.md for evaluation guidance` + ); + } +} + +// --- Output --- + +if (warnings.length) { + console.warn(`\nWARNINGS (${warnings.length}):`); + warnings.forEach((w) => console.warn(` ⚠ ${w}`)); +} + +if (errors.length) { + console.error(`\nERRORS (${errors.length}):`); + errors.forEach((e) => console.error(` ✗ ${e}`)); + process.exit(1); +} else { + console.log( + `\n✓ Project checks passed for ${dirs.length} skills (${warnings.length} warnings)` + ); +} diff --git a/scripts/validate-skills.js b/scripts/validate-skills.js deleted file mode 100644 index e4759a0..0000000 --- a/scripts/validate-skills.js +++ /dev/null @@ -1,156 +0,0 @@ -#!/usr/bin/env node -// Validates SKILL.md files for structural correctness. -// Checks frontmatter fields, required sections, and code block annotations. -// Run: node scripts/validate-skills.js - -import { readFileSync, existsSync } from "fs"; -import { join } from "path"; -import { readAllSkills, SKILLS_DIR } from "./lib/parse-skill.js"; - -const REQUIRED_FRONTMATTER = [ - "name", - "title", - "category", - "description", -]; - -// Known categories — warn on unknown to catch typos, but don't block. -// To add a new category: add it here, in skill.schema.json, and in src/components/Icons.tsx. -const KNOWN_CATEGORIES = [ - "DeFi", - "Tokens", - "Auth", - "Architecture", - "Integration", - "Governance", - "Frontend", - "Security", - "Infrastructure", - "Wallet", -]; - -// Recommended sections (## heading text) — warn if missing, don't block -const RECOMMENDED_SECTIONS = [ - "What This Is", - "Prerequisites", - "Implementation", -]; - -const errors = []; -const warnings = []; - -function error(skill, msg) { - errors.push(`${skill}: ${msg}`); -} - -function warn(skill, msg) { - warnings.push(`${skill}: ${msg}`); -} - -const skills = readAllSkills(); - -// Load JSON schema for allowed categories (read from schema if it exists) -let schema = null; -try { - schema = JSON.parse( - readFileSync(join(SKILLS_DIR, "skill.schema.json"), "utf-8") - ); -} catch { - // Schema is optional for validation -} - -for (const skill of skills) { - const { dir, meta, body } = skill; - const label = `${dir}/SKILL.md`; - - // --- Frontmatter validation --- - - for (const field of REQUIRED_FRONTMATTER) { - if (meta[field] === undefined || meta[field] === "") { - error(label, `missing required frontmatter field: ${field}`); - } - } - - // name must match directory name - if (meta.name && meta.name !== dir) { - error(label, `frontmatter name "${meta.name}" does not match directory "${dir}"`); - } - - // name format - if (meta.name && !/^[a-z][a-z0-9-]*$/.test(meta.name)) { - error(label, `name "${meta.name}" must be lowercase alphanumeric with hyphens`); - } - - // category — warn on unknown to catch typos - if (meta.category && !KNOWN_CATEGORIES.includes(meta.category)) { - warn( - label, - `unknown category "${meta.category}" — known categories: ${KNOWN_CATEGORIES.join(", ")}` - ); - } - - // --- Section validation --- - - // Extract all ## headings from body - const headings = []; - for (const line of body.split("\n")) { - const match = line.match(/^## (.+)$/); - if (match) headings.push(match[1].trim()); - } - - for (const section of RECOMMENDED_SECTIONS) { - if (!headings.includes(section)) { - warn(label, `missing recommended section: "## ${section}"`); - } - } - - // --- Code block validation --- - - // Check that opening code blocks have language annotations. - // Track open/close state: odd occurrences of ``` are openers, even are closers. - const codeBlockMarkers = body.match(/^```.*$/gm) || []; - let insideBlock = false; - for (const marker of codeBlockMarkers) { - if (!insideBlock) { - // This is an opening marker — must have a language annotation - if (marker === "```") { - warn(label, `code block without language annotation found`); - break; // One warning per file is enough - } - } - insideBlock = !insideBlock; - } - - // --- Recommended fields --- - - if (!meta.license) { - warn(label, `missing "license" field in frontmatter`); - } - - if (!meta.compatibility) { - warn(label, `missing "compatibility" field in frontmatter`); - } - - // --- Evals validation --- - const evalsDir = join(SKILLS_DIR, "..", "evaluations"); - if (!existsSync(join(evalsDir, `${dir}.json`))) { - warn(label, `missing evaluations/${dir}.json — see CONTRIBUTING.md for evaluation guidance`); - } -} - -// --- Output --- - -if (warnings.length) { - console.warn(`\nWARNINGS (${warnings.length}):`); - warnings.forEach((w) => console.warn(` ⚠ ${w}`)); -} - -if (errors.length) { - console.error(`\nERRORS (${errors.length}):`); - errors.forEach((e) => console.error(` ✗ ${e}`)); - process.exit(1); -} else { - console.log( - `\n✓ All ${skills.length} skills passed validation (${warnings.length} warnings)` - ); -} From 3dbed1528c25d3e93aa89830e379db0a1f0f7ade Mon Sep 17 00:00:00 2001 From: Marco Walz Date: Mon, 23 Mar 2026 15:21:53 +0100 Subject: [PATCH 2/3] refine: scope PR validation to changed skills only - Detect changed skill directories via git diff against base branch - Run skill-validator and project checks only on affected skills - Skip validation comment when no skills were changed - Add CLI filter support to check-project.js --- .github/workflows/_checks.yml | 49 +++++++++++++++++++++++++++++++---- scripts/check-project.js | 6 ++++- 2 files changed, 49 insertions(+), 6 deletions(-) diff --git a/.github/workflows/_checks.yml b/.github/workflows/_checks.yml index cebadab..fa33df2 100644 --- a/.github/workflows/_checks.yml +++ b/.github/workflows/_checks.yml @@ -11,6 +11,8 @@ jobs: pull-requests: write steps: - uses: actions/checkout@v4 + with: + fetch-depth: 0 - uses: actions/setup-node@v4 with: @@ -23,13 +25,38 @@ jobs: run: | curl -sL "https://github.com/agent-ecosystem/skill-validator/releases/download/v${SKILL_VALIDATOR_VERSION}/skill-validator_${SKILL_VALIDATOR_VERSION}_linux_amd64.tar.gz" | tar xz -C /usr/local/bin skill-validator + - name: Detect changed skills + id: changed + run: | + if [ "${{ github.event_name }}" = "pull_request" ]; then + dirs=$(git diff --name-only origin/${{ github.base_ref }}...HEAD -- 'skills/' \ + | grep -oP '^skills/\K[^/]+' \ + | sort -u \ + | grep -v '^_' \ + | while read -r d; do [ -f "skills/$d/SKILL.md" ] && echo "$d"; done) + if [ -n "$dirs" ]; then + # Build space-separated paths for skill-validator + paths=$(echo "$dirs" | sed 's|^|skills/|' | tr '\n' ' ') + echo "paths=$paths" >> "$GITHUB_OUTPUT" + echo "names=$(echo "$dirs" | tr '\n' ' ')" >> "$GITHUB_OUTPUT" + echo "found=true" >> "$GITHUB_OUTPUT" + else + echo "found=false" >> "$GITHUB_OUTPUT" + fi + else + echo "paths=skills/" >> "$GITHUB_OUTPUT" + echo "names=" >> "$GITHUB_OUTPUT" + echo "found=true" >> "$GITHUB_OUTPUT" + fi + - run: npm ci - name: Run skill-validator checks id: skill-validator + if: steps.changed.outputs.found == 'true' run: | set +e - report=$(skill-validator check skills/ -o markdown 2>/dev/null) + report=$(skill-validator check ${{ steps.changed.outputs.paths }} -o markdown 2>/dev/null) exit_code=$? echo "$report" >> "$GITHUB_STEP_SUMMARY" delimiter=$(openssl rand -hex 8) @@ -40,14 +67,15 @@ jobs: } >> "$GITHUB_OUTPUT" echo "exit_code=$exit_code" >> "$GITHUB_OUTPUT" # Re-run with annotations sent to stdout/stderr for GitHub to process - skill-validator check skills/ --emit-annotations > /dev/null + skill-validator check ${{ steps.changed.outputs.paths }} --emit-annotations > /dev/null exit 0 - name: Run project-specific checks id: check-project + if: steps.changed.outputs.found == 'true' run: | set +e - output=$(node scripts/check-project.js 2>&1) + output=$(node scripts/check-project.js ${{ steps.changed.outputs.names }} 2>&1) exit_code=$? echo "$output" delimiter=$(openssl rand -hex 8) @@ -59,7 +87,7 @@ jobs: echo "exit_code=$exit_code" >> "$GITHUB_OUTPUT" - name: Post PR comment - if: github.event_name == 'pull_request' && always() + if: github.event_name == 'pull_request' && steps.changed.outputs.found == 'true' && always() uses: marocchino/sticky-pull-request-comment@v2 with: header: skill-validator @@ -74,8 +102,19 @@ jobs: ${{ steps.check-project.outputs.report }} ``` + - name: Post PR comment (no skills changed) + if: github.event_name == 'pull_request' && steps.changed.outputs.found == 'false' + uses: marocchino/sticky-pull-request-comment@v2 + with: + header: skill-validator + recreate: true + message: | + ## Skill Validation Report + + No skill files were changed in this PR — validation skipped. + - name: Fail on validation errors - if: steps.skill-validator.outputs.exit_code == '1' || steps.check-project.outputs.exit_code == '1' + if: steps.changed.outputs.found == 'true' && (steps.skill-validator.outputs.exit_code == '1' || steps.check-project.outputs.exit_code == '1') run: | echo "::error::Validation found errors" exit 1 diff --git a/scripts/check-project.js b/scripts/check-project.js index 584735f..76956cf 100644 --- a/scripts/check-project.js +++ b/scripts/check-project.js @@ -22,7 +22,11 @@ const KNOWN_CATEGORIES = [ ]; const evalsDir = join(SKILLS_DIR, "..", "evaluations"); -const dirs = listSkillDirs(); +const filterArgs = process.argv.slice(2); +const allDirs = listSkillDirs(); +const dirs = filterArgs.length > 0 + ? allDirs.filter((d) => filterArgs.includes(d)) + : allDirs; const errors = []; const warnings = []; From 1d7da0fc733374d253537ca74afded40f8f279a5 Mon Sep 17 00:00:00 2001 From: Marco Walz Date: Mon, 23 Mar 2026 15:25:00 +0100 Subject: [PATCH 3/3] fix: add required permissions to PR workflow caller The reusable _checks.yml needs pull-requests: write for the sticky comment. The caller must grant at least the same permissions. --- .github/workflows/ci.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b4c66b7..2aa6630 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,6 +7,13 @@ concurrency: group: ci-${{ github.ref }} cancel-in-progress: true +permissions: + contents: read + pull-requests: write + jobs: checks: uses: ./.github/workflows/_checks.yml + permissions: + contents: read + pull-requests: write