feat: replace custom validator with skill-validator CLI

.claude/CLAUDE.md

@@ -26,7 +26,7 @@ No rigid structure — organize content to best serve agents. The validator warn
 
 ```bash
 npm install          # Install dependencies
-npm run validate     # Validate all skills (frontmatter, sections)
+npm run validate     # Runs skill-validator check + project-specific checks
 npm run dev          # Validate + start Astro dev server
 npm run build        # Validate + Astro production build
 ```
@@ -37,7 +37,15 @@ After modifying any SKILL.md file, ALWAYS run:
 ```bash
 npm run validate     # Fix all errors before committing. Warnings are acceptable.
 ```
-Validate runs in CI and blocks deployment on errors.
+Validate uses [skill-validator](https://github.com/agent-ecosystem/skill-validator) for structure, links, content analysis, and contamination checks. It runs in CI and blocks deployment on errors.
+
+## LLM Quality Scoring
+
+Before submitting a PR, run LLM scoring locally to check skill quality:
+```bash
+skill-validator score evaluate --provider claude-cli skills/<skill-name>
+```
+Uses the locally authenticated `claude` CLI — no API key needed. Low novelty scores indicate the skill may restate common knowledge. Results are cached in `.score_cache/` (gitignored).
 
 ## Evaluations
 
@@ -77,7 +85,7 @@ skills/*/SKILL.md             # Skill source files (the content)
 skills/skill.schema.json      # JSON Schema for frontmatter
 skills/_template/              # Skeleton for new skills
 scripts/lib/parse-skill.js    # Shared parsing utilities
-scripts/validate-skills.js    # Structural validation (CI)
+scripts/check-project.js      # Project-specific checks: metadata, evals (CI)
 src/                           # Astro site source
   data/skills.ts              # Build-time skill loader
   data/constants.ts           # Static data (frameworks list)

.github/workflows/_checks.yml

@@ -8,11 +8,113 @@ jobs:
     runs-on: ubuntu-latest
     permissions:
       contents: read
+      pull-requests: write
     steps:
       - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
       - uses: actions/setup-node@v4
         with:
-          node-version: 20
+          node-version: 22
           cache: npm
+
+      - name: Install skill-validator
+        env:
+          SKILL_VALIDATOR_VERSION: "1.4.0"
+        run: |
+          curl -sL "https://github.com/agent-ecosystem/skill-validator/releases/download/v${SKILL_VALIDATOR_VERSION}/skill-validator_${SKILL_VALIDATOR_VERSION}_linux_amd64.tar.gz" | tar xz -C /usr/local/bin skill-validator
+
+      - name: Detect changed skills
+        id: changed
+        run: |
+          if [ "${{ github.event_name }}" = "pull_request" ]; then
+            dirs=$(git diff --name-only origin/${{ github.base_ref }}...HEAD -- 'skills/' \
+              | grep -oP '^skills/\K[^/]+' \
+              | sort -u \
+              | grep -v '^_' \
+              | while read -r d; do [ -f "skills/$d/SKILL.md" ] && echo "$d"; done)
+            if [ -n "$dirs" ]; then
+              # Build space-separated paths for skill-validator
+              paths=$(echo "$dirs" | sed 's|^|skills/|' | tr '\n' ' ')
+              echo "paths=$paths" >> "$GITHUB_OUTPUT"
+              echo "names=$(echo "$dirs" | tr '\n' ' ')" >> "$GITHUB_OUTPUT"
+              echo "found=true" >> "$GITHUB_OUTPUT"
+            else
+              echo "found=false" >> "$GITHUB_OUTPUT"
+            fi
+          else
+            echo "paths=skills/" >> "$GITHUB_OUTPUT"
+            echo "names=" >> "$GITHUB_OUTPUT"
+            echo "found=true" >> "$GITHUB_OUTPUT"
+          fi
+
       - run: npm ci
-      - run: node scripts/validate-skills.js
+
+      - name: Run skill-validator checks
+        id: skill-validator
+        if: steps.changed.outputs.found == 'true'
+        run: |
+          set +e
+          report=$(skill-validator check ${{ steps.changed.outputs.paths }} -o markdown 2>/dev/null)
+          exit_code=$?
+          echo "$report" >> "$GITHUB_STEP_SUMMARY"
+          delimiter=$(openssl rand -hex 8)
+          {
+            echo "report<<${delimiter}"
+            echo "$report"
+            echo "${delimiter}"
+          } >> "$GITHUB_OUTPUT"
+          echo "exit_code=$exit_code" >> "$GITHUB_OUTPUT"
+          # Re-run with annotations sent to stdout/stderr for GitHub to process
+          skill-validator check ${{ steps.changed.outputs.paths }} --emit-annotations > /dev/null
+          exit 0
+
+      - name: Run project-specific checks
+        id: check-project
+        if: steps.changed.outputs.found == 'true'
+        run: |
+          set +e
+          output=$(node scripts/check-project.js ${{ steps.changed.outputs.names }} 2>&1)
+          exit_code=$?
+          echo "$output"
+          delimiter=$(openssl rand -hex 8)
+          {
+            echo "report<<${delimiter}"
+            echo "$output"
+            echo "${delimiter}"
+          } >> "$GITHUB_OUTPUT"
+          echo "exit_code=$exit_code" >> "$GITHUB_OUTPUT"
+
+      - name: Post PR comment
+        if: github.event_name == 'pull_request' && steps.changed.outputs.found == 'true' && always()
+        uses: marocchino/sticky-pull-request-comment@v2
+        with:
+          header: skill-validator
+          message: |
+            ## Skill Validation Report
+
+            ${{ steps.skill-validator.outputs.report }}
+
+            ### Project Checks
+
+            ```
+            ${{ steps.check-project.outputs.report }}
+            ```
+
+      - name: Post PR comment (no skills changed)
+        if: github.event_name == 'pull_request' && steps.changed.outputs.found == 'false'
+        uses: marocchino/sticky-pull-request-comment@v2
+        with:
+          header: skill-validator
+          recreate: true
+          message: |
+            ## Skill Validation Report
+
+            No skill files were changed in this PR — validation skipped.
+
+      - name: Fail on validation errors
+        if: steps.changed.outputs.found == 'true' && (steps.skill-validator.outputs.exit_code == '1' || steps.check-project.outputs.exit_code == '1')
+        run: |
+          echo "::error::Validation found errors"
+          exit 1

.github/workflows/ci.yml

@@ -7,6 +7,13 @@ concurrency:
   group: ci-${{ github.ref }}
   cancel-in-progress: true
 
+permissions:
+  contents: read
+  pull-requests: write
+
 jobs:
   checks:
     uses: ./.github/workflows/_checks.yml
+    permissions:
+      contents: read
+      pull-requests: write

.github/workflows/deploy-ic.yml

@@ -26,7 +26,7 @@ jobs:
 
       - uses: actions/setup-node@v4
         with:
-          node-version: 20
+          node-version: 22
           cache: npm
 
       - run: npm ci

.gitignore

@@ -11,4 +11,5 @@ public/llms.txt
 lighthouse-*
 .eval-tmp
 evaluations/results/
+.score_cache
 _drafts

CONTRIBUTING.md

@@ -14,8 +14,12 @@ If you're not sure whether something is wrong, open an issue. We'd rather invest
 ## Setup
 
 ```bash
-node -v   # Requires Node.js >= 20
+node -v   # Requires Node.js >= 22
 npm ci    # Install dependencies
+
+# Install skill-validator (required for npm run validate)
+brew tap agent-ecosystem/homebrew-tap && brew install skill-validator
+# or: go install github.com/agent-ecosystem/skill-validator/cmd/skill-validator@latest
 ```
 
 ---
@@ -121,12 +125,22 @@ Use whatever headings fit your skill. A security skill might use `## Security Pi
 ### 3. Validate
 
 ```bash
-npm run validate     # Check frontmatter and sections
+npm run validate     # Runs skill-validator + evals file check
+```
+
+This runs automatically in CI and blocks deployment on errors. Under the hood it runs [`skill-validator check`](https://github.com/agent-ecosystem/skill-validator) (structure, links, content analysis, contamination detection) plus a project-specific check for evaluation files.
+
+### 4. Run LLM quality scoring (recommended)
+
+Before submitting a PR, run LLM scoring locally to check your skill's quality:
+
+```bash
+skill-validator score evaluate --provider claude-cli skills/<skill-name>
 ```
 
-This runs automatically in CI and blocks deployment on errors.
+This uses the locally authenticated `claude` CLI — no API key needed. Low novelty scores indicate the skill may restate common knowledge rather than providing genuinely new information. See the [skill-validator docs](https://github.com/agent-ecosystem/skill-validator#score-evaluate) for interpreting scores.
 
-### 4. Add evaluation cases
+### 5. Add evaluation cases
 
 Create `evaluations/<skill-name>.json` with test cases that verify the skill works. The eval file has two sections:
 
@@ -150,16 +164,17 @@ This sends each prompt to Claude with and without the skill, then has a judge sc
 
 Including a summary of eval results in your PR description is recommended but not required — running evals needs `claude` CLI access and costs API credits.
 
-### 5. That's it — the website auto-discovers skills
+### 6. That's it — the website auto-discovers skills
 
 The website is automatically generated from the SKILL.md frontmatter at build time. You do **not** need to edit any source file. Astro reads all `skills/*/SKILL.md` files, parses their frontmatter, and generates the site pages, `llms.txt`, discovery endpoints, and other files.
 
 Stats (skill count, categories) all update automatically.
 
-### 6. Submit a PR
+### 7. Submit a PR
 
 - One skill per PR
 - Include a brief description of what the skill covers and why it's needed
+- Include LLM scoring output in your PR description if you ran it locally (see step 4)
 - Make sure the SKILL.md is tested — code examples should compile and deploy
 - **All PRs require approval from a repo admin before merge.** No skill additions or updates go live without review.
 
@@ -169,7 +184,8 @@ Stats (skill count, categories) all update automatically.
 
 1. Edit the `SKILL.md` content
 2. Run `npm run validate`
-3. Submit a PR with a summary of what changed
+3. Optionally run LLM scoring (see step 4 above)
+4. Submit a PR with a summary of what changed
 
 The website auto-generates from SKILL.md frontmatter — no need to edit any source files.
 
@@ -191,4 +207,4 @@ Use an existing category when possible. The validator warns on unknown categorie
 
 Current categories: **DeFi**, **Tokens**, **Auth**, **Architecture**, **Integration**, **Governance**, **Frontend**, **Security**, **Infrastructure**, **Wallet**
 
-To add a new category: update `KNOWN_CATEGORIES` in `scripts/validate-skills.js`, the description in `skills/skill.schema.json`, and the icon in `src/components/Icons.tsx`.
+To add a new category: update the enum in `skills/skill.schema.json` and the icon in `src/components/Icons.tsx`.

README.md

@@ -84,7 +84,7 @@ See [CONTRIBUTING.md](CONTRIBUTING.md) for how to add or update skills.
 - **Site**: [Astro](https://astro.build/) — static site generator, zero JS by default. Interactive islands with [Preact](https://preactjs.com/) (~18kb gzipped total)
 - **Hosting**: IC asset canister at [`skills.internetcomputer.org`](https://skills.internetcomputer.org)
 - **Skills**: Plain markdown files in `skills/*/SKILL.md`
-- **Validation**: Structural linter for frontmatter and code blocks (`npm run validate`)
+- **Validation**: [`skill-validator`](https://github.com/agent-ecosystem/skill-validator) for structure, links, content analysis, and contamination checks (`npm run validate`)
 - **Evaluation**: Per-skill eval cases with LLM-as-judge scoring (`node scripts/evaluate-skills.js <skill>`)
 - **Schema**: JSON Schema for frontmatter at `skills/skill.schema.json`
 - **SEO**: Per-skill meta tags, JSON-LD (TechArticle), sitemap, canonical URLs

package.json

@@ -6,7 +6,7 @@
   "license": "Apache-2.0",
   "type": "module",
   "scripts": {
-    "validate": "node scripts/validate-skills.js",
+    "validate": "skill-validator check skills/ && node scripts/check-project.js",
     "dev": "npm run validate && astro dev",
     "build": "npm run validate && astro build",
     "preview": "astro preview"

scripts/check-project.js

@@ -0,0 +1,76 @@
+#!/usr/bin/env node
+// Project-specific checks that complement skill-validator's generic validation.
+// Covers icskills requirements not part of the Agent Skills spec:
+//   - Required metadata fields (title, category) used by the Astro site
+//   - Evaluation file existence
+
+import { existsSync } from "fs";
+import { join } from "path";
+import { listSkillDirs, readSkill, SKILLS_DIR } from "./lib/parse-skill.js";
+
+const KNOWN_CATEGORIES = [
+  "DeFi",
+  "Tokens",
+  "Auth",
+  "Architecture",
+  "Integration",
+  "Governance",
+  "Frontend",
+  "Security",
+  "Infrastructure",
+  "Wallet",
+];
+
+const evalsDir = join(SKILLS_DIR, "..", "evaluations");
+const filterArgs = process.argv.slice(2);
+const allDirs = listSkillDirs();
+const dirs = filterArgs.length > 0
+  ? allDirs.filter((d) => filterArgs.includes(d))
+  : allDirs;
+const errors = [];
+const warnings = [];
+
+for (const dir of dirs) {
+  const skill = readSkill(dir);
+  if (!skill) continue;
+  const label = `${dir}/SKILL.md`;
+
+  // Required metadata fields (used by the Astro site)
+  if (!skill.meta.title) {
+    errors.push(`${label}: missing required frontmatter field: title`);
+  }
+  if (!skill.meta.category) {
+    errors.push(`${label}: missing required frontmatter field: category`);
+  }
+
+  // Category typo detection
+  if (skill.meta.category && !KNOWN_CATEGORIES.includes(skill.meta.category)) {
+    warnings.push(
+      `${label}: unknown category "${skill.meta.category}" — known categories: ${KNOWN_CATEGORIES.join(", ")}`
+    );
+  }
+
+  // Evaluation file existence
+  if (!existsSync(join(evalsDir, `${dir}.json`))) {
+    warnings.push(
+      `${label}: missing evaluations/${dir}.json — see CONTRIBUTING.md for evaluation guidance`
+    );
+  }
+}
+
+// --- Output ---
+
+if (warnings.length) {
+  console.warn(`\nWARNINGS (${warnings.length}):`);
+  warnings.forEach((w) => console.warn(`  ⚠ ${w}`));
+}
+
+if (errors.length) {
+  console.error(`\nERRORS (${errors.length}):`);
+  errors.forEach((e) => console.error(`  ✗ ${e}`));
+  process.exit(1);
+} else {
+  console.log(
+    `\n✓ Project checks passed for ${dirs.length} skills (${warnings.length} warnings)`
+  );
+}