diff --git a/schema/done.schema.json b/schema/done.schema.json index 4b9fe0d..8033858 100644 --- a/schema/done.schema.json +++ b/schema/done.schema.json @@ -139,6 +139,40 @@ "description": "Minimum number of entries. Default 1." } } + }, + { + "type": "object", + "additionalProperties": false, + "required": ["id", "type", "glob", "pattern"], + "properties": { + "id": { "type": "string" }, + "description": { "type": "string" }, + "type": { "const": "no-new" }, + "glob": { "type": "string", "description": "Files to scan." }, + "pattern": { "type": "string", "description": "Regex whose match count must not increase versus the base ref." }, + "flags": { "type": "string" }, + "ignore": { + "type": "array", + "items": { "type": "string" }, + "description": "Extra globs to exclude." + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": ["id", "type", "glob"], + "properties": { + "id": { "type": "string" }, + "description": { "type": "string" }, + "type": { "const": "no-deleted" }, + "glob": { "type": "string", "description": "Files that existed at the base ref and must still exist." }, + "ignore": { + "type": "array", + "items": { "type": "string" }, + "description": "Extra globs to exclude from the must-still-exist set." + } + } } ] } diff --git a/src/cli.ts b/src/cli.ts index 4f6e6d5..56c27e2 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -2,11 +2,12 @@ import fs from "node:fs"; import path from "node:path"; import os from "node:os"; -import { findSpecPath, loadSpec } from "./spec.js"; -import { runGates } from "./core.js"; +import { findSpecPath, loadSpec, parseSpec, DEFAULT_SPEC_PATHS, type Spec } from "./spec.js"; +import { runGates, decideCommand } from "./core.js"; import { checkDrift, formatDiff, DEFAULT_THRESHOLD, discover, pickCanonical } from "./drift.js"; import { runSync } from "./link.js"; import { runScaffold, listTemplates } from "./scaffold.js"; +import { resolveBaseRef, mergeBase, readFileAtRef, repoRoot } from "./git.js"; const C = { reset: "\x1b[0m", @@ -68,6 +69,7 @@ function help(): void { Usage: skillgate audit one-shot read-only audit of this repo (no config needed) skillgate check [spec] run gates, exit 1 if any fail + skillgate gate allow/block one command (any harness); exit 2 = block skillgate init write an example .skillgate/done.yaml skillgate scaffold [--template] generate .skillgate/evidence/ with stack templates skillgate drift report AI instruction-file drift, exit 1 if drifted @@ -77,8 +79,14 @@ Usage: skillgate --version Flags: - --json machine-readable output (check, drift) + --json machine-readable output (check, gate, drift) --cwd run against another directory + --pin check/gate: read the spec from the base ref, not the + working tree, so a change can't loosen its own gate + --base ref for diff-aware gates (no-new, no-deleted) and, with + --pin, the pinned spec. Default: SKILLGATE_BASE or origin/HEAD + --command "" gate: the command to judge (else read from stdin) + --allow-on-error gate: allow instead of fail-closed if evaluation errors --threshold <0..1> drift: similarity required to count as in sync (default 0.95) --dry-run sync: show what would change without writing --symlink sync: use symlinks instead of pointer files and copies @@ -114,6 +122,81 @@ const cwd = cwdIdx >= 0 ? path.resolve(args[cwdIdx + 1]) : process.cwd(); const updateAgents = args.includes("--update-agents"); +const baseIdx = args.indexOf("--base"); +const baseArg = baseIdx >= 0 ? args[baseIdx + 1] : undefined; +// Pinning is a deliberate opt-in and CLI/env-only — never a spec field, so a spec +// can't grant itself immunity. `--pin` reads the spec from the base ref; `--base` +// only names which ref (used for diff-aware gates too), it does not pin on its own. +const pin = args.includes("--pin"); + +function die(code: number, msg: string): never { + console.error(c(C.red, `skillgate: ${msg}`)); + process.exit(code); +} + +interface Resolved { + spec: Spec; + /** merge-base ref diff-aware gates compare against, if one could be resolved. */ + gateBase?: string; + /** ref the spec itself was pinned to (only set in --pin mode). */ + pinnedTo?: string; +} + +/** + * Load the spec and the git base the gates judge against. Without `--pin` the spec + * comes from the working tree (a diff base is still resolved best-effort so + * diff-aware gates work). With `--pin` the spec is read from the base ref itself — + * so the change under review cannot edit or delete the policy it is judged by — + * and anything that prevents that (no base, no pinned spec) fails closed. + */ +function resolveSpecAndBase(specPathHint: string | null): Resolved { + const rawBase = resolveBaseRef(cwd, baseArg); + const gateBase = rawBase ? mergeBase(cwd, rawBase) : undefined; + + if (!pin) { + if (!specPathHint) die(2, "no spec found — run `skillgate init` or pass a path"); + return { spec: loadSpec(specPathHint), gateBase }; + } + + if (!rawBase) { + die(2, "--pin: cannot resolve a base ref (set SKILLGATE_BASE or pass --base ) — refusing to run unpinned (fail-closed)"); + } + const root = repoRoot(cwd); + if (!root) die(2, "--pin: not a git repository (fail-closed)"); + const rels = specPathHint + ? [path.relative(root, specPathHint).split(path.sep).join("/")] + : DEFAULT_SPEC_PATHS; + for (const rel of rels) { + const raw = readFileAtRef(cwd, gateBase!, rel); + if (raw != null) { + const label = `${gateBase!.slice(0, 12)}:${rel}`; + return { spec: parseSpec(raw, label, rel.endsWith(".json")), gateBase, pinnedTo: gateBase }; + } + } + return die(2, `--pin: no committed spec at ${gateBase!.slice(0, 12)} (looked for ${rels.join(", ")}) — commit your .skillgate/done.yaml to the base branch first (fail-closed)`); +} + +/** Read the command an agent is about to run from stdin — raw, or from a hook JSON payload. */ +function readStdinCommand(): string { + if (process.stdin.isTTY) return ""; + let raw = ""; + try { + raw = fs.readFileSync(0, "utf8").trim(); + } catch { + return ""; + } + if (!raw) return ""; + if (raw.startsWith("{")) { + try { + const o: any = JSON.parse(raw); + return String(o?.tool_input?.command ?? o?.command ?? o?.params?.command ?? o?.tool_input?.cmd ?? raw); + } catch { + return raw; + } + } + return raw; +} + if (cmd === "init") { const dir = path.join(cwd, ".skillgate"); fs.mkdirSync(dir, { recursive: true }); @@ -184,24 +267,32 @@ if (cmd === "audit") { if (cmd === "check") { const explicit = args[1] && !args[1].startsWith("-") ? path.resolve(cwd, args[1]) : null; const specPath = explicit ?? findSpecPath(cwd); - if (!specPath || !fs.existsSync(specPath)) { + // Without --pin the working-tree spec must exist; with --pin it may have been + // deleted in the change under review — resolveSpecAndBase reads it from the base. + if (!pin && (!specPath || !fs.existsSync(specPath))) { console.error(c(C.red, "skillgate: no spec found") + " — run `skillgate init` or pass a path"); process.exit(2); } let result; + let pinnedTo: string | undefined; try { - result = runGates(loadSpec(specPath), cwd); + const r = resolveSpecAndBase(specPath && fs.existsSync(specPath) ? specPath : null); + pinnedTo = r.pinnedTo; + result = runGates(r.spec, cwd, { baseRef: r.gateBase }); } catch (e: any) { console.error(c(C.red, `skillgate: ${e.message}`)); process.exit(2); } if (json) { - console.log(JSON.stringify(result, null, 2)); + console.log(JSON.stringify({ ...result, pinnedTo }, null, 2)); process.exit(result.passed ? 0 : 1); } + if (pinnedTo) { + console.log(c(C.dim, ` policy pinned to ${pinnedTo.slice(0, 12)} (base ref) — this change cannot loosen it`)); + } for (const r of result.results) { const mark = r.ok ? c(C.green, "✓") : c(C.red, "✗"); console.log(` ${mark} ${r.id} ${c(C.dim, r.reason)}`); @@ -322,6 +413,44 @@ if (cmd === "canonical") { process.exit(0); } +if (cmd === "gate") { + // Harness-neutral entrypoint: pipe in (or pass) the command an agent is about to + // run; get back allow/block. Works from a Claude Code PreToolUse hook, a Cursor/ + // Codex/git wrapper, or a bare shell — enforcement no longer needs one harness's + // plugin API. Exit 0 = allow, 2 = block. Fails closed on error (unless --allow-on-error). + const cIdx = args.indexOf("--command"); + const command = ((cIdx >= 0 ? args[cIdx + 1] : readStdinCommand()) ?? "").trim(); + const allowOnError = args.includes("--allow-on-error"); + const specPath = findSpecPath(cwd); + + if (!pin && !specPath) { + // No definition of done configured: nothing to enforce, let it through. + const d = { decision: "allow", reason: "no skillgate spec — nothing to enforce", command }; + console.log(json ? JSON.stringify(d, null, 2) : c(C.dim, `allow · ${d.reason}`)); + process.exit(0); + } + + try { + const r = resolveSpecAndBase(specPath && fs.existsSync(specPath) ? specPath : null); + const decision = decideCommand(r.spec, cwd, command, { baseRef: r.gateBase }); + if (json) { + console.log(JSON.stringify({ ...decision, pinnedTo: r.pinnedTo }, null, 2)); + } else if (decision.decision === "block") { + console.error(c(C.red, `✗ blocked: `) + decision.reason); + for (const f of decision.result?.failed ?? []) console.error(c(C.dim, ` · ${f.id}: ${f.reason}`)); + } else { + console.log(c(C.green, `✓ allow`) + c(C.dim, ` · ${decision.reason}`)); + } + process.exit(decision.decision === "block" ? 2 : 0); + } catch (e: any) { + const decision = allowOnError ? "allow" : "block"; + const payload = { decision, reason: `error: ${e.message}`, command }; + if (json) console.log(JSON.stringify(payload, null, 2)); + else console.error(c(allowOnError ? C.dim : C.red, `${decision}: ${payload.reason}`)); + process.exit(allowOnError ? 0 : 2); + } +} + console.error(`unknown command: ${cmd}\n`); help(); process.exit(2); diff --git a/src/core.ts b/src/core.ts index e7ab3a1..0d9a6f7 100644 --- a/src/core.ts +++ b/src/core.ts @@ -4,6 +4,7 @@ import { execSync } from "node:child_process"; import { globSync } from "tinyglobby"; import { type Spec, type Gate, DEFAULT_COMMAND_TIMEOUT_MS, DEFAULT_NOT_EMPTY_MIN } from "./spec.js"; import { checkDrift, DEFAULT_THRESHOLD } from "./drift.js"; +import { readFileAtRef, listFilesAtRef, matchesGlob } from "./git.js"; export interface GateResult { id: string; @@ -18,9 +19,27 @@ export interface RunResult { failed: GateResult[]; } +/** Evaluation context. `baseRef` is the git ref diff-aware gates compare against. */ +export interface RunOptions { + baseRef?: string; +} + const IGNORE = ["**/node_modules/**", "**/.git/**", "dist/**"]; -function checkGate(gate: Gate, cwd: string): GateResult { +/** Count lines of `text` that match `re`; calls `onFirst` with the 0-based index of the first hit. */ +function countMatchingLines(text: string, re: RegExp, onFirst?: (i: number) => void): number { + const lines = text.split("\n"); + let n = 0; + for (let i = 0; i < lines.length; i++) { + if (re.test(lines[i])) { + if (n === 0 && onFirst) onFirst(i); + n++; + } + } + return n; +} + +function checkGate(gate: Gate, cwd: string, opts: RunOptions): GateResult { const base = { id: gate.id, type: gate.type }; try { switch (gate.type) { @@ -105,6 +124,59 @@ function checkGate(gate: Gate, cwd: string): GateResult { if (entries.length < min) return { ...base, ok: false, reason: `directory has ${entries.length} entries, expected at least ${min}` }; return { ...base, ok: true, reason: `directory has ${entries.length} entries` }; } + case "no-new": { + if (!opts.baseRef) { + return { ...base, ok: false, reason: `no git base ref to diff against — pass --base or run in a repo with an upstream (fail-closed)` }; + } + // Fresh regex per line (drop g/y so lastIndex can't advance between .test calls). + const flags = (gate.flags ?? "").replace(/[gy]/g, ""); + const re = () => new RegExp(gate.pattern, flags); + const ignore = gate.ignore ?? []; + const workFiles = globSync(gate.glob, { cwd, dot: true, ignore: [...IGNORE, ...ignore] }); + const baseFiles = listFilesAtRef(cwd, opts.baseRef).filter( + (p) => matchesGlob(p, gate.glob, ignore) && !IGNORE.some((ig) => matchesGlob(p, ig)), + ); + const union = new Set([...workFiles, ...baseFiles]); + let baseCount = 0; + let workCount = 0; + let firstNew = ""; + for (const f of union) { + const baseText = readFileAtRef(cwd, opts.baseRef, f); + baseCount += baseText == null ? 0 : countMatchingLines(baseText, re()); + try { + const t = fs.readFileSync(path.resolve(cwd, f), "utf8"); + workCount += countMatchingLines(t, re(), (i) => { + if (!firstNew) firstNew = `${f}:${i + 1}`; + }); + } catch { + /* file gone in working tree — contributes 0, handled by no-deleted */ + } + } + const short = opts.baseRef.slice(0, 12); + return workCount > baseCount + ? { + ...base, + ok: false, + reason: `+${workCount - baseCount} new /${gate.pattern}/ vs ${short} (base ${baseCount}, now ${workCount})${firstNew ? `, e.g. ${firstNew}` : ""}`, + } + : { ...base, ok: true, reason: `no new /${gate.pattern}/ vs ${short} (${workCount} ≤ ${baseCount})` }; + } + case "no-deleted": { + if (!opts.baseRef) { + return { ...base, ok: false, reason: `no git base ref to diff against — pass --base or run in a repo with an upstream (fail-closed)` }; + } + const ignore = gate.ignore ?? []; + const baseFiles = listFilesAtRef(cwd, opts.baseRef).filter((p) => matchesGlob(p, gate.glob, ignore)); + const missing = baseFiles.filter((f) => !fs.existsSync(path.resolve(cwd, f))); + const short = opts.baseRef.slice(0, 12); + return missing.length + ? { + ...base, + ok: false, + reason: `${missing.length} file(s) matching ${gate.glob} deleted since ${short}: ${missing.slice(0, 3).join(", ")}${missing.length > 3 ? " …" : ""}`, + } + : { ...base, ok: true, reason: `no ${gate.glob} files deleted vs ${short} (${baseFiles.length} present)` }; + } default: return { ...base, ok: false, reason: `unknown gate type` }; } @@ -113,9 +185,9 @@ function checkGate(gate: Gate, cwd: string): GateResult { } } -/** Run every gate in the spec over `cwd`. Pure: same inputs, same verdict. */ -export function runGates(spec: Spec, cwd: string): RunResult { - const results = (spec.gates ?? []).map((g) => checkGate(g, cwd)); +/** Run every gate in the spec over `cwd`. Pure: same inputs (incl. git base), same verdict. */ +export function runGates(spec: Spec, cwd: string, opts: RunOptions = {}): RunResult { + const results = (spec.gates ?? []).map((g) => checkGate(g, cwd, opts)); const failed = results.filter((r) => !r.ok); return { passed: failed.length === 0, results, failed }; } @@ -125,3 +197,33 @@ export function isFinishLine(command: string, patterns: string[] | undefined): b if (!patterns || !patterns.length) return false; return patterns.some((p) => command.includes(p)); } + +export interface Decision { + decision: "allow" | "block"; + reason: string; + command: string; + result?: RunResult; +} + +/** + * The harness-neutral verdict: given a command an agent is about to run, decide + * whether to let it through. Any tool — a Claude Code hook, a Cursor/Codex + * wrapper, a git hook, a bare shell — can call this and read the same answer, so + * enforcement no longer depends on one harness's plugin API. A command that does + * not cross the finish line is always allowed; one that does is blocked unless + * every gate passes. + */ +export function decideCommand(spec: Spec, cwd: string, command: string, opts: RunOptions = {}): Decision { + if (!isFinishLine(command, spec.finishLine)) { + return { decision: "allow", reason: "not a finish-line command", command }; + } + const result = runGates(spec, cwd, opts); + return result.passed + ? { decision: "allow", reason: `all ${result.results.length} gates passed`, command, result } + : { + decision: "block", + reason: `${result.failed.length} of ${result.results.length} gates unmet: ${result.failed.map((f) => f.id).join(", ")}`, + command, + result, + }; +} diff --git a/src/git.ts b/src/git.ts new file mode 100644 index 0000000..e6ec20a --- /dev/null +++ b/src/git.ts @@ -0,0 +1,146 @@ +import { execFileSync } from "node:child_process"; + +/** + * Git plumbing for base-pinned and diff-aware gates. Every call is read-only and + * best-effort: a git failure returns null/[]/false rather than throwing, so the + * caller decides whether "no git" means fail-open or fail-closed. skillgate uses + * these to compare the working tree against the base ref a change forked from — + * so a change can neither loosen its own gate nor silently regress past one. + */ + +const GIT_OPTS = { stdio: "pipe" as const, encoding: "utf8" as const, maxBuffer: 64 * 1024 * 1024 }; + +export function gitAvailable(cwd: string): boolean { + try { + execFileSync("git", ["rev-parse", "--is-inside-work-tree"], { cwd, ...GIT_OPTS }); + return true; + } catch { + return false; + } +} + +/** Absolute path of the repo root containing `cwd`, or null when not a repo. */ +export function repoRoot(cwd: string): string | null { + try { + return execFileSync("git", ["rev-parse", "--show-toplevel"], { cwd, ...GIT_OPTS }).trim(); + } catch { + return null; + } +} + +function refExists(cwd: string, ref: string): boolean { + try { + execFileSync("git", ["rev-parse", "--verify", "--quiet", `${ref}^{commit}`], { cwd, ...GIT_OPTS }); + return true; + } catch { + return false; + } +} + +/** + * Resolve the ref a change should be judged against, in trust order: + * explicit request > `SKILLGATE_BASE` env > origin's default branch > common + * defaults. Returns the first ref that actually resolves to a commit, or null + * when none do (the caller then fails closed). + */ +export function resolveBaseRef(cwd: string, requested?: string): string | null { + const originHead = (() => { + try { + return execFileSync("git", ["symbolic-ref", "refs/remotes/origin/HEAD"], { cwd, ...GIT_OPTS }) + .trim() + .replace(/^refs\/remotes\//, ""); + } catch { + return undefined; + } + })(); + const candidates = [requested, process.env.SKILLGATE_BASE, originHead, "origin/main", "origin/master", "main", "master"] + .map((r) => r?.trim()) + .filter((r): r is string => !!r); + for (const ref of candidates) { + if (refExists(cwd, ref)) return ref; + } + return null; +} + +/** + * The fork point of HEAD and `ref` — the commit the current change actually + * branched from. Pinning to this (not the moving tip of `ref`) is what makes a + * gate immune to the change under review: the same diff cannot edit the policy + * it is judged by. Falls back to `ref` itself if no common ancestor is found. + */ +export function mergeBase(cwd: string, ref: string): string { + try { + return execFileSync("git", ["merge-base", ref, "HEAD"], { cwd, ...GIT_OPTS }).trim(); + } catch { + return ref; + } +} + +/** Contents of `relPath` (repo-root-relative, posix) at `ref`, or null if it did not exist there. */ +export function readFileAtRef(cwd: string, ref: string, relPath: string): string | null { + try { + return execFileSync("git", ["show", `${ref}:${relPath}`], { cwd, ...GIT_OPTS }); + } catch { + return null; + } +} + +/** Every tracked path at `ref` (repo-root-relative, posix), or [] on failure. */ +export function listFilesAtRef(cwd: string, ref: string): string[] { + try { + return execFileSync("git", ["ls-tree", "-r", "--name-only", ref], { cwd, ...GIT_OPTS }) + .split("\n") + .map((s) => s.trim()) + .filter(Boolean); + } catch { + return []; + } +} + +/** + * Compile a shell-style glob to an anchored RegExp for matching paths that only + * exist at a git ref (so they can't be walked on disk). Supports `**`, `*`, `?` + * and `{a,b}` alternation — the subset that appears in gate globs. Kept small and + * dependency-free on purpose; tinyglobby handles the on-disk side. + */ +export function globToRegExp(glob: string): RegExp { + let re = ""; + for (let i = 0; i < glob.length; i++) { + const ch = glob[i]; + if (ch === "*") { + if (glob[i + 1] === "*") { + re += ".*"; + i++; + if (glob[i + 1] === "/") i++; // consume the slash after ** so **/x matches x at root + } else { + re += "[^/]*"; + } + } else if (ch === "?") { + re += "[^/]"; + } else if (ch === "{") { + const end = glob.indexOf("}", i); + if (end === -1) { + re += "\\{"; + } else { + const inner = glob + .slice(i + 1, end) + .split(",") + .map((s) => s.replace(/[.+^${}()|[\]\\]/g, "\\$&")) + .join("|"); + re += `(?:${inner})`; + i = end; + } + } else if (".+^$()|[]\\".includes(ch)) { + re += "\\" + ch; + } else { + re += ch; + } + } + return new RegExp("^" + re + "$"); +} + +/** True when `path` matches `glob` and none of the `ignore` globs. */ +export function matchesGlob(p: string, glob: string, ignore: string[] = []): boolean { + if (!globToRegExp(glob).test(p)) return false; + return !ignore.some((ig) => globToRegExp(ig).test(p)); +} diff --git a/src/spec.ts b/src/spec.ts index 53c9db6..c671920 100644 --- a/src/spec.ts +++ b/src/spec.ts @@ -70,6 +70,35 @@ export interface NotEmptyGate extends BaseGate { min?: number; } +/** + * The count of lines matching `pattern` across `glob` must not INCREASE versus + * the base ref. Catches an agent that games a green gate by adding skips, `xit`, + * `// eslint-disable`, TODOs or debug artifacts — a passing suite can still hide + * a regression. Deterministic and diff-aware: it compares the working tree to the + * commit the change forked from, so it needs a resolvable git base (else it fails + * closed). See `no-deleted` for the removal side. + */ +export interface NoNewGate extends BaseGate { + type: "no-new"; + glob: string; + pattern: string; + flags?: string; + /** Extra globs to exclude (fixtures, the spec file itself). */ + ignore?: string[]; +} + +/** + * Every file matching `glob` that existed at the base ref must still exist. Catches + * an agent that makes a gate pass by deleting the tests (or docs, or migrations) + * that were holding it. Diff-aware; fails closed without a resolvable git base. + */ +export interface NoDeletedGate extends BaseGate { + type: "no-deleted"; + glob: string; + /** Extra globs to exclude from the "must still exist" set. */ + ignore?: string[]; +} + export type Gate = | FileExistsGate | FileContainsGate @@ -77,7 +106,12 @@ export type Gate = | CommandGate | EvidenceGate | InstructionSyncGate - | NotEmptyGate; + | NotEmptyGate + | NoNewGate + | NoDeletedGate; + +/** Gate types that compare the working tree to a git base ref. */ +export const DIFF_GATE_TYPES = new Set(["no-new", "no-deleted"]); export interface Spec { /** @@ -119,21 +153,29 @@ export function findSpecPath(dir: string): string | null { return null; } -export function loadSpec(specPath: string): Spec { - const raw = fs.readFileSync(specPath, "utf8"); - const data = specPath.endsWith(".json") ? JSON.parse(raw) : parseYaml(raw); +/** + * Parse and validate spec text. Split out from {@link loadSpec} so a base-pinned + * spec read straight from git (never touching disk) goes through the exact same + * validation. `label` names the source in errors (a path, or `:`). + */ +export function parseSpec(raw: string, label: string, isJson: boolean): Spec { + const data = isJson ? JSON.parse(raw) : parseYaml(raw); if (!data || !Array.isArray(data.gates)) { - throw new Error(`invalid spec ${specPath}: missing "gates" array`); + throw new Error(`invalid spec ${label}: missing "gates" array`); } if (data.version != null) { if (typeof data.version !== "number" || !Number.isInteger(data.version)) { - throw new Error(`invalid spec ${specPath}: "version" must be an integer`); + throw new Error(`invalid spec ${label}: "version" must be an integer`); } if (data.version > SPEC_VERSION) { console.warn( - `skillgate: spec ${specPath} declares version ${data.version} but this build understands up to ${SPEC_VERSION} — upgrade skillgate; some gates may be misread`, + `skillgate: spec ${label} declares version ${data.version} but this build understands up to ${SPEC_VERSION} — upgrade skillgate; some gates may be misread`, ); } } return data as Spec; } + +export function loadSpec(specPath: string): Spec { + return parseSpec(fs.readFileSync(specPath, "utf8"), specPath, specPath.endsWith(".json")); +} diff --git a/test/antibypass.test.ts b/test/antibypass.test.ts new file mode 100644 index 0000000..2c1ebd7 --- /dev/null +++ b/test/antibypass.test.ts @@ -0,0 +1,208 @@ +// Unit tests for the anti-bypass / diff-aware layer: git plumbing, the no-new and +// no-deleted gates (which compare the working tree to a git base ref), and the +// harness-neutral decideCommand verdict. Fixtures are real throwaway git repos. +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { execFileSync } from "node:child_process"; +import { runGates, decideCommand } from "../src/core.js"; +import { + globToRegExp, + matchesGlob, + resolveBaseRef, + mergeBase, + readFileAtRef, + listFilesAtRef, + gitAvailable, + repoRoot, +} from "../src/git.js"; +import type { Spec } from "../src/spec.js"; + +function git(dir: string, args: string[]): string { + return execFileSync("git", ["-c", "user.email=t@t.dev", "-c", "user.name=t", ...args], { + cwd: dir, + stdio: "pipe", + encoding: "utf8", + }); +} + +/** A throwaway git repo on branch `main` with `files` committed as the base. */ +function gitProject(files: Record): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "skillgate-git-")); + git(dir, ["init", "-q", "-b", "main"]); + write(dir, files); + git(dir, ["add", "-A"]); + git(dir, ["commit", "-q", "-m", "base"]); + return dir; +} + +function write(dir: string, files: Record): void { + for (const [rel, content] of Object.entries(files)) { + const full = path.join(dir, rel); + fs.mkdirSync(path.dirname(full), { recursive: true }); + fs.writeFileSync(full, content); + } +} + +// ---- globToRegExp / matchesGlob --------------------------------------------- + +test("globToRegExp: **, *, ?, and {a,b} alternation", () => { + assert.ok(globToRegExp("**/*.test.ts").test("test/deep/a.test.ts")); + assert.ok(globToRegExp("**/*.ts").test("a.ts")); // ** collapses at root + assert.ok(globToRegExp("src/*.ts").test("src/a.ts")); + assert.ok(!globToRegExp("src/*.ts").test("src/deep/a.ts")); // * stops at / + assert.ok(globToRegExp("src/**/*.{ts,js}").test("src/x/y.js")); + assert.ok(!globToRegExp("src/**/*.{ts,js}").test("src/x/y.py")); + assert.ok(globToRegExp("a?c.ts").test("abc.ts")); + assert.ok(!globToRegExp("a?c.ts").test("a/c.ts")); +}); + +test("matchesGlob: honours ignore globs", () => { + assert.ok(matchesGlob("test/a.test.ts", "test/**/*.test.ts")); + assert.ok(!matchesGlob("test/a.test.ts", "test/**/*.test.ts", ["**/a.test.ts"])); +}); + +// ---- git plumbing ------------------------------------------------------------ + +test("git plumbing: availability, root, read + list at a ref", () => { + const dir = gitProject({ "src/a.ts": "export const a = 1\n", "README.md": "hi\n" }); + assert.equal(gitAvailable(dir), true); + assert.equal(repoRoot(dir), fs.realpathSync(dir)); + assert.equal(readFileAtRef(dir, "HEAD", "src/a.ts"), "export const a = 1\n"); + assert.equal(readFileAtRef(dir, "HEAD", "does/not/exist.ts"), null); + const files = listFilesAtRef(dir, "HEAD"); + assert.ok(files.includes("src/a.ts") && files.includes("README.md")); +}); + +test("git plumbing: non-repo dir is safely empty, never throws", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "skillgate-nogit-")); + assert.equal(gitAvailable(dir), false); + assert.equal(repoRoot(dir), null); + assert.equal(resolveBaseRef(dir), null); + assert.equal(readFileAtRef(dir, "HEAD", "x"), null); + assert.deepEqual(listFilesAtRef(dir, "HEAD"), []); +}); + +test("resolveBaseRef: explicit request wins; mergeBase finds the fork point", () => { + const dir = gitProject({ "a.txt": "1\n" }); + assert.equal(resolveBaseRef(dir, "HEAD"), "HEAD"); + // branch off, advance main, then merge-base(main, HEAD) == the fork commit. + const fork = git(dir, ["rev-parse", "HEAD"]).trim(); + git(dir, ["checkout", "-q", "-b", "feature"]); + write(dir, { "b.txt": "2\n" }); + git(dir, ["add", "-A"]); + git(dir, ["commit", "-q", "-m", "feature work"]); + assert.equal(mergeBase(dir, "main"), fork); +}); + +test("resolveBaseRef: SKILLGATE_BASE env is honoured", () => { + const dir = gitProject({ "a.txt": "1\n" }); + const prev = process.env.SKILLGATE_BASE; + process.env.SKILLGATE_BASE = "main"; + try { + assert.equal(resolveBaseRef(dir), "main"); + } finally { + if (prev === undefined) delete process.env.SKILLGATE_BASE; + else process.env.SKILLGATE_BASE = prev; + } +}); + +// ---- no-new gate ------------------------------------------------------------- + +const skipSpec: Spec = { + gates: [{ id: "no-new-skips", type: "no-new", glob: "**/*.test.ts", pattern: "\\.(skip|only)\\(" }], +}; + +test("no-new: passes when a skip count does not increase", () => { + const dir = gitProject({ "a.test.ts": "test('x', () => {})\n" }); + const r = runGates(skipSpec, dir, { baseRef: "HEAD" }); + assert.equal(r.passed, true, r.failed[0]?.reason); +}); + +test("no-new: fails when the working tree adds a skip (modified file)", () => { + const dir = gitProject({ "a.test.ts": "test('x', () => {})\n" }); + write(dir, { "a.test.ts": "test.skip('x', () => {})\n" }); + const r = runGates(skipSpec, dir, { baseRef: "HEAD" }); + assert.equal(r.passed, false); + assert.match(r.failed[0].reason, /\+1 new/); + assert.match(r.failed[0].reason, /a\.test\.ts:1/); +}); + +test("no-new: fails when a brand-new untracked file introduces a skip", () => { + const dir = gitProject({ "a.test.ts": "test('x', () => {})\n" }); + write(dir, { "b.test.ts": "test.skip('y', () => {})\n" }); // untracked, not in base + const r = runGates(skipSpec, dir, { baseRef: "HEAD" }); + assert.equal(r.passed, false); + assert.match(r.failed[0].reason, /b\.test\.ts/); +}); + +test("no-new: removing a skip keeps it passing (count decreased)", () => { + const dir = gitProject({ "a.test.ts": "test.skip('x', () => {})\n" }); + write(dir, { "a.test.ts": "test('x', () => {})\n" }); + assert.equal(runGates(skipSpec, dir, { baseRef: "HEAD" }).passed, true); +}); + +test("no-new: fails closed with no base ref", () => { + const dir = gitProject({ "a.test.ts": "test('x', () => {})\n" }); + const r = runGates(skipSpec, dir, {}); + assert.equal(r.passed, false); + assert.match(r.failed[0].reason, /fail-closed/); +}); + +// ---- no-deleted gate --------------------------------------------------------- + +const noDeletedSpec: Spec = { + gates: [{ id: "keep-tests", type: "no-deleted", glob: "test/**/*.test.ts" }], +}; + +test("no-deleted: fails when a base test file is removed", () => { + const dir = gitProject({ "test/a.test.ts": "1\n", "test/b.test.ts": "2\n" }); + fs.rmSync(path.join(dir, "test/a.test.ts")); + const r = runGates(noDeletedSpec, dir, { baseRef: "HEAD" }); + assert.equal(r.passed, false); + assert.match(r.failed[0].reason, /a\.test\.ts/); +}); + +test("no-deleted: passes when nothing matching was removed", () => { + const dir = gitProject({ "test/a.test.ts": "1\n", "src/x.ts": "2\n" }); + fs.rmSync(path.join(dir, "src/x.ts")); // not in the glob + assert.equal(runGates(noDeletedSpec, dir, { baseRef: "HEAD" }).passed, true); +}); + +test("no-deleted: fails closed with no base ref", () => { + const dir = gitProject({ "test/a.test.ts": "1\n" }); + const r = runGates(noDeletedSpec, dir, {}); + assert.equal(r.passed, false); + assert.match(r.failed[0].reason, /fail-closed/); +}); + +// ---- decideCommand (harness-neutral entrypoint) ------------------------------ + +const finishSpec: Spec = { + finishLine: ["git commit"], + gates: [{ id: "no-new-skips", type: "no-new", glob: "**/*.test.ts", pattern: "\\.skip\\(" }], +}; + +test("decideCommand: allows a command that is not a finish line", () => { + const dir = gitProject({ "a.test.ts": "test.skip('x', () => {})\n" }); + const d = decideCommand(finishSpec, dir, "ls -la", { baseRef: "HEAD" }); + assert.equal(d.decision, "allow"); + assert.match(d.reason, /not a finish-line/); +}); + +test("decideCommand: blocks a finish-line command when a gate fails", () => { + const dir = gitProject({ "a.test.ts": "test('x', () => {})\n" }); + write(dir, { "a.test.ts": "test.skip('x', () => {})\n" }); + const d = decideCommand(finishSpec, dir, "git commit -m wip", { baseRef: "HEAD" }); + assert.equal(d.decision, "block"); + assert.match(d.reason, /no-new-skips/); +}); + +test("decideCommand: allows a finish-line command when every gate passes", () => { + const dir = gitProject({ "a.test.ts": "test('x', () => {})\n" }); + const d = decideCommand(finishSpec, dir, "git commit -m ok", { baseRef: "HEAD" }); + assert.equal(d.decision, "allow"); + assert.match(d.reason, /gates passed/); +}); diff --git a/test/e2e-antibypass.test.ts b/test/e2e-antibypass.test.ts new file mode 100644 index 0000000..a70d9c3 --- /dev/null +++ b/test/e2e-antibypass.test.ts @@ -0,0 +1,166 @@ +// End-to-end tests for the anti-bypass CLI surface: `check --base` diff-aware +// gates, `check --pin` (policy read from the base ref, immune to the change under +// review), and the `gate` neutral entrypoint. Fixtures are real git repos. +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { execFileSync } from "node:child_process"; + +const CLI = new URL("../src/cli.js", import.meta.url).pathname; + +interface Run { + status: number; + stdout: string; + stderr: string; +} + +function sg(args: string[], cwd: string, input?: string): Run { + try { + const stdout = execFileSync(process.execPath, [CLI, ...args], { + cwd, + encoding: "utf8", + input, + stdio: [input === undefined ? "ignore" : "pipe", "pipe", "pipe"], + }); + return { status: 0, stdout, stderr: "" }; + } catch (e: any) { + return { status: e.status ?? 1, stdout: String(e.stdout ?? ""), stderr: String(e.stderr ?? "") }; + } +} + +function gitCmd(dir: string, args: string[]): void { + execFileSync("git", ["-c", "user.email=t@t.dev", "-c", "user.name=t", ...args], { cwd: dir, stdio: "pipe" }); +} + +function write(dir: string, files: Record): void { + for (const [rel, content] of Object.entries(files)) { + const full = path.join(dir, rel); + fs.mkdirSync(path.dirname(full), { recursive: true }); + fs.writeFileSync(full, content); + } +} + +/** Throwaway git repo on `main` with `files` committed as the base. */ +function gitProject(files: Record): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "skillgate-e2e-ab-")); + gitCmd(dir, ["init", "-q", "-b", "main"]); + write(dir, files); + gitCmd(dir, ["add", "-A"]); + gitCmd(dir, ["commit", "-q", "-m", "base"]); + return dir; +} + +test("check --base: no-new gate fails when the working tree adds a skip", () => { + const dir = gitProject({ + "a.test.ts": "test('x', () => {})\n", + ".skillgate/done.yaml": + "gates:\n - id: no-new-skips\n type: no-new\n glob: '**/*.test.ts'\n pattern: '\\.skip\\('\n", + }); + write(dir, { "a.test.ts": "test.skip('x', () => {})\n" }); + const r = sg(["check", "--base", "HEAD", "--json"], dir); + assert.equal(r.status, 1); + const out = JSON.parse(r.stdout); + assert.equal(out.passed, false); + assert.equal(out.failed[0].id, "no-new-skips"); +}); + +test("check --base: no-new gate passes when nothing regressed", () => { + const dir = gitProject({ + "a.test.ts": "test('x', () => {})\n", + ".skillgate/done.yaml": + "gates:\n - id: no-new-skips\n type: no-new\n glob: '**/*.test.ts'\n pattern: '\\.skip\\('\n", + }); + const r = sg(["check", "--base", "HEAD"], dir); + assert.equal(r.status, 0); +}); + +test("check --pin: a change cannot loosen the gate it is judged by", () => { + const dir = gitProject({ + "README.md": "hi\n", + ".skillgate/done.yaml": "gates:\n - id: needs-license\n type: file-exists\n file: LICENSE\n", + }); + // Agent edits the working-tree spec to delete the failing gate. + write(dir, { ".skillgate/done.yaml": "gates:\n - id: trivial\n type: file-exists\n file: README.md\n" }); + + // Unpinned: the loosened working-tree spec wins → passes. + assert.equal(sg(["check"], dir).status, 0); + // Pinned to the base ref: the original strict spec is enforced → still fails. + const pinned = sg(["check", "--pin", "--base", "HEAD"], dir); + assert.equal(pinned.status, 1); + assert.match(pinned.stdout, /needs-license/); +}); + +test("check --pin: still enforces even if the spec is deleted in the working tree", () => { + const dir = gitProject({ + "README.md": "hi\n", + ".skillgate/done.yaml": "gates:\n - id: needs-license\n type: file-exists\n file: LICENSE\n", + }); + fs.rmSync(path.join(dir, ".skillgate/done.yaml")); // agent deletes the policy + const pinned = sg(["check", "--pin", "--base", "HEAD"], dir); + assert.equal(pinned.status, 1); + assert.match(pinned.stdout, /needs-license/); +}); + +test("check --pin: fails closed when no base ref can be resolved", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "skillgate-nogit-")); + write(dir, { ".skillgate/done.yaml": "gates:\n - id: t\n type: file-exists\n file: README.md\n" }); + const r = sg(["check", "--pin"], dir); + assert.equal(r.status, 2); + assert.match(r.stderr, /fail-closed/); +}); + +test("gate: allows a command that is not a finish line", () => { + const dir = gitProject({ + "a.txt": "1\n", + ".skillgate/done.yaml": + "finishLine:\n - 'git commit'\ngates:\n - id: needs-license\n type: file-exists\n file: LICENSE\n", + }); + const r = sg(["gate", "--command", "ls -la", "--json"], dir); + assert.equal(r.status, 0); + assert.equal(JSON.parse(r.stdout).decision, "allow"); +}); + +test("gate: blocks a finish-line command when a gate fails (exit 2)", () => { + const dir = gitProject({ + "a.txt": "1\n", + ".skillgate/done.yaml": + "finishLine:\n - 'git commit'\ngates:\n - id: needs-license\n type: file-exists\n file: LICENSE\n", + }); + const r = sg(["gate", "--command", "git commit -m wip", "--json"], dir); + assert.equal(r.status, 2); + const out = JSON.parse(r.stdout); + assert.equal(out.decision, "block"); + assert.match(out.reason, /needs-license/); +}); + +test("gate: reads a Claude Code hook JSON payload from stdin", () => { + const dir = gitProject({ + "a.txt": "1\n", + ".skillgate/done.yaml": + "finishLine:\n - 'git commit'\ngates:\n - id: needs-license\n type: file-exists\n file: LICENSE\n", + }); + const payload = JSON.stringify({ tool_input: { command: "git commit -m x" } }); + const r = sg(["gate", "--json"], dir, payload); + assert.equal(r.status, 2); + assert.equal(JSON.parse(r.stdout).decision, "block"); +}); + +test("gate: no spec means nothing to enforce (allow)", () => { + const dir = gitProject({ "a.txt": "1\n" }); + const r = sg(["gate", "--command", "git commit -m x", "--json"], dir); + assert.equal(r.status, 0); + assert.equal(JSON.parse(r.stdout).decision, "allow"); +}); + +test("gate: human output blocks with a reason on stderr", () => { + const dir = gitProject({ + "a.txt": "1\n", + ".skillgate/done.yaml": + "finishLine:\n - 'git commit'\ngates:\n - id: needs-license\n type: file-exists\n file: LICENSE\n", + }); + const r = sg(["gate", "--command", "git commit"], dir); + assert.equal(r.status, 2); + assert.match(r.stderr, /blocked/); +});