ci(migrate): fixtures integrity check + pricing-cache staleness report

migrate/plugins/migration-to-aws/tools/fixtures-check.ts

@@ -0,0 +1,214 @@
+// fixtures-check.ts — structural integrity gate for the committed replay fixtures.
+//
+// WHY: the fixture sets under `fixtures/` are the plugin's regression harness — canned
+// captures, mid-pipeline seeds, expected-assertion documents, and stdlib asserters that
+// fresh-agent replays are checked against. They rot silently in ways `mise run build`
+// never sees: a capture manifest can reference files that were never committed (a
+// repo-root `.gitignore` rule once swallowed three of them), a seed `.phase-status.json`
+// can lag behind the phases a skill declares, and an asserter can carry syntax only a
+// newer Python accepts. Each of those failures is invisible until someone replays the
+// fixture by hand. This check makes them a CI failure instead.
+//
+// Checks (all offline, zero-dep, read-only):
+//   1. every fixtures/**/*.json (dotfiles included) parses
+//   2. every fixtures/**/*.py byte-compiles under the ambient python3
+//   3. every capture manifest's file references resolve:
+//        - `captures[]` / `api[]` entries with status "ok" must have their file on disk
+//          (failed/skipped entries may legitimately have none)
+//        - `build.files[]` must all exist unless build.method is "unavailable"
+//   4. no file under fixtures/ is gitignored (exists locally but would never commit)
+//   5. every fixtures/**/.phase-status.json is schema-shaped (migration_id/last_updated/
+//      phases; statuses in the enum) and its phases match the owning skill's declared
+//      phase set exactly (owning skill inferred from the fixture dir's first name
+//      segment -> skills/<segment>-to-aws; cross-check skipped when no such skill)
+//
+// Usage:
+//   node fixtures-check.ts            # check the repo's own fixtures (mise task)
+//   node fixtures-check.ts <root>     # check another checkout (e.g. a PR worktree)
+//
+// Zero-dep: runs under Node 24 native TS type-stripping (same as the other tools).
+
+import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
+import { basename, dirname, join, relative } from "node:path";
+import { spawnSync } from "node:child_process";
+
+const ROOT = process.argv[2] ?? ".";
+const PLUGIN = join(ROOT, "migrate/plugins/migration-to-aws");
+const FIXTURES = join(PLUGIN, "fixtures");
+const SKILLS = join(PLUGIN, "skills");
+
+const problems: string[] = [];
+const notes: string[] = [];
+
+/** Recursively list files under a dir (paths relative to it), or [] when absent. */
+function walk(root: string, rel = ""): string[] {
+  const abs = join(root, rel);
+  if (!existsSync(abs)) return [];
+  const out: string[] = [];
+  for (const entry of readdirSync(abs)) {
+    const r = rel ? join(rel, entry) : entry;
+    if (statSync(join(root, r)).isDirectory()) out.push(...walk(root, r));
+    else out.push(r);
+  }
+  return out;
+}
+
+const files = walk(FIXTURES);
+const jsonFiles = files.filter((f) => f.endsWith(".json"));
+const pyFiles = files.filter((f) => f.endsWith(".py"));
+const manifests = jsonFiles.filter((f) => basename(f) === "manifest.json");
+const phaseStatusFiles = jsonFiles.filter((f) => basename(f) === ".phase-status.json");
+
+// ---- 1. JSON parses -------------------------------------------------------
+const parsed = new Map<string, unknown>();
+for (const rel of jsonFiles) {
+  try {
+    parsed.set(rel, JSON.parse(readFileSync(join(FIXTURES, rel), "utf8")));
+  } catch (e) {
+    problems.push(`invalid JSON: fixtures/${rel} (${(e as Error).message})`);
+  }
+}
+
+// ---- 2. asserters parse as Python -------------------------------------------
+// ast.parse (not py_compile): same syntax guarantee, but writes no __pycache__
+// bytecode into the fixture tree. Checked one file at a time so every broken
+// asserter is reported, not just the first.
+if (pyFiles.length > 0) {
+  const probe = spawnSync("python3", ["--version"], { encoding: "utf8" });
+  if (probe.error) {
+    notes.push(`python3 unavailable — skipped syntax-checking ${pyFiles.length} asserter(s)`);
+  } else {
+    for (const f of pyFiles) {
+      const py = spawnSync(
+        "python3",
+        ["-c", "import ast,sys\nsrc=open(sys.argv[1],'rb').read()\nast.parse(src, sys.argv[1])", join(FIXTURES, f)],
+        { encoding: "utf8" },
+      );
+      if (py.status !== 0) {
+        problems.push(`asserter does not parse: fixtures/${f}\n${(py.stderr || py.stdout).trim()}`);
+      }
+    }
+  }
+}
+
+// ---- 3. manifest reference integrity ---------------------------------------
+type CaptureEntry = { file?: unknown; status?: unknown };
+function checkEntries(manifestRel: string, entries: unknown, label: string): number {
+  if (!Array.isArray(entries)) return 0;
+  let checked = 0;
+  for (const raw of entries) {
+    const e = raw as CaptureEntry;
+    if (typeof e?.file !== "string") continue;
+    checked++;
+    const target = join(FIXTURES, dirname(manifestRel), e.file);
+    if (e.status === "ok" && !existsSync(target)) {
+      problems.push(
+        `fixtures/${manifestRel}: ${label} entry '${e.file}' has status "ok" but the file is missing`,
+      );
+    }
+  }
+  return checked;
+}
+
+let manifestRefs = 0;
+for (const rel of manifests) {
+  const m = parsed.get(rel) as Record<string, unknown> | undefined;
+  if (!m) continue; // parse failure already reported
+  manifestRefs += checkEntries(rel, m["captures"], "captures[]");
+  manifestRefs += checkEntries(rel, m["api"], "api[]");
+  const build = m["build"] as { method?: unknown; files?: unknown } | undefined;
+  if (build && build.method !== "unavailable" && Array.isArray(build.files)) {
+    for (const f of build.files) {
+      if (typeof f !== "string") continue;
+      manifestRefs++;
+      if (!existsSync(join(FIXTURES, dirname(rel), f))) {
+        problems.push(
+          `fixtures/${rel}: build.files entry '${f}' is missing (build.method is "${String(build.method)}")`,
+        );
+      }
+    }
+  }
+}
+
+// ---- 4. gitignore detection -------------------------------------------------
+if (files.length > 0) {
+  const relToRepo = files.map((f) => join(relative(ROOT, FIXTURES), f)).join("\n");
+  const ci = spawnSync("git", ["-C", ROOT, "check-ignore", "--stdin"], {
+    input: relToRepo,
+    encoding: "utf8",
+  });
+  if (ci.error) {
+    notes.push("git unavailable — skipped the gitignore check");
+  } else {
+    // exit 0 = some ignored (listed on stdout), 1 = none ignored, 128 = error
+    if (ci.status !== null && ci.status > 1) {
+      notes.push(`git check-ignore failed (${(ci.stderr || "").trim()}) — skipped the gitignore check`);
+    } else {
+      for (const line of ci.stdout.split("\n").filter(Boolean)) {
+        problems.push(`gitignored fixture: ${line} exists locally but will never be committed`);
+      }
+    }
+  }
+}
+
+// ---- 5. .phase-status.json shape + phase-set cross-check --------------------
+const STATUS_ENUM = new Set(["pending", "in_progress", "completed"]);
+
+/** Declared phases of a skill: the `_phase:` frontmatter values under references/phases/. */
+function declaredPhases(skill: string): Set<string> | null {
+  const phasesDir = join(SKILLS, skill, "references/phases");
+  if (!existsSync(phasesDir)) return null;
+  const out = new Set<string>();
+  for (const rel of walk(phasesDir).filter((f) => f.endsWith(".md"))) {
+    const head = readFileSync(join(phasesDir, rel), "utf8").slice(0, 400);
+    const m = head.match(/^_phase:\s*([a-z0-9_-]+)\s*$/m);
+    if (m) out.add(m[1]);
+  }
+  return out.size > 0 ? out : null;
+}
+
+for (const rel of phaseStatusFiles) {
+  const ps = parsed.get(rel) as Record<string, unknown> | undefined;
+  if (!ps) continue;
+  const where = `fixtures/${rel}`;
+  for (const req of ["migration_id", "last_updated", "phases"]) {
+    if (!(req in ps)) problems.push(`${where}: missing required key '${req}'`);
+  }
+  const phases = (ps["phases"] ?? {}) as Record<string, unknown>;
+  for (const [name, status] of Object.entries(phases)) {
+    if (typeof status !== "string" || !STATUS_ENUM.has(status)) {
+      problems.push(`${where}: phases.${name} has invalid status '${String(status)}'`);
+    }
+  }
+  // owning skill: first segment of the fixture dir name -> skills/<segment>-to-aws
+  const fixtureDir = rel.split("/")[0] ?? "";
+  const skill = `${fixtureDir.split("-")[0]}-to-aws`;
+  const declared = declaredPhases(skill);
+  if (!declared) {
+    notes.push(`${where}: no declared-phase set found for inferred skill '${skill}' — cross-check skipped`);
+    continue;
+  }
+  const have = new Set(Object.keys(phases));
+  for (const p of declared) {
+    if (!have.has(p)) problems.push(`${where}: skill '${skill}' declares phase '${p}' but the seed omits it`);
+  }
+  for (const p of have) {
+    if (!declared.has(p)) problems.push(`${where}: seed lists phase '${p}' which skill '${skill}' does not declare`);
+  }
+  const current = ps["current_phase"];
+  if (typeof current === "string" && current !== "complete" && !declared.has(current)) {
+    problems.push(`${where}: current_phase '${current}' is not a declared phase of '${skill}'`);
+  }
+}
+
+// ---- report -----------------------------------------------------------------
+for (const n of notes) console.log(`note: ${n}`);
+if (problems.length > 0) {
+  console.error(`fixtures check: FAILED (${problems.length} problem(s))`);
+  for (const p of problems) console.error(`  - ${p}`);
+  process.exit(1);
+}
+console.log(
+  `fixtures check: OK (${jsonFiles.length} json, ${pyFiles.length} asserter(s), ` +
+    `${manifests.length} manifest(s) / ${manifestRefs} reference(s), ${phaseStatusFiles.length} phase-status seed(s))`,
+);

migrate/plugins/migration-to-aws/tools/frontmatter-validator/node-shims.d.ts

@@ -1,8 +1,9 @@
 // node-shims.d.ts
 //
-// Minimal ambient declarations for the slice of Node's stdlib this validator uses.
-// Runs under Node 24 (native type-stripping); this file exists ONLY so `tsc` can
-// type-check without pulling @types/node — keeping the validator zero-dependency.
+// Minimal ambient declarations for the slice of Node's stdlib the plugin's tools use
+// (this validator, fixtures-check.ts, pricing-staleness.ts). Runs under Node 24
+// (native type-stripping); this file exists ONLY so `tsc` can type-check without
+// pulling @types/node — keeping the tools zero-dependency.
 
 declare module "node:fs" {
   export function readFileSync(path: string, encoding: "utf8"): string;
@@ -19,6 +20,16 @@ declare module "node:path" {
   export function join(...parts: string[]): string;
   export function resolve(...parts: string[]): string;
   export function dirname(p: string): string;
+  export function basename(p: string): string;
+  export function relative(from: string, to: string): string;
+}
+
+declare module "node:child_process" {
+  export function spawnSync(
+    command: string,
+    args: string[],
+    options?: { input?: string; encoding: "utf8" },
+  ): { status: number | null; stdout: string; stderr: string; error?: Error };
 }
 
 declare module "node:os" {

migrate/plugins/migration-to-aws/tools/pricing-staleness.ts

@@ -0,0 +1,102 @@
+// pricing-staleness.ts — surface stale pricing caches before users do.
+//
+// WHY: the estimate phases price from vendored caches (`aws-infra-pricing.json`,
+// the per-skill markdown rate cards). Every cache declares its own freshness
+// contract — JSON caches carry `_meta.last_updated` + `_meta.staleness_days`,
+// markdown caches carry a `**Last updated:** YYYY-MM-DD` line and document a
+// 30-day window — but nothing enforced it: caches have quietly crossed their own
+// threshold and every estimate silently degraded to `cached_fallback` accuracy.
+// This check reads each cache's OWN declared date and window and reports drift.
+//
+// Modes:
+//   node pricing-staleness.ts            # report; ALWAYS exit 0 (safe in `build`
+//                                        # — a stale cache must not fail unrelated PRs)
+//   node pricing-staleness.ts --strict   # exit 1 when any cache is stale (for a
+//                                        # scheduled freshness workflow)
+//
+// Zero-dep: runs under Node 24 native TS type-stripping (same as the other tools).
+
+import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
+import { join } from "node:path";
+
+const strict = process.argv.includes("--strict");
+const PLUGIN = "migrate/plugins/migration-to-aws";
+const SKILLS = join(PLUGIN, "skills");
+const DEFAULT_WINDOW_DAYS = 30;
+
+type CacheStatus = { path: string; lastUpdated: string | null; windowDays: number; staleDays: number };
+
+/** Recursively list files under a dir (paths relative to it), or [] when absent. */
+function walk(root: string, rel = ""): string[] {
+  const abs = join(root, rel);
+  if (!existsSync(abs)) return [];
+  const out: string[] = [];
+  for (const entry of readdirSync(abs)) {
+    const r = rel ? join(rel, entry) : entry;
+    if (statSync(join(root, r)).isDirectory()) out.push(...walk(root, r));
+    else out.push(r);
+  }
+  return out;
+}
+
+function daysSince(isoDate: string): number {
+  const then = Date.parse(`${isoDate}T00:00:00Z`);
+  return Math.floor((Date.now() - then) / 86_400_000);
+}
+
+const caches: CacheStatus[] = [];
+
+for (const rel of walk(SKILLS)) {
+  const path = join(SKILLS, rel);
+  if (rel.endsWith("aws-infra-pricing.json")) {
+    try {
+      const meta = (JSON.parse(readFileSync(path, "utf8"))["_meta"] ?? {}) as {
+        last_updated?: string;
+        staleness_days?: number;
+      };
+      const windowDays = typeof meta.staleness_days === "number" ? meta.staleness_days : DEFAULT_WINDOW_DAYS;
+      const lastUpdated = typeof meta.last_updated === "string" ? meta.last_updated : null;
+      caches.push({
+        path,
+        lastUpdated,
+        windowDays,
+        staleDays: lastUpdated ? Math.max(0, daysSince(lastUpdated) - windowDays) : -1,
+      });
+    } catch {
+      caches.push({ path, lastUpdated: null, windowDays: DEFAULT_WINDOW_DAYS, staleDays: -1 });
+    }
+  } else if (/pricing-cache\.md$/.test(rel) || rel.endsWith("heroku-pricing-cache.md")) {
+    const m = readFileSync(path, "utf8").match(/\*\*Last updated:\*\*\s*(\d{4}-\d{2}-\d{2})/);
+    const lastUpdated = m ? m[1] : null;
+    caches.push({
+      path,
+      lastUpdated,
+      windowDays: DEFAULT_WINDOW_DAYS,
+      staleDays: lastUpdated ? Math.max(0, daysSince(lastUpdated) - DEFAULT_WINDOW_DAYS) : -1,
+    });
+  }
+}
+
+let staleCount = 0;
+for (const c of caches) {
+  if (c.lastUpdated === null) {
+    console.log(`pricing cache: NO DATE FOUND  ${c.path} — cannot assess freshness`);
+    staleCount++;
+  } else if (c.staleDays > 0) {
+    console.log(
+      `pricing cache: STALE  ${c.path} — last updated ${c.lastUpdated}, ` +
+        `${c.staleDays} day(s) past its own ${c.windowDays}-day window (estimates degrade to fallback accuracy)`,
+    );
+    staleCount++;
+  } else {
+    console.log(`pricing cache: fresh  ${c.path} (last updated ${c.lastUpdated}, window ${c.windowDays}d)`);
+  }
+}
+
+if (caches.length === 0) console.log("pricing staleness: no pricing caches found");
+else if (staleCount > 0) {
+  console.log(`pricing staleness: ${staleCount}/${caches.length} cache(s) stale or unassessable`);
+  if (strict) process.exit(1);
+} else {
+  console.log(`pricing staleness: OK (${caches.length} cache(s) fresh)`);
+}

migrate/plugins/migration-to-aws/tsconfig.json

@@ -12,6 +12,8 @@
   },
   "include": [
     "tools/frontmatter-validator/**/*.ts",
+    "tools/fixtures-check.ts",
+    "tools/pricing-staleness.ts",
     "tests/tools/**/*.ts"
   ]
 }

mise.toml

@@ -66,13 +66,23 @@ run = "node migrate/plugins/migration-to-aws/tools/sync-vendored-shared.ts --wri
 description = "Verify each skill's vendored shared files are byte-identical to canonical skills/shared/ (CI)"
 run = "node migrate/plugins/migration-to-aws/tools/sync-vendored-shared.ts"
 
+[tasks."fixtures:check"]
+description = "Structural integrity of committed replay fixtures (JSON parse, asserter byte-compile, manifest file references, gitignore traps, seed phase-status vs declared phases)"
+run = "node migrate/plugins/migration-to-aws/tools/fixtures-check.ts"
+
+[tasks."pricing:staleness"]
+description = "Report pricing caches past their own declared freshness window (warn-only; use --strict in a scheduled workflow to fail)"
+run = "node migrate/plugins/migration-to-aws/tools/pricing-staleness.ts"
+
 [tasks.lint]
 description = "Run all linters"
 run = [
   { task = "lint:md" },
   { task = "lint:types" },
   { task = "lint:frontmatter" },
   { task = "shared:check" },
+  { task = "fixtures:check" },
+  { task = "pricing:staleness" },
   { task = "test" },
 ]