diff --git a/scripts/analyze.mjs b/scripts/analyze.mjs index d3df067..fb0a4b4 100755 --- a/scripts/analyze.mjs +++ b/scripts/analyze.mjs @@ -5196,26 +5196,65 @@ function resolveEvidence(ref, inv) { const path = ref.replace(/:\d+(-\d+)?$/, ""); return (inv.files ?? []).some((f) => f.path === path) || features.some((f) => (f.files ?? []).includes(path)); } +function readTodoPairs(outDir) { + try { + const todo = JSON.parse(readFileSync12(join14(outDir, "VERIFY.todo.json"), "utf8")); + if (!Array.isArray(todo?.pairs)) return void 0; + const byClaim = /* @__PURE__ */ new Map(); + for (const p of todo.pairs) if (p && typeof p.claimId === "string") byClaim.set(p.claimId, p); + return byClaim.size ? byClaim : void 0; + } catch { + return void 0; + } +} function applyVerdicts(outDir, verdictsPath) { const raw = JSON.parse(readFileSync12(verdictsPath, "utf8")); - const list = Array.isArray(raw) ? raw : Array.isArray(raw?.pairs) ? raw.pairs : []; + const list = Array.isArray(raw) ? raw : Array.isArray(raw?.pairs) ? raw.pairs : Array.isArray(raw?.verdicts) ? raw.verdicts : []; + if (list.length === 0) { + throw new Error(`${verdictsPath}: no verdict rows found \u2014 expected a bare array, { "pairs": [...] } or { "verdicts": [...] } with at least one row.`); + } + const todo = readTodoPairs(outDir); + const problems = []; + const unknown = []; const verdicts = []; - for (const v of list) { - if (!v || typeof v.claimId !== "string") continue; + for (const [i, v] of list.entries()) { + if (!v || typeof v.claimId !== "string") { + problems.push(`row ${i + 1}: missing claimId`); + continue; + } + if (v.verdict != null && !VALID.includes(v.verdict)) { + problems.push(`row ${i + 1} (${v.claimId}): invalid verdict "${String(v.verdict)}" \u2014 expected ${VALID.join("|")} or null`); + continue; + } + const base = todo?.get(v.claimId); + if (todo && !base) { + unknown.push(v.claimId); + continue; + } const verdict = VALID.includes(v.verdict) ? v.verdict : void 0; const confidence = VALID_CONFIDENCE.includes(v.confidence) ? v.confidence : void 0; verdicts.push({ claimId: v.claimId, - claim: typeof v.claim === "string" ? v.claim : "", - feature: typeof v.feature === "string" ? v.feature : "", - evidenceRef: typeof v.evidenceRef === "string" ? v.evidenceRef : "", - digest: typeof v.digest === "string" ? v.digest : "", + claim: typeof v.claim === "string" ? v.claim : base?.claim ?? "", + feature: typeof v.feature === "string" ? v.feature : base?.feature ?? "", + evidenceRef: typeof v.evidenceRef === "string" ? v.evidenceRef : base?.evidenceRef ?? "", + digest: typeof v.digest === "string" ? v.digest : base?.digest ?? "", verdict, note: typeof v.note === "string" ? v.note : "", ...confidence ? { confidence } : {} }); } + if (problems.length) { + throw new Error(`${verdictsPath}: ${problems.length} malformed row(s) \u2014 fix them and re-apply (fail-closed): + - ${problems.join("\n - ")}`); + } + if (verdicts.length === 0) { + throw new Error( + `${verdictsPath}: every row cites a claimId unknown to ${join14(outDir, "VERIFY.todo.json")} (${unknown.join(", ")}) \u2014 stale fragment? Re-run --verify and re-adjudicate.` + ); + } const result = reduceVerdicts(verdicts, readInventoryIfPresent(outDir)); + if (unknown.length) result.ignored = unknown; writeFileSync2(join14(outDir, "VERIFY.json"), JSON.stringify({ ...result, verdicts }, null, 2)); return result; } @@ -5280,6 +5319,12 @@ function foldSemantic(outDir, check, opts = {}) { return; } const fresh = reduceVerdicts(sem.verdicts, readInventoryIfPresent(outDir)); + if (fresh.adjudicated === 0) { + skip( + "--semantic: VERIFY.json carries 0 adjudicated verdicts \u2014 the requirement gate never engaged (re-run --verify then --verify --apply with valid verdict tokens)" + ); + return; + } if (!fresh.ok) { check.errors.push( `semantic verification failed: ${fresh.failures.length} requirement(s) refuted, unsupported or citing unresolvable evidence (see VERIFY.json)` @@ -5308,6 +5353,9 @@ function formatVerifyReport(r) { if (r.unadjudicated.length) { lines.push(` \u26A0 ${r.unadjudicated.length} requirement(s) not fully adjudicated: ${r.unadjudicated.join(", ")}`); } + if (r.ignored?.length) { + lines.push(` \u26A0 ${r.ignored.length} ignored (unknown id): ${r.ignored.join(", ")} \u2014 not in VERIFY.todo.json (stale fragment?)`); + } lines.push(r.ok ? ` \u2713 every requirement traces to the original source` : ` \u2717 some requirements are refuted or unsupported (invented)`); return lines.join("\n"); } @@ -5918,7 +5966,7 @@ function agentContracts(outAbs, engineAbs) { You draft ONE feature of a reconstruction at a time, to full PRD depth \u2014 the MAP half of the enrichment map-reduce (\`references/orchestration.md\`, Phase 1). -Worklist: \`${join17(outAbs, "inventory.json")}\` (\`features[]\` \u2014 each entry carries \`slug\`, \`files\`, \`routes\`, \`interfaces\`, \`entities\`, \`writes\`). Handle ONLY the features whose \`slug\` is named in your prompt (\`ITEMS=\`). +Worklist: \`${join17(outAbs, "inventory.json")}\` (\`features[]\` \u2014 each entry carries \`slug\`, \`files\`, \`routes\`, \`interfaces\`, \`entities\`, \`writes\`). Handle ONLY the features whose \`slug\` is named in your prompt (\`ITEMS=\`). If an ITEMS id is no longer in the worklist, skip it and say so in your note. For EACH of your features: @@ -5938,7 +5986,7 @@ ${footer}`, You are a FINDER of the AI buildability review \u2014 one adversarial reviewer per flagged feature (\`references/orchestration.md\`, Phase 2 step B; rubric: \`references/ai-review-rubric.md\`). -Worklist: \`${join17(outAbs, "REVIEW.todo.json")}\` (\`units[]\`; the flagged ones carry \`needsReview: true\`). Handle ONLY the features named in your prompt (\`ITEMS=\`). +Worklist: \`${join17(outAbs, "REVIEW.todo.json")}\` (\`units[]\`; the flagged ones carry \`needsReview: true\`). Handle ONLY the features named in your prompt (\`ITEMS=\`). If an ITEMS id is no longer in the worklist, skip it and say so in your note. For EACH of your features: @@ -5952,7 +6000,7 @@ ${footer}`, You are an INDEPENDENT VERIFIER of the review loop \u2014 one fresh, adversarial agent per open blocker (\`references/orchestration.md\`, Phase 2 step C). A finding "counts" only when you confirm it. -Worklist: \`${join17(outAbs, "REVIEW.json")}\` (\`failures[]\` \u2014 the open blockers, each \`{ id, feature, category, problem, fix }\`). Handle ONLY the blockers whose \`id\` is named in your prompt (\`ITEMS=\`). +Worklist: \`${join17(outAbs, "REVIEW.json")}\` (\`failures[]\` \u2014 the open blockers, each \`{ id, feature, category, problem, fix }\`). Handle ONLY the blockers whose \`id\` is named in your prompt (\`ITEMS=\`). If an ITEMS id is no longer in the worklist, skip it and say so in your note. For EACH of your blockers: @@ -5966,7 +6014,7 @@ ${footer}`, You adjudicate the requirement\u2194source verification gate of a reconstruction \u2014 judging whether each PRD requirement TRACES to the original code (faithful inference) or was invented. -Worklist: \`${join17(outAbs, "VERIFY.todo.json")}\` (\`pairs[]\`, each \`{ claimId, claim, feature, evidenceRef, digest }\`). Handle ONLY the pairs whose \`claimId\` is named in your prompt (\`ITEMS=\`). +Worklist: \`${join17(outAbs, "VERIFY.todo.json")}\` (\`pairs[]\`, each \`{ claimId, claim, feature, evidenceRef, digest }\`). Handle ONLY the pairs whose \`claimId\` is named in your prompt (\`ITEMS=\`). If an ITEMS id is no longer in the worklist, skip it and say so in your note. For EACH of your pairs: diff --git a/skills/reconstruct/scripts/analyze.mjs b/skills/reconstruct/scripts/analyze.mjs index d3df067..fb0a4b4 100755 --- a/skills/reconstruct/scripts/analyze.mjs +++ b/skills/reconstruct/scripts/analyze.mjs @@ -5196,26 +5196,65 @@ function resolveEvidence(ref, inv) { const path = ref.replace(/:\d+(-\d+)?$/, ""); return (inv.files ?? []).some((f) => f.path === path) || features.some((f) => (f.files ?? []).includes(path)); } +function readTodoPairs(outDir) { + try { + const todo = JSON.parse(readFileSync12(join14(outDir, "VERIFY.todo.json"), "utf8")); + if (!Array.isArray(todo?.pairs)) return void 0; + const byClaim = /* @__PURE__ */ new Map(); + for (const p of todo.pairs) if (p && typeof p.claimId === "string") byClaim.set(p.claimId, p); + return byClaim.size ? byClaim : void 0; + } catch { + return void 0; + } +} function applyVerdicts(outDir, verdictsPath) { const raw = JSON.parse(readFileSync12(verdictsPath, "utf8")); - const list = Array.isArray(raw) ? raw : Array.isArray(raw?.pairs) ? raw.pairs : []; + const list = Array.isArray(raw) ? raw : Array.isArray(raw?.pairs) ? raw.pairs : Array.isArray(raw?.verdicts) ? raw.verdicts : []; + if (list.length === 0) { + throw new Error(`${verdictsPath}: no verdict rows found \u2014 expected a bare array, { "pairs": [...] } or { "verdicts": [...] } with at least one row.`); + } + const todo = readTodoPairs(outDir); + const problems = []; + const unknown = []; const verdicts = []; - for (const v of list) { - if (!v || typeof v.claimId !== "string") continue; + for (const [i, v] of list.entries()) { + if (!v || typeof v.claimId !== "string") { + problems.push(`row ${i + 1}: missing claimId`); + continue; + } + if (v.verdict != null && !VALID.includes(v.verdict)) { + problems.push(`row ${i + 1} (${v.claimId}): invalid verdict "${String(v.verdict)}" \u2014 expected ${VALID.join("|")} or null`); + continue; + } + const base = todo?.get(v.claimId); + if (todo && !base) { + unknown.push(v.claimId); + continue; + } const verdict = VALID.includes(v.verdict) ? v.verdict : void 0; const confidence = VALID_CONFIDENCE.includes(v.confidence) ? v.confidence : void 0; verdicts.push({ claimId: v.claimId, - claim: typeof v.claim === "string" ? v.claim : "", - feature: typeof v.feature === "string" ? v.feature : "", - evidenceRef: typeof v.evidenceRef === "string" ? v.evidenceRef : "", - digest: typeof v.digest === "string" ? v.digest : "", + claim: typeof v.claim === "string" ? v.claim : base?.claim ?? "", + feature: typeof v.feature === "string" ? v.feature : base?.feature ?? "", + evidenceRef: typeof v.evidenceRef === "string" ? v.evidenceRef : base?.evidenceRef ?? "", + digest: typeof v.digest === "string" ? v.digest : base?.digest ?? "", verdict, note: typeof v.note === "string" ? v.note : "", ...confidence ? { confidence } : {} }); } + if (problems.length) { + throw new Error(`${verdictsPath}: ${problems.length} malformed row(s) \u2014 fix them and re-apply (fail-closed): + - ${problems.join("\n - ")}`); + } + if (verdicts.length === 0) { + throw new Error( + `${verdictsPath}: every row cites a claimId unknown to ${join14(outDir, "VERIFY.todo.json")} (${unknown.join(", ")}) \u2014 stale fragment? Re-run --verify and re-adjudicate.` + ); + } const result = reduceVerdicts(verdicts, readInventoryIfPresent(outDir)); + if (unknown.length) result.ignored = unknown; writeFileSync2(join14(outDir, "VERIFY.json"), JSON.stringify({ ...result, verdicts }, null, 2)); return result; } @@ -5280,6 +5319,12 @@ function foldSemantic(outDir, check, opts = {}) { return; } const fresh = reduceVerdicts(sem.verdicts, readInventoryIfPresent(outDir)); + if (fresh.adjudicated === 0) { + skip( + "--semantic: VERIFY.json carries 0 adjudicated verdicts \u2014 the requirement gate never engaged (re-run --verify then --verify --apply with valid verdict tokens)" + ); + return; + } if (!fresh.ok) { check.errors.push( `semantic verification failed: ${fresh.failures.length} requirement(s) refuted, unsupported or citing unresolvable evidence (see VERIFY.json)` @@ -5308,6 +5353,9 @@ function formatVerifyReport(r) { if (r.unadjudicated.length) { lines.push(` \u26A0 ${r.unadjudicated.length} requirement(s) not fully adjudicated: ${r.unadjudicated.join(", ")}`); } + if (r.ignored?.length) { + lines.push(` \u26A0 ${r.ignored.length} ignored (unknown id): ${r.ignored.join(", ")} \u2014 not in VERIFY.todo.json (stale fragment?)`); + } lines.push(r.ok ? ` \u2713 every requirement traces to the original source` : ` \u2717 some requirements are refuted or unsupported (invented)`); return lines.join("\n"); } @@ -5918,7 +5966,7 @@ function agentContracts(outAbs, engineAbs) { You draft ONE feature of a reconstruction at a time, to full PRD depth \u2014 the MAP half of the enrichment map-reduce (\`references/orchestration.md\`, Phase 1). -Worklist: \`${join17(outAbs, "inventory.json")}\` (\`features[]\` \u2014 each entry carries \`slug\`, \`files\`, \`routes\`, \`interfaces\`, \`entities\`, \`writes\`). Handle ONLY the features whose \`slug\` is named in your prompt (\`ITEMS=\`). +Worklist: \`${join17(outAbs, "inventory.json")}\` (\`features[]\` \u2014 each entry carries \`slug\`, \`files\`, \`routes\`, \`interfaces\`, \`entities\`, \`writes\`). Handle ONLY the features whose \`slug\` is named in your prompt (\`ITEMS=\`). If an ITEMS id is no longer in the worklist, skip it and say so in your note. For EACH of your features: @@ -5938,7 +5986,7 @@ ${footer}`, You are a FINDER of the AI buildability review \u2014 one adversarial reviewer per flagged feature (\`references/orchestration.md\`, Phase 2 step B; rubric: \`references/ai-review-rubric.md\`). -Worklist: \`${join17(outAbs, "REVIEW.todo.json")}\` (\`units[]\`; the flagged ones carry \`needsReview: true\`). Handle ONLY the features named in your prompt (\`ITEMS=\`). +Worklist: \`${join17(outAbs, "REVIEW.todo.json")}\` (\`units[]\`; the flagged ones carry \`needsReview: true\`). Handle ONLY the features named in your prompt (\`ITEMS=\`). If an ITEMS id is no longer in the worklist, skip it and say so in your note. For EACH of your features: @@ -5952,7 +6000,7 @@ ${footer}`, You are an INDEPENDENT VERIFIER of the review loop \u2014 one fresh, adversarial agent per open blocker (\`references/orchestration.md\`, Phase 2 step C). A finding "counts" only when you confirm it. -Worklist: \`${join17(outAbs, "REVIEW.json")}\` (\`failures[]\` \u2014 the open blockers, each \`{ id, feature, category, problem, fix }\`). Handle ONLY the blockers whose \`id\` is named in your prompt (\`ITEMS=\`). +Worklist: \`${join17(outAbs, "REVIEW.json")}\` (\`failures[]\` \u2014 the open blockers, each \`{ id, feature, category, problem, fix }\`). Handle ONLY the blockers whose \`id\` is named in your prompt (\`ITEMS=\`). If an ITEMS id is no longer in the worklist, skip it and say so in your note. For EACH of your blockers: @@ -5966,7 +6014,7 @@ ${footer}`, You adjudicate the requirement\u2194source verification gate of a reconstruction \u2014 judging whether each PRD requirement TRACES to the original code (faithful inference) or was invented. -Worklist: \`${join17(outAbs, "VERIFY.todo.json")}\` (\`pairs[]\`, each \`{ claimId, claim, feature, evidenceRef, digest }\`). Handle ONLY the pairs whose \`claimId\` is named in your prompt (\`ITEMS=\`). +Worklist: \`${join17(outAbs, "VERIFY.todo.json")}\` (\`pairs[]\`, each \`{ claimId, claim, feature, evidenceRef, digest }\`). Handle ONLY the pairs whose \`claimId\` is named in your prompt (\`ITEMS=\`). If an ITEMS id is no longer in the worklist, skip it and say so in your note. For EACH of your pairs: diff --git a/src/orchestrate-templates.ts b/src/orchestrate-templates.ts index 7342d1d..7c0caad 100644 --- a/src/orchestrate-templates.ts +++ b/src/orchestrate-templates.ts @@ -273,7 +273,7 @@ export function agentContracts(outAbs: string, engineAbs: string): Record\`). +Worklist: \`${join(outAbs, "inventory.json")}\` (\`features[]\` — each entry carries \`slug\`, \`files\`, \`routes\`, \`interfaces\`, \`entities\`, \`writes\`). Handle ONLY the features whose \`slug\` is named in your prompt (\`ITEMS=\`). If an ITEMS id is no longer in the worklist, skip it and say so in your note. For EACH of your features: @@ -293,7 +293,7 @@ ${footer}`, You are a FINDER of the AI buildability review — one adversarial reviewer per flagged feature (\`references/orchestration.md\`, Phase 2 step B; rubric: \`references/ai-review-rubric.md\`). -Worklist: \`${join(outAbs, "REVIEW.todo.json")}\` (\`units[]\`; the flagged ones carry \`needsReview: true\`). Handle ONLY the features named in your prompt (\`ITEMS=\`). +Worklist: \`${join(outAbs, "REVIEW.todo.json")}\` (\`units[]\`; the flagged ones carry \`needsReview: true\`). Handle ONLY the features named in your prompt (\`ITEMS=\`). If an ITEMS id is no longer in the worklist, skip it and say so in your note. For EACH of your features: @@ -307,7 +307,7 @@ ${footer}`, You are an INDEPENDENT VERIFIER of the review loop — one fresh, adversarial agent per open blocker (\`references/orchestration.md\`, Phase 2 step C). A finding "counts" only when you confirm it. -Worklist: \`${join(outAbs, "REVIEW.json")}\` (\`failures[]\` — the open blockers, each \`{ id, feature, category, problem, fix }\`). Handle ONLY the blockers whose \`id\` is named in your prompt (\`ITEMS=\`). +Worklist: \`${join(outAbs, "REVIEW.json")}\` (\`failures[]\` — the open blockers, each \`{ id, feature, category, problem, fix }\`). Handle ONLY the blockers whose \`id\` is named in your prompt (\`ITEMS=\`). If an ITEMS id is no longer in the worklist, skip it and say so in your note. For EACH of your blockers: @@ -321,7 +321,7 @@ ${footer}`, You adjudicate the requirement↔source verification gate of a reconstruction — judging whether each PRD requirement TRACES to the original code (faithful inference) or was invented. -Worklist: \`${join(outAbs, "VERIFY.todo.json")}\` (\`pairs[]\`, each \`{ claimId, claim, feature, evidenceRef, digest }\`). Handle ONLY the pairs whose \`claimId\` is named in your prompt (\`ITEMS=\`). +Worklist: \`${join(outAbs, "VERIFY.todo.json")}\` (\`pairs[]\`, each \`{ claimId, claim, feature, evidenceRef, digest }\`). Handle ONLY the pairs whose \`claimId\` is named in your prompt (\`ITEMS=\`). If an ITEMS id is no longer in the worklist, skip it and say so in your note. For EACH of your pairs: diff --git a/src/types.ts b/src/types.ts index 4add3ce..2895dc9 100644 --- a/src/types.ts +++ b/src/types.ts @@ -637,6 +637,8 @@ export interface VerifyResult { verdicts?: Verdict[]; /** Aggregated confidence labels, when the adjudicator stamped any. */ confidence?: { confirmed: number; inferred: number; gap: number; unlabeled: number }; + /** Verdict rows whose claimId is not in the run's VERIFY.todo.json — ignored, never folded. */ + ignored?: string[]; } export interface VerifyWorklist { diff --git a/src/verify.ts b/src/verify.ts index 7326030..fa3b1ea 100644 --- a/src/verify.ts +++ b/src/verify.ts @@ -222,29 +222,83 @@ export function resolveEvidence(ref: string, inv: Inventory): boolean { return (inv.files ?? []).some((f) => f.path === path) || features.some((f) => (f.files ?? []).includes(path)); } -// Phase B — read an agent-filled verdicts file (`{ pairs: Verdict[] }` or a bare -// array), reduce to a VerifyResult (re-resolving each citation against the -// inventory), and persist VERIFY.json. +/** + * Read the run's VERIFY.todo.json pairs keyed by claimId — the worklist the + * orchestrate-emitted adjudicator fragments answer. A missing/unreadable + * worklist only disables backfill and unknown-id detection (hand-rolled + * verdicts files stay self-contained). + */ +function readTodoPairs(outDir: string): Map | undefined { + try { + const todo = JSON.parse(readFileSync(join(outDir, "VERIFY.todo.json"), "utf8")); + if (!Array.isArray(todo?.pairs)) return undefined; + const byClaim = new Map(); + for (const p of todo.pairs) if (p && typeof p.claimId === "string") byClaim.set(p.claimId, p as ClaimEvidencePair); + return byClaim.size ? byClaim : undefined; + } catch { + return undefined; + } +} + +// Phase B — read an agent-filled verdicts file (`{ pairs: Verdict[] }`, a +// `{ verdicts: Verdict[] }` object — the shape the orchestrate-emitted +// adjudicator fragments return — or a bare array), validate it FAIL-CLOSED, +// reduce to a VerifyResult (re-resolving each citation against the inventory), +// and persist VERIFY.json. Fragment rows carrying only claimId/verdict/note/ +// confidence are backfilled (claim, feature, evidenceRef, digest) from the +// run's VERIFY.todo.json by claimId, so the citation guard stays engaged. export function applyVerdicts(outDir: string, verdictsPath: string): VerifyResult { const raw = JSON.parse(readFileSync(verdictsPath, "utf8")); - const list: any[] = Array.isArray(raw) ? raw : Array.isArray(raw?.pairs) ? raw.pairs : []; + const list: any[] = Array.isArray(raw) ? raw : Array.isArray(raw?.pairs) ? raw.pairs : Array.isArray(raw?.verdicts) ? raw.verdicts : []; + // Fail closed: a parseable file that yields no rows means the fold never + // engaged — writing an empty (vacuously ok:true) VERIFY.json here is exactly + // the fail-open path `--check --semantic` exists to prevent. + if (list.length === 0) { + throw new Error(`${verdictsPath}: no verdict rows found — expected a bare array, { "pairs": [...] } or { "verdicts": [...] } with at least one row.`); + } + const todo = readTodoPairs(outDir); + const problems: string[] = []; + const unknown: string[] = []; const verdicts: Verdict[] = []; - for (const v of list) { - if (!v || typeof v.claimId !== "string") continue; + for (const [i, v] of (list as any[]).entries()) { + if (!v || typeof v.claimId !== "string") { + problems.push(`row ${i + 1}: missing claimId`); + continue; + } + // An explicit-but-unknown token is a typo, not an un-adjudication: hard-error + // beats silently downgrading a "REFUTED!!" to a warning-level unadjudicated. + if (v.verdict != null && !VALID.includes(v.verdict)) { + problems.push(`row ${i + 1} (${v.claimId}): invalid verdict "${String(v.verdict)}" — expected ${VALID.join("|")} or null`); + continue; + } + const base = todo?.get(v.claimId); + if (todo && !base) { + unknown.push(v.claimId); + continue; + } const verdict = VALID.includes(v.verdict) ? (v.verdict as VerdictKind) : (undefined as unknown as VerdictKind); const confidence = VALID_CONFIDENCE.includes(v.confidence) ? (v.confidence as ConfidenceKind) : undefined; verdicts.push({ claimId: v.claimId, - claim: typeof v.claim === "string" ? v.claim : "", - feature: typeof v.feature === "string" ? v.feature : "", - evidenceRef: typeof v.evidenceRef === "string" ? v.evidenceRef : "", - digest: typeof v.digest === "string" ? v.digest : "", + claim: typeof v.claim === "string" ? v.claim : (base?.claim ?? ""), + feature: typeof v.feature === "string" ? v.feature : (base?.feature ?? ""), + evidenceRef: typeof v.evidenceRef === "string" ? v.evidenceRef : (base?.evidenceRef ?? ""), + digest: typeof v.digest === "string" ? v.digest : (base?.digest ?? ""), verdict, note: typeof v.note === "string" ? v.note : "", ...(confidence ? { confidence } : {}), }); } + if (problems.length) { + throw new Error(`${verdictsPath}: ${problems.length} malformed row(s) — fix them and re-apply (fail-closed):\n - ${problems.join("\n - ")}`); + } + if (verdicts.length === 0) { + throw new Error( + `${verdictsPath}: every row cites a claimId unknown to ${join(outDir, "VERIFY.todo.json")} (${unknown.join(", ")}) — stale fragment? Re-run --verify and re-adjudicate.`, + ); + } const result = reduceVerdicts(verdicts, readInventoryIfPresent(outDir)); + if (unknown.length) result.ignored = unknown; writeFileSync(join(outDir, "VERIFY.json"), JSON.stringify({ ...result, verdicts }, null, 2)); return result; } @@ -323,6 +377,15 @@ export function foldSemantic(outDir: string, check: CheckResult, opts: { allowUn return; } const fresh = reduceVerdicts(sem.verdicts, readInventoryIfPresent(outDir)); + // A green semantic exit must mean the gate ENGAGED: an empty verdicts[] (the + // old fail-open fold wrote one for fragments it could not read) or a ledger + // whose verdicts were all dropped leaves 0 adjudications — a bypass, not a pass. + if (fresh.adjudicated === 0) { + skip( + "--semantic: VERIFY.json carries 0 adjudicated verdicts — the requirement gate never engaged (re-run --verify then --verify --apply with valid verdict tokens)", + ); + return; + } if (!fresh.ok) { check.errors.push( `semantic verification failed: ${fresh.failures.length} requirement(s) refuted, unsupported or citing unresolvable evidence (see VERIFY.json)`, @@ -352,6 +415,9 @@ export function formatVerifyReport(r: VerifyResult): string { if (r.unadjudicated.length) { lines.push(` ⚠ ${r.unadjudicated.length} requirement(s) not fully adjudicated: ${r.unadjudicated.join(", ")}`); } + if (r.ignored?.length) { + lines.push(` ⚠ ${r.ignored.length} ignored (unknown id): ${r.ignored.join(", ")} — not in VERIFY.todo.json (stale fragment?)`); + } lines.push(r.ok ? ` ✓ every requirement traces to the original source` : ` ✗ some requirements are refuted or unsupported (invented)`); return lines.join("\n"); } diff --git a/tests/__snapshots__/orchestrate.test.ts.snap b/tests/__snapshots__/orchestrate.test.ts.snap index ae01bd4..ed5590c 100644 --- a/tests/__snapshots__/orchestrate.test.ts.snap +++ b/tests/__snapshots__/orchestrate.test.ts.snap @@ -76,7 +76,7 @@ exports[`orchestrate — contracts & runbook > golden shape (paths normalized) > You draft ONE feature of a reconstruction at a time, to full PRD depth — the MAP half of the enrichment map-reduce (\`references/orchestration.md\`, Phase 1). -Worklist: \`/inventory.json\` (\`features[]\` — each entry carries \`slug\`, \`files\`, \`routes\`, \`interfaces\`, \`entities\`, \`writes\`). Handle ONLY the features whose \`slug\` is named in your prompt (\`ITEMS=\`). +Worklist: \`/inventory.json\` (\`features[]\` — each entry carries \`slug\`, \`files\`, \`routes\`, \`interfaces\`, \`entities\`, \`writes\`). Handle ONLY the features whose \`slug\` is named in your prompt (\`ITEMS=\`). If an ITEMS id is no longer in the worklist, skip it and say so in your note. For EACH of your features: diff --git a/tests/orchestrate.test.ts b/tests/orchestrate.test.ts index 943b735..7673b1f 100644 --- a/tests/orchestrate.test.ts +++ b/tests/orchestrate.test.ts @@ -278,6 +278,15 @@ describe("orchestrate — contracts & runbook", () => { } }); + it("every worklist-driven contract carries the family stale-id rule", () => { + const dir = fullState(); + orchestrateRun(dir, ENGINE); + for (const role of ["drafter", "finder", "verifier", "adjudicator"]) { + const md = readFileSync(join(dir, "orchestration", "agents", `${role}.md`), "utf8"); + expect(md, role).toContain("If an ITEMS id is no longer in the worklist, skip it and say so in your note"); + } + }); + it("adjudicator carries the verdict kinds + confidence labels and the harsher-verdict rule", () => { const dir = fullState(); orchestrateRun(dir, ENGINE); diff --git a/tests/verify-fold-e2e.test.ts b/tests/verify-fold-e2e.test.ts new file mode 100644 index 0000000..28c4797 --- /dev/null +++ b/tests/verify-fold-e2e.test.ts @@ -0,0 +1,93 @@ +import { describe, expect, it } from "vitest"; +import { spawnSync } from "node:child_process"; +import { mkdtempSync, rmSync, writeFileSync, readFileSync, readdirSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { fileURLToPath } from "node:url"; + +// Fold-integration tests for the adjudication gate: the EXACT fragment shape the +// orchestrate-emitted adjudicator contract/schema returns — +// `{ "verdicts": [{ claimId, verdict, note, confidence }] }` — driven through the +// REAL bundled CLI (`--verify` then `--verify --apply`), the same subprocess the +// skill and the emitted workflows invoke. This is the round-trip that used to +// fold 0/0 with exit 0 (the fail-open path on the flagship gate). + +const BUNDLE = fileURLToPath(new URL("../scripts/analyze.mjs", import.meta.url)); +const FIXTURE = fileURLToPath(new URL("./fixtures/sample-app", import.meta.url)); +const NODE = process.execPath; + +function cli(dir: string, args: string[]) { + const r = spawnSync(NODE, [BUNDLE, ...args, "--out", dir], { encoding: "utf8" }); + return { status: r.status ?? 0, stdout: r.stdout ?? "", stderr: r.stderr ?? "" }; +} + +/** Generate a real tree, give one feature real requirement bullets, run --verify. */ +function generateWithWorklist(): { dir: string; todo: { pairs: any[] } } { + const dir = mkdtempSync(join(tmpdir(), "rc-fold-e2e-")); + const gen = spawnSync(NODE, [BUNDLE, "--repo", FIXTURE, "--out", dir], { encoding: "utf8" }); + if (gen.status !== 0) throw new Error(`generate failed: ${gen.stderr}`); + const slug = readdirSync(join(dir, "features"))[0]!; + const prdPath = join(dir, "features", slug, "PRD.md"); + const bullets = ["- The layout renders the application shell from app/layout.tsx", "- The home page lists items from app/page.tsx"].join("\n"); + writeFileSync(prdPath, readFileSync(prdPath, "utf8").replace("## Functional requirements", `## Functional requirements\n${bullets}`)); + const v = cli(dir, ["--verify"]); + if (v.status !== 0) throw new Error(`--verify failed: ${v.stderr}`); + const todo = JSON.parse(readFileSync(join(dir, "VERIFY.todo.json"), "utf8")); + return { dir, todo }; +} + +/** Save the schema-shaped fragment (claimId/verdict/note/confidence ONLY) and run the real fold. */ +function applyFragment(dir: string, todo: { pairs: any[] }, map: Record = {}) { + const verdicts = todo.pairs.map((p) => ({ + claimId: p.claimId, + verdict: map[p.claimId] ?? "supported", + note: "traces to the captured source", + confidence: "confirmed", + })); + const f = join(dir, "verdicts.json"); + writeFileSync(f, JSON.stringify({ verdicts })); + return cli(dir, ["--verify", "--apply", f]); +} + +describe("--verify --apply round-trips the orchestrate adjudicator fragment (real CLI)", () => { + it("supported fragment: adjudicates every pair and exits 0 (never the vacuous 0/0)", () => { + const { dir, todo } = generateWithWorklist(); + expect(todo.pairs.length).toBeGreaterThan(0); + const r = applyFragment(dir, todo); + expect(r.status, r.stderr || r.stdout).toBe(0); + expect(r.stdout).toContain(`${todo.pairs.length}/${todo.pairs.length} requirement(s) adjudicated`); + expect(r.stdout).not.toContain("0/0"); + const sem = JSON.parse(readFileSync(join(dir, "VERIFY.json"), "utf8")); + expect(sem.ok).toBe(true); + expect(sem.adjudicated).toBe(todo.pairs.length); + // evidence backfilled from the worklist, so the citation guard stays engaged + for (const [i, v] of sem.verdicts.entries()) { + expect(v.evidenceRef).toBe(todo.pairs[i].evidenceRef); + expect(v.evidenceRef.length).toBeGreaterThan(0); + } + rmSync(dir, { recursive: true, force: true }); + }); + + it("unsupported fragment verdict: the fold fails the gate (exit 1) with the failure named", () => { + const { dir, todo } = generateWithWorklist(); + const first = todo.pairs[0]!.claimId as string; + const r = applyFragment(dir, todo, { [first]: "unsupported" }); + expect(r.status).toBe(1); + expect(r.stdout).toContain(first); + expect(r.stdout).toContain("unsupported"); + const sem = JSON.parse(readFileSync(join(dir, "VERIFY.json"), "utf8")); + expect(sem.ok).toBe(false); + expect(sem.failures.some((f: any) => f.claimId === first && f.verdict === "unsupported")).toBe(true); + rmSync(dir, { recursive: true, force: true }); + }); + + it("a fragment yielding zero rows fails closed (non-zero exit, no ok ledger)", () => { + const { dir } = generateWithWorklist(); + const f = join(dir, "verdicts.json"); + writeFileSync(f, JSON.stringify({ verdicts: [] })); + const r = cli(dir, ["--verify", "--apply", f]); + expect(r.status).not.toBe(0); + expect(r.stderr).toMatch(/no verdict rows/i); + rmSync(dir, { recursive: true, force: true }); + }); +}); diff --git a/tests/verify.test.ts b/tests/verify.test.ts index 870a2b2..7fcdef7 100644 --- a/tests/verify.test.ts +++ b/tests/verify.test.ts @@ -117,6 +117,176 @@ describe("applyVerdicts (gate)", () => { }); }); +describe("orchestrate adjudicator fragments (ADJUDICATE_SCHEMA shape)", () => { + function setup(): string { + const dir = scratch(); + tree(dir, PRD); + runVerify(dir); + return dir; + } + + /** + * The EXACT shape the emitted adjudicate contract/schema returns: + * `{ verdicts: [{ claimId, verdict, note, confidence }] }` — NO evidenceRef, + * claim, feature or digest (those live in VERIFY.todo.json). + */ + function fragment(dir: string, map: Record = {}): string { + const todo = JSON.parse(readFileSync(join(dir, "VERIFY.todo.json"), "utf8")); + const verdicts = todo.pairs.map((p: any) => ({ + claimId: p.claimId, + verdict: map[p.claimId] ?? "supported", + note: "grounded in the captured evidence", + confidence: "confirmed", + })); + const f = join(dir, "verdicts.json"); + writeFileSync(f, JSON.stringify({ verdicts })); + return f; + } + + it("folds the schema-shaped fragment, backfilling evidence from VERIFY.todo.json (supported)", () => { + const dir = setup(); + const r = applyVerdicts(dir, fragment(dir)); + expect(r.ok).toBe(true); + expect(r.adjudicated).toBe(3); // NOT the 0/0 vacuous fold + expect(r.supported).toBe(3); + const sem = JSON.parse(readFileSync(join(dir, "VERIFY.json"), "utf8")); + const todo = JSON.parse(readFileSync(join(dir, "VERIFY.todo.json"), "utf8")); + for (const [i, v] of sem.verdicts.entries()) { + expect(v.claimId).toBe(todo.pairs[i].claimId); + expect(v.evidenceRef).toBe(todo.pairs[i].evidenceRef); // backfilled, so resolveEvidence still guards it + expect(v.evidenceRef.length).toBeGreaterThan(0); + expect(v.claim).toBe(todo.pairs[i].claim); + expect(v.feature).toBe(todo.pairs[i].feature); + expect(v.digest).toBe(todo.pairs[i].digest); + expect(v.confidence).toBe("confirmed"); + } + rmSync(dir, { recursive: true, force: true }); + }); + + it("adjudicates an unsupported fragment verdict into a real gate failure", () => { + const dir = setup(); + const r = applyVerdicts(dir, fragment(dir, { C1: "unsupported" })); + expect(r.ok).toBe(false); + expect(r.failures.some((f) => f.claimId === "C1" && f.verdict === "unsupported")).toBe(true); + // and the persisted ledger gates check --semantic + const check = checkOutput(dir); + const before = check.errors.length; + foldSemantic(dir, check); + expect(check.errors.length).toBeGreaterThan(before); + rmSync(dir, { recursive: true, force: true }); + }); + + it("the vacuous-pass repro now FAILS: a file yielding zero rows throws instead of writing an ok ledger", () => { + const dir = setup(); + const f = join(dir, "verdicts.json"); + writeFileSync(f, JSON.stringify({ verdicts: [] })); + expect(() => applyVerdicts(dir, f)).toThrow(/no verdict rows/i); + expect(existsSync(join(dir, "VERIFY.json"))).toBe(false); + // an object under any OTHER key is just as empty — never a silent 0/0 ok:true + writeFileSync(f, JSON.stringify({ judgements: [{ claimId: "C1", verdict: "supported" }] })); + expect(() => applyVerdicts(dir, f)).toThrow(/no verdict rows/i); + expect(existsSync(join(dir, "VERIFY.json"))).toBe(false); + rmSync(dir, { recursive: true, force: true }); + }); + + it("ignores unknown claimIds with a report, and errors when ALL are unknown", () => { + const dir = setup(); + const f = join(dir, "verdicts.json"); + writeFileSync( + f, + JSON.stringify({ + verdicts: [ + { claimId: "C1", verdict: "supported", note: "ok", confidence: "confirmed" }, + { claimId: "C999", verdict: "supported", note: "ghost of a stale worklist", confidence: "confirmed" }, + ], + }), + ); + const r = applyVerdicts(dir, f); + expect(r.pairs).toBe(1); + expect(r.ignored).toEqual(["C999"]); + expect(formatVerifyReport(r)).toMatch(/1 ignored \(unknown id\)/); + writeFileSync(f, JSON.stringify({ verdicts: [{ claimId: "C888", verdict: "supported", note: "x", confidence: "confirmed" }] })); + expect(() => applyVerdicts(dir, f)).toThrow(/unknown/i); + rmSync(dir, { recursive: true, force: true }); + }); + + it("hard-errors on an explicit-but-invalid verdict token (fail-closed, not unadjudicated)", () => { + const dir = setup(); + const f = join(dir, "verdicts.json"); + writeFileSync(f, JSON.stringify({ verdicts: [{ claimId: "C1", verdict: "SUPPORTED!!", note: "typo", confidence: "confirmed" }] })); + expect(() => applyVerdicts(dir, f)).toThrow(/invalid verdict/i); + rmSync(dir, { recursive: true, force: true }); + }); + + it("hard-errors on a row missing its claimId", () => { + const dir = setup(); + const f = join(dir, "verdicts.json"); + writeFileSync(f, JSON.stringify({ verdicts: [{ verdict: "supported", note: "who am I?" }] })); + expect(() => applyVerdicts(dir, f)).toThrow(/claimId/); + rmSync(dir, { recursive: true, force: true }); + }); + + it("still accepts a null verdict as unadjudicated (mid-loop fragments)", () => { + const dir = setup(); + const f = join(dir, "verdicts.json"); + writeFileSync( + f, + JSON.stringify({ + verdicts: [ + { claimId: "C1", verdict: "supported", note: "", confidence: "confirmed" }, + { claimId: "C2", verdict: null, note: "", confidence: null }, + ], + }), + ); + const r = applyVerdicts(dir, f); + expect(r.adjudicated).toBe(1); + expect(r.unadjudicated).toEqual(["C2"]); + rmSync(dir, { recursive: true, force: true }); + }); +}); + +describe("foldSemantic engagement (vacuous ledgers)", () => { + it("fails when VERIFY.json carries 0 adjudicated verdicts — the gate never engaged", () => { + const dir = scratch(); + tree(dir, PRD); + // The exact ledger the old fail-open applyVerdicts wrote when it could not + // read a fragment: ok:true with an empty verdicts[]. + writeFileSync( + join(dir, "VERIFY.json"), + JSON.stringify({ + ok: true, + pairs: 0, + adjudicated: 0, + supported: 0, + partial: 0, + refuted: 0, + unsupported: 0, + failures: [], + unadjudicated: [], + verdicts: [], + }), + ); + const check = checkOutput(dir); + const before = check.errors.length; + foldSemantic(dir, check); + expect(check.errors.length).toBeGreaterThan(before); + expect(check.errors.join(" ")).toMatch(/0 adjudicated|never engaged/i); + rmSync(dir, { recursive: true, force: true }); + }); + + it("downgrades the vacuous ledger to a warning under allowUnverified", () => { + const dir = scratch(); + tree(dir, PRD); + writeFileSync(join(dir, "VERIFY.json"), JSON.stringify({ ok: true, pairs: 0, adjudicated: 0, failures: [], unadjudicated: [], verdicts: [] })); + const check = checkOutput(dir); + const before = check.errors.length; + foldSemantic(dir, check, { allowUnverified: true }); + expect(check.errors.length).toBe(before); + expect(check.warnings.join(" ")).toMatch(/0 adjudicated|never engaged/i); + rmSync(dir, { recursive: true, force: true }); + }); +}); + describe("foldSemantic (--check --semantic composition)", () => { it("adds an error to the check result when VERIFY.json fails", () => { const dir = scratch();