diff --git a/CHANGELOG.md b/CHANGELOG.md index ee55b45..0d65f1f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,10 @@ All notable changes are documented here. The project follows Semantic Versioning cannot multiply the configured timeout across every route. - Added integer-or-`nonzero` exit contracts and consistent replay semantics for negative command assertions. +- Recovered an empty compiler criteria list from the bounded explicit user instruction, + while still requiring independent criticism and criterion-to-execution evidence; the + recovery remains fail-closed when repository evidence does not substantiate the goal + and is recorded in the hashed contract, Receipt, and task UI. ### Reliability diff --git a/README.md b/README.md index 5de27e8..0cb7384 100644 --- a/README.md +++ b/README.md @@ -145,6 +145,12 @@ Median/p95/max were 4/7/9 steps, 10,386/19,736/25,656 provider-reported tokens, 16.011/22.589/27.213 seconds. The one unsolved attempt was an unnecessary UI clarification after the contract compiler returned an invalid empty criteria list; Loop failed closed before mutation rather than accepting unverified work. +That failure remains in the frozen report. The current runtime now treats the bounded +user instruction as the fallback criterion, but only locks it after independent criticism +and deterministic criterion-to-check coverage; a focused regression reproduces both the +recovered UI path and the no-evidence fail-closed path. Recovery provenance is included in +the hashed contract and Receipt and shown in the task UI. A future real-provider matrix, +not the regression alone, will determine the new score. An [archived same-model comparison](./evals/results/deepseek-chat-repository-matrix-v0.2.0.json) also records one-shot, ungated tool loop, and full Loop on the versioned v0.1 matrix. diff --git a/apps/api/app/schemas/contract.py b/apps/api/app/schemas/contract.py index 4e4319a..98d67a9 100644 --- a/apps/api/app/schemas/contract.py +++ b/apps/api/app/schemas/contract.py @@ -104,6 +104,7 @@ class ContractDraft(ContractProposal): checks: list[ContractCheck] = Field(default_factory=list, max_length=96) artifacts: list[str] = Field(default_factory=list, max_length=64) schema_version: Literal["loop.contract-draft/v1"] = "loop.contract-draft/v1" + criteria_recovery: Literal["explicit_user_goal"] | None = None compiler: ContractModelIdentity discovery: RepositoryDiscovery clarifications: list[str] = Field(default_factory=list, max_length=12) diff --git a/apps/api/app/services/agent_react.py b/apps/api/app/services/agent_react.py index 9a33f3c..0eb653a 100644 --- a/apps/api/app/services/agent_react.py +++ b/apps/api/app/services/agent_react.py @@ -1159,7 +1159,12 @@ async def _prepare_project_contract( task.criteria_source = "compiled" task.pending_question = None self._apply_locked_contract(task, compiled.draft) - self._transition(task, LoopEvent.CONTRACT_READY, "contract_locked") + transition_reason = ( + "contract_locked_after_empty_criteria_recovery" + if compiled.draft.criteria_recovery == "explicit_user_goal" + else "contract_locked" + ) + self._transition(task, LoopEvent.CONTRACT_READY, transition_reason) await self._commit() return True diff --git a/apps/api/app/services/contract.py b/apps/api/app/services/contract.py index f3a2eff..41c516f 100644 --- a/apps/api/app/services/contract.py +++ b/apps/api/app/services/contract.py @@ -78,6 +78,7 @@ _MIN_AUTO_CONFIDENCE = 80 _MAX_AUTO_CRITERIA = 8 _MAX_CONTRACT_ATTEMPTS = 3 +_MAX_GOAL_CRITERION_CHARS = 4_000 @dataclass(frozen=True) @@ -193,8 +194,12 @@ async def compile_project_contract( temperature=0.2, token_budget=token_budget, ) + criteria_recovered = False try: - proposal = _proposal_from_result(compiler_result.content) + proposal, criteria_recovered = _proposal_from_result( + compiler_result.content, + fallback_criteria=[goal], + ) proposal = _merge_required_checks(proposal, required_checks or []) except (TypeError, ValueError) as exc: draft = _failed_draft( @@ -242,6 +247,7 @@ async def compile_project_contract( ), ), compiler_result, + criteria_recovered=criteria_recovered, ) return CompiledContract( draft=draft, @@ -289,6 +295,7 @@ async def compile_project_contract( known_clarifications, final_critique, compiler_result, + criteria_recovered=criteria_recovered, ) return CompiledContract( draft=draft, @@ -328,6 +335,7 @@ async def compile_project_contract( known_clarifications, repair_critique, compiler_result, + criteria_recovered=criteria_recovered, ) return CompiledContract( draft=draft, @@ -338,7 +346,11 @@ async def compile_project_contract( ) tokens_spent += repair_result.tokens try: - proposal = _proposal_from_result(repair_result.content) + proposal, repair_recovered = _proposal_from_result( + repair_result.content, + fallback_criteria=[goal], + ) + criteria_recovered = criteria_recovered or repair_recovered proposal = _merge_required_checks(proposal, required_checks or []) except (TypeError, ValueError) as exc: repair_critique = final_critique.model_copy( @@ -355,6 +367,7 @@ async def compile_project_contract( known_clarifications, repair_critique, compiler_result, + criteria_recovered=criteria_recovered, ) return CompiledContract( draft=draft, @@ -420,10 +433,13 @@ def _draft_from_proposal( clarifications: list[str], critique: ContractCritique, compiler_result: LLMResult, + *, + criteria_recovered: bool = False, ) -> ContractDraft: return ContractDraft( **proposal.model_dump(exclude={"checks"}), checks=_effective_checks(proposal, discovery), + criteria_recovery="explicit_user_goal" if criteria_recovered else None, compiler=_compiler_identity(compiler_result), discovery=discovery, clarifications=clarifications, @@ -438,8 +454,11 @@ def _compiler_identity(result: LLMResult | None) -> ContractModelIdentity: def hash_contract(draft: ContractDraft) -> str: + payload = draft.model_dump(mode="json") + if payload.get("criteria_recovery") is None: + payload.pop("criteria_recovery", None) canonical = json.dumps( - draft.model_dump(mode="json"), + payload, ensure_ascii=False, sort_keys=True, separators=(",", ":"), @@ -452,7 +471,11 @@ def verify_contract_hash(draft: dict[str, Any] | ContractDraft, expected: str) - return bool(expected) and hash_contract(parsed) == expected -def _proposal_from_result(content: str) -> ContractProposal: +def _proposal_from_result( + content: str, + *, + fallback_criteria: list[str] | None = None, +) -> tuple[ContractProposal, bool]: parsed = _extract_json(content) if not isinstance(parsed, dict): raise ValueError("contract compiler returned no JSON object") @@ -460,6 +483,10 @@ def _proposal_from_result(content: str) -> ContractProposal: criteria = list( dict.fromkeys(text for item in raw_criteria if (text := _criterion_text(item))) )[:12] + recovered = False + if not criteria: + criteria = _bounded_fallback_criteria(fallback_criteria or []) + recovered = bool(criteria) parsed["criteria"] = criteria count = min(len(criteria), 12) valid_ids = {f"criterion-{index:03d}" for index in range(1, count + 1)} @@ -545,7 +572,20 @@ def _proposal_from_result(content: str) -> ContractProposal: "authority_requests": list(dict.fromkeys(authority_requests)), } ) - return proposal.model_copy(update={"checks": _deduplicate_contract_checks(proposal.checks)}) + return ( + proposal.model_copy(update={"checks": _deduplicate_contract_checks(proposal.checks)}), + recovered, + ) + + +def _bounded_fallback_criteria(values: list[str]) -> list[str]: + return list( + dict.fromkeys( + text + for value in values + if (text := " ".join(value.split())) and len(text) <= _MAX_GOAL_CRITERION_CHARS + ) + )[:12] def _as_list(value: Any) -> list[Any]: diff --git a/apps/api/app/services/prompts.py b/apps/api/app/services/prompts.py index f4bd25b..a527594 100644 --- a/apps/api/app/services/prompts.py +++ b/apps/api/app/services/prompts.py @@ -44,7 +44,7 @@ def contract_compile_prompts( "[DATA] Deterministic read-only repository discovery:\n" f"{discovery.model_dump_json(indent=2)}\n\n" "Return ONLY one JSON object with these keys: criteria (the minimal 1-8 concrete " - "outcomes), " + "outcomes; never return an empty criteria array), " "checks (objects with kind set to command, file_exists, or file_contains; command checks " "include command and expect_exit as an integer or the string 'nonzero'; file checks " "include path, and file_contains also includes text; every check includes criterion_ids), " @@ -82,7 +82,8 @@ def contract_repair_prompts( f"Blocking review issues:\n{json.dumps(issues, ensure_ascii=False)}\n\n" "[DATA] Deterministic read-only repository discovery:\n" f"{discovery.model_dump_json(indent=2)}\n\n" - "Return ONLY one replacement JSON object with the same schema: minimal criteria, checks " + "Return ONLY one replacement JSON object with the same schema: at least one minimal " + "criterion, checks " "whose kind is command, file_exists, or file_contains and whose criterion_ids map every " "criterion; command expect_exit may be an integer or 'nonzero'; artifacts, risk, " "assumptions, confidence, and authority_requests. Copy " diff --git a/apps/api/tests/test_contract.py b/apps/api/tests/test_contract.py index abc87e9..db4b0ad 100644 --- a/apps/api/tests/test_contract.py +++ b/apps/api/tests/test_contract.py @@ -236,6 +236,52 @@ async def complete( return LLMResult(json.dumps(decision), "fixture", 10, model="executor-v1") +class EmptyCriteriaContractLLM: + def __init__(self) -> None: + self.compile_calls = 0 + self.critic_calls = 0 + self.critic_prompt = "" + + async def complete( + self, + system: str, + user: str, + *, + max_tokens: int = 4096, + temperature: float = 0.7, + token_budget: int | None = None, + ) -> LLMResult: + del max_tokens, temperature, token_budget + if "compile one software instruction" in system: + self.compile_calls += 1 + return LLMResult( + json.dumps( + { + "criteria": [], + "checks": [], + "artifacts": [], + "risk": "low", + "assumptions": [], + "confidence": 95, + "authority_requests": [], + } + ), + "fixture", + 20, + model="contract-v1", + ) + if "independent acceptance-contract critic" in system: + self.critic_calls += 1 + self.critic_prompt = user + return LLMResult( + json.dumps({"accepted": True, "issues": [], "question": None}), + "fixture-critic", + 10, + model="critic-v1", + ) + raise AssertionError("unexpected model role") + + @pytest.fixture def project_settings(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: projects = tmp_path / "projects" @@ -343,6 +389,68 @@ async def test_compiler_normalizes_structured_criteria(project_settings: Path) - ] +async def test_compiler_recovers_empty_criteria_from_goal_and_test_evidence() -> None: + root = ROOT / "evals" / "repositories" / "accessible-dialog-ui" + goal = ( + "Finish the dependency-free modal dialog. The HTML needs an accessible dialog labelled " + "by its title. openDialog must show it, remember the previously focused element and " + "focus the close button; closeDialog must hide it and restore focus. Escape closes an " + "open dialog. Keep the exported JavaScript API and make all tests pass." + ) + before = { + path.relative_to(root): path.read_bytes() for path in root.rglob("*") if path.is_file() + } + model = EmptyCriteriaContractLLM() + + compiled = await compile_project_contract( + goal=goal, + root=root, + compiler=model, + critic=model, + granted_capabilities={Capability.FS_READ, Capability.FS_WRITE, Capability.EXEC}, + token_budget=10_000, + ) + + assert compiled.contract_hash + assert compiled.draft.criteria_recovery == "explicit_user_goal" + assert compiled.draft.criteria == [goal] + assert compiled.draft.critique.accepted is True + without_recovery = compiled.draft.model_copy(update={"criteria_recovery": None}) + assert not verify_contract_hash(without_recovery, compiled.contract_hash) + assert model.compile_calls == 1 + assert model.critic_calls == 1 + assert goal in model.critic_prompt + contract_checks = [check for check in compiled.draft.checks if check.source == "contract"] + assert [check.command for check in contract_checks] == ["npm run test"] + assert contract_checks[0].criterion_ids == ["criterion-001"] + assert { + path.relative_to(root): path.read_bytes() for path in root.rglob("*") if path.is_file() + } == before + + +async def test_empty_criteria_recovery_still_fails_without_grounding( + project_settings: Path, +) -> None: + model = EmptyCriteriaContractLLM() + + compiled = await compile_project_contract( + goal="Replace an unavailable remote billing service", + root=project_settings, + compiler=model, + critic=model, + granted_capabilities={Capability.FS_READ, Capability.FS_WRITE, Capability.EXEC}, + token_budget=10_000, + ) + + assert compiled.contract_hash is None + assert compiled.draft.criteria_recovery == "explicit_user_goal" + assert compiled.draft.critique.accepted is False + assert any( + "No execution check substantiates: criterion-001" in issue + for issue in compiled.draft.critique.issues + ) + + async def test_discovered_checks_are_not_duplicated(project_settings: Path) -> None: model = ContractLoopLLM(include_discovered_check=True) compiled = await compile_project_contract( @@ -735,6 +843,20 @@ def test_user_contract_supports_full_advanced_input_bounds(project_settings: Pat assert verify_contract_hash(draft, contract_hash) +def test_contract_hash_accepts_legacy_draft_without_recovery_field( + project_settings: Path, +) -> None: + draft, contract_hash = lock_user_project_contract( + root=project_settings, + criteria=["The requested output exists and is validated."], + required_checks=[], + ) + legacy = draft.model_dump(mode="json") + assert legacy.pop("criteria_recovery") is None + + assert verify_contract_hash(legacy, contract_hash) + + async def test_one_instruction_compiles_repairs_verifies_and_applies( session: AsyncSession, project_settings: Path ) -> None: diff --git a/apps/web/components/authority-panel.test.tsx b/apps/web/components/authority-panel.test.tsx index c02898a..078078c 100644 --- a/apps/web/components/authority-panel.test.tsx +++ b/apps/web/components/authority-panel.test.tsx @@ -143,6 +143,7 @@ describe('ContractPanel', () => { contract_hash: '0123456789abcdef', contract: { schema_version: 'loop.contract-draft/v1', + criteria_recovery: 'explicit_user_goal', compiler: { provider: 'fixture', model: 'contract-v1' }, criteria: ['The greeting is updated'], checks: [ @@ -186,6 +187,7 @@ describe('ContractPanel', () => { ); expect(screen.getByText('Loop compiled')).toBeInTheDocument(); + expect(screen.getByText('criterion recovered from instruction')).toBeInTheDocument(); expect(screen.getByText('locked 0123456789ab')).toBeInTheDocument(); expect(screen.getByText('The greeting is updated')).toBeInTheDocument(); expect(screen.getByText(/12 files discovered/)).toBeInTheDocument(); diff --git a/apps/web/components/contract-panel.tsx b/apps/web/components/contract-panel.tsx index b9b2632..458f8c9 100644 --- a/apps/web/components/contract-panel.tsx +++ b/apps/web/components/contract-panel.tsx @@ -25,6 +25,11 @@ export function ContractPanel({ task }: { task: Task }) { {task.verification_mode} + {draft?.criteria_recovery === 'explicit_user_goal' && ( + + criterion recovered from instruction + + )} {task.contract_status === 'locked' && task.contract_hash && ( locked {task.contract_hash.slice(0, 12)} diff --git a/docs/STRATEGY.md b/docs/STRATEGY.md index 043e878..3cca745 100644 --- a/docs/STRATEGY.md +++ b/docs/STRATEGY.md @@ -218,6 +218,13 @@ acceptances. Median/p95/max were 4/7/9 steps, 10,386/19,736/25,656 tokens, and 16.011/22.589/27.213 seconds. The one failure remains published as an unnecessary, fail-closed UI clarification caused by an invalid empty contract draft. +The empty-contract failure is now a deterministic regression. If a compiler returns no +usable criteria, Loop may preserve the bounded explicit user instruction as the criterion, +but it still runs the independent critic and refuses to lock unless safe execution evidence +substantantiates it. The accessible-dialog fixture proves the discovered test-gate recovery; +an unrelated no-evidence goal proves the same path remains fail-closed. The frozen report is +not retroactively rescored; a new real-provider matrix must confirm the improvement. + The archived v0.1 same-model comparison records one-shot, ungated tool loop, and full Loop on a versioned manifest. It exposed three one-shot false acceptances and the full Loop's earlier contract/convergence failures, which became regression tests and runtime @@ -227,8 +234,7 @@ fixture drift, and waits through API publication throttling. This gate is not marked fully complete because both published repository runs used the reduced-isolation `inline` backend. The same current matrix still needs a clean -Docker/Kubernetes run, and the empty-contract failure remains the next error-analysis -target. +Docker/Kubernetes run, and the empty-contract regression still needs real-provider repeats. - Add realistic fixture repositories covering bug repair, feature work, multi-file refactoring, CLI/API/UI changes, regressions, and incomplete specifications. diff --git a/evals/README.md b/evals/README.md index 5a2cb54..9aa197e 100644 --- a/evals/README.md +++ b/evals/README.md @@ -175,12 +175,15 @@ the evaluated contract can be matched to the repository. ## Recorded repository results [`results/deepseek-chat-full-loop-v0.2.12.json`](./results/deepseek-chat-full-loop-v0.2.12.json) -is the current release-gate report. DeepSeek `deepseek-chat` completed 24/24 cells on -the current matrix: 20/21 deliverable attempts solved (95.24%), 3/3 contradictory +is the frozen release-gate report for its recorded runtime hash. DeepSeek `deepseek-chat` +completed 24/24 cells on the current matrix: 20/21 deliverable attempts solved (95.24%), +3/3 contradictory specifications safely deferred, and zero false acceptances. Median/p95/max were 4/7/9 steps, 10,386/19,736/25,656 provider-reported tokens, and 16.011/22.589/27.213 seconds. The one failure was an unnecessary UI clarification after the compiler returned an invalid empty criteria list; it failed closed at step zero and was not accepted. +The runtime now has a focused evidence-gated recovery regression for that failure, but the +published result remains unchanged until a new real-provider run is completed. [`results/deepseek-chat-repository-matrix-v0.2.0.json`](./results/deepseek-chat-repository-matrix-v0.2.0.json) is the archived same-model comparison on `repository-suite-v0.1.json`. Across 21 diff --git a/packages/api-contract/src/index.ts b/packages/api-contract/src/index.ts index 878b615..b5cd11f 100644 --- a/packages/api-contract/src/index.ts +++ b/packages/api-contract/src/index.ts @@ -75,6 +75,7 @@ export interface ContractCheck { export interface ContractDraft { schema_version: 'loop.contract-draft/v1'; + criteria_recovery?: 'explicit_user_goal' | null; compiler: { provider: string; model: string;