From 5c5310314359509f908daab627e6616fbb6aa319 Mon Sep 17 00:00:00 2001 From: Yichen Wu Date: Wed, 15 Jul 2026 18:26:02 -0700 Subject: [PATCH] Harden artifact verification and publish DeepSeek eval --- .gitignore | 1 + README.md | 21 +- apps/api/app/schemas/task.py | 17 + apps/api/app/services/agent_react.py | 10 +- apps/api/app/services/completion.py | 22 +- apps/api/app/services/evaluation.py | 20 +- apps/api/app/services/task.py | 9 + apps/api/app/services/verification.py | 49 +-- .../scripts/evaluate_verified_completion.py | 25 +- apps/api/tests/test_agent_react.py | 59 ++++ apps/api/tests/test_completion.py | 25 ++ apps/api/tests/test_evaluation.py | 36 ++ apps/api/tests/test_tasks.py | 16 + apps/api/tests/test_verification.py | 13 + .../components/publish-form.desktop.test.tsx | 4 + apps/web/components/publish-form.tsx | 22 +- apps/web/lib/api-client.ts | 1 + docs/loop.md | 22 +- evals/README.md | 25 +- evals/results/deepseek-chat-v0.1.0.json | 322 ++++++++++++++++++ evals/verified-completion.json | 30 +- 21 files changed, 670 insertions(+), 79 deletions(-) create mode 100644 evals/results/deepseek-chat-v0.1.0.json diff --git a/.gitignore b/.gitignore index 08a4b23..dc35a80 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ apps/web/playwright-report/ apps/web/test-results/ /evals/results/* !/evals/results/demo-smoke.json +!/evals/results/deepseek-chat-v0.1.0.json # ---------- Python ---------- __pycache__/ diff --git a/README.md b/README.md index aa0a19c..1910d75 100644 --- a/README.md +++ b/README.md @@ -23,10 +23,11 @@ the evidence, and returns a Receipt anyone can replay.** Most coding agents end when the model says it is done. Loop separates **work** from **acceptance**: -1. You confirm concrete success criteria and may provide exact verification commands. +1. You confirm concrete success criteria, required final artifacts, and optional exact + verification commands. 2. Loop works inside a per-task workspace under a server-enforced capability and token/step envelope. -3. An independent verifier re-runs the checks on a fresh copy of the workspace. +3. An independent verifier re-runs each check on its own fresh workspace snapshot. 4. Every criterion must map to passing execution evidence in strict mode. 5. Loop emits a content-addressed Receipt with the contract, checks, model/runtime provenance, output hashes, and the head of a hash-chained step ledger. @@ -77,9 +78,14 @@ fresh environment → open UI → confirm contract → run → execution verifie The committed [demo smoke report](./evals/results/demo-smoke.json) records `1/1` solved, zero false acceptances, two steps, 24 scripted tokens, and a passing replay. -It proves product wiring, not general model quality. The separate -[12-case real-provider suite](./evals/verified-completion.json) is published without -invented results; running it requires explicit acknowledgement of provider spend. +It proves product wiring, not general model quality. + +A recorded [DeepSeek `deepseek-chat` run](./evals/results/deepseek-chat-v0.1.0.json) +of the [12-case real-provider suite](./evals/verified-completion.json) solved `12/12` +with zero false acceptances: 39 steps, 64,447 provider-reported tokens, and 94.954 +seconds. Every case was execution-verified, fully covered, artifact-complete, +integrity-valid, and replayable. This is one clean local run using visibly reduced +`inline` isolation—not a cross-model quality claim or a production-sandbox result. ## The trust boundary @@ -109,8 +115,9 @@ and residual risks. ## What is implemented - ReAct planning with independent executor/verifier provider selection and fallback. -- Strict acceptance contracts, baseline regression checks, criterion-to-evidence - mappings, Receipt replay, and offline verification. +- Strict acceptance contracts with required final artifacts, independent check + snapshots, baseline regression checks, criterion-to-evidence mappings, Receipt + replay, and offline verification. - Per-task workspaces, container or Kubernetes command isolation, secret redaction, destination-bound egress, and restart-safe approval gates. - Durable Redis Streams workers with visibility leases, stale-message reclaim, diff --git a/apps/api/app/schemas/task.py b/apps/api/app/schemas/task.py index 52f823e..7a0cc92 100644 --- a/apps/api/app/schemas/task.py +++ b/apps/api/app/schemas/task.py @@ -31,6 +31,7 @@ class TaskCreate(BaseModel): goal: str = Field(min_length=4, max_length=4_000) success_criteria: list[str] | None = None verification_commands: list[str] = Field(default_factory=list) + required_artifacts: list[str] = Field(default_factory=list) verification_mode: str | None = Field(default=None, pattern=r"^(strict|judgment)$") project_id: str = Field(default="default", min_length=1, max_length=100, pattern=r"^[\w.-]+$") # Optional path relative to LOOP_LOCAL_PROJECTS_ROOT. The source repository @@ -86,6 +87,22 @@ def validate_verification_commands(cls, value: list[str]) -> list[str]: raise ValueError("each verification command must be at most 1000 characters") return commands + @field_validator("required_artifacts") + @classmethod + def validate_required_artifacts(cls, value: list[str]) -> list[str]: + artifacts = list(dict.fromkeys(item.strip() for item in value if item.strip())) + if len(artifacts) > 32: + raise ValueError("required_artifacts may contain at most 32 paths") + for artifact in artifacts: + parts = artifact.split("/") + if ( + len(artifact) > 500 + or "\\" in artifact + or any(part in {"", ".", ".."} for part in parts) + ): + raise ValueError("required_artifacts must be workspace-relative POSIX paths") + return artifacts + @model_validator(mode="after") def validate_network_authority(self) -> TaskCreate: requested = set(self.capabilities or []) diff --git a/apps/api/app/services/agent_react.py b/apps/api/app/services/agent_react.py index 1699be2..a289018 100644 --- a/apps/api/app/services/agent_react.py +++ b/apps/api/app/services/agent_react.py @@ -59,6 +59,7 @@ attach_baseline, completion_gates_pass, discover_project_checks, + mark_supplementary_agent_checks, merge_completion_checks, regressions, ) @@ -1768,6 +1769,9 @@ async def _handle_finish( await self._run_completion_checks(task, workspace, checks), task.baseline_checks or [], ) + contract_substantiation_authoritative = mark_supplementary_agent_checks( + check_results, len(task.rubric or []) + ) gates_passed = completion_gates_pass(check_results) coverage_complete = execution_coverage_complete(check_results, len(task.rubric or [])) @@ -1822,12 +1826,6 @@ async def _handle_finish( else: score, missing, llm_met, substantiate = 0, ["verifier returned no verdict"], False, True - contract_results = [result for result in check_results if result.source == "contract"] - contract_substantiation_authoritative = bool( - contract_results - and completion_gates_pass(contract_results) - and execution_coverage_complete(contract_results, len(task.rubric or [])) - ) execution_ready = bool( check_results and coverage_complete diff --git a/apps/api/app/services/completion.py b/apps/api/app/services/completion.py index 09786a0..5639ed7 100644 --- a/apps/api/app/services/completion.py +++ b/apps/api/app/services/completion.py @@ -6,7 +6,7 @@ from pathlib import Path from typing import Any -from app.services.verification import CheckResult +from app.services.verification import CheckResult, execution_coverage_complete def discover_project_checks(root: Path) -> list[dict[str, Any]]: @@ -78,6 +78,8 @@ def attach_baseline( def completion_gates_pass(results: list[CheckResult]) -> bool: for result in results: + if not result.gating: + continue if result.passed: continue if result.source == "system" and result.baseline_passed is False: @@ -86,6 +88,24 @@ def completion_gates_pass(results: list[CheckResult]) -> bool: return True +def mark_supplementary_agent_checks(results: list[CheckResult], criterion_count: int) -> bool: + contract = [result for result in results if result.source == "contract"] + authoritative = bool( + contract + and completion_gates_pass(contract) + and execution_coverage_complete(contract, criterion_count) + ) + if not authoritative: + return False + for result in results: + if result.source != "agent": + continue + result.gating = False + if result.definition is not None: + result.definition["gating"] = False + return True + + def regressions(results: list[CheckResult]) -> list[CheckResult]: return [ result diff --git a/apps/api/app/services/evaluation.py b/apps/api/app/services/evaluation.py index d9374af..f8ae71c 100644 --- a/apps/api/app/services/evaluation.py +++ b/apps/api/app/services/evaluation.py @@ -65,21 +65,19 @@ def aggregate_verified_completion(results: list[dict[str, Any]]) -> dict[str, An total = len(results) solved = sum(bool(item.get("solved")) for item in results) false_acceptances = sum(bool(item.get("false_acceptance")) for item in results) + total_steps = sum(int(item.get("steps_used", 0)) for item in results) + total_tokens = sum(int(item.get("tokens_used", 0)) for item in results) + total_duration_seconds = sum(float(item.get("duration_seconds", 0)) for item in results) return { "cases": total, "solved": solved, "solve_rate": solved / total if total else 0.0, "false_acceptances": false_acceptances, "false_acceptance_rate": false_acceptances / total if total else 0.0, - "average_steps": ( - sum(int(item.get("steps_used", 0)) for item in results) / total if total else 0.0 - ), - "average_tokens": ( - sum(int(item.get("tokens_used", 0)) for item in results) / total if total else 0.0 - ), - "average_duration_seconds": ( - sum(float(item.get("duration_seconds", 0)) for item in results) / total - if total - else 0.0 - ), + "total_steps": total_steps, + "total_tokens": total_tokens, + "total_duration_seconds": total_duration_seconds, + "average_steps": total_steps / total if total else 0.0, + "average_tokens": total_tokens / total if total else 0.0, + "average_duration_seconds": total_duration_seconds / total if total else 0.0, } diff --git a/apps/api/app/services/task.py b/apps/api/app/services/task.py index e1f0cdb..c4068ed 100644 --- a/apps/api/app/services/task.py +++ b/apps/api/app/services/task.py @@ -114,6 +114,15 @@ async def publish(self, payload: TaskCreate) -> TaskModel: } for index, command in enumerate(payload.verification_commands, start=1) ] + required_checks.extend( + { + "id": f"contract-artifact-{index:03d}", + "kind": "file_exists", + "path": path, + "source": "contract", + } + for index, path in enumerate(payload.required_artifacts, start=1) + ) task = await self.tasks.create( goal=payload.goal.strip(), owner_id=self.subject, diff --git a/apps/api/app/services/verification.py b/apps/api/app/services/verification.py index 28d46cd..87f06c9 100644 --- a/apps/api/app/services/verification.py +++ b/apps/api/app/services/verification.py @@ -44,6 +44,7 @@ class CheckResult: definition: dict[str, Any] | None = None source: str = "agent" baseline_passed: bool | None = None + gating: bool = True VERIFY_DIR_PREFIX = "verify-" @@ -85,8 +86,8 @@ async def run_checks( docker_workspace_mount: str | None = None, infer_criterion_ids: bool = True, ) -> list[CheckResult]: - """Re-run each check on a fresh copy of the workspace. Never raises — a bad - check definition becomes a failed result the verifier can act on.""" + """Re-run every check on its own fresh copy of the workspace. Never raises — + a bad check definition becomes a failed result the verifier can act on.""" if not checks: return [] @@ -94,34 +95,34 @@ async def run_checks( # than system temp, so re-verification runs in the same sandbox as the agent. tmp_root = source.root.parent / f"{VERIFY_DIR_PREFIX}{uuid.uuid4().hex[:12]}" try: - copy_dir = tmp_root / "ws" - await asyncio.to_thread(shutil.copytree, source.root, copy_dir) - workspace = Workspace(copy_dir) # Verify under the same authority as the task: default-deny egress unless # the task had it (so a check can't reach the network the task couldn't, # and a legit network check isn't blocked in a container). check_envelope = envelope or CapabilityEnvelope.from_tools( None, egress_allowed=egress_allowed ) - executor = ToolExecutor( - workspace, - approval_mode=approval_mode, - command_timeout=command_timeout, - output_limit=output_limit, - envelope=check_envelope, - before_tool=make_egress_guard(check_envelope, workspace), - sandbox_image=sandbox_image, - sandbox_backend=sandbox_backend, - sandbox_memory=sandbox_memory, - sandbox_cpus=sandbox_cpus, - egress_proxy_url=egress_proxy_url, - egress_network=egress_network, - egress_token_factory=egress_token_factory, - docker_workspace_volume=docker_workspace_volume, - docker_workspace_mount=docker_workspace_mount, - ) results: list[CheckResult] = [] for index, check in enumerate(checks, start=1): + copy_dir = tmp_root / f"check-{index:03d}" + await asyncio.to_thread(shutil.copytree, source.root, copy_dir) + workspace = Workspace(copy_dir) + executor = ToolExecutor( + workspace, + approval_mode=approval_mode, + command_timeout=command_timeout, + output_limit=output_limit, + envelope=check_envelope, + before_tool=make_egress_guard(check_envelope, workspace), + sandbox_image=sandbox_image, + sandbox_backend=sandbox_backend, + sandbox_memory=sandbox_memory, + sandbox_cpus=sandbox_cpus, + egress_proxy_url=egress_proxy_url, + egress_network=egress_network, + egress_token_factory=egress_token_factory, + docker_workspace_volume=docker_workspace_volume, + docker_workspace_mount=docker_workspace_mount, + ) mapped_check = dict(check) if infer_criterion_ids and not mapped_check.get("criterion_ids"): if criterion_count == 1: @@ -157,6 +158,7 @@ def make_result(target: str, passed: bool, evidence: str) -> CheckResult: criterion_ids=criterion_ids, definition=definition, source=str(check.get("source") or "agent")[:40], + gating=check.get("gating") is not False, ) if kind == "command": @@ -208,7 +210,8 @@ def checks_summary(results: list[CheckResult]) -> str: baseline = "" if r.baseline_passed is not None: baseline = f"; baseline={'PASS' if r.baseline_passed else 'FAIL'}" - lines.append(f"[{mark}] {r.source} {r.kind} {r.target}: {r.evidence}{baseline}") + gating = "" if r.gating else "; supplementary" + lines.append(f"[{mark}] {r.source} {r.kind} {r.target}: {r.evidence}{baseline}{gating}") return "\n".join(lines) diff --git a/apps/api/scripts/evaluate_verified_completion.py b/apps/api/scripts/evaluate_verified_completion.py index 32daec2..8cd5fce 100644 --- a/apps/api/scripts/evaluate_verified_completion.py +++ b/apps/api/scripts/evaluate_verified_completion.py @@ -2,6 +2,7 @@ from __future__ import annotations import argparse +import hashlib import json import os import time @@ -38,15 +39,27 @@ def _wait_for_task(client: httpx.Client, task_id: str, timeout: float) -> dict[s raise TimeoutError(f"task {task_id} did not finish within {timeout:g}s") -def _evaluate_case(client: httpx.Client, case: dict[str, Any], timeout: float) -> dict[str, Any]: - started = time.monotonic() - payload = { +def _build_task_payload(case: dict[str, Any]) -> dict[str, Any]: + expected_files = [str(path) for path in case.get("expected_files", [])] + success_criteria = list(case["success_criteria"]) + if expected_files: + artifacts = ", ".join(f"`{path}`" for path in expected_files) + success_criteria.append( + f"The final workspace contains all required artifacts: {artifacts}." + ) + return { "goal": case["goal"], - "success_criteria": case["success_criteria"], + "success_criteria": success_criteria, "verification_commands": case["verification_commands"], + "required_artifacts": expected_files, "verification_mode": "strict", "limits": case.get("limits", {"max_steps": 20, "token_budget": 30_000}), } + + +def _evaluate_case(client: httpx.Client, case: dict[str, Any], timeout: float) -> dict[str, Any]: + started = time.monotonic() + payload = _build_task_payload(case) response = client.post("/api/v1/tasks", json=payload) response.raise_for_status() task = _wait_for_task(client, response.json()["id"], timeout) @@ -74,6 +87,9 @@ def _evaluate_case(client: httpx.Client, case: dict[str, Any], timeout: float) - "status": task["status"], "duration_seconds": round(time.monotonic() - started, 3), "model": provenance.get("model"), + "isolation": receipt.get("isolation"), + "ledger_head": receipt.get("ledger_head"), + "receipt_hash": receipt.get("receipt_hash"), **scored, } @@ -123,6 +139,7 @@ def main(argv: list[str] | None = None) -> int: "run_at": datetime.now(UTC).isoformat(), "label": args.label, "manifest": manifest, + "manifest_sha256": hashlib.sha256(args.cases.read_bytes()).hexdigest(), "summary": aggregate_verified_completion(results), "results": results, } diff --git a/apps/api/tests/test_agent_react.py b/apps/api/tests/test_agent_react.py index eedceaf..8de381d 100644 --- a/apps/api/tests/test_agent_react.py +++ b/apps/api/tests/test_agent_react.py @@ -19,6 +19,7 @@ from app.repositories.task import TaskRepository from app.services.agent_react import AgentReactService, _extract_json from app.services.progress import HistoryEntry, ProgressGuard, compact_history +from app.services.task import TaskService from app.tools import ToolStatus, Workspace @@ -764,6 +765,64 @@ async def test_strict_contract_runs_required_check_and_maps_every_criterion( assert steps[-1].thought.startswith("[Loop]") +async def test_complete_contract_overrides_failing_supplementary_agent_check( + session: AsyncSession, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(settings, "agent_sandbox", "off") + plans = [ + { + "thought": "write a generator", + "tool": "write_file", + "args": { + "path": "generate.py", + "content": "from pathlib import Path\nPath('result.txt').write_text('correct')\n", + }, + }, + { + "thought": "generate the contracted output", + "tool": "run_command", + "args": {"command": "python3 generate.py"}, + }, + { + "thought": "finish with redundant evidence", + "tool": "finish", + "args": { + "summary": "ready", + "checks": [{"kind": "file_contains", "path": "result.txt", "text": "wrong"}], + }, + }, + ] + task = await _make_task( + session, + max_steps=8, + token_budget=1_000_000, + rubric=["result.txt exists"], + verification_mode="strict", + required_checks=[ + { + "id": "contract-001", + "kind": "file_exists", + "path": "result.txt", + "source": "contract", + } + ], + ) + await _service( + session, + ScriptedLLM(plans, verify={"score": 100, "met": True, "missing": []}), + ).run(task.id) + + await session.refresh(task) + assert task.stop_reason == StopReason.GOAL_ACHIEVED.value + receipt = json.loads(Workspace(Path(task.workspace_path)).read("receipt.json")) + assert receipt["checks_passed"] is True + assert receipt["checks"][1]["gating"] is False + replay = await TaskService(TaskRepository(session), StepRepository(session)).replay_receipt( + task.id + ) + assert replay["passed"] is True + + async def test_strict_contract_refuses_judgment_only_finish(session: AsyncSession) -> None: plans = [{"thought": "claim done", "tool": "finish", "args": {"summary": "done"}}] task = await _make_task( diff --git a/apps/api/tests/test_completion.py b/apps/api/tests/test_completion.py index 377f8b4..211e5dc 100644 --- a/apps/api/tests/test_completion.py +++ b/apps/api/tests/test_completion.py @@ -6,6 +6,7 @@ attach_baseline, completion_gates_pass, discover_project_checks, + mark_supplementary_agent_checks, merge_completion_checks, regressions, ) @@ -88,3 +89,27 @@ def test_failed_mapped_check_does_not_count_as_execution_coverage() -> None: ) assert completion_gates_pass([failed]) is True assert execution_coverage_complete([failed], 1) is False + + +def test_complete_contract_makes_agent_checks_supplementary() -> None: + contract = CheckResult( + "file_exists", + "result.json", + True, + "found", + criterion_ids=("criterion-001",), + definition={"kind": "file_exists", "path": "result.json"}, + source="contract", + ) + agent = CheckResult( + "file_contains", + "result.json", + False, + "text not found", + definition={"kind": "file_contains", "path": "result.json", "text": "wrong"}, + ) + + assert mark_supplementary_agent_checks([contract, agent], 1) is True + assert agent.gating is False + assert agent.definition["gating"] is False + assert completion_gates_pass([contract, agent]) is True diff --git a/apps/api/tests/test_evaluation.py b/apps/api/tests/test_evaluation.py index e75b6d3..e686857 100644 --- a/apps/api/tests/test_evaluation.py +++ b/apps/api/tests/test_evaluation.py @@ -1,7 +1,15 @@ from __future__ import annotations +import json +import re +from pathlib import Path + +from scripts.evaluate_verified_completion import _build_task_payload + from app.services.evaluation import aggregate_verified_completion, score_verified_completion +EVAL_MANIFEST = Path(__file__).parents[3] / "evals" / "verified-completion.json" + def _fixture() -> tuple[dict[str, object], dict[str, object], dict[str, object]]: task: dict[str, object] = { @@ -59,6 +67,34 @@ def test_aggregate_reports_cost_and_duration() -> None: ] ) + assert summary["total_steps"] == 6 + assert summary["total_tokens"] == 400 + assert summary["total_duration_seconds"] == 4 assert summary["average_steps"] == 3 assert summary["average_tokens"] == 200 assert summary["average_duration_seconds"] == 2 + + +def test_expected_artifacts_are_part_of_the_published_task_contract() -> None: + payload = _build_task_payload( + { + "goal": "Produce a result.", + "success_criteria": ["The result is correct."], + "verification_commands": ["test -f result.json"], + "expected_files": ["result.json", "audit.log"], + } + ) + + assert payload["success_criteria"] == [ + "The result is correct.", + "The final workspace contains all required artifacts: `result.json`, `audit.log`.", + ] + assert payload["required_artifacts"] == ["result.json", "audit.log"] + + +def test_eval_commands_use_portable_standard_library_python() -> None: + cases = json.loads(EVAL_MANIFEST.read_text())["cases"] + commands = [command for case in cases for command in case["verification_commands"]] + + assert all(not re.search(r"(^|[;&|]\s*)python(?:\s|$)", command) for command in commands) + assert all("pytest" not in command for command in commands) diff --git a/apps/api/tests/test_tasks.py b/apps/api/tests/test_tasks.py index 68bf727..68ff881 100644 --- a/apps/api/tests/test_tasks.py +++ b/apps/api/tests/test_tasks.py @@ -40,6 +40,7 @@ async def test_user_acceptance_contract_round_trips(client: AsyncClient) -> None "goal": "make a rigorously checked result", "success_criteria": ["The output is present", "The complete test suite passes"], "verification_commands": ["pytest -q"], + "required_artifacts": ["dist/result.json"], }, ) assert response.status_code == 201 @@ -48,6 +49,21 @@ async def test_user_acceptance_contract_round_trips(client: AsyncClient) -> None assert task["criteria_source"] == "user" assert task["verification_mode"] == "strict" assert task["required_checks"][0]["source"] == "contract" + assert task["required_checks"][1] == { + "id": "contract-artifact-001", + "kind": "file_exists", + "path": "dist/result.json", + "source": "contract", + } + + +async def test_required_artifact_paths_are_workspace_relative(client: AsyncClient) -> None: + response = await client.post( + "/api/v1/tasks", + json={"goal": "produce an artifact", "required_artifacts": ["../result.json"]}, + ) + + assert response.status_code == 422 async def test_goal_too_short_is_validation_error(client: AsyncClient) -> None: diff --git a/apps/api/tests/test_verification.py b/apps/api/tests/test_verification.py index 2d7dd25..1b68e05 100644 --- a/apps/api/tests/test_verification.py +++ b/apps/api/tests/test_verification.py @@ -49,6 +49,19 @@ async def test_checks_run_on_a_copy_not_the_original(tmp_path: Path) -> None: assert ("sentinel.txt", 0) not in ws.list_files() +async def test_each_check_gets_an_independent_workspace_copy(tmp_path: Path) -> None: + ws = Workspace(tmp_path / "ws") + results = await run_checks( + [ + {"kind": "command", "command": "touch generated.txt"}, + {"kind": "file_exists", "path": "generated.txt"}, + ], + ws, + ) + + assert [result.passed for result in results] == [True, False] + + def test_sweep_orphaned_verify_dirs(tmp_path: Path) -> None: from app.services.verification import sweep_orphaned_verify_dirs diff --git a/apps/web/components/publish-form.desktop.test.tsx b/apps/web/components/publish-form.desktop.test.tsx index a690507..286121c 100644 --- a/apps/web/components/publish-form.desktop.test.tsx +++ b/apps/web/components/publish-form.desktop.test.tsx @@ -65,11 +65,15 @@ describe('PublishForm desktop binding', () => { fireEvent.change(screen.getByLabelText('Acceptance contract'), { target: { value: 'The requested change is present\nThe complete test suite passes' }, }); + fireEvent.change(screen.getByLabelText('Required final artifacts'), { + target: { value: 'dist/report.json\ndist/audit.log' }, + }); fireEvent.click(screen.getByRole('button', { name: /Run the agent/ })); await waitFor(() => expect(publish).toHaveBeenCalledOnce()); expect(publish.mock.calls[0]?.[0]).toMatchObject({ project_path: '.', + required_artifacts: ['dist/report.json', 'dist/audit.log'], success_criteria: ['The requested change is present', 'The complete test suite passes'], verification_mode: 'strict', }); diff --git a/apps/web/components/publish-form.tsx b/apps/web/components/publish-form.tsx index e186e39..905cef6 100644 --- a/apps/web/components/publish-form.tsx +++ b/apps/web/components/publish-form.tsx @@ -49,6 +49,7 @@ export function PublishForm({ const [goal, setGoal] = useState(''); const [successCriteria, setSuccessCriteria] = useState(''); const [verificationCommands, setVerificationCommands] = useState(''); + const [requiredArtifacts, setRequiredArtifacts] = useState(''); const [maxSteps, setMaxSteps] = useState(d.max_steps_default); const [tokenBudget, setTokenBudget] = useState(d.token_budget_default); const [files, setFiles] = useState([]); @@ -125,6 +126,10 @@ export function PublishForm({ .split('\n') .map((item) => item.trim()) .filter(Boolean), + required_artifacts: requiredArtifacts + .split('\n') + .map((item) => item.trim()) + .filter(Boolean), verification_mode: hasProject || criteria.length > 0 ? ('strict' as const) : ('judgment' as const), project_path: projectPath.trim() || null, @@ -225,9 +230,22 @@ export function PublishForm({ placeholder="One command per line, e.g. pnpm test" className="mt-1 w-full resize-y rounded-lg border border-black/10 bg-transparent px-3 py-2 font-mono text-xs outline-none focus:border-blue-500/60 dark:border-white/15" /> + +