From a4f00aa87e48a10191e98da56c4a5aa45b8631d6 Mon Sep 17 00:00:00 2001 From: Eva Date: Mon, 1 Jun 2026 12:47:51 +0700 Subject: [PATCH 1/2] Add hybrid app handoff gate --- docs/AGENT_GRADE_APP_TESTABILITY.md | 30 ++ qa/SCORECARD.md | 4 + qa/app_handoff_gate.py | 803 ++++++++++++++++++++++++++++ qa/app_handoff_hooks.js | 159 ++++++ qa/export_app_evidence.py | 313 ++++++++++- qa/test_app_handoff_gate.py | 139 +++++ qa/test_export_app_evidence.py | 78 ++- qa/test_macos_app_static.py | 2 + qa/ui_playtest_app.sh | 15 + 9 files changed, 1530 insertions(+), 13 deletions(-) create mode 100644 qa/app_handoff_gate.py create mode 100644 qa/app_handoff_hooks.js create mode 100644 qa/test_app_handoff_gate.py diff --git a/docs/AGENT_GRADE_APP_TESTABILITY.md b/docs/AGENT_GRADE_APP_TESTABILITY.md index 5e23c682..dd4ea9e2 100644 --- a/docs/AGENT_GRADE_APP_TESTABILITY.md +++ b/docs/AGENT_GRADE_APP_TESTABILITY.md @@ -217,6 +217,14 @@ Required contents for #485: smoke/playtest run into this bundle shape. `--app-status-url ` remains supported for live read-only export from a running app. +Each exported `manifest.json` also carries a normalized +`review_entrypoint` object. That object is the first file a reviewing agent +should open: it repeats the command, repo, branch, commit SHA, dirty state, app +build SHA, provider, gate kind, run id, timestamps, verdict, failure bucket, +art status, and indexed pointers for screenshots, app-status snapshots, +session-surface snapshots, moves, provider trace, console logs, network logs, +and action logs. + Bundles are evidence, not source. Do not commit them. Screenshots may contain private art and must remain in `/Volumes/LEXAR/Codex` unless the owner explicitly chooses to publish a redacted excerpt. @@ -226,6 +234,28 @@ chooses to publish a redacted excerpt. Issue #486 defines three distinct gates. They must not be collapsed into one score. +## 100/100 Handoff Gate + +`qa/app_handoff_gate.py` is the fast hybrid gate for Codex-led GUI work. It is +the command a main implementation agent should run before spending budget on +longer exploratory/persona playtests. + +The handoff gate writes +`/Volumes/LEXAR/Codex/worldos-agent-grade-app-testability//handoff.json` +with `schema: worldos.app-handoff.v1`. `handoff_score` is `100` only when every +mandatory gate passes on the same clean commit SHA: + +- web deterministic scripted smoke. +- built `dist/WorldOS.app` deterministic scripted smoke. +- built `dist/WorldOS.app` short Codex-provider playtest. +- bounded hook probe for launcher/resume, table actions, free-text move, + settings provider status, modal/error/status hook presence. +- evidence manifests with no blocking gaps. + +This score means the GUI implementation agent has a trustworthy fast loop for +app wiring and core controls. It does not mean the product is release-ready. +Full non-partial five-persona RRI remains the release verdict. + ### 1. Deterministic Built-App Smoke Purpose: fast, repeatable app wiring proof. diff --git a/qa/SCORECARD.md b/qa/SCORECARD.md index fac4fda0..322fd082 100644 --- a/qa/SCORECARD.md +++ b/qa/SCORECARD.md @@ -9,6 +9,10 @@ > Diagnostic product evidence from the shipped Mac surface. These rows prove built-app behavior but do not > replace the RRI release sweep below. +> The hybrid handoff gate (`qa/app_handoff_gate.py`) is an implementation-velocity gate: it can score +> `handoff_score: 100` only when web deterministic smoke, built-app deterministic smoke, and a short +> built-app Codex playtest all pass on the same clean SHA. That still is not a release verdict; the +> Release Sweep Ledger remains the RRI source of truth. | Run | Date | app/code SHA | Surface | Provider | Evidence | Result / notes | |---|---|---|---|---|---|---| diff --git a/qa/app_handoff_gate.py b/qa/app_handoff_gate.py new file mode 100644 index 00000000..a900393d --- /dev/null +++ b/qa/app_handoff_gate.py @@ -0,0 +1,803 @@ +#!/usr/bin/env python3 +"""Hybrid handoff gate for WorldOS GUI implementation velocity. + +This orchestrates fast evidence gates for the app handoff lane. It is deliberately +not the release verdict: full five-persona RRI remains owned by +qa/release_readiness.py. +""" +from __future__ import annotations + +import argparse +import json +import os +import shutil +import subprocess +import sys +import time +import urllib.error +import urllib.parse +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from qa import app_smoke_scripted as smoke # noqa: E402 +from qa.app_failure_buckets import APP_FAILURE_BUCKETS # noqa: E402 + + +DEFAULT_OUTPUT_ROOT = Path("/Volumes/LEXAR/Codex/worldos-agent-grade-app-testability") +DEFAULT_ART_ROOT = Path("/Users/lume/ClawDnD-val") + + +def utc_stamp() -> str: + return datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + + +def json_dump(path: Path, payload: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def read_json(path: Path) -> dict[str, Any]: + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return {} + return payload if isinstance(payload, dict) else {} + + +def append_ndjson(path: Path, payload: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a", encoding="utf-8") as fh: + fh.write(json.dumps(payload, sort_keys=True) + "\n") + + +def repo_sha(short: bool = True) -> str: + args = ["git", "-C", str(ROOT), "rev-parse"] + if short: + args.extend(["--short", "HEAD"]) + else: + args.append("HEAD") + proc = subprocess.run(args, text=True, capture_output=True, check=False, timeout=5) + return (proc.stdout or "").strip() or "unknown" + + +def repo_dirty() -> bool: + proc = subprocess.run(["git", "-C", str(ROOT), "status", "--porcelain"], text=True, capture_output=True, check=False, timeout=5) + return bool((proc.stdout or "").strip()) + + +def command_text(args: list[str]) -> str: + return " ".join(subprocess.list2cmdline([part]) for part in args) + + +def run_logged(cmd: list[str], *, cwd: Path, env: dict[str, str], log_path: Path, timeout: float | None = None) -> int: + log_path.parent.mkdir(parents=True, exist_ok=True) + with log_path.open("w", encoding="utf-8") as log: + log.write(f"$ {command_text(cmd)}\n") + log.flush() + try: + proc = subprocess.run(cmd, cwd=cwd, env=env, stdout=log, stderr=subprocess.STDOUT, text=True, check=False, timeout=timeout) + except subprocess.TimeoutExpired as exc: + log.write(f"\n[timeout after {exc.timeout}s]\n") + log.write("[exit 124]\n") + log.flush() + return 124 + log.write(f"\n[exit {proc.returncode}]\n") + return int(proc.returncode) + + +def failure(bucket: str, detail: str) -> tuple[str, str]: + if bucket not in APP_FAILURE_BUCKETS: + bucket = "no_provider" + return bucket, detail + + +def build_matches(reported: str, expected: str) -> bool: + reported = (reported or "").strip() + expected = (expected or "").strip() + if not reported or not expected or reported == "unknown": + return False + return reported == expected or expected.startswith(reported) or reported.startswith(expected) + + +def validate_app_status( + status: dict[str, Any], + *, + expected_port: int | None, + expected_sha: str, + require_ready_for_play: bool = True, +) -> tuple[str, str]: + if not status: + return failure("no_launcher", "app-status JSON is missing") + if status.get("schema") != "worldos.app-status.v1": + return failure("no_launcher", "app-status schema is missing or wrong") + viewer = status.get("viewer") if isinstance(status.get("viewer"), dict) else {} + build = status.get("build") if isinstance(status.get("build"), dict) else {} + art = status.get("art") if isinstance(status.get("art"), dict) else {} + live = status.get("live") if isinstance(status.get("live"), dict) else {} + readiness = status.get("readiness") if isinstance(status.get("readiness"), dict) else {} + health = status.get("health") if isinstance(status.get("health"), dict) else {} + actor = live.get("actor") if isinstance(live.get("actor"), dict) else {} + + if expected_port is not None and int(viewer.get("port") or 0) != int(expected_port): + return failure("no_launcher", f"app-status answered for port {viewer.get('port')} instead of expected same port {expected_port}") + if expected_sha and not build_matches(str(build.get("sha") or ""), expected_sha): + return failure("no_app", f"app-status build SHA {build.get('sha') or 'missing'} does not match expected {expected_sha}") + for source in (readiness, health): + bucket = source.get("failure_bucket") + if isinstance(bucket, str) and bucket and bucket != "none": + return failure(bucket, str(source.get("failure_detail") or "app-status readiness failed")) + if art.get("private_root_present") is not True: + return failure("no_art", "private art root is missing from app-status") + if require_ready_for_play and readiness.get("ready_for_play") is not True: + return failure("no_provider", "app-status did not report ready_for_play:true") + if readiness.get("ready_for_smoke") is not True: + return failure("no_provider", "app-status did not report ready_for_smoke:true") + if live.get("can_act") is not True: + return failure("no_provider", "app-status did not report can_act:true") + if not (actor.get("id") or actor.get("name")): + return failure("no_actor", "app-status did not report an active player actor") + if int(live.get("enabled_action_count") or 0) <= 0: + return failure("no_actions", "app-status reported no enabled player actions") + if int(viewer.get("chat_lines") or 0) <= 0: + return failure("no_narration", "app-status reported no chat/narration") + return "", "" + + +def provider_trace_summary(play_state: Path, provider: str) -> dict[str, Any]: + provider_dir = play_state / f"{provider}-provider" + summary_path = provider_dir / "summary.json" + if summary_path.exists(): + payload = read_json(summary_path) + if payload: + payload.setdefault("provider", provider) + payload.setdefault("failed_or_error_count", 0) + return payload + + failed = 0 + parsed = 0 + samples: list[str] = [] + patterns = ("*stdout*.jsonl", "*stderr*.log", "*.ndjson", "*.jsonl", "*.log", "*.txt") + seen: set[Path] = set() + for pattern in patterns: + for path in provider_dir.glob(pattern): + if path in seen or not path.is_file(): + continue + seen.add(path) + try: + lines = path.read_text(encoding="utf-8", errors="ignore").splitlines() + except OSError: + continue + for line in lines: + if not line.strip(): + continue + parsed += 1 + parsed_payload: dict[str, Any] | None = None + if path.suffix in {".jsonl", ".ndjson"}: + try: + payload = json.loads(line) + except json.JSONDecodeError: + payload = None + if isinstance(payload, dict): + parsed_payload = payload + if parsed_payload is not None: + item = parsed_payload.get("item") if isinstance(parsed_payload.get("item"), dict) else {} + status = str(item.get("status") or parsed_payload.get("status") or "").lower() + error = item.get("error") if "error" in item else parsed_payload.get("error") + event_type = str(parsed_payload.get("type") or item.get("type") or "").lower() + is_bad = bool(error) or status in {"failed", "error", "cancelled", "canceled"} or event_type in {"turn.failed", "turn.error"} + if is_bad: + failed += 1 + if len(samples) < 5: + samples.append(line[:300]) + continue + lower = line.lower() + is_bad = any(marker in lower for marker in ( + '"is_error":true', + "extra_forbidden", + "validation error", + "cancelled", + "canceled", + "safety", + '"status":"failed"', + "traceback", + )) + if is_bad: + failed += 1 + if len(samples) < 5: + samples.append(line[:300]) + return { + "schema": "worldos.provider-trace-summary.v1", + "provider": provider, + "trace_dir": str(provider_dir), + "trace_exists": provider_dir.is_dir(), + "line_count": parsed, + "failed_or_error_count": failed, + "samples": samples, + } + + +def summarize_hook_probe(path: Path) -> tuple[bool, str, dict[str, Any]]: + payload = read_json(path) + if not payload: + return False, "hook probe did not write JSON", {} + missing = payload.get("missing_required") if isinstance(payload.get("missing_required"), list) else [] + errors = int(payload.get("console_errors") or 0) + if missing: + return False, "missing hooks: " + ", ".join(str(item) for item in missing), payload + if errors: + return False, f"hook probe saw console_errors={errors}", payload + return bool(payload.get("ok")), "" if payload.get("ok") else "hook probe reported ok:false", payload + + +def evidence_gap_count(payload: dict[str, Any]) -> int: + gaps = payload.get("evidence_gaps") + return len(gaps) if isinstance(gaps, list) else 0 + + +@dataclass +class GateResult: + name: str + provider: str + surface: str + required: bool = True + status: str = "failed" + failure_bucket: str = "no_provider" + failure_detail: str = "" + evidence_dir: str = "" + evidence_manifest: str = "" + run_id: str = "" + app_status_url: str = "" + build_sha: str = "" + port: int | None = None + command: list[str] = field(default_factory=list) + evidence_gaps: list[Any] = field(default_factory=list) + details: dict[str, Any] = field(default_factory=dict) + + def pass_(self) -> None: + self.status = "passed" + self.failure_bucket = "" + self.failure_detail = "" + + def fail(self, bucket: str, detail: str) -> None: + self.status = "failed" + self.failure_bucket, self.failure_detail = failure(bucket, detail) + + def as_dict(self) -> dict[str, Any]: + return { + "name": self.name, + "provider": self.provider, + "surface": self.surface, + "required": self.required, + "status": self.status, + "failure_bucket": self.failure_bucket, + "failure_detail": self.failure_detail, + "evidence_dir": self.evidence_dir, + "evidence_manifest": self.evidence_manifest, + "run_id": self.run_id, + "app_status_url": self.app_status_url, + "build_sha": self.build_sha, + "port": self.port, + "command": self.command, + "evidence_gaps": self.evidence_gaps, + "details": self.details, + } + + +def finalize_handoff(*, run_id: str, out: Path, gates: list[GateResult], started_at: str, expected_sha: str) -> dict[str, Any]: + required = [gate for gate in gates if gate.required] + passed = [gate for gate in required if gate.status == "passed"] + failed = [gate for gate in required if gate.status != "passed"] + same_sha = all(build_matches(gate.build_sha, expected_sha) for gate in required if gate.build_sha) + dirty = repo_dirty() + status = "passed" if len(passed) == len(required) and same_sha and not dirty else "failed" + blockers = [] + for gate in failed: + blockers.append({"gate": gate.name, "bucket": gate.failure_bucket, "detail": gate.failure_detail}) + if not same_sha: + blockers.append({"gate": "same_sha", "bucket": "no_app", "detail": "mandatory gates did not prove the same build SHA"}) + if dirty: + blockers.append({"gate": "repo_dirty", "bucket": "no_app", "detail": "handoff score cannot be 100 from a dirty checkout"}) + return { + "schema": "worldos.app-handoff.v1", + "run_id": run_id, + "status": status, + "handoff_score": 100 if status == "passed" else 0, + "release_verdict": False, + "release_verdict_note": "Full non-partial five-persona RRI remains the release verdict.", + "repo": str(ROOT), + "branch": subprocess.run(["git", "-C", str(ROOT), "branch", "--show-current"], text=True, capture_output=True, check=False).stdout.strip(), + "commit_sha": expected_sha, + "dirty": dirty, + "started_at": started_at, + "finished_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), + "evidence_root": str(out), + "gates": [gate.as_dict() for gate in gates], + "blockers": blockers, + "next_action": "handoff to main GUI implementation agent" if status == "passed" else "fix failing handoff gate before long GUI/RRI runs", + } + + +def copy_native_run(ui_run: Path, gate_dir: Path) -> None: + if not ui_run.exists(): + return + for rel in ("run.json", "backend.log", "score.json", "summary.md", "session_surface.final.json"): + src = ui_run / rel + if src.exists() and src.is_file(): + dst = gate_dir / rel + dst.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src, dst) + native = ui_run / "native" + if native.exists(): + dst = gate_dir / "native" + if dst.exists(): + shutil.rmtree(dst) + shutil.copytree(native, dst) + + +def cleanup_run(run_id: str, port: int | None) -> None: + if run_id: + patterns = [ + f"play-state/{run_id}/", + f"play_party.sh .* {run_id}", + f"play.sh .* {run_id}", + f" {run_id} ", + ] + for pattern in patterns: + subprocess.run(["pkill", "-f", pattern], check=False, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + if port: + subprocess.run(["pkill", "-f", f"server.py .* {port}$"], check=False, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + +def export_evidence( + *, + gate_dir: Path, + run_dir: Path, + app_status_url: str, + transition_file: Path | None, + command: list[str], + gate_kind: str, + provider: str, + started_at: str = "", + verdict: str = "", +) -> tuple[str, dict[str, Any]]: + out = gate_dir / "app-evidence" + cmd = [ + sys.executable, + str(ROOT / "qa" / "export_app_evidence.py"), + "--run-dir", + str(run_dir), + "--out", + str(out), + "--command-json", + json.dumps(command), + "--gate-kind", + gate_kind, + "--provider", + provider, + "--commit-sha", + repo_sha(short=False), + ] + if started_at: + cmd.extend(["--started-at", started_at]) + if verdict: + cmd.extend(["--verdict", verdict]) + if app_status_url: + cmd.extend(["--app-status-url", app_status_url]) + if transition_file and transition_file.exists(): + cmd.extend(["--transition-file", str(transition_file)]) + manifest_path = out / "manifest.json" + try: + proc = subprocess.run(cmd, cwd=ROOT, text=True, capture_output=True, check=False, timeout=60) + except subprocess.TimeoutExpired as exc: + reason = f"export_app_evidence timed out after {exc.timeout}s" + return str(manifest_path), { + "schema": "worldos.app-evidence.v1", + "evidence_gaps": [{"source": "export_app_evidence", "kind": "manifest", "path": str(manifest_path), "reason": reason}], + "failure": {"failure_bucket": "no_provider", "failure_detail": reason}, + "handoff_gate": {"ok": False, "blocking_reasons": [reason]}, + } + if proc.returncode != 0: + reason = f"export_app_evidence exited {proc.returncode}: {(proc.stderr or proc.stdout)[-1000:]}" + return str(manifest_path), { + "schema": "worldos.app-evidence.v1", + "evidence_gaps": [{"source": "export_app_evidence", "kind": "manifest", "path": str(manifest_path), "reason": reason}], + "failure": {"failure_bucket": "no_provider", "failure_detail": reason}, + "handoff_gate": {"ok": False, "blocking_reasons": [reason]}, + } + if not manifest_path.exists(): + reason = "export_app_evidence did not write manifest.json" + return str(manifest_path), { + "schema": "worldos.app-evidence.v1", + "evidence_gaps": [{"source": "export_app_evidence", "kind": "manifest", "path": str(manifest_path), "reason": reason}], + "failure": {"failure_bucket": "no_provider", "failure_detail": reason}, + "handoff_gate": {"ok": False, "blocking_reasons": [reason]}, + } + manifest = read_json(manifest_path) + if not manifest: + reason = "export_app_evidence wrote invalid or empty manifest.json" + return str(manifest_path), { + "schema": "worldos.app-evidence.v1", + "evidence_gaps": [{"source": "export_app_evidence", "kind": "manifest", "path": str(manifest_path), "reason": reason}], + "failure": {"failure_bucket": "no_provider", "failure_detail": reason}, + "handoff_gate": {"ok": False, "blocking_reasons": [reason]}, + } + return str(manifest_path), manifest + + +def run_hook_probe(base_url: str, gate_dir: Path) -> tuple[bool, str, dict[str, Any]]: + out = gate_dir / "hook-probe.json" + cmd = ["node", str(ROOT / "qa" / "app_handoff_hooks.js"), base_url] + try: + proc = subprocess.run(cmd, cwd=ROOT, text=True, capture_output=True, check=False, timeout=45) + except subprocess.TimeoutExpired as exc: + payload = { + "schema": "worldos.app-handoff-hooks.v1", + "ok": False, + "exit_code": "timeout", + "stderr": str(exc), + "stdout": (exc.stdout or "")[-2000:] if isinstance(exc.stdout, str) else "", + } + json_dump(out, payload) + return False, f"hook probe timed out: {exc}", payload + if proc.returncode != 0: + payload = { + "schema": "worldos.app-handoff-hooks.v1", + "ok": False, + "exit_code": proc.returncode, + "stderr": proc.stderr[-2000:], + "stdout": proc.stdout[-2000:], + } + json_dump(out, payload) + return False, payload["stderr"] or f"hook probe exited {proc.returncode}", payload + try: + payload = json.loads(proc.stdout) + except json.JSONDecodeError: + payload = {"ok": False, "stdout": proc.stdout[-2000:], "stderr": proc.stderr[-2000:]} + json_dump(out, payload) + return summarize_hook_probe(out) + + +def drive_moves( + *, + base_url: str, + gate_dir: Path, + run_id: str, + provider: str, + beats: int, + timeout: float, + expected_sha: str, + expected_port: int, +) -> tuple[bool, str, str, dict[str, Any]]: + screenshots: list[str] = [] + gaps: list[dict[str, str]] = [] + (gate_dir / "screenshots").mkdir(parents=True, exist_ok=True) + (gate_dir / "a11y").mkdir(parents=True, exist_ok=True) + for rel in ("console.ndjson", "network.ndjson", "actions.ndjson", "moves.ndjson"): + (gate_dir / rel).write_text("", encoding="utf-8") + status = smoke.wait_for_status(base_url, gate_dir, timeout=timeout) + json_dump(gate_dir / "app-status.initial.json", status) + smoke.write_text_snapshot(gate_dir / "a11y" / "initial.html", smoke.html_text(base_url)) + smoke.capture_openworlds_screenshot(base_url=base_url, out=gate_dir, port=expected_port, label="initial", gaps=gaps, screenshots=screenshots) + bucket, detail = validate_app_status(status, expected_port=expected_port, expected_sha=expected_sha) + if bucket: + return False, bucket, detail, {"screenshots": screenshots, "evidence_gaps": gaps} + hook_ok, hook_detail, hook_payload = run_hook_probe(base_url, gate_dir) + if not hook_ok: + return False, "no_actions", hook_detail, {"screenshots": screenshots, "evidence_gaps": gaps, "hook_probe": hook_payload} + + try: + surface, _ = smoke.fetch_json(smoke.surface_url(base_url, status)) + json_dump(gate_dir / "session-surface.initial.json", surface) + except Exception as exc: # noqa: BLE001 + return False, "no_provider", f"initial session-surface fetch failed: {exc}", {"screenshots": screenshots, "evidence_gaps": gaps} + + last_chat_lines = int(((status.get("viewer") or {}).get("chat_lines") or 0) if isinstance(status.get("viewer"), dict) else 0) + move_url = urllib.parse.urljoin(base_url, "/move") + for beat in range(1, beats + 1): + move = { + "kind": "do", + "text": f"handoff {provider} gate beat {beat}: check the table wiring and continue.", + } + append_ndjson(gate_dir / "moves.ndjson", {"at": time.time(), "beat": beat, "request": move}) + append_ndjson(gate_dir / "actions.ndjson", {"at": time.time(), "beat": beat, "action": "post_move", "url": move_url}) + try: + response, http_status = smoke.post_json(move_url, move, timeout=5) + except (OSError, urllib.error.URLError, ValueError) as exc: + append_ndjson(gate_dir / "network.ndjson", {"at": time.time(), "method": "POST", "url": move_url, "error": str(exc)}) + return False, "move_rejected", str(exc), {"screenshots": screenshots, "evidence_gaps": gaps} + append_ndjson(gate_dir / "network.ndjson", {"at": time.time(), "method": "POST", "url": move_url, "status": http_status}) + if not response.get("ok"): + return False, "move_rejected", str(response.get("reason") or response), {"screenshots": screenshots, "evidence_gaps": gaps} + + deadline = time.time() + timeout + advanced = False + while time.time() < deadline: + status = smoke.wait_for_status(base_url, gate_dir, timeout=3) + chat_lines = int(((status.get("viewer") or {}).get("chat_lines") or 0) if isinstance(status.get("viewer"), dict) else 0) + if chat_lines > last_chat_lines: + if provider != "scripted": + advanced = True + last_chat_lines = chat_lines + break + summary = smoke.provider_summary(ROOT / "play-state" / run_id) + if int(summary.get("resolved_move_count") or summary.get("move_resolved_count") or 0) >= beat: + advanced = True + last_chat_lines = chat_lines + break + time.sleep(1 if provider != "scripted" else 0.5) + json_dump(gate_dir / f"app-status.beat-{beat}.json", status) + try: + surface, _ = smoke.fetch_json(smoke.surface_url(base_url, status)) + json_dump(gate_dir / f"session-surface.beat-{beat}.json", surface) + except Exception as exc: # noqa: BLE001 + return False, "no_provider", f"session-surface fetch failed after beat {beat}: {exc}", {"screenshots": screenshots, "evidence_gaps": gaps} + smoke.write_text_snapshot(gate_dir / "a11y" / f"beat-{beat}.html", smoke.html_text(base_url)) + smoke.capture_openworlds_screenshot(base_url=base_url, out=gate_dir, port=expected_port, label=f"beat-{beat:03d}", gaps=gaps, screenshots=screenshots) + if not advanced: + return False, "no_narration", f"narration did not advance after {provider} beat {beat}", {"screenshots": screenshots, "evidence_gaps": gaps} + + final_status = smoke.wait_for_status(base_url, gate_dir, timeout=5) + json_dump(gate_dir / "app-status.final.json", final_status) + try: + final_surface, _ = smoke.fetch_json(smoke.surface_url(base_url, final_status)) + json_dump(gate_dir / "session-surface.final.json", final_surface) + except Exception as exc: # noqa: BLE001 + return False, "no_provider", f"final session-surface fetch failed: {exc}", {"screenshots": screenshots, "evidence_gaps": gaps} + smoke.write_text_snapshot(gate_dir / "a11y" / "final.html", smoke.html_text(base_url)) + smoke.capture_openworlds_screenshot(base_url=base_url, out=gate_dir, port=expected_port, label="final", gaps=gaps, screenshots=screenshots) + smoke.copy_play_state(run_id, gate_dir) + trace = provider_trace_summary(ROOT / "play-state" / run_id, provider) + json_dump(gate_dir / "provider-trace-summary.json", trace) + if provider == "codex" and int(trace.get("failed_or_error_count") or 0) > 0: + return False, "no_provider", "Codex provider trace reported failed/error/cancellation events", {"screenshots": screenshots, "evidence_gaps": gaps, "provider_trace": trace} + if gaps: + return False, "no_provider", "required evidence capture has gaps", {"screenshots": screenshots, "evidence_gaps": gaps, "provider_trace": trace} + return True, "", "", {"screenshots": screenshots, "evidence_gaps": gaps, "provider_trace": trace} + + +def run_web_scripted(args: argparse.Namespace, out: Path, expected_sha: str) -> GateResult: + gate_dir = out / "web-scripted" + gate = GateResult(name="web_scripted_smoke", provider="scripted", surface="web", evidence_dir=str(gate_dir), run_id=f"{args.run_id}-web-scripted", build_sha=expected_sha) + cmd = [ + sys.executable, + str(ROOT / "qa" / "app_smoke_scripted.py"), + "--beats", + str(args.web_beats), + "--port", + str(args.web_port), + "--run-id", + gate.run_id, + "--out", + str(gate_dir), + "--timeout", + str(args.timeout), + ] + if args.art_root: + cmd.extend(["--art-root", args.art_root]) + gate.command = cmd + rc = run_logged(cmd, cwd=ROOT, env=os.environ.copy(), log_path=gate_dir / "handoff-command.log") + smoke_json = read_json(gate_dir / "smoke.json") + final_status = read_json(gate_dir / "app-status.final.json") + gate.port = int(args.web_port) + gate.app_status_url = f"http://127.0.0.1:{args.web_port}/app-status" + gate.evidence_gaps = smoke_json.get("evidence_gaps") if isinstance(smoke_json.get("evidence_gaps"), list) else [] + gate.details["smoke"] = smoke_json + if rc != 0 or smoke_json.get("status") != "passed": + gate.fail(str(smoke_json.get("failure_bucket") or "no_provider"), str(smoke_json.get("failure_detail") or f"web scripted smoke exited {rc}")) + else: + bucket, detail = validate_app_status(final_status, expected_port=int(args.web_port), expected_sha=expected_sha) + if bucket: + gate.fail(bucket, detail) + else: + gate.pass_() + manifest_path, manifest = export_evidence( + gate_dir=gate_dir, + run_dir=gate_dir, + app_status_url="", + transition_file=None, + command=cmd, + gate_kind=gate.name, + provider=gate.provider, + verdict=gate.status, + ) + gate.evidence_manifest = manifest_path + if gate.status == "passed" and evidence_gap_count(manifest): + gate.fail("no_provider", "web scripted evidence manifest has gaps") + gate.evidence_gaps = manifest.get("evidence_gaps", []) + elif gate.status != "passed": + gate.evidence_gaps = manifest.get("evidence_gaps", []) if isinstance(manifest.get("evidence_gaps"), list) else gate.evidence_gaps + return gate + + +def run_native_provider_gate( + args: argparse.Namespace, + out: Path, + *, + provider: str, + beats: int, + budget: str, + expected_sha: str, +) -> GateResult: + name = "built_app_scripted_smoke" if provider == "scripted" else "built_app_codex_playtest" + gate_dir = out / name + native_run = f"{args.run_id}-{provider}-native" + gate = GateResult(name=name, provider=provider, surface="dist/WorldOS.app", evidence_dir=str(gate_dir), run_id=native_run, build_sha=expected_sha) + env = os.environ.copy() + env.update({ + "WOS_APP_PART": "A", + "WOS_APP_KEEP_MINTED_BACKEND": "1", + "WOS_APP_SELECTED_PROVIDER": provider, + }) + if args.art_root: + env["WORLDOS_ART_REPO_ROOT"] = args.art_root + env["CLAWDND_ART_REPO_ROOT"] = args.art_root + if provider == "scripted": + env["WORLDOS_ENABLE_SCRIPTED_PROVIDER"] = "1" + cmd = ["bash", str(ROOT / "qa" / "ui_playtest_app.sh"), native_run, args.world, "newbie", "1", budget] + gate.command = cmd + rc = run_logged(cmd, cwd=ROOT, env=env, log_path=gate_dir / "ui_playtest_app.log") + ui_run = ROOT / "qa" / "ui_playtest_runs" / native_run + copy_native_run(ui_run, gate_dir) + run_json = read_json(ui_run / "run.json") + transition = read_json(ui_run / "native" / "transition.json") + part_a = run_json.get("part_a") if isinstance(run_json.get("part_a"), dict) else {} + port = part_a.get("minted_port") + minted_run = str(part_a.get("minted_run_dir") or "") + gate.details["ui_run_dir"] = str(ui_run) + gate.details["transition"] = transition + gate.details["run_json"] = run_json + if isinstance(port, int): + gate.port = port + elif isinstance(port, str) and port.isdigit(): + gate.port = int(port) + gate.run_id = minted_run or native_run + if rc != 0 or part_a.get("result") != "PASS": + gate.fail(str(part_a.get("failure_bucket") or transition.get("failure_bucket") or "no_launcher"), str(part_a.get("failure_detail") or transition.get("failure_detail") or f"native provider launch exited {rc}")) + export_path, manifest = export_evidence( + gate_dir=gate_dir, + run_dir=gate_dir, + app_status_url="", + transition_file=gate_dir / "native" / "transition.json", + command=cmd, + gate_kind=gate.name, + provider=gate.provider, + verdict=gate.status, + ) + gate.evidence_manifest = export_path + gate.evidence_gaps = manifest.get("evidence_gaps", []) + return gate + if part_a.get("kept_backend_alive") is not True or part_a.get("first_turn_ready") is not True: + gate.fail("no_provider", "native Part A did not keep a first-turn-ready backend alive") + export_path, manifest = export_evidence( + gate_dir=gate_dir, + run_dir=gate_dir, + app_status_url="", + transition_file=gate_dir / "native" / "transition.json", + command=cmd, + gate_kind=gate.name, + provider=gate.provider, + verdict=gate.status, + ) + gate.evidence_manifest = export_path + gate.evidence_gaps = manifest.get("evidence_gaps", []) + return gate + if not gate.port or not minted_run: + gate.fail("no_launcher", "native Part A did not report minted port and run id") + export_path, manifest = export_evidence( + gate_dir=gate_dir, + run_dir=gate_dir, + app_status_url="", + transition_file=gate_dir / "native" / "transition.json", + command=cmd, + gate_kind=gate.name, + provider=gate.provider, + verdict=gate.status, + ) + gate.evidence_manifest = export_path + gate.evidence_gaps = manifest.get("evidence_gaps", []) + return gate + + base_url = f"http://127.0.0.1:{gate.port}" + gate.app_status_url = f"{base_url}/app-status" + try: + ok, bucket, detail, details = drive_moves( + base_url=base_url, + gate_dir=gate_dir, + run_id=minted_run, + provider=provider, + beats=beats, + timeout=args.timeout if provider == "scripted" else args.codex_timeout, + expected_sha=expected_sha, + expected_port=gate.port, + ) + gate.details.update(details) + if not ok: + gate.fail(bucket, detail) + else: + gate.pass_() + except Exception as exc: # noqa: BLE001 - bucketed in handoff.json. + gate.fail("no_provider", f"{provider} gate crashed: {exc}") + finally: + export_path, manifest = export_evidence( + gate_dir=gate_dir, + run_dir=gate_dir, + app_status_url=gate.app_status_url, + transition_file=gate_dir / "native" / "transition.json", + command=cmd, + gate_kind=gate.name, + provider=gate.provider, + verdict=gate.status, + ) + gate.evidence_manifest = export_path + gate.evidence_gaps = manifest.get("evidence_gaps", []) if isinstance(manifest.get("evidence_gaps"), list) else [] + cleanup_run(minted_run, gate.port) + if gate.status == "passed" and evidence_gap_count(read_json(Path(gate.evidence_manifest))): + gate.fail("no_provider", "native evidence manifest has gaps") + return gate + + +def parse_args(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Run the WorldOS 100/100 hybrid handoff gate.") + parser.add_argument("--run-id", default=f"handoff-{utc_stamp()}-{repo_sha(short=True)}") + parser.add_argument("--out", default="") + parser.add_argument("--world", default="baldurs-gate") + parser.add_argument("--web-port", type=int, default=8899) + parser.add_argument("--web-beats", type=int, default=5) + parser.add_argument("--built-beats", type=int, default=5) + parser.add_argument("--codex-moves", type=int, default=1) + parser.add_argument("--timeout", type=float, default=60.0) + parser.add_argument("--codex-timeout", type=float, default=180.0) + parser.add_argument("--codex-budget", default="3.00") + parser.add_argument("--scripted-budget", default="1.00") + parser.add_argument("--art-root", default=os.environ.get("WORLDOS_ART_REPO_ROOT") or os.environ.get("CLAWDND_ART_REPO_ROOT") or (str(DEFAULT_ART_ROOT) if DEFAULT_ART_ROOT.exists() else "")) + parser.add_argument("--skip-web", action="store_true") + parser.add_argument("--skip-built-scripted", action="store_true") + parser.add_argument("--skip-codex", action="store_true") + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv if argv is not None else sys.argv[1:]) + out = (Path(args.out).expanduser() if args.out else DEFAULT_OUTPUT_ROOT / args.run_id).resolve() + out.mkdir(parents=True, exist_ok=True) + started_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + expected_sha = repo_sha(short=True) + gates: list[GateResult] = [] + try: + if args.skip_web: + g = GateResult(name="web_scripted_smoke", provider="scripted", surface="web") + g.fail("no_provider", "web deterministic smoke was skipped") + gates.append(g) + else: + gates.append(run_web_scripted(args, out, expected_sha)) + if args.skip_built_scripted: + g = GateResult(name="built_app_scripted_smoke", provider="scripted", surface="dist/WorldOS.app") + g.fail("no_provider", "built-app deterministic smoke was skipped") + gates.append(g) + else: + gates.append(run_native_provider_gate(args, out, provider="scripted", beats=int(args.built_beats), budget=args.scripted_budget, expected_sha=expected_sha)) + if args.skip_codex: + g = GateResult(name="built_app_codex_playtest", provider="codex", surface="dist/WorldOS.app") + g.fail("no_provider", "short Codex provider playtest was skipped") + gates.append(g) + else: + gates.append(run_native_provider_gate(args, out, provider="codex", beats=int(args.codex_moves), budget=args.codex_budget, expected_sha=expected_sha)) + finally: + # Keep a tidy desktop even if one gate crashes. + subprocess.run(["pkill", "-x", "WorldOSApp"], check=False, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + handoff = finalize_handoff(run_id=args.run_id, out=out, gates=gates, started_at=started_at, expected_sha=expected_sha) + json_dump(out / "handoff.json", handoff) + print(str(out / "handoff.json")) + return 0 if handoff["status"] == "passed" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/qa/app_handoff_hooks.js b/qa/app_handoff_hooks.js new file mode 100644 index 00000000..b462ff5b --- /dev/null +++ b/qa/app_handoff_hooks.js @@ -0,0 +1,159 @@ +#!/usr/bin/env node +// Bounded OpenWorlds hook probe for the handoff gate. +// +// This probes the same live viewer port that /app-status came from, but reads +// source modules instead of launching another browser. The handoff gate already +// captures screenshots and drives /move; this probe is for exact hook coverage +// without adding a Playwright or Chrome dependency. + +const http = require('http'); +const https = require('https'); + +const baseUrl = process.argv[2] || ''; +if (!baseUrl) { + console.error('usage: node qa/app_handoff_hooks.js '); + process.exit(2); +} + +const FILES = { + launcher: 'screen-launcher.jsx', + table: 'screen-table.jsx', + settings: 'screen-settings.jsx', + toast: 'toast.jsx', + modal: 'camp-sidebar.jsx', + chrome: 'chrome.jsx', +}; + +const CHECKS = { + launcher: { + file: FILES.launcher, + required: ['worldos-launcher', 'chronicle-start-flow', 'campaign-row'], + optional: ['continue-banner', 'chronicle-resume', 'chronicle-resume-detail', 'error-banner'], + }, + table: { + file: FILES.table, + required: [ + 'openworlds-root', + 'app-status-banner', + 'narration-log', + 'active-player', + 'action-palette', + 'action-button', + 'move-input', + 'move-submit', + ], + optional: ['move-composer', 'dice-button', 'error-banner'], + requiredSourceMarkers: ['data-worldos-action-id'], + }, + settings: { + file: FILES.settings, + required: ['settings-root', 'settings-tab', 'provider-status', 'provider-controls'], + optional: ['provider-card', 'provider-start', 'provider-stop', 'error-banner'], + requiredSourceMarkers: ['data-worldos-tab-id'], + }, + modal: { + file: FILES.modal, + required: ['modal-close'], + optional: [], + }, + toast: { + file: FILES.toast, + required: ['error-banner'], + optional: ['toast-region', 'toast'], + }, + chrome: { + file: FILES.chrome, + required: ['primary-navigation', 'screen-tabs', 'screen-tab'], + optional: [], + }, +}; + +function fetchText(url) { + return new Promise((resolve, reject) => { + const client = url.startsWith('https:') ? https : http; + const req = client.get(url, { timeout: 8000 }, (res) => { + const chunks = []; + res.setEncoding('utf8'); + res.on('data', (chunk) => chunks.push(chunk)); + res.on('end', () => { + if (res.statusCode < 200 || res.statusCode >= 300) { + reject(new Error(`${url} returned HTTP ${res.statusCode}`)); + return; + } + resolve(chunks.join('')); + }); + }); + req.on('timeout', () => { + req.destroy(new Error(`${url} timed out`)); + }); + req.on('error', reject); + }); +} + +function escapeRegExp(value) { + return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +function countTestId(source, testId) { + const dataRe = new RegExp(`data-worldos-testid=(?:["']${escapeRegExp(testId)}["']|\\{[^}]*["']${escapeRegExp(testId)}["'][^}]*\\})`, 'g'); + const propRe = new RegExp(`\\btestId=["']${escapeRegExp(testId)}["']`, 'g'); + return (source.match(dataRe) || []).length + (source.match(propRe) || []).length; +} + +function sourceHas(source, marker) { + return source.includes(marker); +} + +async function runScreen(name, spec, sources) { + const url = `${baseUrl.replace(/\/$/, '')}/openworlds/${spec.file}`; + const source = await fetchText(url); + sources[spec.file] = source.length; + const required = Object.fromEntries(spec.required.map((testId) => [testId, countTestId(source, testId)])); + const optional = Object.fromEntries(spec.optional.map((testId) => [testId, countTestId(source, testId)])); + const missingRequired = Object.entries(required) + .filter(([, count]) => !count) + .map(([testId]) => testId); + for (const marker of spec.requiredSourceMarkers || []) { + if (!sourceHas(source, marker)) missingRequired.push(marker); + } + return { + screen: name, + url, + ok: missingRequired.length === 0, + missing_required: missingRequired, + console_errors: 0, + console_error_samples: [], + observed: { + required, + optional, + source_bytes: source.length, + source_markers: Object.fromEntries((spec.requiredSourceMarkers || []).map((marker) => [marker, sourceHas(source, marker)])), + }, + }; +} + +(async () => { + const appStatusUrl = `${baseUrl.replace(/\/$/, '')}/app-status`; + const sources = {}; + const appStatus = JSON.parse(await fetchText(appStatusUrl)); + const screens = []; + for (const [name, spec] of Object.entries(CHECKS)) { + screens.push(await runScreen(name, spec, sources)); + } + const missing = screens.flatMap((screen) => screen.missing_required.map((testId) => `${screen.screen}:${testId}`)); + console.log(JSON.stringify({ + schema: 'worldos.app-handoff-hooks.v1', + ok: missing.length === 0, + probe_mode: 'same-port-source-http', + base_url: baseUrl, + app_status_schema: appStatus.schema || '', + app_status_port: appStatus.viewer?.port || null, + missing_required: missing, + console_errors: 0, + fetched_sources: sources, + screens, + }, null, 2)); +})().catch((error) => { + console.error(`app_handoff_hooks fatal: ${error?.stack || error}`); + process.exit(3); +}); diff --git a/qa/export_app_evidence.py b/qa/export_app_evidence.py index 60fd11c2..885bc901 100644 --- a/qa/export_app_evidence.py +++ b/qa/export_app_evidence.py @@ -11,6 +11,7 @@ import json import os import shutil +import subprocess import sys import urllib.error import urllib.parse @@ -21,6 +22,7 @@ DEFAULT_OUTPUT_ROOT = Path("/Volumes/LEXAR/Codex") +ROOT = Path(__file__).resolve().parents[1] MAX_HTTP_BYTES = 8 * 1024 * 1024 MAX_LOCAL_FILE_BYTES = 64 * 1024 * 1024 RUN_DIR_PATTERNS = ( @@ -55,9 +57,26 @@ "native/*.png", "native/*.json", "native/*.log", + "hook-probe.json", + "provider-trace-summary.json", + "handoff-command.log", + "ui_playtest_app.log", "scripted-provider/*.json", "scripted-provider/*.ndjson", "scripted-provider/*.log", + "codex-provider/*.json", + "codex-provider/*.jsonl", + "codex-provider/*.ndjson", + "codex-provider/*.log", + "codex-provider/*.txt", + "play-state/*.jsonl", + "play-state/scripted-provider/*.json", + "play-state/scripted-provider/*.ndjson", + "play-state/codex-provider/*.json", + "play-state/codex-provider/*.jsonl", + "play-state/codex-provider/*.ndjson", + "play-state/codex-provider/*.log", + "play-state/codex-provider/*.txt", ) @@ -398,12 +417,231 @@ def run_dir_failure(bundle: Path) -> dict[str, str]: return {"failure_bucket": "", "failure_detail": "", "source": ""} +def copied_kinds(copied_files: list[dict[str, Any]]) -> list[str]: + values: set[str] = set() + for entry in copied_files: + kind = str(entry.get("kind") or "") + path = str(entry.get("path") or "") + if kind: + values.add(kind) + if path: + values.add(path) + return sorted(values) + + +def git_text(args: list[str]) -> str: + proc = subprocess.run( + ["git", "-C", str(ROOT), *args], + text=True, + capture_output=True, + check=False, + timeout=5, + ) + return (proc.stdout or "").strip() + + +def repo_snapshot() -> dict[str, Any]: + status = git_text(["status", "--porcelain"]) + return { + "path": str(ROOT), + "branch": git_text(["branch", "--show-current"]), + "commit_sha": git_text(["rev-parse", "HEAD"]), + "dirty": bool(status), + } + + +def command_from_arg(value: str) -> list[str]: + if not value: + return [] + try: + parsed = json.loads(value) + except json.JSONDecodeError: + return [value] + if isinstance(parsed, list): + return [str(item) for item in parsed] + return [str(parsed)] + + +def copied_paths(copied_files: list[dict[str, Any]]) -> list[str]: + paths = [] + for entry in copied_files: + path = str(entry.get("path") or "") + if path: + paths.append(path) + return sorted(paths) + + +def evidence_index(copied_files: list[dict[str, Any]], sources: dict[str, Any]) -> dict[str, list[str]]: + paths = copied_paths(copied_files) + + def matching(*needles: str) -> list[str]: + lowered = tuple(needle.lower() for needle in needles) + return sorted(path for path in paths if any(needle in path.lower() for needle in lowered)) + + app_status = matching("app-status") + session_surface = matching("session-surface") + for key in ("app_status", "app_status_snapshot"): + if (sources.get(key) or {}).get("path"): + app_status.insert(0, str((sources.get(key) or {}).get("path"))) + for key in ("session_surface", "session_surface_snapshot"): + if (sources.get(key) or {}).get("path"): + session_surface.insert(0, str((sources.get(key) or {}).get("path"))) + return { + "screenshots": sorted(path for path in paths if "/screenshots/" in f"/{path}" or path.lower().endswith((".png", ".jpg", ".jpeg"))), + "app_status_snapshots": sorted(dict.fromkeys(app_status)), + "session_surface_snapshots": sorted(dict.fromkeys(session_surface)), + "moves": matching("moves", "player_moves"), + "provider_trace": matching("provider-trace", "scripted-provider", "codex-provider"), + "console_logs": matching("console.ndjson", "console.log", "console"), + "network_logs": matching("network.ndjson", "network.log", "network"), + "action_logs": matching("actions.ndjson", "actions.log", "actions"), + "all_copied": paths, + } + + +def first_bundle_json(bundle: Path, patterns: tuple[str, ...]) -> tuple[dict[str, Any], str]: + for pattern in patterns: + for path in sorted((bundle / "run-dir").glob(pattern)): + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + continue + if isinstance(payload, dict): + return payload, source_display(path, bundle) + return {}, "" + + +def review_verdict(handoff_gate: dict[str, Any], failure: dict[str, str], gaps: list[dict[str, str]]) -> str: + if failure.get("failure_bucket"): + return "failed" + if handoff_gate.get("ok") is True: + return "passed" + if gaps: + return "failed" + return "incomplete" + + +def build_review_entrypoint( + *, + args: argparse.Namespace, + created_at: str, + build: dict[str, str], + art: dict[str, Any], + live: dict[str, Any], + failure: dict[str, str], + sources: dict[str, Any], + copied_files: list[dict[str, Any]], + gaps: list[dict[str, str]], + handoff_gate: dict[str, Any], +) -> dict[str, Any]: + repo = repo_snapshot() + index = evidence_index(copied_files, sources) + provider = args.provider or str(live.get("provider") or "") + verdict = args.verdict or review_verdict(handoff_gate, failure, gaps) + return { + "schema": "worldos.app-evidence-review-entrypoint.v1", + "command": command_from_arg(args.command_json), + "repo": repo["path"], + "branch": repo["branch"], + "commit_sha": args.commit_sha or repo["commit_sha"], + "dirty": repo["dirty"], + "app_build_sha": build.get("sha") or "", + "provider": provider, + "gate_kind": args.gate_kind or "", + "run_id": str(live.get("run_id") or ""), + "started_at": args.started_at or "", + "ended_at": args.ended_at or created_at, + "verdict": verdict, + "failure_bucket": str(failure.get("failure_bucket") or ""), + "failure_detail": str(failure.get("failure_detail") or ""), + "art_status": art, + "handoff_gate_ok": bool(handoff_gate.get("ok")), + "evidence_gaps": gaps, + "files": index, + } + + +def build_handoff_gate( + *, + build: dict[str, str], + art: dict[str, Any], + live: dict[str, Any], + failure: dict[str, str], + sources: dict[str, Any], + copied_files: list[dict[str, Any]], + gaps: list[dict[str, str]], +) -> dict[str, Any]: + copied = copied_kinds(copied_files) + app_status_ok = bool((sources.get("app_status") or {}).get("ok") or (sources.get("app_status_snapshot") or {}).get("ok")) + session_surface_ok = bool((sources.get("session_surface") or {}).get("ok") or (sources.get("session_surface_snapshot") or {}).get("ok")) + run_dir_source = sources.get("run_dir") if isinstance(sources.get("run_dir"), dict) else None + run_dir_ok = None if run_dir_source is None else bool(run_dir_source.get("ok")) + private_art_present = art.get("private_root_present") + can_act = live.get("can_act") + enabled_action_count = live.get("enabled_action_count") + try: + enabled_count_int = int(enabled_action_count or 0) + except (TypeError, ValueError): + enabled_count_int = 0 + move_sink_present = any( + "moves" in item or item.endswith("moves.ndjson") or item.endswith("player_moves.jsonl") + for item in copied + ) + bucket = str(failure.get("failure_bucket") or "") + detail = str(failure.get("failure_detail") or "") + + blocking: list[str] = [] + if not build.get("sha"): + blocking.append("missing build SHA") + if (sources.get("app_status") is not None or sources.get("app_status_snapshot") is not None) and not app_status_ok: + blocking.append("app-status fetch failed") + if (sources.get("session_surface") is not None or sources.get("session_surface_snapshot") is not None) and not session_surface_ok: + blocking.append("session-surface fetch failed") + if run_dir_source is not None and not run_dir_ok: + blocking.append("run-dir copy failed") + if private_art_present is not True: + blocking.append("private art not proven present") + if not live.get("campaign_id"): + blocking.append("campaign id missing") + if can_act is not True: + blocking.append("can_act not true") + if enabled_count_int < 1: + blocking.append("no enabled actions") + if not move_sink_present: + blocking.append("move sink evidence missing") + if bucket: + blocking.append(f"failure bucket: {bucket}") + if gaps: + blocking.append(f"evidence gaps: {len(gaps)}") + + return { + "schema": "worldos.app-evidence-handoff.v1", + "ok": len(blocking) == 0, + "build_sha": str(build.get("sha") or ""), + "app_status_ok": app_status_ok, + "session_surface_ok": session_surface_ok, + "run_dir_ok": run_dir_ok, + "private_art_present": private_art_present if isinstance(private_art_present, bool) else None, + "campaign_id": str(live.get("campaign_id") or ""), + "run_id": str(live.get("run_id") or ""), + "can_act": can_act if isinstance(can_act, bool) else None, + "enabled_action_count": enabled_action_count, + "move_sink_present": move_sink_present, + "copied_kinds": copied, + "failure_bucket": bucket, + "failure_detail": detail, + "evidence_gap_count": len(gaps), + "blocking_reasons": blocking, + } + + def exporter_manifest(args: argparse.Namespace, bundle: Path) -> tuple[dict[str, Any], int]: gaps: list[dict[str, str]] = [] sources: dict[str, Any] = {} copied_files: list[dict[str, Any]] = [] app_status: dict[str, Any] = {} exit_code = 0 + created_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") if args.app_status_url: try: @@ -462,6 +700,14 @@ def exporter_manifest(args: argparse.Namespace, bundle: Path) -> tuple[dict[str, if run_source: sources["run_dir"] = run_source copied_files.extend(run_copied) + if not app_status: + app_status, app_status_snapshot = first_bundle_json(bundle, ("app-status.final.json", "app-status*.json")) + if app_status_snapshot: + sources["app_status_snapshot"] = {"path": app_status_snapshot, "ok": True} + if "session_surface" not in sources: + _surface, session_surface_snapshot = first_bundle_json(bundle, ("session-surface.final.json", "session-surface*.json")) + if session_surface_snapshot: + sources["session_surface_snapshot"] = {"path": session_surface_snapshot, "ok": True} live = app_status.get("live") if isinstance(app_status.get("live"), dict) else {} failure = { @@ -477,24 +723,64 @@ def exporter_manifest(args: argparse.Namespace, bundle: Path) -> tuple[dict[str, } if not failure["failure_bucket"] and run_source: failure = run_dir_failure(bundle) + build = build_info(app_status) + art = art_status(app_status) + live_summary = { + "campaign_id": str(live.get("campaign_id") or ""), + "attached_campaign_id": str(live.get("attached_campaign_id") or ""), + "run_id": str(live.get("run_id") or ""), + "can_act": bool(live.get("can_act")) if "can_act" in live else None, + "enabled_action_count": live.get("enabled_action_count"), + } + handoff_gate = build_handoff_gate( + build=build, + art=art, + live=live_summary, + failure=failure, + sources=sources, + copied_files=copied_files, + gaps=gaps, + ) + review_entrypoint = build_review_entrypoint( + args=args, + created_at=created_at, + build=build, + art=art, + live=live_summary, + failure=failure, + sources=sources, + copied_files=copied_files, + gaps=gaps, + handoff_gate=handoff_gate, + ) manifest = { "schema": "worldos.app-evidence.v1", - "created_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), + "created_at": created_at, "app_status_url": args.app_status_url or "", "bundle_dir": str(bundle), - "build": build_info(app_status), - "art": art_status(app_status), - "live": { - "campaign_id": str(live.get("campaign_id") or ""), - "attached_campaign_id": str(live.get("attached_campaign_id") or ""), - "run_id": str(live.get("run_id") or ""), - "can_act": bool(live.get("can_act")) if "can_act" in live else None, - "enabled_action_count": live.get("enabled_action_count"), - }, + "command": review_entrypoint["command"], + "repo": review_entrypoint["repo"], + "branch": review_entrypoint["branch"], + "commit_sha": review_entrypoint["commit_sha"], + "dirty": review_entrypoint["dirty"], + "app_build_sha": review_entrypoint["app_build_sha"], + "provider": review_entrypoint["provider"], + "gate_kind": review_entrypoint["gate_kind"], + "run_id": review_entrypoint["run_id"], + "started_at": review_entrypoint["started_at"], + "ended_at": review_entrypoint["ended_at"], + "verdict": review_entrypoint["verdict"], + "failure_bucket": review_entrypoint["failure_bucket"], + "build": build, + "art": art, + "live": live_summary, "failure": failure, "sources": sources, "copied_files": copied_files, "evidence_gaps": gaps, + "review_entrypoint": review_entrypoint, + "evidence_files": review_entrypoint["files"], + "handoff_gate": handoff_gate, } return manifest, exit_code @@ -505,6 +791,13 @@ def parse_args(argv: list[str]) -> argparse.Namespace: parser.add_argument("--out", default="", help="Evidence bundle directory (default: /Volumes/LEXAR/Codex/worldos-app-evidence/)") parser.add_argument("--transition-file", default="", help="Optional local JSON file with failure bucket/detail") parser.add_argument("--run-dir", default="", help="Optional app playtest run directory to copy into the evidence bundle") + parser.add_argument("--command-json", default="", help="JSON command argv that produced this evidence") + parser.add_argument("--gate-kind", default="", help="Gate kind such as web_scripted_smoke or built_app_codex_playtest") + parser.add_argument("--provider", default="", help="Provider under test") + parser.add_argument("--started-at", default="", help="Gate start timestamp") + parser.add_argument("--ended-at", default="", help="Gate end timestamp (defaults to manifest creation time)") + parser.add_argument("--verdict", default="", help="Optional explicit gate verdict") + parser.add_argument("--commit-sha", default="", help="Optional repo commit SHA override") return parser.parse_args(argv) diff --git a/qa/test_app_handoff_gate.py b/qa/test_app_handoff_gate.py new file mode 100644 index 00000000..e90f6511 --- /dev/null +++ b/qa/test_app_handoff_gate.py @@ -0,0 +1,139 @@ +import json +import os +import sys +import tempfile +import unittest +from pathlib import Path + +from qa import app_handoff_gate as gate + + +class AppHandoffGateTests(unittest.TestCase): + def test_repo_sha_short_resolves_head(self): + short = gate.repo_sha(short=True) + full = gate.repo_sha(short=False) + + self.assertNotEqual(short, "unknown") + self.assertTrue(full.startswith(short)) + self.assertGreaterEqual(len(short), 7) + + def test_run_logged_returns_timeout_exit_code(self): + with tempfile.TemporaryDirectory() as td: + log = Path(td) / "timeout.log" + + rc = gate.run_logged( + [sys.executable, "-c", "import time; time.sleep(1)"], + cwd=Path(td), + env=os.environ.copy(), + log_path=log, + timeout=0.05, + ) + + text = log.read_text(encoding="utf-8") + self.assertEqual(rc, 124) + self.assertIn("[timeout after", text) + self.assertIn("[exit 124]", text) + + def test_handoff_score_requires_all_mandatory_gates(self): + with tempfile.TemporaryDirectory() as td: + out = Path(td) + web = gate.GateResult(name="web_scripted_smoke", provider="scripted", surface="web", build_sha="abc1234") + web.pass_() + built = gate.GateResult(name="built_app_scripted_smoke", provider="scripted", surface="dist/WorldOS.app", build_sha="abc1234") + built.fail("no_app", "dist/WorldOS.app missing") + + verdict = gate.finalize_handoff( + run_id="fixture", + out=out, + gates=[web, built], + started_at="2026-06-01T00:00:00Z", + expected_sha="abc1234", + ) + + self.assertEqual(verdict["status"], "failed") + self.assertEqual(verdict["handoff_score"], 0) + self.assertEqual(verdict["blockers"][0]["gate"], "built_app_scripted_smoke") + self.assertEqual(verdict["blockers"][0]["bucket"], "no_app") + + def test_app_status_wrong_port_is_no_launcher(self): + status = { + "schema": "worldos.app-status.v1", + "build": {"sha": "abc1234"}, + "viewer": {"port": 8898, "chat_lines": 1}, + "art": {"private_root_present": True}, + "live": { + "can_act": True, + "actor": {"id": "char_1", "name": "Abby"}, + "enabled_action_count": 5, + }, + "readiness": {"ready_for_smoke": True, "ready_for_play": True, "failure_bucket": "none"}, + "health": {"failure_bucket": "none"}, + } + + bucket, detail = gate.validate_app_status(status, expected_port=8899, expected_sha="abc1234") + + self.assertEqual(bucket, "no_launcher") + self.assertIn("expected same port 8899", detail) + + def test_codex_provider_trace_cancellations_fail_summary(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + provider = root / "codex-provider" + provider.mkdir() + (provider / "codex-dm.stderr.log").write_text("tool validation error: extra_forbidden\n", encoding="utf-8") + + summary = gate.provider_trace_summary(root, "codex") + + self.assertEqual(summary["provider"], "codex") + self.assertGreater(summary["failed_or_error_count"], 0) + + def test_codex_provider_trace_ignores_failed_word_inside_command_output(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + provider = root / "codex-provider" + provider.mkdir() + (provider / "codex-dm.stdout.jsonl").write_text( + json.dumps( + { + "type": "item.completed", + "item": { + "type": "command_execution", + "status": "completed", + "aggregated_output": "a README says a historical check failed, but this command succeeded", + "error": None, + }, + } + ) + + "\n", + encoding="utf-8", + ) + + summary = gate.provider_trace_summary(root, "codex") + + self.assertEqual(summary["failed_or_error_count"], 0) + + def test_hook_probe_summary_reports_exact_missing_controls(self): + with tempfile.TemporaryDirectory() as td: + path = Path(td) / "hook-probe.json" + path.write_text( + json.dumps( + { + "schema": "worldos.app-handoff-hooks.v1", + "ok": False, + "missing_required": ["table:move-submit", "settings:provider-status"], + "console_errors": 0, + } + ), + encoding="utf-8", + ) + + ok, detail, payload = gate.summarize_hook_probe(path) + + self.assertFalse(ok) + self.assertIn("table:move-submit", detail) + self.assertIn("settings:provider-status", detail) + self.assertEqual(payload["schema"], "worldos.app-handoff-hooks.v1") + + +if __name__ == "__main__": + unittest.main() diff --git a/qa/test_export_app_evidence.py b/qa/test_export_app_evidence.py index 9deb7e2a..67a680af 100644 --- a/qa/test_export_app_evidence.py +++ b/qa/test_export_app_evidence.py @@ -69,11 +69,21 @@ def test_creates_manifest_with_status_surface_and_local_evidence_files(self): moves = tmp / "player_moves.jsonl" chat.write_text('{"role":"dm","text":"Opening."}\n', encoding="utf-8") moves.write_text('{"text":"Continue."}\n', encoding="utf-8") + run = tmp / "smoke-run" + (run / "screenshots").mkdir(parents=True) + (run / "scripted-provider").mkdir() + (run / "screenshots" / "beat-001.png").write_bytes(b"\x89PNG\r\n\x1a\nfixture") + (run / "app-status.final.json").write_text(json.dumps({"schema": "worldos.app-status.v1"}), encoding="utf-8") + (run / "session-surface.final.json").write_text(json.dumps({"schema": "worldos.session-surface.v1"}), encoding="utf-8") + (run / "console.ndjson").write_text("", encoding="utf-8") + (run / "network.ndjson").write_text('{"status":200}\n', encoding="utf-8") + (run / "actions.ndjson").write_text('{"action":"post_move"}\n', encoding="utf-8") + (run / "scripted-provider" / "summary.json").write_text('{"provider":"scripted"}\n', encoding="utf-8") app_status = { "schema": "worldos.app-status.v1", "build": {"sha": "abc1234", "version": "v1-test"}, "viewer": {"chat_path": str(chat), "transcript_path": ""}, - "live": {"moves_path": str(moves), "campaign_id": "camp_test"}, + "live": {"moves_path": str(moves), "campaign_id": "camp_test", "run_id": "run_test", "can_act": True, "enabled_action_count": 5}, "art": {"private_root": str(tmp / "art"), "private_root_present": True}, "endpoints": {"session_surface": "/session-surface"}, } @@ -85,7 +95,24 @@ def test_creates_manifest_with_status_surface_and_local_evidence_files(self): server, url = self.serve(app_status, session_surface) out = tmp / "bundle" try: - rc, text, payload = self.run_exporter(out, url) + rc, text, payload = self.run_exporter( + out, + url, + [ + "--run-dir", + str(run), + "--gate-kind", + "web_scripted_smoke", + "--provider", + "scripted", + "--command-json", + json.dumps(["python3", "qa/app_smoke_scripted.py", "--beats", "5"]), + "--started-at", + "2026-06-01T00:00:00Z", + "--verdict", + "passed", + ], + ) finally: server.shutdown() @@ -96,10 +123,48 @@ def test_creates_manifest_with_status_surface_and_local_evidence_files(self): self.assertEqual(payload["sources"]["app_status"]["path"], "app-status.json") self.assertEqual(payload["sources"]["session_surface"]["path"], "session-surface.json") copied = {entry["kind"]: entry for entry in payload["copied_files"]} - self.assertEqual(set(copied), {"chat", "moves"}) + self.assertIn("chat", copied) + self.assertIn("moves", copied) self.assertEqual((out / copied["chat"]["path"]).read_text(encoding="utf-8"), chat.read_text(encoding="utf-8")) self.assertEqual((out / copied["moves"]["path"]).read_text(encoding="utf-8"), moves.read_text(encoding="utf-8")) self.assertEqual(payload["evidence_gaps"], []) + self.assertEqual(payload["handoff_gate"]["schema"], "worldos.app-evidence-handoff.v1") + self.assertEqual(payload["handoff_gate"]["ok"], True) + self.assertEqual(payload["handoff_gate"]["build_sha"], "abc1234") + self.assertEqual(payload["handoff_gate"]["private_art_present"], True) + self.assertEqual(payload["handoff_gate"]["campaign_id"], "camp_test") + self.assertEqual(payload["handoff_gate"]["can_act"], True) + self.assertEqual(payload["handoff_gate"]["run_id"], "run_test") + self.assertEqual(payload["handoff_gate"]["enabled_action_count"], 5) + self.assertEqual(payload["handoff_gate"]["move_sink_present"], True) + self.assertEqual(payload["handoff_gate"]["evidence_gap_count"], 0) + self.assertEqual(payload["command"], ["python3", "qa/app_smoke_scripted.py", "--beats", "5"]) + self.assertEqual(payload["gate_kind"], "web_scripted_smoke") + self.assertEqual(payload["provider"], "scripted") + self.assertEqual(payload["run_id"], "run_test") + self.assertEqual(payload["app_build_sha"], "abc1234") + self.assertEqual(payload["verdict"], "passed") + self.assertEqual(payload["started_at"], "2026-06-01T00:00:00Z") + review = payload["review_entrypoint"] + self.assertEqual(review["schema"], "worldos.app-evidence-review-entrypoint.v1") + self.assertEqual(review["failure_bucket"], "") + for category in ( + "screenshots", + "app_status_snapshots", + "session_surface_snapshots", + "moves", + "provider_trace", + "network_logs", + "action_logs", + ): + self.assertIn(category, review["files"]) + self.assertIn("run-dir/screenshots/beat-001.png", review["files"]["screenshots"]) + self.assertIn("app-status.json", review["files"]["app_status_snapshots"]) + self.assertIn("session-surface.json", review["files"]["session_surface_snapshots"]) + self.assertIn("local-files/moves.jsonl", review["files"]["moves"]) + self.assertIn("run-dir/scripted-provider/summary.json", review["files"]["provider_trace"]) + self.assertIn("run-dir/network.ndjson", review["files"]["network_logs"]) + self.assertIn("run-dir/actions.ndjson", review["files"]["action_logs"]) def test_missing_optional_local_files_are_recorded_as_evidence_gaps(self): with tempfile.TemporaryDirectory() as td: @@ -127,6 +192,10 @@ def test_missing_optional_local_files_are_recorded_as_evidence_gaps(self): gaps = {(gap["source"], gap["kind"]) for gap in payload["evidence_gaps"]} self.assertIn(("local_file", "chat"), gaps) self.assertIn(("local_file", "moves"), gaps) + self.assertEqual(payload["handoff_gate"]["ok"], False) + self.assertEqual(payload["handoff_gate"]["evidence_gap_count"], len(payload["evidence_gaps"])) + self.assertIn("private art not proven present", payload["handoff_gate"]["blocking_reasons"]) + self.assertIn(f"evidence gaps: {len(payload['evidence_gaps'])}", payload["handoff_gate"]["blocking_reasons"]) def test_run_dir_mode_copies_allowlisted_artifacts_and_failure_bucket(self): with tempfile.TemporaryDirectory() as td: @@ -167,6 +236,9 @@ def test_run_dir_mode_copies_allowlisted_artifacts_and_failure_bucket(self): self.assertIn("run-dir/scripted-provider/trace.ndjson", copied) self.assertIn("run-dir/native/transition.json", copied) self.assertEqual(payload["evidence_gaps"], []) + self.assertEqual(payload["handoff_gate"]["ok"], False) + self.assertEqual(payload["handoff_gate"]["failure_bucket"], "move_rejected") + self.assertIn("failure bucket: move_rejected", payload["handoff_gate"]["blocking_reasons"]) if __name__ == "__main__": diff --git a/qa/test_macos_app_static.py b/qa/test_macos_app_static.py index 12046438..e4137e32 100644 --- a/qa/test_macos_app_static.py +++ b/qa/test_macos_app_static.py @@ -54,6 +54,8 @@ def test_built_app_playtest_can_keep_minted_backend_for_manual_gameplay(self): harness = self.read("qa/ui_playtest_app.sh") self.assertIn("WOS_APP_KEEP_MINTED_BACKEND=1", harness) + self.assertIn("WOS_APP_SELECTED_PROVIDER=codex|scripted|claude|openclaw", harness) + self.assertIn('defaults write dev.clawdnd.app selectedProvider "$SELECTED_PROVIDER"', harness) self.assertIn("requires WOS_APP_PART=A", harness) self.assertIn('KEEP_MINTED_BACKEND="${WOS_APP_KEEP_MINTED_BACKEND:-0}"', harness) self.assertIn("keeping minted backend alive for gameplay proof", harness) diff --git a/qa/ui_playtest_app.sh b/qa/ui_playtest_app.sh index ebf106ba..b7a7712a 100755 --- a/qa/ui_playtest_app.sh +++ b/qa/ui_playtest_app.sh @@ -41,6 +41,8 @@ # WOS_APP_SKIP_BUILD=1 reuse an already-running .app (skip pkill/rebuild) — for fast inner loop. # WOS_APP_NO_GLOBAL_KILL=1 do not pkill other WorldOSApp processes (used for takeover smoke). # WOS_APP_PART=A|B|AB run only part A, only part B, or both (default AB). +# WOS_APP_SELECTED_PROVIDER=codex|scripted|claude|openclaw +# set the native app's provider preference before minting a session. # WOS_APP_KEEP_MINTED_BACKEND=1 # part A only: leave the .app-minted provider backend alive so # an operator can continue a short built-app gameplay playtest. @@ -64,6 +66,7 @@ BEATS="${4:-6}" BUDGET="${5:-4.00}" PART="$(worldos_env APP_PART "${WOS_APP_PART:-AB}")" KEEP_MINTED_BACKEND="${WOS_APP_KEEP_MINTED_BACKEND:-0}" +SELECTED_PROVIDER="${WOS_APP_SELECTED_PROVIDER:-}" if [ "$KEEP_MINTED_BACKEND" = "1" ] && [ "$PART" != "A" ]; then printf '[uipt-app] WOS_APP_KEEP_MINTED_BACKEND=1 requires WOS_APP_PART=A; refusing to mix kept native backend with part B.\n' >&2 exit 2 @@ -89,6 +92,18 @@ VERSION="$( ([ -f "$ROOT/VERSION" ] && cat "$ROOT/VERSION") \ log() { printf '[uipt-app] %s\n' "$*"; } log "run=$RUN world=$WORLD persona=$PERSONA beats=$BEATS budget=\$$BUDGET part=$PART" log "build_sha=$BUILD_SHA version=$VERSION repo=$ROOT" +if [ -n "$SELECTED_PROVIDER" ]; then + case "$SELECTED_PROVIDER" in + claude|codex|openclaw|scripted) + defaults write dev.clawdnd.app selectedProvider "$SELECTED_PROVIDER" >/dev/null 2>&1 || true + log "selected provider preference set to $SELECTED_PROVIDER" + ;; + *) + printf '[uipt-app] WOS_APP_SELECTED_PROVIDER must be claude, codex, openclaw, or scripted (got %s)\n' "$SELECTED_PROVIDER" >&2 + exit 2 + ;; + esac +fi # Agent-readable failure buckets for built-app smoke. Keep these crisp and stable; the # detailed shell/native result still travels separately as original_result. From c3d37ae489d541b53d35ea4113d7655c182bb12e Mon Sep 17 00:00:00 2001 From: Eva Date: Mon, 1 Jun 2026 13:34:54 +0700 Subject: [PATCH 2/2] Harden handoff evidence recovery --- qa/app_handoff_gate.py | 43 ++++++++++++---------------------- qa/export_app_evidence.py | 16 ++++++++++++- qa/test_app_handoff_gate.py | 37 +++++++++++++++++++++++++++++ qa/test_export_app_evidence.py | 41 ++++++++++++++++++++++++++++++++ 4 files changed, 108 insertions(+), 29 deletions(-) diff --git a/qa/app_handoff_gate.py b/qa/app_handoff_gate.py index a900393d..0c5bd482 100644 --- a/qa/app_handoff_gate.py +++ b/qa/app_handoff_gate.py @@ -393,41 +393,28 @@ def export_evidence( if transition_file and transition_file.exists(): cmd.extend(["--transition-file", str(transition_file)]) manifest_path = out / "manifest.json" - try: - proc = subprocess.run(cmd, cwd=ROOT, text=True, capture_output=True, check=False, timeout=60) - except subprocess.TimeoutExpired as exc: - reason = f"export_app_evidence timed out after {exc.timeout}s" - return str(manifest_path), { + + def persist_failure(reason: str) -> tuple[str, dict[str, Any]]: + payload = { "schema": "worldos.app-evidence.v1", "evidence_gaps": [{"source": "export_app_evidence", "kind": "manifest", "path": str(manifest_path), "reason": reason}], "failure": {"failure_bucket": "no_provider", "failure_detail": reason}, "handoff_gate": {"ok": False, "blocking_reasons": [reason]}, } + json_dump(manifest_path, payload) + return str(manifest_path), payload + + try: + proc = subprocess.run(cmd, cwd=ROOT, text=True, capture_output=True, check=False, timeout=60) + except subprocess.TimeoutExpired as exc: + return persist_failure(f"export_app_evidence timed out after {exc.timeout}s") if proc.returncode != 0: - reason = f"export_app_evidence exited {proc.returncode}: {(proc.stderr or proc.stdout)[-1000:]}" - return str(manifest_path), { - "schema": "worldos.app-evidence.v1", - "evidence_gaps": [{"source": "export_app_evidence", "kind": "manifest", "path": str(manifest_path), "reason": reason}], - "failure": {"failure_bucket": "no_provider", "failure_detail": reason}, - "handoff_gate": {"ok": False, "blocking_reasons": [reason]}, - } + return persist_failure(f"export_app_evidence exited {proc.returncode}: {(proc.stderr or proc.stdout)[-1000:]}") if not manifest_path.exists(): - reason = "export_app_evidence did not write manifest.json" - return str(manifest_path), { - "schema": "worldos.app-evidence.v1", - "evidence_gaps": [{"source": "export_app_evidence", "kind": "manifest", "path": str(manifest_path), "reason": reason}], - "failure": {"failure_bucket": "no_provider", "failure_detail": reason}, - "handoff_gate": {"ok": False, "blocking_reasons": [reason]}, - } + return persist_failure("export_app_evidence did not write manifest.json") manifest = read_json(manifest_path) if not manifest: - reason = "export_app_evidence wrote invalid or empty manifest.json" - return str(manifest_path), { - "schema": "worldos.app-evidence.v1", - "evidence_gaps": [{"source": "export_app_evidence", "kind": "manifest", "path": str(manifest_path), "reason": reason}], - "failure": {"failure_bucket": "no_provider", "failure_detail": reason}, - "handoff_gate": {"ok": False, "blocking_reasons": [reason]}, - } + return persist_failure("export_app_evidence wrote invalid or empty manifest.json") return str(manifest_path), manifest @@ -555,8 +542,8 @@ def drive_moves( smoke.copy_play_state(run_id, gate_dir) trace = provider_trace_summary(ROOT / "play-state" / run_id, provider) json_dump(gate_dir / "provider-trace-summary.json", trace) - if provider == "codex" and int(trace.get("failed_or_error_count") or 0) > 0: - return False, "no_provider", "Codex provider trace reported failed/error/cancellation events", {"screenshots": screenshots, "evidence_gaps": gaps, "provider_trace": trace} + if provider == "codex" and (not trace.get("trace_exists") or int(trace.get("failed_or_error_count") or 0) > 0): + return False, "no_provider", "Codex provider trace missing or reported failed/error/cancellation events", {"screenshots": screenshots, "evidence_gaps": gaps, "provider_trace": trace} if gaps: return False, "no_provider", "required evidence capture has gaps", {"screenshots": screenshots, "evidence_gaps": gaps, "provider_trace": trace} return True, "", "", {"screenshots": screenshots, "evidence_gaps": gaps, "provider_trace": trace} diff --git a/qa/export_app_evidence.py b/qa/export_app_evidence.py index 885bc901..05bf99cc 100644 --- a/qa/export_app_evidence.py +++ b/qa/export_app_evidence.py @@ -643,6 +643,13 @@ def exporter_manifest(args: argparse.Namespace, bundle: Path) -> tuple[dict[str, exit_code = 0 created_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + def clear_http_gap(source: str) -> None: + gaps[:] = [ + gap + for gap in gaps + if not (gap.get("source") == source and gap.get("kind") == "http_json") + ] + if args.app_status_url: try: app_status, meta = fetch_json(args.app_status_url) @@ -704,10 +711,17 @@ def exporter_manifest(args: argparse.Namespace, bundle: Path) -> tuple[dict[str, app_status, app_status_snapshot = first_bundle_json(bundle, ("app-status.final.json", "app-status*.json")) if app_status_snapshot: sources["app_status_snapshot"] = {"path": app_status_snapshot, "ok": True} - if "session_surface" not in sources: + if isinstance(sources.get("app_status"), dict): + sources["app_status"]["recovered_by"] = "app_status_snapshot" + clear_http_gap("app_status") + exit_code = 0 + if not (sources.get("session_surface") or {}).get("ok"): _surface, session_surface_snapshot = first_bundle_json(bundle, ("session-surface.final.json", "session-surface*.json")) if session_surface_snapshot: sources["session_surface_snapshot"] = {"path": session_surface_snapshot, "ok": True} + if isinstance(sources.get("session_surface"), dict): + sources["session_surface"]["recovered_by"] = "session_surface_snapshot" + clear_http_gap("session_surface") live = app_status.get("live") if isinstance(app_status.get("live"), dict) else {} failure = { diff --git a/qa/test_app_handoff_gate.py b/qa/test_app_handoff_gate.py index e90f6511..6ce8cadb 100644 --- a/qa/test_app_handoff_gate.py +++ b/qa/test_app_handoff_gate.py @@ -1,9 +1,11 @@ import json import os +import subprocess import sys import tempfile import unittest from pathlib import Path +from unittest import mock from qa import app_handoff_gate as gate @@ -111,6 +113,41 @@ def test_codex_provider_trace_ignores_failed_word_inside_command_output(self): summary = gate.provider_trace_summary(root, "codex") self.assertEqual(summary["failed_or_error_count"], 0) + self.assertEqual(summary["trace_exists"], True) + + def test_codex_provider_trace_missing_is_explicit(self): + with tempfile.TemporaryDirectory() as td: + summary = gate.provider_trace_summary(Path(td), "codex") + + self.assertEqual(summary["provider"], "codex") + self.assertEqual(summary["trace_exists"], False) + self.assertEqual(summary["failed_or_error_count"], 0) + + def test_export_evidence_persists_failure_manifest(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + gate_dir = root / "gate" + with mock.patch.object(gate, "repo_sha", return_value="abc1234"): + with mock.patch.object( + gate.subprocess, + "run", + return_value=subprocess.CompletedProcess(args=[], returncode=17, stdout="", stderr="export broke"), + ): + manifest_path, payload = gate.export_evidence( + gate_dir=gate_dir, + run_dir=gate_dir, + app_status_url="", + transition_file=None, + command=["fixture"], + gate_kind="fixture_gate", + provider="scripted", + ) + + persisted = json.loads(Path(manifest_path).read_text(encoding="utf-8")) + + self.assertEqual(payload["failure"]["failure_bucket"], "no_provider") + self.assertIn("export_app_evidence exited 17", payload["failure"]["failure_detail"]) + self.assertEqual(persisted, payload) def test_hook_probe_summary_reports_exact_missing_controls(self): with tempfile.TemporaryDirectory() as td: diff --git a/qa/test_export_app_evidence.py b/qa/test_export_app_evidence.py index 67a680af..c6f841fa 100644 --- a/qa/test_export_app_evidence.py +++ b/qa/test_export_app_evidence.py @@ -240,6 +240,47 @@ def test_run_dir_mode_copies_allowlisted_artifacts_and_failure_bucket(self): self.assertEqual(payload["handoff_gate"]["failure_bucket"], "move_rejected") self.assertIn("failure bucket: move_rejected", payload["handoff_gate"]["blocking_reasons"]) + def test_run_dir_snapshots_recover_dead_app_status_url(self): + with tempfile.TemporaryDirectory() as td: + tmp = Path(td) + run = tmp / "smoke-run" + run.mkdir() + (run / "moves.ndjson").write_text('{"text":"Continue."}\n', encoding="utf-8") + (run / "app-status.final.json").write_text( + json.dumps( + { + "schema": "worldos.app-status.v1", + "build": {"sha": "abc1234", "version": "v1-test"}, + "art": {"private_root_present": True}, + "live": { + "campaign_id": "camp_test", + "run_id": "run_test", + "can_act": True, + "enabled_action_count": 3, + }, + } + ), + encoding="utf-8", + ) + (run / "session-surface.final.json").write_text( + json.dumps({"schema": "worldos.session-surface.v1", "campaign_id": "camp_test"}), + encoding="utf-8", + ) + out = tmp / "bundle" + + rc, text, payload = self.run_exporter( + out, + "http://127.0.0.1:1/app-status", + ["--run-dir", str(run), "--gate-kind", "built_app_scripted_smoke", "--provider", "scripted"], + ) + + self.assertEqual(rc, 0, text) + self.assertEqual(payload["evidence_gaps"], []) + self.assertEqual(payload["sources"]["app_status"]["recovered_by"], "app_status_snapshot") + self.assertEqual(payload["sources"]["app_status_snapshot"]["ok"], True) + self.assertEqual(payload["sources"]["session_surface_snapshot"]["ok"], True) + self.assertEqual(payload["handoff_gate"]["ok"], True) + if __name__ == "__main__": unittest.main()