diff --git a/docs/AGENT_GRADE_APP_TESTABILITY.md b/docs/AGENT_GRADE_APP_TESTABILITY.md index 22d99660..5e23c682 100644 --- a/docs/AGENT_GRADE_APP_TESTABILITY.md +++ b/docs/AGENT_GRADE_APP_TESTABILITY.md @@ -80,31 +80,26 @@ Current v1 minimum, implemented first so agents can stop guessing: } ``` -Target expansion for #481/#483: +V2 readiness/health expansion for #481/#483: ```json { - "status": "booting|ready|degraded|blocked", - "ready_for_smoke": false, - "ready_for_play": false, - "build": {"app_sha": "string", "bundle_id": "dev.clawdnd.app"}, - "surface_detail": { - "kind": "built_app|dev_viewer", - "route": "/openworlds/", - "viewer_url": "http://127.0.0.1:/openworlds/", - "native_window_ready": false - }, - "provider_detail": { - "id": "deterministic-smoke|codex|claude|...", - "mode": "deterministic|real", - "ready": false, - "dev_test_provider_enabled": false + "readiness": { + "status": "ready|degraded", + "ready_for_smoke": false, + "ready_for_play": false, + "failure_bucket": "none|no_app|no_launcher|no_provider|no_art|no_actor|no_actions|move_rejected|no_narration|console_error|permission_prompt", + "failure_detail": "string" }, "health": { + "same_port_alive": true, + "route_loaded": true, "console_errors": 0, "network_failures": 0, - "last_error": "string|null", - "failure_bucket": "none|app_not_running|viewer_unreachable|openworlds_not_loaded|provider_unavailable|test_provider_disabled|session_missing|player_not_seated|palette_disabled|move_sink_missing|private_art_missing|image_probe_failed|console_error|network_error|timeout|unknown" + "provider_ready": false, + "image_probe_ok": false, + "failure_bucket": "none|no_app|no_launcher|no_provider|no_art|no_actor|no_actions|move_rejected|no_narration|console_error|permission_prompt", + "failure_detail": "string" } } ``` @@ -119,7 +114,8 @@ Behavioral rules: real provider or an explicitly enabled deterministic test provider, and it must report no blocking console/network failures. - `degraded` means the app is observable but not fully playable; include a - failure bucket. `blocked` means the harness cannot continue safely. + failure bucket. Harnesses that cannot continue safely should stop with the + appropriate stable failure bucket rather than inventing another status value. - Status must never expose private art file contents, secrets, model keys, or operator-only VM details. Paths may be omitted or redacted when not needed for diagnosis. @@ -132,26 +128,26 @@ real providers. Contract: -- Provider id: `deterministic-smoke`. +- Provider id: `scripted`. - No network calls, model calls, randomness without a recorded seed, or external auth. - Enabled only when an explicit dev/test gate is set, for example - `WORLDOS_ENABLE_TEST_PROVIDERS=1`. If requested without the gate, app-status - reports `failure_bucket: "test_provider_disabled"` and the app refuses to run - it. + `WORLDOS_ENABLE_SCRIPTED_PROVIDER=1`. If requested without the gate, the app + refuses to launch it. - Uses the normal engine/player architecture. It may script the DM response, but campaign state is still written only by the engine and player input still enters as `/move`. - Seats a living canon player, emits visible DM narration, exposes enabled - actions, accepts one representative `/move`, resolves a deterministic follow-up - turn, and leaves `/session-surface` actionable. + actions, accepts representative `/move` intents, resolves deterministic + follow-up turns, writes `scripted-provider/summary.json`, and leaves + `/session-surface` actionable. - Is never release proof by itself. It is a wiring smoke for #482/#483/#486. ## Accessibility and Driving Hooks Agents should prefer semantic accessibility over implementation-specific DOM -shape. `data-testid` is allowed when role/name is ambiguous or when copy changes -would make tests brittle. +shape. `data-worldos-testid` is allowed when role/name is ambiguous or when copy +changes would make tests brittle. Policy: @@ -160,11 +156,12 @@ Policy: - Important regions expose stable landmarks or labels: launcher, campaign shelf, OpenWorlds root, narration log, active-player panel, action palette, move composer, provider/status banner, modal/dialog layer. -- `data-testid` values are stable public test hooks, not CSS hooks. Do not rename - or remove one without updating the harness in the same change. +- `data-worldos-testid` values are stable public test hooks, not CSS hooks. Do + not rename or remove one without updating the harness in the same change. - Prefer generic test ids plus state attributes for repeated controls, for - example `data-testid="action-button"` with `data-action-id="say"` rather than - embedding volatile label text in the test id. + example `data-worldos-testid="action-button"` with + `data-worldos-action-id="say"` rather than embedding volatile label text in + the test id. - Narration/progress surfaces use `aria-live` or an equivalent observable update marker so an agent can tell whether a turn advanced. - Hooks must not reveal private art paths, secrets, or internal provider prompts. @@ -176,11 +173,14 @@ Minimum hook set for #484: - `chronicle-resume` - `openworlds-root` - `app-status-banner` +- `error-banner` +- `provider-status` - `narration-log` - `active-player` - `action-palette` - `action-button` plus `data-action-id` -- `move-composer` +- `move-input` +- `move-submit` - `turn-progress` ## Evidence Bundle @@ -213,6 +213,10 @@ Required contents for #485: - Gate-specific score output: `smoke.json`, `provider_playtest.json`, or `RRI.json`. +`qa/export_app_evidence.py --run-dir --out ` copies a completed +smoke/playtest run into this bundle shape. `--app-status-url ` remains +supported for live read-only export from a running app. + Bundles are evidence, not source. Do not commit them. Screenshots may contain private art and must remain in `/Volumes/LEXAR/Codex` unless the owner explicitly chooses to publish a redacted excerpt. @@ -226,18 +230,17 @@ score. Purpose: fast, repeatable app wiring proof. -Surface: rebuilt `dist/WorldOS.app`, deterministic smoke provider, app-status -v1, stable hooks, one `/move`. +Surface: rebuilt `dist/WorldOS.app`, deterministic scripted provider, +app-status v1, stable hooks, five to eight `/move` beats. Pass requires: - Native app launches and serves `/openworlds/`. -- Current v1: `app-status.live.can_act` is true, `moves_writable` is true, an - actor is seated, enabled actions are non-empty, and private art is present. - Future expansion: `app-status` reaches `ready_for_smoke: true`. +- `app-status.readiness.ready_for_smoke` is true on the same port that serves + `/openworlds/`. - Private art probe succeeds without committing art. -- A living player is seated, narration is visible, actions are enabled, and one - `/move` is accepted and resolved. +- A living player is seated, narration is visible, actions are enabled, and + every scripted `/move` beat is accepted and advances narration. - Evidence bundle exists with no missing required files. This gate catches wiring failures early. It does not prove provider quality, diff --git a/qa/app_failure_buckets.py b/qa/app_failure_buckets.py new file mode 100644 index 00000000..e84feae6 --- /dev/null +++ b/qa/app_failure_buckets.py @@ -0,0 +1,257 @@ +#!/usr/bin/env python3 +"""WorldOS built-app failure bucket classification. + +These buckets are intentionally small and stable so agents can route failures +without parsing screenshots. Callers may pass partial evidence; missing evidence +falls back to the crispest safe bucket instead of crashing. +""" +from __future__ import annotations + +import argparse +import json +import re +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Any + + +APP_FAILURE_BUCKETS = ( + "no_app", + "no_launcher", + "no_provider", + "no_art", + "no_actor", + "no_actions", + "move_rejected", + "no_narration", + "console_error", + "permission_prompt", +) + + +@dataclass(frozen=True) +class Classification: + bucket: str + detail: str + + def as_pair(self) -> str: + detail = self.detail.replace("|", "/").replace("\n", " ").replace("\r", " ").strip() + return f"{self.bucket}|{detail}" + + +def load_json(value: str | None) -> dict[str, Any]: + if not value: + return {} + try: + payload = json.loads(value) + except json.JSONDecodeError: + return {} + return payload if isinstance(payload, dict) else {} + + +def truthy(value: Any) -> bool: + if isinstance(value, bool): + return value + if isinstance(value, str): + return value.strip().lower() in {"1", "true", "yes", "y"} + return bool(value) + + +def bucket_pair(bucket: str, detail: str) -> Classification: + if bucket not in APP_FAILURE_BUCKETS: + bucket = "no_provider" + return Classification(bucket=bucket, detail=detail) + + +def classify_native_failure( + *, + result: str, + can_act: Any, + surface: dict[str, Any] | None = None, + app_status: dict[str, Any] | None = None, +) -> Classification: + surface = surface or {} + app_status = app_status or {} + readiness = app_status.get("readiness") if isinstance(app_status.get("readiness"), dict) else {} + health = app_status.get("health") if isinstance(app_status.get("health"), dict) else {} + if isinstance(readiness.get("failure_bucket"), str) and readiness.get("failure_bucket") in APP_FAILURE_BUCKETS: + return bucket_pair(str(readiness["failure_bucket"]), str(readiness.get("failure_detail") or "app-status readiness check failed")) + if isinstance(health.get("failure_bucket"), str) and health.get("failure_bucket") in APP_FAILURE_BUCKETS: + return bucket_pair(str(health["failure_bucket"]), str(health.get("failure_detail") or "app-status health check failed")) + + art = app_status.get("art") if isinstance(app_status.get("art"), dict) else {} + live = app_status.get("live") if isinstance(app_status.get("live"), dict) else {} + actor = live.get("actor") if isinstance(live.get("actor"), dict) else {} + enabled_count = live.get("enabled_action_count") + viewer = app_status.get("viewer") if isinstance(app_status.get("viewer"), dict) else {} + chat_lines = viewer.get("chat_lines") + + if result in {"build_failed", "app_not_running"}: + return bucket_pair("no_app", "WorldOS.app did not build, launch, or remain running") + if result == "no_launcher": + return bucket_pair("no_launcher", "launcher viewer did not answer /openworlds/ and /app-status on the same port") + if app_status and app_status.get("ok") is False: + return bucket_pair("no_launcher", "app-status reported an unhealthy launcher/viewer") + if art.get("private_root_present") is False: + return bucket_pair("no_art", "private art root was not present in app-status") + if not truthy(can_act): + return bucket_pair("no_provider", "no minted live provider viewer reported can_act:true") + if not actor.get("id") and not actor.get("name"): + return bucket_pair("no_actor", "app-status did not report an active player actor") + if enabled_count == 0: + return bucket_pair("no_actions", "app-status reported zero enabled player actions") + if chat_lines == 0: + return bucket_pair("no_narration", "app-status reported no chat/narration lines") + + surface_actor = ((surface.get("actionModel") or {}).get("actor") or {}) if isinstance(surface, dict) else {} + if truthy(can_act) and not (surface_actor.get("id") or surface_actor.get("name") or actor.get("id") or actor.get("name")): + return bucket_pair("no_actor", "session-surface did not report an active player actor") + return bucket_pair("no_provider", "native transition failed without a more specific bucket") + + +def classify_part_b_readiness_failure(*, saw_canact: Any, saw_pc: Any, chat_lines: int) -> Classification: + if not truthy(saw_canact): + return bucket_pair("no_provider", "faithful backend never exposed can_act:true") + if not truthy(saw_pc): + return bucket_pair("no_actor", "faithful backend never seated a player character") + if int(chat_lines or 0) <= 0: + return bucket_pair("no_narration", "faithful backend produced no opening narration") + return bucket_pair("no_actions", "faithful backend was not player-ready") + + +def _grep_any(paths: list[Path], pattern: str) -> bool: + rx = re.compile(pattern, re.IGNORECASE) + for path in paths: + if not path.exists() or path.is_dir(): + continue + try: + if rx.search(path.read_text(encoding="utf-8", errors="ignore")): + return True + except OSError: + continue + return False + + +def classify_part_b_failure_from_artifacts(run_dir: Path, fallback_result: str = "FAIL") -> Classification: + paths = [ + run_dir / "console.ndjson", + run_dir / "network.ndjson", + run_dir / "actions.ndjson", + run_dir / "summary.md", + run_dir / "player" / "console.ndjson", + run_dir / "player" / "network.ndjson", + ] + if _grep_any(paths, r"permission|not authorized|accessibility|screen recording|AXIsProcessTrusted"): + return bucket_pair("permission_prompt", "macOS permission prompt or accessibility/screen-recording denial appeared") + if _grep_any(paths, r"console_error|pageerror|uncaught|exception"): + return bucket_pair("console_error", "browser console/page error recorded during app playtest") + if _grep_any(paths, r"move_rejected|/move.*(4[0-9][0-9]|5[0-9][0-9])|move not sent|rejected"): + return bucket_pair("move_rejected", "player move was rejected or failed to reach /move") + return bucket_pair("no_provider", f"part B failed: {fallback_result}") + + +def classify_part_b_score_failure(score_path: Path) -> Classification: + try: + score = json.loads(score_path.read_text(encoding="utf-8")) + except Exception as exc: # noqa: BLE001 - surfaced in the failure detail. + return bucket_pair("no_provider", f"score.json pass=false and score could not be read: {exc}") + if not isinstance(score, dict): + return bucket_pair("no_provider", "score.json pass=false and score was not an object") + + console_errors = int(score.get("console_errors") or 0) + critical = int(score.get("bug_reports_critical") or 0) + satisfaction = score.get("persona_satisfaction") + if console_errors > 0: + return bucket_pair("console_error", f"score.json failed: console_errors={console_errors}") + if critical > 0: + return bucket_pair("no_provider", f"score.json failed: critical_bug_reports={critical}") + if not score.get("completed_intro_flow"): + if score.get("reached_play_screen"): + return bucket_pair("no_actions", "score.json failed: player reached the table but submitted no in-story turn") + return bucket_pair("no_actions", "score.json failed: player never reached the playable table") + if score.get("gave_up"): + detail = str(score.get("give_up_reason") or "player gave up").strip() + return bucket_pair("no_provider", f"score.json failed: {detail}") + if isinstance(satisfaction, (int, float)) and satisfaction < 6: + return bucket_pair("no_provider", f"score.json failed: satisfaction={satisfaction}/10") + return bucket_pair("no_provider", "score.json pass=false without a more specific signal") + + +def classify_browser_probe(*, tab_url: str, app_status_ok: Any, status_url: str = "") -> Classification | None: + """Return no_launcher when a visible browser tab is stale or unbacked. + + A screenshot of /openworlds/ is only evidence when the same localhost port + answers /app-status. Without that probe, a cached rendered page can fool the + harness into accepting a dead app. + """ + if truthy(app_status_ok): + return None + detail = "browser tab is visible but same-port /app-status is unreachable" + if tab_url: + detail += f" (tab={tab_url})" + if status_url: + detail += f" (status={status_url})" + return bucket_pair("no_launcher", detail) + + +def parse_args(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Classify WorldOS built-app harness failures.") + sub = parser.add_subparsers(dest="command", required=True) + + native = sub.add_parser("native") + native.add_argument("--result", required=True) + native.add_argument("--can-act", default="false") + native.add_argument("--surface-json", default="{}") + native.add_argument("--app-status-json", default="{}") + + ready = sub.add_parser("part-b-readiness") + ready.add_argument("--saw-canact", default="0") + ready.add_argument("--saw-pc", default="0") + ready.add_argument("--chat-lines", type=int, default=0) + + artifacts = sub.add_parser("part-b-artifacts") + artifacts.add_argument("--run-dir", required=True) + artifacts.add_argument("--fallback-result", default="FAIL") + + score = sub.add_parser("part-b-score") + score.add_argument("--score-json", required=True) + + browser = sub.add_parser("browser-probe") + browser.add_argument("--tab-url", default="") + browser.add_argument("--status-url", default="") + browser.add_argument("--app-status-ok", default="false") + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv if argv is not None else sys.argv[1:]) + if args.command == "native": + result = classify_native_failure( + result=args.result, + can_act=args.can_act, + surface=load_json(args.surface_json), + app_status=load_json(args.app_status_json), + ) + elif args.command == "part-b-readiness": + result = classify_part_b_readiness_failure( + saw_canact=args.saw_canact, + saw_pc=args.saw_pc, + chat_lines=args.chat_lines, + ) + elif args.command == "part-b-artifacts": + result = classify_part_b_failure_from_artifacts(Path(args.run_dir), args.fallback_result) + elif args.command == "part-b-score": + result = classify_part_b_score_failure(Path(args.score_json)) + else: + result = classify_browser_probe( + tab_url=args.tab_url, + status_url=args.status_url, + app_status_ok=args.app_status_ok, + ) or bucket_pair("no_provider", "browser probe passed") + print(result.as_pair()) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/qa/app_smoke_scripted.py b/qa/app_smoke_scripted.py new file mode 100644 index 00000000..9919e144 --- /dev/null +++ b/qa/app_smoke_scripted.py @@ -0,0 +1,446 @@ +#!/usr/bin/env python3 +"""Deterministic multi-beat WorldOS scripted-provider smoke. + +This is a fast wiring proof for the real viewer/provider path. It does not +replace the release RRI gate. It launches the dev-gated scripted provider, +validates same-port /app-status, submits deterministic /move intents, and writes +one disk-backed evidence bundle. +""" +from __future__ import annotations + +import argparse +import json +import os +import shutil +import signal +import subprocess +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from qa.app_failure_buckets import classify_browser_probe # noqa: E402 + +DEFAULT_ROOT = Path("/Volumes/LEXAR/Codex/worldos-agent-grade-app-testability") +MAX_HTTP_BYTES = 8 * 1024 * 1024 + + +def utc_stamp() -> str: + return datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + + +def json_dump(path: Path, payload: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def append_ndjson(path: Path, payload: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a", encoding="utf-8") as fh: + fh.write(json.dumps(payload, sort_keys=True) + "\n") + + +def fetch_json(url: str, *, timeout: float = 3.0) -> tuple[dict[str, Any], int]: + req = urllib.request.Request(url, headers={"Accept": "application/json"}) + with urllib.request.urlopen(req, timeout=timeout) as resp: + data = resp.read(MAX_HTTP_BYTES + 1) + if len(data) > MAX_HTTP_BYTES: + raise ValueError(f"{url} response exceeded {MAX_HTTP_BYTES} bytes") + payload = json.loads(data.decode("utf-8")) + if not isinstance(payload, dict): + raise ValueError(f"{url} returned non-object JSON") + return payload, int(getattr(resp, "status", 0) or 0) + + +def post_json(url: str, payload: dict[str, Any], *, timeout: float = 5.0) -> tuple[dict[str, Any], int]: + data = json.dumps(payload).encode("utf-8") + req = urllib.request.Request( + url, + data=data, + headers={"Content-Type": "application/json", "Accept": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(req, timeout=timeout) as resp: + body = resp.read(MAX_HTTP_BYTES + 1) + parsed = json.loads(body.decode("utf-8")) if body else {} + return parsed if isinstance(parsed, dict) else {}, int(getattr(resp, "status", 0) or 0) + + +def wait_for_status(base_url: str, out: Path, timeout: float = 60.0) -> dict[str, Any]: + deadline = time.time() + timeout + last_error = "" + url = urllib.parse.urljoin(base_url, "/app-status") + while time.time() < deadline: + try: + payload, status = fetch_json(url) + append_ndjson(out / "network.ndjson", {"at": time.time(), "method": "GET", "url": url, "status": status}) + return payload + except Exception as exc: # noqa: BLE001 - evidence bucket wants the raw reason. + last_error = str(exc) + append_ndjson(out / "network.ndjson", {"at": time.time(), "method": "GET", "url": url, "error": last_error}) + time.sleep(0.5) + raise RuntimeError(f"/app-status did not answer on {url}: {last_error}") + + +def surface_url(base_url: str, status: dict[str, Any]) -> str: + endpoint = ((status.get("endpoints") or {}).get("session_surface") or "/session-surface") if isinstance(status.get("endpoints"), dict) else "/session-surface" + url = urllib.parse.urljoin(base_url, str(endpoint)) + campaign = ((status.get("live") or {}).get("campaign_id") or "") if isinstance(status.get("live"), dict) else "" + parsed = urllib.parse.urlparse(url) + if campaign and not parsed.query: + return urllib.parse.urlunparse(parsed._replace(query=urllib.parse.urlencode({"campaign": campaign}))) + return url + + +def provider_summary(play_state: Path) -> dict[str, Any]: + path = play_state / "scripted-provider" / "summary.json" + if path.exists(): + try: + payload = json.loads(path.read_text(encoding="utf-8")) + return payload if isinstance(payload, dict) else {} + except (OSError, json.JSONDecodeError): + return {} + trace = play_state / "scripted-provider" / "trace.ndjson" + if not trace.exists(): + return {} + events: list[dict[str, Any]] = [] + for line in trace.read_text(encoding="utf-8", errors="ignore").splitlines(): + try: + payload = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(payload, dict): + events.append(payload) + return { + "schema": "worldos.scripted-provider-summary.v1", + "provider": "scripted", + "deterministic": True, + "model_free": True, + "trace_exists": True, + "event_count": len(events), + "move_resolved_count": sum(1 for event in events if event.get("event") == "move_resolved"), + "first_event": events[0] if events else None, + "last_event": events[-1] if events else None, + } + + +def copy_play_state(run_id: str, out: Path) -> None: + play_state = ROOT / "play-state" / run_id + if not play_state.exists(): + return + dest = out / "play-state" + for rel in ("chat.jsonl", "player_moves.jsonl", "viewer.log", "scripted-provider/trace.ndjson", "scripted-provider/summary.json"): + source = play_state / rel + if source.exists() and source.is_file(): + target = dest / rel + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, target) + summary = provider_summary(play_state) + if summary: + json_dump(out / "scripted-provider" / "summary.json", summary) + if (play_state / "scripted-provider" / "trace.ndjson").exists(): + (out / "scripted-provider").mkdir(parents=True, exist_ok=True) + shutil.copy2(play_state / "scripted-provider" / "trace.ndjson", out / "scripted-provider" / "trace.ndjson") + + +def write_text_snapshot(path: Path, payload: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(payload, encoding="utf-8") + + +def html_text(base_url: str) -> str: + try: + with urllib.request.urlopen(urllib.parse.urljoin(base_url, "/openworlds/"), timeout=5) as resp: + return resp.read(MAX_HTTP_BYTES).decode("utf-8", errors="replace") + except Exception as exc: # noqa: BLE001 + return f"unavailable: {exc}\n" + + +def chrome_binary() -> str: + for candidate in ( + "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", + "/Applications/Chromium.app/Contents/MacOS/Chromium", + ): + if Path(candidate).exists(): + return candidate + for name in ("google-chrome", "chromium", "chromium-browser"): + found = shutil.which(name) + if found: + return found + return "" + + +def capture_openworlds_screenshot( + *, + base_url: str, + out: Path, + port: int, + label: str, + gaps: list[dict[str, str]], + screenshots: list[str], +) -> None: + chrome = chrome_binary() + target = out / "screenshots" / f"{label}.png" + if not chrome: + gaps.append({"source": "screenshot", "kind": label, "path": str(target), "reason": "chrome_not_found"}) + return + profile = out / ".chrome-profile" / f"{port}-{label}" + profile.mkdir(parents=True, exist_ok=True) + url = f"{base_url}/openworlds/#table" + cmd = [ + chrome, + "--headless=new", + "--disable-gpu", + "--hide-scrollbars", + "--force-device-scale-factor=1", + "--window-size=1512,982", + f"--user-data-dir={profile}", + "--no-first-run", + "--no-default-browser-check", + "--disable-background-networking", + "--disable-component-update", + "--disable-default-apps", + "--disable-sync", + "--virtual-time-budget=5000", + f"--screenshot={target}", + url, + ] + proc = subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, text=True) + deadline = time.time() + 12 + while time.time() < deadline: + if target.exists() and target.stat().st_size > 200: + screenshots.append(str(target.relative_to(out))) + proc.terminate() + try: + proc.wait(timeout=2) + except subprocess.TimeoutExpired: + proc.kill() + shutil.rmtree(profile, ignore_errors=True) + return + if proc.poll() is not None: + break + time.sleep(0.25) + if target.exists() and target.stat().st_size > 200: + screenshots.append(str(target.relative_to(out))) + shutil.rmtree(profile, ignore_errors=True) + return + try: + proc.terminate() + proc.wait(timeout=2) + except (ProcessLookupError, subprocess.TimeoutExpired): + proc.kill() + finally: + shutil.rmtree(profile, ignore_errors=True) + if target.exists() and target.stat().st_size > 200: + screenshots.append(str(target.relative_to(out))) + return + reason = f"chrome_exit={proc.returncode}" + gaps.append({"source": "screenshot", "kind": label, "path": str(target), "reason": reason}) + + +def classify_status(status: dict[str, Any]) -> tuple[str, str]: + readiness = status.get("readiness") if isinstance(status.get("readiness"), dict) else {} + if readiness.get("failure_bucket") and readiness.get("failure_bucket") != "none": + return str(readiness.get("failure_bucket")), str(readiness.get("failure_detail") or "app-status readiness failed") + live = status.get("live") if isinstance(status.get("live"), dict) else {} + art = status.get("art") if isinstance(status.get("art"), dict) else {} + viewer = status.get("viewer") if isinstance(status.get("viewer"), dict) else {} + actor = live.get("actor") if isinstance(live.get("actor"), dict) else {} + if art.get("private_root_present") is False: + return "no_art", "private art root missing" + if not live.get("can_act"): + return "no_provider", "scripted provider did not expose can_act:true" + if not actor.get("id") and not actor.get("name"): + return "no_actor", "no active actor in app-status" + if int(live.get("enabled_action_count") or 0) <= 0: + return "no_actions", "no enabled actions in app-status" + if int(viewer.get("chat_lines") or 0) <= 0: + return "no_narration", "no narration/chat lines in app-status" + return "", "" + + +def run_smoke(args: argparse.Namespace) -> int: + run_id = args.run_id or f"scripted-smoke-{utc_stamp()}" + out = (Path(args.out).expanduser() if args.out else DEFAULT_ROOT / run_id).resolve() + out.mkdir(parents=True, exist_ok=True) + for rel in ("screenshots", "a11y"): + (out / rel).mkdir(parents=True, exist_ok=True) + (out / "console.ndjson").write_text("", encoding="utf-8") + (out / "network.ndjson").write_text("", encoding="utf-8") + (out / "actions.ndjson").write_text("", encoding="utf-8") + (out / "moves.ndjson").write_text("", encoding="utf-8") + + base_url = f"http://127.0.0.1:{int(args.port)}" + env = os.environ.copy() + env.update({ + "WORLDOS_ENABLE_SCRIPTED_PROVIDER": "1", + "CLAWDND_RUN_ID": run_id, + "CLAWDND_WORLD": args.world, + "CLAWDND_PLAY_PORT": str(args.port), + }) + if args.art_root: + env["WORLDOS_ART_REPO_ROOT"] = args.art_root + env["CLAWDND_ART_REPO_ROOT"] = args.art_root + + log = (out / "run.log").open("w", encoding="utf-8") + proc = subprocess.Popen( + [str(ROOT / "scripts" / "play_scripted_dm.sh")], + cwd=ROOT, + env=env, + stdout=log, + stderr=subprocess.STDOUT, + text=True, + start_new_session=True, + ) + verdict = { + "schema": "worldos.scripted-app-smoke.v1", + "run_id": run_id, + "world": args.world, + "port": int(args.port), + "beats_requested": int(args.beats), + "status": "failed", + "failure_bucket": "", + "failure_detail": "", + "screenshots": [], + "evidence_gaps": [], + "started_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), + } + evidence_gaps: list[dict[str, str]] = verdict["evidence_gaps"] + screenshots: list[str] = verdict["screenshots"] + + def fail(bucket: str, detail: str) -> int: + verdict.update({"failure_bucket": bucket, "failure_detail": detail}) + json_dump(out / "smoke.json", verdict) + return 1 + + try: + try: + status = wait_for_status(base_url, out, timeout=args.timeout) + except Exception as exc: # noqa: BLE001 + stale = classify_browser_probe(tab_url=f"{base_url}/openworlds/", status_url=f"{base_url}/app-status", app_status_ok=False) + return fail(stale.bucket if stale else "no_launcher", str(exc)) + + json_dump(out / "app-status.initial.json", status) + write_text_snapshot(out / "a11y" / "initial.html", html_text(base_url)) + capture_openworlds_screenshot(base_url=base_url, out=out, port=int(args.port), label="initial", gaps=evidence_gaps, screenshots=screenshots) + try: + surface, _ = fetch_json(surface_url(base_url, status)) + except (OSError, urllib.error.URLError, ValueError) as exc: + return fail("no_provider", f"initial session-surface fetch failed: {exc}") + json_dump(out / "session-surface.initial.json", surface) + bucket, detail = classify_status(status) + if bucket: + return fail(bucket, detail) + + last_chat_lines = int(((status.get("viewer") or {}).get("chat_lines") or 0) if isinstance(status.get("viewer"), dict) else 0) + move_url = urllib.parse.urljoin(base_url, "/move") + for beat in range(1, int(args.beats) + 1): + move = {"kind": "do", "text": f"scripted smoke beat {beat}: inspect the lantern and keep moving."} + append_ndjson(out / "moves.ndjson", {"at": time.time(), "beat": beat, "request": move}) + append_ndjson(out / "actions.ndjson", {"at": time.time(), "beat": beat, "action": "post_move", "url": move_url}) + try: + response, http_status = post_json(move_url, move) + except (OSError, urllib.error.URLError, ValueError) as exc: + append_ndjson(out / "network.ndjson", {"at": time.time(), "method": "POST", "url": move_url, "error": str(exc)}) + verdict.update({"failure_bucket": "move_rejected", "failure_detail": str(exc)}) + json_dump(out / "smoke.json", verdict) + return 1 + append_ndjson(out / "network.ndjson", {"at": time.time(), "method": "POST", "url": move_url, "status": http_status}) + if not response.get("ok"): + verdict.update({"failure_bucket": "move_rejected", "failure_detail": str(response.get("reason") or response)}) + json_dump(out / "smoke.json", verdict) + return 1 + deadline = time.time() + args.timeout + advanced = False + while time.time() < deadline: + try: + status = wait_for_status(base_url, out, timeout=3) + except Exception as exc: # noqa: BLE001 + return fail("no_launcher", f"app-status dropped during beat {beat}: {exc}") + chat_lines = int(((status.get("viewer") or {}).get("chat_lines") or 0) if isinstance(status.get("viewer"), dict) else 0) + summary = provider_summary(ROOT / "play-state" / run_id) + if chat_lines > last_chat_lines and int(summary.get("resolved_move_count") or summary.get("move_resolved_count") or 0) >= beat: + advanced = True + last_chat_lines = chat_lines + break + time.sleep(0.5) + json_dump(out / f"app-status.beat-{beat}.json", status) + try: + surface, _ = fetch_json(surface_url(base_url, status)) + except (OSError, urllib.error.URLError, ValueError) as exc: + return fail("no_provider", f"session-surface fetch failed after beat {beat}: {exc}") + json_dump(out / f"session-surface.beat-{beat}.json", surface) + write_text_snapshot(out / "a11y" / f"beat-{beat}.html", html_text(base_url)) + capture_openworlds_screenshot(base_url=base_url, out=out, port=int(args.port), label=f"beat-{beat:03d}", gaps=evidence_gaps, screenshots=screenshots) + if not advanced: + return fail("no_narration", f"narration did not advance after beat {beat}") + + try: + final_status = wait_for_status(base_url, out, timeout=5) + except Exception as exc: # noqa: BLE001 + return fail("no_launcher", f"final app-status fetch failed: {exc}") + try: + final_surface, _ = fetch_json(surface_url(base_url, final_status)) + except (OSError, urllib.error.URLError, ValueError) as exc: + return fail("no_provider", f"final session-surface fetch failed: {exc}") + json_dump(out / "app-status.final.json", final_status) + json_dump(out / "session-surface.final.json", final_surface) + write_text_snapshot(out / "a11y" / "final.html", html_text(base_url)) + capture_openworlds_screenshot(base_url=base_url, out=out, port=int(args.port), label="final", gaps=evidence_gaps, screenshots=screenshots) + copy_play_state(run_id, out) + verdict.update({ + "status": "passed", + "failure_bucket": "", + "failure_detail": "", + "finished_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), + "beats_completed": int(args.beats), + "app_status_url": f"{base_url}/app-status", + "evidence_dir": str(out), + }) + json_dump(out / "smoke.json", verdict) + return 0 + finally: + try: + os.killpg(proc.pid, signal.SIGTERM) + except ProcessLookupError: + pass + except OSError: + proc.terminate() + try: + proc.wait(timeout=10) + except subprocess.TimeoutExpired: + proc.kill() + log.close() + copy_play_state(run_id, out) + + +def parse_args(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Run deterministic multi-beat scripted WorldOS app smoke.") + parser.add_argument("--beats", type=int, default=5) + parser.add_argument("--port", type=int, default=8899) + parser.add_argument("--out", default="") + parser.add_argument("--run-id", default="") + parser.add_argument("--world", default="baldurs-gate") + parser.add_argument("--timeout", type=float, default=60.0) + parser.add_argument("--art-root", default=os.environ.get("WORLDOS_ART_REPO_ROOT") or os.environ.get("CLAWDND_ART_REPO_ROOT") or "") + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv if argv is not None else sys.argv[1:]) + if args.beats < 1: + raise SystemExit("--beats must be at least 1") + return run_smoke(args) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/qa/export_app_evidence.py b/qa/export_app_evidence.py index 9c450341..60fd11c2 100644 --- a/qa/export_app_evidence.py +++ b/qa/export_app_evidence.py @@ -23,6 +23,42 @@ DEFAULT_OUTPUT_ROOT = Path("/Volumes/LEXAR/Codex") MAX_HTTP_BYTES = 8 * 1024 * 1024 MAX_LOCAL_FILE_BYTES = 64 * 1024 * 1024 +RUN_DIR_PATTERNS = ( + "run.json", + "smoke.json", + "provider_playtest.json", + "RRI.json", + "score.json", + "summary.md", + "run.log", + "backend.log", + "viewer.log", + "console.ndjson", + "network.ndjson", + "actions.ndjson", + "bugs.ndjson", + "moves.ndjson", + "app-status*.json", + "session-surface*.json", + "screenshots/*.png", + "screenshots/*.jpg", + "a11y/*", + "player/console.ndjson", + "player/network.ndjson", + "player/actions.ndjson", + "player/bugs.ndjson", + "player/summary.md", + "player/score.json", + "player/screenshots/*.png", + "player/screenshots/*.jpg", + "player/a11y/*", + "native/*.png", + "native/*.json", + "native/*.log", + "scripted-provider/*.json", + "scripted-provider/*.ndjson", + "scripted-provider/*.log", +) def utc_stamp() -> str: @@ -194,6 +230,75 @@ def copy_local_file(kind: str, value: Any, bundle: Path, gaps: list[dict[str, st } +def copy_evidence_file(kind: str, source: Path, dest: Path, bundle: Path, gaps: list[dict[str, str]]) -> dict[str, Any] | None: + try: + resolved = source.resolve(strict=True) + except FileNotFoundError: + gaps.append({"source": "run_dir", "kind": kind, "path": str(source), "reason": "missing"}) + return None + except OSError as exc: + gaps.append({"source": "run_dir", "kind": kind, "path": str(source), "reason": f"unreadable_path: {exc}"}) + return None + try: + stat = resolved.stat() + except OSError as exc: + gaps.append({"source": "run_dir", "kind": kind, "path": str(resolved), "reason": f"stat_failed: {exc}"}) + return None + if not resolved.is_file(): + return None + if stat.st_size > MAX_LOCAL_FILE_BYTES: + gaps.append({"source": "run_dir", "kind": kind, "path": str(resolved), "reason": f"too_large: {stat.st_size} bytes"}) + return None + dest.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(resolved, dest) + return { + "kind": kind, + "source": str(resolved), + "path": source_display(dest, bundle), + "bytes": stat.st_size, + } + + +def copy_run_dir_evidence(run_dir_value: str, bundle: Path, gaps: list[dict[str, str]]) -> tuple[dict[str, Any], list[dict[str, Any]]]: + if not run_dir_value: + return {}, [] + run_dir = Path(run_dir_value).expanduser() + try: + resolved = run_dir.resolve(strict=True) + except FileNotFoundError: + gaps.append({"source": "run_dir", "kind": "directory", "path": str(run_dir), "reason": "missing"}) + return {"path": str(run_dir), "ok": False}, [] + except OSError as exc: + gaps.append({"source": "run_dir", "kind": "directory", "path": str(run_dir), "reason": f"unreadable_path: {exc}"}) + return {"path": str(run_dir), "ok": False}, [] + if not resolved.is_dir(): + gaps.append({"source": "run_dir", "kind": "directory", "path": str(resolved), "reason": "not_a_directory"}) + return {"path": str(resolved), "ok": False}, [] + + copied: list[dict[str, Any]] = [] + seen: set[Path] = set() + for pattern in RUN_DIR_PATTERNS: + for source in resolved.glob(pattern): + try: + rel = source.relative_to(resolved) + except ValueError: + continue + if source in seen: + continue + seen.add(source) + kind = rel.as_posix().replace("/", "__") + dest = bundle / "run-dir" / rel + entry = copy_evidence_file(kind, source, dest, bundle, gaps) + if entry is not None: + copied.append(entry) + return { + "path": str(resolved), + "ok": True, + "copied_count": len(copied), + "patterns": list(RUN_DIR_PATTERNS), + }, copied + + def local_file_candidates(app_status: dict[str, Any]) -> list[tuple[str, Any]]: viewer = app_status.get("viewer") if isinstance(app_status.get("viewer"), dict) else {} live = app_status.get("live") if isinstance(app_status.get("live"), dict) else {} @@ -272,6 +377,27 @@ def read_transition_file(path_value: str, bundle: Path, gaps: list[dict[str, str } +def run_dir_failure(bundle: Path) -> dict[str, str]: + candidates = ( + bundle / "run-dir" / "smoke.json", + bundle / "run-dir" / "native" / "transition.json", + bundle / "run-dir" / "transition.json", + bundle / "run-dir" / "run.json", + ) + for path in candidates: + if not path.exists(): + continue + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + continue + bucket = find_first_string(payload, ("failure_bucket", "failureBucket", "bucket")) + detail = find_first_string(payload, ("failure_detail", "failureDetail", "detail", "error")) + if bucket or detail: + return {"failure_bucket": bucket, "failure_detail": detail, "source": source_display(path, bundle)} + return {"failure_bucket": "", "failure_detail": "", "source": ""} + + def exporter_manifest(args: argparse.Namespace, bundle: Path) -> tuple[dict[str, Any], int]: gaps: list[dict[str, str]] = [] sources: dict[str, Any] = {} @@ -279,26 +405,27 @@ def exporter_manifest(args: argparse.Namespace, bundle: Path) -> tuple[dict[str, app_status: dict[str, Any] = {} exit_code = 0 - try: - app_status, meta = fetch_json(args.app_status_url) - json_dump(bundle / "app-status.json", app_status) - sources["app_status"] = { - **meta, - "path": "app-status.json", - "ok": True, - } - except (OSError, urllib.error.URLError, ValueError) as exc: - gaps.append({ - "source": "app_status", - "kind": "http_json", - "path": args.app_status_url, - "reason": str(exc), - }) - sources["app_status"] = { - "url": args.app_status_url, - "ok": False, - } - exit_code = 1 + if args.app_status_url: + try: + app_status, meta = fetch_json(args.app_status_url) + json_dump(bundle / "app-status.json", app_status) + sources["app_status"] = { + **meta, + "path": "app-status.json", + "ok": True, + } + except (OSError, urllib.error.URLError, ValueError) as exc: + gaps.append({ + "source": "app_status", + "kind": "http_json", + "path": args.app_status_url, + "reason": str(exc), + }) + sources["app_status"] = { + "url": args.app_status_url, + "ok": False, + } + exit_code = 1 if app_status: surface_url = session_surface_url(args.app_status_url, app_status) @@ -331,11 +458,29 @@ def exporter_manifest(args: argparse.Namespace, bundle: Path) -> tuple[dict[str, if transition: sources["transition"] = transition + run_source, run_copied = copy_run_dir_evidence(args.run_dir, bundle, gaps) + if run_source: + sources["run_dir"] = run_source + copied_files.extend(run_copied) + live = app_status.get("live") if isinstance(app_status.get("live"), dict) else {} + failure = { + "failure_bucket": "", + "failure_detail": "", + "source": "", + } + if transition: + failure = { + "failure_bucket": str(transition.get("failure_bucket") or ""), + "failure_detail": str(transition.get("failure_detail") or ""), + "source": str(transition.get("path") or ""), + } + if not failure["failure_bucket"] and run_source: + failure = run_dir_failure(bundle) manifest = { "schema": "worldos.app-evidence.v1", "created_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), - "app_status_url": args.app_status_url, + "app_status_url": args.app_status_url or "", "bundle_dir": str(bundle), "build": build_info(app_status), "art": art_status(app_status), @@ -346,6 +491,7 @@ def exporter_manifest(args: argparse.Namespace, bundle: Path) -> tuple[dict[str, "can_act": bool(live.get("can_act")) if "can_act" in live else None, "enabled_action_count": live.get("enabled_action_count"), }, + "failure": failure, "sources": sources, "copied_files": copied_files, "evidence_gaps": gaps, @@ -355,9 +501,10 @@ def exporter_manifest(args: argparse.Namespace, bundle: Path) -> tuple[dict[str, def parse_args(argv: list[str]) -> argparse.Namespace: parser = argparse.ArgumentParser(description="Export a read-only WorldOS app evidence bundle.") - parser.add_argument("--app-status-url", required=True, help="URL for the app /app-status endpoint") + parser.add_argument("--app-status-url", default="", help="URL for the app /app-status endpoint") parser.add_argument("--out", default="", help="Evidence bundle directory (default: /Volumes/LEXAR/Codex/worldos-app-evidence/)") parser.add_argument("--transition-file", default="", help="Optional local JSON file with failure bucket/detail") + parser.add_argument("--run-dir", default="", help="Optional app playtest run directory to copy into the evidence bundle") return parser.parse_args(argv) diff --git a/qa/release_readiness.py b/qa/release_readiness.py index 594f5aeb..554789f7 100644 --- a/qa/release_readiness.py +++ b/qa/release_readiness.py @@ -142,12 +142,19 @@ def main() -> int: rj = read_json(rd / "run.json") sc = read_json(rd / "score.json") if not sc: - part_b = (rj.get("part_b") or {}).get("persona_loop") + part_a_obj = rj.get("part_a") or {} + part_b_obj = rj.get("part_b") or {} + part_b = part_b_obj.get("persona_loop") harness_failures.append({ "run": rd.name, "persona": expected_for_run or infer_persona(rd), "missing": "score.json", + "part_a": part_a_obj.get("result") or "n/a", "part_b": part_b or "n/a", + "part_a_failure_bucket": part_a_obj.get("failure_bucket") or "", + "part_a_failure_detail": part_a_obj.get("failure_detail") or "", + "part_b_failure_bucket": part_b_obj.get("failure_bucket") or "", + "part_b_failure_detail": part_b_obj.get("failure_detail") or "", }) continue rate, ok, total, image_source = image_render_rate(rd, sc) @@ -169,6 +176,11 @@ def main() -> int: "run_build_sha": rj.get("build_sha") or "", "part_b_result": (rj.get("part_b") or {}).get("persona_loop") or "n/a", "part_b_score_pass": bool((rj.get("part_b") or {}).get("score_pass")), + "part_a_result": (rj.get("part_a") or {}).get("result") or "n/a", + "part_a_failure_bucket": (rj.get("part_a") or {}).get("failure_bucket") or "", + "part_a_failure_detail": (rj.get("part_a") or {}).get("failure_detail") or "", + "part_b_failure_bucket": (rj.get("part_b") or {}).get("failure_bucket") or "", + "part_b_failure_detail": (rj.get("part_b") or {}).get("failure_detail") or "", }) if not expected_personas: @@ -199,11 +211,16 @@ def main() -> int: # native gate: read part_a from any run.json present native = "" + native_detail = "" for rd in run_dirs: rj = read_json(rd / "run.json") - pa = (rj.get("part_a") or {}).get("result") + part_a_obj = rj.get("part_a") or {} + pa = part_a_obj.get("result") if pa: native = pa + bucket = part_a_obj.get("failure_bucket") or "" + detail = part_a_obj.get("failure_detail") or "" + native_detail = f" failure_bucket={bucket} failure_detail={detail}".strip() if bucket or detail else "" break evidence_gaps = [] @@ -245,17 +262,26 @@ def main() -> int: "detail": missing_detail, }) for h in harness_failures: + buckets = ", ".join( + value for value in ( + f"part_a_bucket={h.get('part_a_failure_bucket')}" if h.get("part_a_failure_bucket") else "", + f"part_b_bucket={h.get('part_b_failure_bucket')}" if h.get("part_b_failure_bucket") else "", + ) + if value + ) evidence_gaps.append({ "gate": "cross_persona_sat", "missing": f"{h['run']}/score.json", - "detail": f"persona={h.get('persona') or 'unknown'} part_b={h.get('part_b') or 'n/a'}", + "detail": f"persona={h.get('persona') or 'unknown'} part_a={h.get('part_a') or 'n/a'} part_b={h.get('part_b') or 'n/a'} {buckets}".strip(), }) failed_part_b = [p for p in persona_scores if p.get("part_b_result") != "PASS"] for p in failed_part_b: + bucket = p.get("part_b_failure_bucket") or "" + detail = p.get("part_b_failure_detail") or "" evidence_gaps.append({ "gate": "arc_completed", "missing": f"{p['run']}/run.json part_b PASS", - "detail": f"persona={p.get('persona') or 'unknown'} part_b={p.get('part_b_result')} score_pass={p.get('part_b_score_pass')}", + "detail": f"persona={p.get('persona') or 'unknown'} part_b={p.get('part_b_result')} score_pass={p.get('part_b_score_pass')} failure_bucket={bucket} failure_detail={detail}".strip(), }) if not native: evidence_gaps.append({ @@ -263,6 +289,12 @@ def main() -> int: "missing": "run.json part_a.result", "detail": "no persona run recorded native built-app transition evidence", }) + elif native != "PASS": + evidence_gaps.append({ + "gate": "native_gate", + "missing": "run.json part_a PASS", + "detail": f"part_a={native} {native_detail}".strip(), + }) if not args.story: evidence_gaps.append({"gate": "story_craft", "missing": "--story", "detail": "story lens path not supplied"}) elif "overall" not in story: @@ -300,7 +332,7 @@ def main() -> int: # ---- the 11 gates (each contributes to RRI; all must hold for 10/10) ---- gates = { "native_gate": (native == "PASS" and "native_gate" not in evidence_gap_gates, - f"part_a={native or 'n/a'}"), + f"part_a={native or 'n/a'} {native_detail}".strip()), "arc_completed": (any_completed and "arc_completed" not in evidence_gap_gates, f"completed_intro_flow on >=1 persona"), "cross_persona_sat": (not missing_release_personas and expected_complete and avg_sat >= 7.0, diff --git a/qa/test_app_failure_buckets.py b/qa/test_app_failure_buckets.py new file mode 100644 index 00000000..f961d6e8 --- /dev/null +++ b/qa/test_app_failure_buckets.py @@ -0,0 +1,117 @@ +import tempfile +import unittest +from pathlib import Path + +from qa.app_failure_buckets import ( + APP_FAILURE_BUCKETS, + classify_browser_probe, + classify_native_failure, + classify_part_b_failure_from_artifacts, + classify_part_b_readiness_failure, + classify_part_b_score_failure, +) + + +class AppFailureBucketTests(unittest.TestCase): + def test_native_buckets_cover_stable_external_contract(self): + self.assertEqual( + APP_FAILURE_BUCKETS, + ( + "no_app", + "no_launcher", + "no_provider", + "no_art", + "no_actor", + "no_actions", + "move_rejected", + "no_narration", + "console_error", + "permission_prompt", + ), + ) + + def test_native_build_and_launcher_failures(self): + self.assertEqual(classify_native_failure(result="build_failed", can_act=False).bucket, "no_app") + self.assertEqual(classify_native_failure(result="app_not_running", can_act=False).bucket, "no_app") + self.assertEqual(classify_native_failure(result="no_launcher", can_act=False).bucket, "no_launcher") + + def test_native_status_payload_drives_specific_buckets(self): + base = { + "art": {"private_root_present": True}, + "viewer": {"chat_lines": 1}, + "live": { + "actor": {"id": "hero", "name": "Hero"}, + "enabled_action_count": 5, + }, + } + missing_art = {**base, "art": {"private_root_present": False}} + self.assertEqual(classify_native_failure(result="FAIL", can_act=True, app_status=missing_art).bucket, "no_art") + + self.assertEqual(classify_native_failure(result="FAIL", can_act=False, app_status=base).bucket, "no_provider") + + no_actor = {**base, "live": {"actor": {}, "enabled_action_count": 5}} + self.assertEqual(classify_native_failure(result="FAIL", can_act=True, app_status=no_actor).bucket, "no_actor") + + no_actions = {**base, "live": {"actor": {"name": "Hero"}, "enabled_action_count": 0}} + self.assertEqual(classify_native_failure(result="FAIL", can_act=True, app_status=no_actions).bucket, "no_actions") + + no_narration = {**base, "viewer": {"chat_lines": 0}} + self.assertEqual(classify_native_failure(result="FAIL", can_act=True, app_status=no_narration).bucket, "no_narration") + + def test_app_status_readiness_bucket_takes_precedence(self): + status = { + "readiness": { + "failure_bucket": "no_actions", + "failure_detail": "readiness found no enabled actions", + } + } + result = classify_native_failure(result="FAIL", can_act=False, app_status=status) + self.assertEqual(result.bucket, "no_actions") + self.assertIn("readiness", result.detail) + + def test_part_b_readiness_buckets(self): + self.assertEqual(classify_part_b_readiness_failure(saw_canact=0, saw_pc=1, chat_lines=1).bucket, "no_provider") + self.assertEqual(classify_part_b_readiness_failure(saw_canact=1, saw_pc=0, chat_lines=1).bucket, "no_actor") + self.assertEqual(classify_part_b_readiness_failure(saw_canact=1, saw_pc=1, chat_lines=0).bucket, "no_narration") + self.assertEqual(classify_part_b_readiness_failure(saw_canact=1, saw_pc=1, chat_lines=1).bucket, "no_actions") + + def test_part_b_artifact_buckets(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + (root / "console.ndjson").write_text("pageerror: uncaught exception\n", encoding="utf-8") + self.assertEqual(classify_part_b_failure_from_artifacts(root, "FAIL").bucket, "console_error") + + with tempfile.TemporaryDirectory() as td: + root = Path(td) + (root / "actions.ndjson").write_text("POST /move returned 500\n", encoding="utf-8") + self.assertEqual(classify_part_b_failure_from_artifacts(root, "FAIL").bucket, "move_rejected") + + with tempfile.TemporaryDirectory() as td: + root = Path(td) + (root / "summary.md").write_text("AXIsProcessTrusted false; screen recording permission missing\n", encoding="utf-8") + self.assertEqual(classify_part_b_failure_from_artifacts(root, "FAIL").bucket, "permission_prompt") + + def test_part_b_score_failure_maps_to_stable_bucket_contract(self): + with tempfile.TemporaryDirectory() as td: + score = Path(td) / "score.json" + score.write_text('{"pass": false, "completed_intro_flow": true, "reached_play_screen": true, "persona_satisfaction": 5}\n', encoding="utf-8") + result = classify_part_b_score_failure(score) + + self.assertEqual(result.bucket, "no_provider") + self.assertIn("satisfaction=5/10", result.detail) + + def test_visible_stale_browser_without_same_port_status_is_no_launcher(self): + result = classify_browser_probe( + tab_url="http://127.0.0.1:8899/openworlds/", + status_url="http://127.0.0.1:8899/app-status", + app_status_ok=False, + ) + self.assertIsNotNone(result) + self.assertEqual(result.bucket, "no_launcher") + self.assertIn("same-port /app-status", result.detail) + + self.assertIsNone(classify_browser_probe(tab_url="http://127.0.0.1:8899/openworlds/", app_status_ok=True)) + + +if __name__ == "__main__": + unittest.main() diff --git a/qa/test_app_smoke_scripted.py b/qa/test_app_smoke_scripted.py new file mode 100644 index 00000000..35cbdc5e --- /dev/null +++ b/qa/test_app_smoke_scripted.py @@ -0,0 +1,61 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from qa import app_smoke_scripted as smoke + + +class AppSmokeScriptedTests(unittest.TestCase): + def test_surface_url_preserves_campaign_from_app_status(self): + status = { + "live": {"campaign_id": "camp_123"}, + "endpoints": {"session_surface": "/session-surface"}, + } + + self.assertEqual( + smoke.surface_url("http://127.0.0.1:8899/openworlds/", status), + "http://127.0.0.1:8899/session-surface?campaign=camp_123", + ) + + def test_classify_status_returns_stable_failure_buckets(self): + self.assertEqual(smoke.classify_status({"art": {"private_root_present": False}})[0], "no_art") + self.assertEqual( + smoke.classify_status({ + "art": {"private_root_present": True}, + "live": {"can_act": False}, + })[0], + "no_provider", + ) + self.assertEqual( + smoke.classify_status({ + "art": {"private_root_present": True}, + "live": {"can_act": True, "actor": {}, "enabled_action_count": 5}, + "viewer": {"chat_lines": 1}, + })[0], + "no_actor", + ) + + def test_provider_summary_synthesizes_trace_counts(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + trace = root / "scripted-provider" / "trace.ndjson" + trace.parent.mkdir() + trace.write_text( + "\n".join([ + json.dumps({"event": "bootstrap"}), + json.dumps({"event": "move_resolved", "beat": 1}), + json.dumps({"event": "move_resolved", "beat": 2}), + ]) + "\n", + encoding="utf-8", + ) + + summary = smoke.provider_summary(root) + + self.assertEqual(summary["schema"], "worldos.scripted-provider-summary.v1") + self.assertEqual(summary["move_resolved_count"], 2) + self.assertEqual(summary["event_count"], 3) + + +if __name__ == "__main__": + unittest.main() diff --git a/qa/test_export_app_evidence.py b/qa/test_export_app_evidence.py index 913ae9ee..9deb7e2a 100644 --- a/qa/test_export_app_evidence.py +++ b/qa/test_export_app_evidence.py @@ -46,15 +46,17 @@ def serve(self, app_status: dict, session_surface: dict) -> tuple[HTTPServer, st thread.start() return server, f"http://127.0.0.1:{server.server_port}/app-status?campaign=camp_test" - def run_exporter(self, out: Path, app_status_url: str) -> tuple[int, str, dict]: + def run_exporter(self, out: Path, app_status_url: str = "", extra_args: list[str] | None = None) -> tuple[int, str, dict]: cmd = [ sys.executable, str(SCRIPT), - "--app-status-url", - app_status_url, "--out", str(out), ] + if app_status_url: + cmd.extend(["--app-status-url", app_status_url]) + if extra_args: + cmd.extend(extra_args) proc = subprocess.run(cmd, cwd=ROOT, text=True, capture_output=True, check=False) manifest = out / "manifest.json" payload = json.loads(manifest.read_text(encoding="utf-8")) if manifest.exists() else {} @@ -126,6 +128,46 @@ def test_missing_optional_local_files_are_recorded_as_evidence_gaps(self): self.assertIn(("local_file", "chat"), gaps) self.assertIn(("local_file", "moves"), gaps) + def test_run_dir_mode_copies_allowlisted_artifacts_and_failure_bucket(self): + with tempfile.TemporaryDirectory() as td: + tmp = Path(td) + run = tmp / "smoke-run" + (run / "screenshots").mkdir(parents=True) + (run / "a11y").mkdir() + (run / "scripted-provider").mkdir() + (run / "native").mkdir() + (run / "smoke.json").write_text( + json.dumps( + { + "schema": "worldos.scripted-app-smoke.v1", + "status": "failed", + "failure_bucket": "move_rejected", + "failure_detail": "POST /move returned 500", + } + ), + encoding="utf-8", + ) + (run / "screenshots" / "beat-001.png").write_bytes(b"\x89PNG\r\n\x1a\nfixture") + (run / "a11y" / "beat-001.html").write_text("
fixture
\n", encoding="utf-8") + (run / "actions.ndjson").write_text('{"action":"post_move"}\n', encoding="utf-8") + (run / "scripted-provider" / "summary.json").write_text('{"provider":"scripted"}\n', encoding="utf-8") + (run / "scripted-provider" / "trace.ndjson").write_text('{"event":"move_resolved"}\n', encoding="utf-8") + (run / "native" / "transition.json").write_text('{"failure_bucket":"no_launcher"}\n', encoding="utf-8") + out = tmp / "bundle" + + rc, text, payload = self.run_exporter(out, extra_args=["--run-dir", str(run)]) + + self.assertEqual(rc, 0, text) + self.assertEqual(payload["sources"]["run_dir"]["ok"], True) + self.assertEqual(payload["failure"]["failure_bucket"], "move_rejected") + copied = {entry["path"] for entry in payload["copied_files"]} + self.assertIn("run-dir/screenshots/beat-001.png", copied) + self.assertIn("run-dir/a11y/beat-001.html", copied) + self.assertIn("run-dir/scripted-provider/summary.json", copied) + self.assertIn("run-dir/scripted-provider/trace.ndjson", copied) + self.assertIn("run-dir/native/transition.json", copied) + self.assertEqual(payload["evidence_gaps"], []) + if __name__ == "__main__": unittest.main() diff --git a/qa/test_macos_app_static.py b/qa/test_macos_app_static.py index 98e5e7e6..12046438 100644 --- a/qa/test_macos_app_static.py +++ b/qa/test_macos_app_static.py @@ -125,7 +125,6 @@ def test_built_app_playtest_emits_split_failure_buckets(self): "no_narration", "console_error", "permission_prompt", - "score_failed", ): self.assertIn(f'"{bucket}"', harness) @@ -142,7 +141,7 @@ def test_built_app_playtest_emits_split_failure_buckets(self): self.assertIn("original_result", harness) self.assertIn("move_rejected", harness) self.assertIn("console_error", harness) - self.assertIn("score_failed", harness) + self.assertNotIn('"score_failed"', harness) def test_provider_viewer_stays_attached_during_native_restarts(self): root_view = self.read("macos/WorldOSApp/Sources/WorldOSApp/Views/RootView.swift") diff --git a/qa/test_release_readiness.py b/qa/test_release_readiness.py index 224e766f..5ef94702 100644 --- a/qa/test_release_readiness.py +++ b/qa/test_release_readiness.py @@ -608,7 +608,12 @@ def test_existing_score_does_not_override_part_b_failure(self): ) part_b = {"persona_loop": "PASS", "score_pass": True} if persona == "veteran": - part_b = {"persona_loop": "FAIL", "score_pass": False} + part_b = { + "persona_loop": "FAIL", + "score_pass": False, + "failure_bucket": "move_rejected", + "failure_detail": "POST /move returned 500", + } (run / "run.json").write_text( json.dumps({"build_sha": "deadbee", "part_a": {"result": "PASS"}, "part_b": part_b}), encoding="utf-8", @@ -660,6 +665,7 @@ def test_existing_score_does_not_override_part_b_failure(self): self.assertFalse(payload["release_ready"]) self.assertIn("arc_completed", {gap["gate"] for gap in payload["evidence_gaps"]}) self.assertIn("veteran", " ".join(gap["detail"] for gap in payload["evidence_gaps"])) + self.assertIn("move_rejected", " ".join(gap["detail"] for gap in payload["evidence_gaps"])) def test_mixed_build_sha_blocks_release_ready(self): with tempfile.TemporaryDirectory() as td: diff --git a/qa/test_ui_playtest_app_buckets.py b/qa/test_ui_playtest_app_buckets.py index 8a6d6b52..9fcb403d 100644 --- a/qa/test_ui_playtest_app_buckets.py +++ b/qa/test_ui_playtest_app_buckets.py @@ -50,7 +50,7 @@ def test_part_b_score_failure_gets_explicit_bucket(self): out = self.run_classifier(f'classify_part_b_score_failure "{score}"') - self.assertEqual(out, "score_failed|score.json failed: satisfaction=5/10") + self.assertEqual(out, "no_provider|score.json failed: satisfaction=5/10") def test_part_b_artifact_classifier_prefers_console_error(self): with tempfile.TemporaryDirectory() as td: diff --git a/qa/ui_playtest_app.sh b/qa/ui_playtest_app.sh index 7dae248e..ebf106ba 100755 --- a/qa/ui_playtest_app.sh +++ b/qa/ui_playtest_app.sh @@ -92,92 +92,34 @@ log "build_sha=$BUILD_SHA version=$VERSION repo=$ROOT" # Agent-readable failure buckets for built-app smoke. Keep these crisp and stable; the # detailed shell/native result still travels separately as original_result. -APP_FAILURE_BUCKETS_JSON='["no_app","no_launcher","no_provider","no_art","no_actor","no_actions","move_rejected","no_narration","console_error","permission_prompt","score_failed"]' +APP_FAILURE_BUCKETS_JSON='["no_app","no_launcher","no_provider","no_art","no_actor","no_actions","move_rejected","no_narration","console_error","permission_prompt"]' bucket_pair() { printf '%s|%s\n' "$1" "$2"; } classify_native_failure() { # $1=result $2=can_act $3=surface_json $4=app_status_json - python3 - "$1" "$2" "${3:-{}}" "${4:-{}}" <<'PY' -import json, sys -result, can_act, surf_raw, status_raw = sys.argv[1:5] -try: surf = json.loads(surf_raw or "{}") -except Exception: surf = {} -try: status = json.loads(status_raw or "{}") -except Exception: status = {} -art = status.get("art") if isinstance(status.get("art"), dict) else {} -live = status.get("live") if isinstance(status.get("live"), dict) else {} -actor = live.get("actor") if isinstance(live.get("actor"), dict) else {} -enabled_count = live.get("enabled_action_count") -chat_lines = (status.get("viewer") or {}).get("chat_lines") if isinstance(status.get("viewer"), dict) else None -if result in ("build_failed", "app_not_running"): - print("no_app|WorldOS.app did not build, launch, or remain running") -elif result == "no_launcher": - print("no_launcher|launcher viewer did not answer /openworlds/") -elif art.get("private_root_present") is False: - print("no_art|private art root was not present in app-status") -elif can_act != "true": - print("no_provider|no minted live provider viewer reported can_act:true") -elif not actor.get("id") and not actor.get("name"): - print("no_actor|app-status did not report an active player actor") -elif enabled_count == 0: - print("no_actions|app-status reported zero enabled player actions") -elif chat_lines == 0: - print("no_narration|app-status reported no chat/narration lines") -else: - print("no_provider|native transition failed without a more specific bucket") -PY + python3 "$ROOT/qa/app_failure_buckets.py" native \ + --result "$1" \ + --can-act "${2:-false}" \ + --surface-json "${3:-{}}" \ + --app-status-json "${4:-{}}" } classify_part_b_readiness_failure() { # $1=saw_canact $2=saw_pc $3=chat_lines - local saw_canact="${1:-0}" saw_pc="${2:-0}" chat_lines="${3:-0}" - if [ "$saw_canact" != "1" ]; then bucket_pair "no_provider" "faithful backend never exposed can_act:true"; return 0; fi - if [ "$saw_pc" != "1" ]; then bucket_pair "no_actor" "faithful backend never seated a player character"; return 0; fi - if [ "${chat_lines:-0}" -le 0 ]; then bucket_pair "no_narration" "faithful backend produced no opening narration"; return 0; fi - bucket_pair "no_actions" "faithful backend was not player-ready" + python3 "$ROOT/qa/app_failure_buckets.py" part-b-readiness \ + --saw-canact "${1:-0}" \ + --saw-pc "${2:-0}" \ + --chat-lines "${3:-0}" } classify_part_b_failure_from_artifacts() { # $1=run_dir $2=fallback_result - local dir="$1" fallback="${2:-FAIL}" - if grep -RqiE 'permission|not authorized|accessibility|screen recording|AXIsProcessTrusted' "$dir" 2>/dev/null; then - bucket_pair "permission_prompt" "macOS permission prompt or accessibility/screen-recording denial appeared" - elif grep -RqiE 'console_error|pageerror|uncaught|exception' "$dir/console.ndjson" "$dir/player/console.ndjson" 2>/dev/null; then - bucket_pair "console_error" "browser console/page error recorded during app playtest" - elif grep -RqiE 'move_rejected|/move.*(4[0-9][0-9]|5[0-9][0-9])|move not sent|rejected' "$dir/actions.ndjson" "$dir/network.ndjson" "$dir/player/network.ndjson" "$dir/summary.md" 2>/dev/null; then - bucket_pair "move_rejected" "player move was rejected or failed to reach /move" - else - bucket_pair "no_provider" "part B failed: $fallback" - fi + python3 "$ROOT/qa/app_failure_buckets.py" part-b-artifacts \ + --run-dir "$1" \ + --fallback-result "${2:-FAIL}" } classify_part_b_score_failure() { # $1=score_json - python3 - "${1:-}" <<'PY' -import json, sys -path = sys.argv[1] -try: - score = json.load(open(path, encoding="utf-8")) -except Exception as exc: - print(f"score_failed|score.json pass=false and score could not be read: {exc}") - raise SystemExit(0) -console_errors = int(score.get("console_errors") or 0) -critical = int(score.get("bug_reports_critical") or 0) -satisfaction = score.get("persona_satisfaction") -if console_errors > 0: - print(f"console_error|score.json failed: console_errors={console_errors}") -elif critical > 0: - print(f"score_failed|score.json failed: critical_bug_reports={critical}") -elif not score.get("completed_intro_flow"): - if score.get("reached_play_screen"): - print("no_actions|score.json failed: player reached the table but submitted no in-story turn") - else: - print("no_actions|score.json failed: player never reached the playable table") -elif score.get("gave_up"): - detail = str(score.get("give_up_reason") or "player gave up").strip() - print(f"score_failed|score.json failed: {detail}") -elif isinstance(satisfaction, (int, float)) and satisfaction < 6: - print(f"score_failed|score.json failed: satisfaction={satisfaction}/10") -else: - print("score_failed|score.json pass=false without a more specific signal") -PY + python3 "$ROOT/qa/app_failure_buckets.py" part-b-score \ + --score-json "${1:-}" } set_bucket_pair() { # $1=A|B $2='bucket|detail' diff --git a/scripts/play_scripted_dm.sh b/scripts/play_scripted_dm.sh index 5ae41596..01e33fc4 100755 --- a/scripts/play_scripted_dm.sh +++ b/scripts/play_scripted_dm.sh @@ -39,6 +39,7 @@ MOVES="$STATE_DIR/player_moves.jsonl" CHAT="$STATE_DIR/chat.jsonl" VIEWER_LOG="$STATE_DIR/viewer.log" TRACE="$TRACE_DIR/trace.ndjson" +SUMMARY="$TRACE_DIR/summary.json" VPID_FILE="$STATE_DIR/.viewer.pid" mkdir -p "$TRACE_DIR" @@ -64,6 +65,47 @@ with open(path, "a", encoding="utf-8") as fh: PY } +trace_json() { + python3 - "$TRACE" "$1" "$2" "$3" <<'PY' +import json, sys, time +path, event, beat, detail = sys.argv[1:5] +try: + parsed_detail = json.loads(detail) +except json.JSONDecodeError: + parsed_detail = detail +with open(path, "a", encoding="utf-8") as fh: + fh.write(json.dumps({ + "event": event, + "beat": int(beat), + "detail": parsed_detail, + "at": time.time(), + }) + "\n") +PY +} + +write_summary() { + python3 - "$SUMMARY" "$CAMPAIGN_ID" "$PLAYER_NAME" "$WORLD" "$RUN" "$PORT" "$processed" <<'PY' +import json, sys, time +path, campaign_id, player_name, world, run_id, port, processed = sys.argv[1:8] +payload = { + "schema": "worldos.scripted-provider-summary.v1", + "provider": "scripted", + "deterministic": True, + "model_free": True, + "world": world, + "run_id": run_id, + "campaign_id": campaign_id, + "player": player_name, + "port": int(port), + "resolved_move_count": int(processed), + "updated_at": time.time(), +} +with open(path, "w", encoding="utf-8") as fh: + json.dump(payload, fh, indent=2, sort_keys=True) + fh.write("\n") +PY +} + move_chat_text() { python3 - "$1" <<'PY' import json, sys @@ -145,12 +187,17 @@ OPENING="$(python3 -c 'import json,sys;print(json.loads(sys.stdin.read())["openi PLAYER_NAME="$(python3 -c 'import json,sys;print((json.loads(sys.stdin.read())["player"].get("name") or "Hero"))' <<<"$BOOTSTRAP_JSON")" json_append "$CHAT" "dm" "$OPENING" trace "bootstrap" "campaign=$CAMPAIGN_ID player=$PLAYER_NAME" +processed=0 +write_summary viewer_supervisor() { while :; do WORLDOS_STATE_DIR="$STATE_DIR" CLAWDND_STATE_DIR="$STATE_DIR" \ WORLDOS_VIEWER_CHAT="$CHAT" CLAWDND_VIEWER_CHAT="$CHAT" \ WORLDOS_PLAYER_MOVES="$MOVES" CLAWDND_PLAYER_MOVES="$MOVES" \ + WORLDOS_PROVIDER=scripted CLAWDND_PROVIDER=scripted \ + WORLDOS_BROWSER_CONSOLE_LOG="${WORLDOS_BROWSER_CONSOLE_LOG:-}" \ + WORLDOS_BROWSER_NETWORK_LOG="${WORLDOS_BROWSER_NETWORK_LOG:-}" \ python3 viewer/server.py "" "$PORT" >> "$VIEWER_LOG" 2>&1 & local vp=$! echo "$vp" > "$VPID_FILE" @@ -169,7 +216,6 @@ trap cleanup EXIT trap 'cleanup; exit 0' INT TERM trace "viewer_started" "port=$PORT" -processed=0 while :; do count="$(grep -c . "$MOVES" 2>/dev/null || true)" @@ -195,10 +241,12 @@ server.log_event(campaign_id, "narration", reply) print(reply) PY )" + beat=$((processed + 1)) json_append "$CHAT" "player" "$(move_chat_text "$line")" json_append "$CHAT" "dm" "$reply" - trace "move_resolved" "$line" + trace_json "move_resolved" "$beat" "$line" processed=$((processed + 1)) + write_summary done < <(sed -n "$((processed + 1)),${count}p" "$MOVES") processed="$count" fi diff --git a/viewer/index.html b/viewer/index.html index b9158fb7..b69e33a1 100644 --- a/viewer/index.html +++ b/viewer/index.html @@ -1,183 +1,42 @@ - - -WorldOS — Play View + + + +WorldOS - OpenWorlds Redirect + -
-

WorldOS

- - -
-
-

Map

-

Party

-

In the scene

-

Quests

-

Factions

-

What's happening

-

Agent activity idle

-
- +
+

Deprecated viewer entry

+

The old browser MVP at viewer/index.html is retired. The supported WorldOS app surface is OpenWorlds.

+
diff --git a/viewer/openworlds/screen-table.jsx b/viewer/openworlds/screen-table.jsx index c72130ce..ef80ddeb 100644 --- a/viewer/openworlds/screen-table.jsx +++ b/viewer/openworlds/screen-table.jsx @@ -594,7 +594,8 @@ function ScreenTable({ onNavigate, state, setState, liveSession }) {
str: return "" +def _app_status_image_probe(surface: dict) -> bool: + scene = surface.get("scene") if isinstance(surface.get("scene"), dict) else {} + scope = scene.get("imageScope") if isinstance(scene, dict) else "" + return bool(scope and _latest_descriptor(str(scope))) + + +def _browser_health_counts(console_log: str | None, network_log: str | None) -> tuple[int, int]: + def iter_ndjson(path_value: str | None): + if not path_value: + return + try: + path = Path(path_value).expanduser().resolve(strict=True) + except (OSError, FileNotFoundError): + return + try: + with path.open(encoding="utf-8", errors="ignore") as fh: + for line in fh: + line = line.strip() + if not line: + continue + try: + payload = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(payload, dict): + yield payload + except OSError: + return + + console_errors = 0 + for item in iter_ndjson(console_log) or (): + kind = str(item.get("type") or item.get("level") or "").lower() + text = str(item.get("text") or item.get("message") or item.get("error") or "").lower() + if kind in {"error", "pageerror", "fatal"} or "uncaught" in text or "exception" in text: + console_errors += 1 + + network_failures = 0 + for item in iter_ndjson(network_log) or (): + status = item.get("status") or item.get("status_code") + try: + status_int = int(status) + except (TypeError, ValueError): + status_int = 0 + if item.get("error") or item.get("failed") or status_int >= 400: + network_failures += 1 + return console_errors, network_failures + + +def _app_status_readiness(*, live: dict | None, moves: Path | None, is_live_view: bool, + actor: dict, enabled_actions: list[str], + art_root: Path, chat_lines: int, surface: dict, + provider: str, console_errors: int = 0, + network_failures: int = 0) -> tuple[dict, dict]: + same_port_alive = True # This payload was generated by the same port that answered /app-status. + route_loaded = True + moves_writable = bool(live and moves is not None) + can_act = bool(surface.get("can_act")) + provider_ready = bool(live and provider.strip()) + actor_present = bool(actor.get("id") or actor.get("name")) + actions_present = bool(enabled_actions) + recent = surface.get("recentEvents") if isinstance(surface.get("recentEvents"), list) else [] + recent_narration = sum( + 1 for item in recent + if isinstance(item, dict) and item.get("kind") in {"narration", "dialogue"} + ) + narration_present = chat_lines > 0 or recent_narration > 0 + art_present = art_root.is_dir() + image_probe_ok = _app_status_image_probe(surface) + console_errors = max(0, int(console_errors or 0)) + network_failures = max(0, int(network_failures or 0)) + + failure_bucket = "none" + failure_detail = "" + if not same_port_alive or not route_loaded: + failure_bucket = "no_launcher" + failure_detail = "viewer route is not live on the same localhost port as app-status" + elif not art_present or not image_probe_ok: + failure_bucket = "no_art" + failure_detail = "private art root or representative image probe is missing" + elif not provider_ready or not moves_writable or not is_live_view or not can_act: + failure_bucket = "no_provider" + failure_detail = "live provider move sink is not ready" + elif not actor_present: + failure_bucket = "no_actor" + failure_detail = "no active player actor is seated" + elif not actions_present: + failure_bucket = "no_actions" + failure_detail = "no enabled player actions are exposed" + elif not narration_present: + failure_bucket = "no_narration" + failure_detail = "no visible narration/chat has been observed" + elif console_errors: + failure_bucket = "console_error" + failure_detail = "browser console errors were reported" + elif network_failures: + failure_bucket = "console_error" + failure_detail = "browser network failures were reported" + + ready_for_smoke = failure_bucket == "none" + ready_for_play = ready_for_smoke and provider.strip().lower() in {"codex", "claude", "openclaw", "scripted"} + status = "ready" if ready_for_smoke else "degraded" + health = { + "same_port_alive": same_port_alive, + "route_loaded": route_loaded, + "console_errors": console_errors, + "network_failures": network_failures, + "provider_ready": provider_ready, + "image_probe_ok": image_probe_ok, + "failure_bucket": failure_bucket, + "failure_detail": failure_detail, + } + readiness = { + "status": status, + "ready_for_smoke": ready_for_smoke, + "ready_for_play": ready_for_play, + "failure_bucket": failure_bucket, + "failure_detail": failure_detail, + } + return readiness, health + + def _app_status_payload(*, port: int, attached_campaign_id: str, viewed_campaign_id: str, transcript_path: str, chat_path: str) -> dict: """Machine-readable app/test harness truth for agents. @@ -5403,6 +5524,25 @@ def _app_status_payload(*, port: int, attached_campaign_id: str, viewed_campaign actor = (surface.get("actionModel") or {}).get("actor") or {} art_root = _ingested_images_root() state_root = _state_dir() + provider = env_var("PROVIDER", "") or "" + chat_lines = _file_line_count(chat_path) + console_errors, network_failures = _browser_health_counts( + env_var("BROWSER_CONSOLE_LOG", ""), + env_var("BROWSER_NETWORK_LOG", ""), + ) + readiness, health = _app_status_readiness( + live=live, + moves=moves, + is_live_view=is_live_view, + actor=actor, + enabled_actions=enabled_actions, + art_root=art_root, + chat_lines=chat_lines, + surface=surface, + provider=provider, + console_errors=console_errors, + network_failures=network_failures, + ) return { "ok": True, "schema": "worldos.app-status.v1", @@ -5417,10 +5557,10 @@ def _app_status_payload(*, port: int, attached_campaign_id: str, viewed_campaign "port": int(port), "repo_root": _resolved(_REPO_ROOT), "state_root": _resolved(state_root), - "provider": env_var("PROVIDER", "") or "", + "provider": provider, "transcript_path": transcript_path, "chat_path": chat_path, - "chat_lines": _file_line_count(chat_path), + "chat_lines": chat_lines, }, "art": { "repo_root": _resolved(_art_repo_root()), @@ -5444,6 +5584,8 @@ def _app_status_payload(*, port: int, attached_campaign_id: str, viewed_campaign "enabled_action_ids": enabled_actions, "enabled_action_count": len(enabled_actions), }, + "readiness": readiness, + "health": health, "endpoints": { "app_status": "/app-status", "session_surface": "/session-surface", @@ -6127,12 +6269,10 @@ def do_GET(self) -> None: # noqa: N802 # The root used to serve the pre-OpenWorlds raw-DOM dashboard (viewer/index.html). # OpenWorlds (/openworlds/) is the real, current UI the desktop app loads, so the # root now redirects there — hitting 127.0.0.1:/ in a browser shows the same - # app the native shell does, never the legacy view. The old dashboards stay - # reachable at their explicit /dashboard and /legacy paths for reference. + # app the native shell does, never the retired raw-DOM MVP. self._redirect(f"{_OPENWORLDS_ROUTE}/") elif route in ("/legacy", "/legacy.html"): - html = (_HERE / "index.html").read_bytes() - self._send(200, html, "text/html; charset=utf-8") + self._redirect(f"{_OPENWORLDS_ROUTE}/") elif route in ("/dashboard", "/dashboard.html"): html = (_HERE / "dashboard.html").read_bytes() self._send(200, html, "text/html; charset=utf-8") diff --git a/viewer/tests/test_openworlds_static.py b/viewer/tests/test_openworlds_static.py index 8e47f7fe..a9524486 100644 --- a/viewer/tests/test_openworlds_static.py +++ b/viewer/tests/test_openworlds_static.py @@ -106,6 +106,24 @@ def test_openworlds_without_trailing_slash_redirects_to_directory_route(self): self.assertEqual(headers.get("Location"), "/openworlds/") self.assertEqual(body, b"") + def test_root_and_legacy_routes_redirect_to_openworlds(self): + for route in ("/", "/index.html", "/legacy", "/legacy.html"): + with self.subTest(route=route): + status, headers, body = self._get_with_headers(route) + + self.assertEqual(status, 302) + self.assertEqual(headers.get("Location"), "/openworlds/") + self.assertEqual(body, b"") + + def test_deprecated_static_index_redirects_to_openworlds(self): + source = (server._HERE / "index.html").read_text(encoding="utf-8") + + self.assertIn("Deprecated viewer entry", source) + self.assertIn("url=/openworlds/", source) + self.assertIn('window.location.replace("/openworlds/")', source) + self.assertNotIn('id="grid"', source) + self.assertNotIn('fetch("/state")', source) + def test_openworlds_index_uses_local_runtime_assets(self): status, ctype, body = self._get("/openworlds/") @@ -194,8 +212,39 @@ def test_app_status_route_exposes_agent_probe_contract(self): self.assertTrue(payload["live"]["can_act"]) self.assertEqual(payload["live"]["actor"]["name"], "Probe Hero") self.assertIn("continue", payload["live"]["enabled_action_ids"]) + self.assertIn("readiness", payload) + self.assertIn("health", payload) + self.assertIn(payload["readiness"]["status"], ("ready", "degraded")) + self.assertIn("ready_for_smoke", payload["readiness"]) + self.assertIn("ready_for_play", payload["readiness"]) + self.assertTrue(payload["health"]["same_port_alive"]) + self.assertTrue(payload["health"]["route_loaded"]) + self.assertIn("provider_ready", payload["health"]) + self.assertIn("image_probe_ok", payload["health"]) + self.assertIn("failure_bucket", payload["health"]) self.assertEqual(payload["endpoints"]["session_surface"], "/session-surface") + def test_app_status_browser_health_counts_console_and_network_logs(self): + console = self._tmp / "console.ndjson" + network = self._tmp / "network.ndjson" + console.write_text( + "\n".join([ + json.dumps({"type": "warning", "text": "benign"}), + json.dumps({"type": "pageerror", "text": "Uncaught ReferenceError"}), + ]) + "\n", + encoding="utf-8", + ) + network.write_text( + "\n".join([ + json.dumps({"status": 200, "url": "/app-status"}), + json.dumps({"status": 500, "url": "/move"}), + json.dumps({"error": "requestfailed", "url": "/chat"}), + ]) + "\n", + encoding="utf-8", + ) + + self.assertEqual(server._browser_health_counts(str(console), str(network)), (1, 2)) + def test_openworlds_static_assets_are_same_origin_and_local(self): status, ctype, body = self._get("/openworlds/vendor/google-fonts.css") @@ -355,7 +404,8 @@ def test_openworlds_agent_driving_hooks_are_stable(self): for hook in ( 'data-worldos-testid="openworlds-root"', - 'data-worldos-testid="session-surface-status"', + 'data-worldos-testid="app-status-banner"', + 'data-worldos-status-scope="session-surface"', 'data-worldos-testid="narration-log"', 'data-worldos-testid="active-player"', 'data-worldos-testid="action-palette"',