From 3ece300a0400c66abab73fa80edb490f82c9f2d7 Mon Sep 17 00:00:00 2001 From: Eva Date: Sun, 7 Jun 2026 04:32:57 +0700 Subject: [PATCH 1/2] docs: package Codex GPT-DM fair-test preflight --- WorldOS-RUNBOOK.md | 40 +++++++++++++++++ qa/support_vm_preflight.py | 76 ++++++++++++++++++++++++++++++++- qa/test_support_vm_preflight.py | 43 +++++++++++++++++++ scripts/codex_qa_home.sh | 44 +++++++++++++++++++ 4 files changed, 201 insertions(+), 2 deletions(-) create mode 100755 scripts/codex_qa_home.sh diff --git a/WorldOS-RUNBOOK.md b/WorldOS-RUNBOOK.md index bb886d94..2a1f4278 100644 --- a/WorldOS-RUNBOOK.md +++ b/WorldOS-RUNBOOK.md @@ -248,6 +248,46 @@ critical/high** adversarial defects. defect** — both AI player and DM drift to roleplay, so combat is rarely formally run. The wandering-encounter system + combat-seeking personas force real fights. +### Codex GPT-DM Fair-Test Lane (Mac) + +Use this lane only to answer the provider decision question: **can GPT, when run through +Codex's native tool loop, match the Opus DM quality bar?** It is not an RRI/release gate and +does not use the OpenClaw gateway. The release Opus sweep remains the release signal unless +fresh scored evidence explicitly changes that decision. + +Preflight, from the Mac where Codex CLI is logged in: + +```bash +cd /Users/lume/ClawDnD-val +scripts/codex_qa_home.sh ~/.codex-worldos-qa /Users/lume/ClawDnD-val +CODEX_HOME=~/.codex-worldos-qa codex login status +CODEX_HOME=~/.codex-worldos-qa codex --version +``` + +Codex CLI `>=0.128.0` rejects stale `service_tier = "default"` in `config.toml`; the value +must be absent, `fast`, or `flex`. `qa/support_vm_preflight.py` now records this as +`tools.codex_auth.config` and blocks Codex persona readiness when the effective config is +stale. The effective config is `CODEX_HOME/config.toml` when `CODEX_HOME` is set, otherwise +`~/.codex/config.toml`. + +Fair-test shape: + +- DM provider: `CODEX_HOME=~/.codex-worldos-qa WORLDOS_CODEX_MODEL=gpt-5.5` or `gpt-5.4` + through `scripts/play_codex_dm.sh`, which wires engine/rules/voice MCP per `codex exec -c`. +- Player: Sonnet via the constrained `clawdnd-player` facade, using a combat-seeking persona + when the question is mechanical viability. +- Scoring: Sonnet `qa/score.sh` on Tolkien story and Angry-DM 5e fidelity, plus + `qa/assert_behavioral.py` on the transcoded Codex tool stream. +- Evidence stays private under `/Volumes/LEXAR/Codex`; do not commit raw transcripts, + private art, or credentials. + +Current #691 result on `93df5d2` (private Lexar evidence, 2026-06-07): native Codex GPT is +mechanically capable enough to use real tools, but the scored fair-test runs did **not** +green-light the OpenClaw gateway plugin build. `gpt-5.5` scored Tolkien `3.1`, Angry-DM +`3.5`, behavioral `RED`; `gpt-5.4` scored Tolkien `2.4`, Angry-DM `3.3`, behavioral `RED`. +Therefore #690 remains gated off unless a later same-method rerun reaches the Opus comparison +bar with a GREEN behavioral gate. + --- ## AGENT DELEGATION diff --git a/qa/support_vm_preflight.py b/qa/support_vm_preflight.py index 4bf9f0ac..11e7d1cb 100644 --- a/qa/support_vm_preflight.py +++ b/qa/support_vm_preflight.py @@ -29,6 +29,8 @@ CANONICAL_PERSONAS = ["newbie", "veteran", "adversarial", "narrative", "optimizer"] MIN_SHA_MATCH_CHARS = 7 MIN_CODEX_MCP_OVERRIDE_VERSION = (0, 120, 0) +MIN_CODEX_CONFIG_DRIFT_VERSION = (0, 128, 0) +ALLOWED_CODEX_SERVICE_TIERS = ("fast", "flex") BASE_REQUIRED_TOOLS = [ "git", "python3", @@ -169,6 +171,55 @@ def supports_codex_mcp_overrides(version_text: str) -> bool: return bool(version and version >= MIN_CODEX_MCP_OVERRIDE_VERSION) +def codex_config_path(env: dict[str, str] | None = None) -> Path: + """Return the effective Codex config path without reading or printing secrets.""" + env = env or os.environ + home = (env.get("CODEX_HOME") or "").strip() + if home: + return Path(home).expanduser() / "config.toml" + return Path.home() / ".codex" / "config.toml" + + +def parse_codex_service_tier(config_text: str) -> str: + """Best-effort top-level service_tier parse for the 0.128 config-drift guard.""" + for line in config_text.splitlines(): + stripped = line.strip() + if not stripped or stripped.startswith("#"): + continue + match = re.match(r"""service_tier\s*=\s*(['"]?)([^'"\s#]+)\1""", stripped) + if match: + return match.group(2).strip() + return "" + + +def inspect_codex_config(version_text: str, env: dict[str, str] | None = None) -> dict: + """Guard Codex CLI >=0.128 against stale service_tier='default' config drift.""" + version = parse_semver(version_text) + path = codex_config_path(env) + info = { + "checked": bool(version and version >= MIN_CODEX_CONFIG_DRIFT_VERSION), + "min_version": ".".join(str(part) for part in MIN_CODEX_CONFIG_DRIFT_VERSION), + "path": str(path), + "source": "CODEX_HOME/config.toml" if (env or os.environ).get("CODEX_HOME") else "~/.codex/config.toml", + "present": path.exists(), + "service_tier": "", + "service_tier_allowed": True, + "blocking": False, + } + if not info["checked"] or not path.exists(): + return info + try: + service_tier = parse_codex_service_tier(path.read_text(encoding="utf-8", errors="replace")) + except OSError as exc: + info.update({"read_error": redact(str(exc)), "service_tier_allowed": False, "blocking": True}) + return info + info["service_tier"] = service_tier + if service_tier and service_tier not in ALLOWED_CODEX_SERVICE_TIERS: + info["service_tier_allowed"] = False + info["blocking"] = True + return info + + def run_command(cmd: Sequence[str], cwd: Path | None = None, timeout: int = 8) -> dict: try: proc = subprocess.run( @@ -395,6 +446,7 @@ def inspect_tools( runner: CommandRunner, which: WhichFn, required_tools: Sequence[str], + env: dict[str, str] | None = None, ) -> tuple[dict, list[str], list[str]]: blockers: list[str] = [] warnings: list[str] = [] @@ -472,10 +524,23 @@ def inspect_tools( codex_version = tools.get("codex", {}).get("version") or "" codex["mcp_override_min_version"] = ".".join(str(part) for part in MIN_CODEX_MCP_OVERRIDE_VERSION) codex["mcp_override_supported"] = supports_codex_mcp_overrides(codex_version) if codex_path else False + codex["config"] = inspect_codex_config(codex_version, env) if codex_path else { + "checked": False, + "present": False, + "service_tier": "", + "service_tier_allowed": True, + "blocking": False, + } if codex_required and codex_path and not codex["mcp_override_supported"]: blockers.append( "Codex CLI version does not prove support for codex exec -c mcp_servers.* overrides; require >= 0.120.0" ) + if codex_required and codex_path and codex["config"].get("blocking"): + tier = codex["config"].get("service_tier") or "" + blockers.append( + "Codex CLI config drift: service_tier must be unset, 'fast', or 'flex' for codex-cli >=0.128.0 " + f"(found {tier!r})" + ) if codex_path: codex["auth_status"] = "not_proven" codex["auth_probe_command"] = "codex login status" @@ -712,6 +777,9 @@ def readiness_summary( artifact_return_ready = bool(config.artifact_return_target.strip()) provider_auth_ready = lane_auth_ready(config.provider, tools) player_agent_auth_ready = lane_auth_ready(config.player_agent, tools) + codex_config_ready = True + if config.provider == "codex" or config.player_agent == "codex": + codex_config_ready = not bool((tools.get("codex_auth") or {}).get("config", {}).get("blocking")) host_memory = host.get("memory_total_gb") host_capacity_ready = config.min_memory_gb <= 0 or ( host_memory is not None and host_memory >= config.min_memory_gb @@ -722,6 +790,7 @@ def readiness_summary( "required_tools": required_tools_ready, "provider_auth": provider_auth_ready, "player_agent_auth": player_agent_auth_ready, + "codex_config": codex_config_ready, "persona_briefs": persona_briefs_ready, "private_art": private_art_ready, "artifact_return": artifact_return_ready, @@ -737,6 +806,7 @@ def readiness_summary( "player_agent": config.player_agent, "provider_auth_ready": provider_auth_ready, "player_agent_auth_ready": player_agent_auth_ready, + "codex_config_ready": codex_config_ready, "required_tools_ready": required_tools_ready, "persona_briefs_ready": persona_briefs_ready, "private_art_ready": private_art_ready, @@ -765,7 +835,8 @@ def build_report( required_tools = required_tools_for(config) repo, repo_blockers, repo_warnings = inspect_repo(config.repo, config.expected_sha, runner) - tools, tool_blockers, tool_warnings = inspect_tools(config.repo, runner, which, required_tools) + effective_env = env or dict(os.environ) + tools, tool_blockers, tool_warnings = inspect_tools(config.repo, runner, which, required_tools, effective_env) art, art_blockers, art_warnings = inspect_private_art(config.art_root, config.private_art_mode) repo_files, file_blockers, file_warnings = inspect_required_repo_files( config.repo, @@ -810,7 +881,7 @@ def build_report( host=host, required_tools=required_tools, ), - "environment": env_snapshot(env or dict(os.environ)), + "environment": env_snapshot(effective_env), "rri_plan": { "expected_personas": config.personas, "canonical_personas": CANONICAL_PERSONAS, @@ -879,6 +950,7 @@ def markdown_report(report: dict) -> str: "## Readiness", "", f"- Safe to run personas: `{str(report.get('readiness', {}).get('safe_to_run_personas')).lower()}`", + f"- Codex config ready: `{str(report.get('readiness', {}).get('codex_config_ready')).lower()}`", f"- Blocking categories: `{','.join(report.get('readiness', {}).get('blocking_categories') or []) or 'none'}`", "", "## Blockers", diff --git a/qa/test_support_vm_preflight.py b/qa/test_support_vm_preflight.py index cdc41bad..2ff7881a 100644 --- a/qa/test_support_vm_preflight.py +++ b/qa/test_support_vm_preflight.py @@ -166,6 +166,48 @@ def test_build_sha_matches_requires_seven_characters(self): self.assertTrue(preflight.build_sha_matches("deadbeefcafebabe", "deadbee")) self.assertTrue(preflight.build_sha_matches("deadbee", "deadbeefcafebabe")) + def test_codex_service_tier_parser_ignores_commented_default(self): + text = '\n# service_tier = "default"\nservice_tier = "fast"\n' + self.assertEqual(preflight.parse_codex_service_tier(text), "fast") + + def test_codex_cli_0128_blocks_stale_default_service_tier(self): + with tempfile.TemporaryDirectory() as td: + config = make_config(Path(td)) + codex_home = Path(td) / "codex-home" + codex_home.mkdir() + (codex_home / "config.toml").write_text('service_tier = "default"\n', encoding="utf-8") + + report = preflight.build_report( + config, + runner=FakeRunner(config.repo, codex_version="codex-cli 0.128.0"), + which=fake_which, + env={"CODEX_HOME": str(codex_home)}, + ) + + self.assertFalse(report["ready_for_rri"]) + self.assertFalse(report["readiness"]["codex_config_ready"]) + self.assertIn("codex_config", report["readiness"]["blocking_categories"]) + self.assertIn("service_tier", "\n".join(report["blockers"])) + self.assertEqual(report["tools"]["codex_auth"]["config"]["service_tier"], "default") + + def test_codex_cli_0128_allows_fast_or_unset_service_tier(self): + with tempfile.TemporaryDirectory() as td: + config = make_config(Path(td)) + codex_home = Path(td) / "codex-home" + codex_home.mkdir() + (codex_home / "config.toml").write_text('service_tier = "flex"\n', encoding="utf-8") + + report = preflight.build_report( + config, + runner=FakeRunner(config.repo, codex_version="codex-cli 0.128.0"), + which=fake_which, + env={"CODEX_HOME": str(codex_home)}, + ) + + self.assertTrue(report["ready_for_rri"]) + self.assertTrue(report["readiness"]["codex_config_ready"]) + self.assertEqual(report["tools"]["codex_auth"]["config"]["service_tier"], "flex") + def test_private_art_required_blocks_and_optional_warns(self): with tempfile.TemporaryDirectory() as td: missing_root = Path(td) / "missing-art" @@ -261,6 +303,7 @@ def test_report_includes_redacted_readiness_summary_for_agent_routing(self): self.assertTrue(readiness["same_sha_ready"]) self.assertTrue(readiness["provider_auth_ready"]) self.assertTrue(readiness["player_agent_auth_ready"]) + self.assertTrue(readiness["codex_config_ready"]) self.assertTrue(readiness["required_tools_ready"]) self.assertTrue(readiness["persona_briefs_ready"]) self.assertTrue(readiness["private_art_ready"]) diff --git a/scripts/codex_qa_home.sh b/scripts/codex_qa_home.sh new file mode 100755 index 00000000..7fe6f93e --- /dev/null +++ b/scripts/codex_qa_home.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# Build the isolated Codex home used by the WorldOS GPT-DM fair-test lane. +# +# This restores a lean Codex CLI environment after --ignore-user-config was +# removed. It writes only a minimal config and symlinks the operator's existing +# auth.json in place. It never copies credentials. +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +DEST="${1:-$HOME/.codex-worldos-qa}" +REPO="${2:-$ROOT}" + +mkdir -p "$DEST" + +python3 - "$DEST/config.toml" "$REPO" <<'PY' +import json +import sys +from pathlib import Path + +out = Path(sys.argv[1]).expanduser() +repo = str(Path(sys.argv[2]).expanduser().resolve(strict=False)) +out.write_text( + "\n".join( + [ + 'approval_policy = "never"', + 'sandbox_mode = "read-only"', + "", + f"[projects.{json.dumps(repo)}]", + 'trust_level = "trusted"', + "", + ] + ), + encoding="utf-8", +) +PY + +if [ -f "$HOME/.codex/auth.json" ]; then + ln -sf "$HOME/.codex/auth.json" "$DEST/auth.json" +else + echo "[codex-qa-home] warning: $HOME/.codex/auth.json not found; run codex login before fair-test runs" >&2 +fi + +echo "CODEX_HOME=$DEST" +echo "Run: CODEX_HOME=$DEST codex login status" From 693371a77f56dde0a855601e63337d91d71479a0 Mon Sep 17 00:00:00 2001 From: Eva Date: Sun, 7 Jun 2026 04:42:08 +0700 Subject: [PATCH 2/2] fix: tighten Codex config preflight guard --- WorldOS-RUNBOOK.md | 3 +- qa/support_vm_preflight.py | 10 +++++-- qa/test_support_vm_preflight.py | 53 ++++++++++++++++++++++----------- 3 files changed, 45 insertions(+), 21 deletions(-) diff --git a/WorldOS-RUNBOOK.md b/WorldOS-RUNBOOK.md index 2a1f4278..3467affd 100644 --- a/WorldOS-RUNBOOK.md +++ b/WorldOS-RUNBOOK.md @@ -281,7 +281,8 @@ Fair-test shape: - Evidence stays private under `/Volumes/LEXAR/Codex`; do not commit raw transcripts, private art, or credentials. -Current #691 result on `93df5d2` (private Lexar evidence, 2026-06-07): native Codex GPT is +Current #691 result on `93df5d2` (private Lexar evidence from 2026-06-06 UTC / 2026-06-07 +local): native Codex GPT is mechanically capable enough to use real tools, but the scored fair-test runs did **not** green-light the OpenClaw gateway plugin build. `gpt-5.5` scored Tolkien `3.1`, Angry-DM `3.5`, behavioral `RED`; `gpt-5.4` scored Tolkien `2.4`, Angry-DM `3.3`, behavioral `RED`. diff --git a/qa/support_vm_preflight.py b/qa/support_vm_preflight.py index 11e7d1cb..2c5a8a29 100644 --- a/qa/support_vm_preflight.py +++ b/qa/support_vm_preflight.py @@ -173,7 +173,8 @@ def supports_codex_mcp_overrides(version_text: str) -> bool: def codex_config_path(env: dict[str, str] | None = None) -> Path: """Return the effective Codex config path without reading or printing secrets.""" - env = env or os.environ + if env is None: + env = os.environ home = (env.get("CODEX_HOME") or "").strip() if home: return Path(home).expanduser() / "config.toml" @@ -186,6 +187,8 @@ def parse_codex_service_tier(config_text: str) -> str: stripped = line.strip() if not stripped or stripped.startswith("#"): continue + if re.match(r"^\s*\[.*\]", line): + break match = re.match(r"""service_tier\s*=\s*(['"]?)([^'"\s#]+)\1""", stripped) if match: return match.group(2).strip() @@ -195,12 +198,13 @@ def parse_codex_service_tier(config_text: str) -> str: def inspect_codex_config(version_text: str, env: dict[str, str] | None = None) -> dict: """Guard Codex CLI >=0.128 against stale service_tier='default' config drift.""" version = parse_semver(version_text) + effective_env = os.environ if env is None else env path = codex_config_path(env) info = { "checked": bool(version and version >= MIN_CODEX_CONFIG_DRIFT_VERSION), "min_version": ".".join(str(part) for part in MIN_CODEX_CONFIG_DRIFT_VERSION), "path": str(path), - "source": "CODEX_HOME/config.toml" if (env or os.environ).get("CODEX_HOME") else "~/.codex/config.toml", + "source": "CODEX_HOME/config.toml" if effective_env.get("CODEX_HOME") else "~/.codex/config.toml", "present": path.exists(), "service_tier": "", "service_tier_allowed": True, @@ -835,7 +839,7 @@ def build_report( required_tools = required_tools_for(config) repo, repo_blockers, repo_warnings = inspect_repo(config.repo, config.expected_sha, runner) - effective_env = env or dict(os.environ) + effective_env = dict(os.environ) if env is None else env tools, tool_blockers, tool_warnings = inspect_tools(config.repo, runner, which, required_tools, effective_env) art, art_blockers, art_warnings = inspect_private_art(config.art_root, config.private_art_mode) repo_files, file_blockers, file_warnings = inspect_required_repo_files( diff --git a/qa/test_support_vm_preflight.py b/qa/test_support_vm_preflight.py index 2ff7881a..47f9b20a 100644 --- a/qa/test_support_vm_preflight.py +++ b/qa/test_support_vm_preflight.py @@ -1,7 +1,9 @@ import json +import os import tempfile import unittest from pathlib import Path +from unittest import mock import qa.support_vm_preflight as preflight @@ -170,6 +172,16 @@ def test_codex_service_tier_parser_ignores_commented_default(self): text = '\n# service_tier = "default"\nservice_tier = "fast"\n' self.assertEqual(preflight.parse_codex_service_tier(text), "fast") + def test_codex_service_tier_parser_stops_at_first_table(self): + text = '\n[profiles.default]\nservice_tier = "default"\n' + self.assertEqual(preflight.parse_codex_service_tier(text), "") + + def test_codex_config_path_respects_explicit_empty_env(self): + with tempfile.TemporaryDirectory() as td: + host_home = Path(td) / "host-codex-home" + with mock.patch.dict(os.environ, {"CODEX_HOME": str(host_home)}, clear=False): + self.assertEqual(preflight.codex_config_path({}), Path.home() / ".codex" / "config.toml") + def test_codex_cli_0128_blocks_stale_default_service_tier(self): with tempfile.TemporaryDirectory() as td: config = make_config(Path(td)) @@ -190,23 +202,30 @@ def test_codex_cli_0128_blocks_stale_default_service_tier(self): self.assertIn("service_tier", "\n".join(report["blockers"])) self.assertEqual(report["tools"]["codex_auth"]["config"]["service_tier"], "default") - def test_codex_cli_0128_allows_fast_or_unset_service_tier(self): - with tempfile.TemporaryDirectory() as td: - config = make_config(Path(td)) - codex_home = Path(td) / "codex-home" - codex_home.mkdir() - (codex_home / "config.toml").write_text('service_tier = "flex"\n', encoding="utf-8") - - report = preflight.build_report( - config, - runner=FakeRunner(config.repo, codex_version="codex-cli 0.128.0"), - which=fake_which, - env={"CODEX_HOME": str(codex_home)}, - ) - - self.assertTrue(report["ready_for_rri"]) - self.assertTrue(report["readiness"]["codex_config_ready"]) - self.assertEqual(report["tools"]["codex_auth"]["config"]["service_tier"], "flex") + def test_codex_cli_0128_allows_fast_flex_or_unset_service_tier(self): + cases = ( + ("fast", 'service_tier = "fast"\n'), + ("flex", 'service_tier = "flex"\n'), + ("", ""), + ) + for expected_tier, config_text in cases: + with self.subTest(expected_tier=expected_tier or "unset"): + with tempfile.TemporaryDirectory() as td: + config = make_config(Path(td)) + codex_home = Path(td) / "codex-home" + codex_home.mkdir() + (codex_home / "config.toml").write_text(config_text, encoding="utf-8") + + report = preflight.build_report( + config, + runner=FakeRunner(config.repo, codex_version="codex-cli 0.128.0"), + which=fake_which, + env={"CODEX_HOME": str(codex_home)}, + ) + + self.assertTrue(report["ready_for_rri"]) + self.assertTrue(report["readiness"]["codex_config_ready"]) + self.assertEqual(report["tools"]["codex_auth"]["config"]["service_tier"], expected_tier) def test_private_art_required_blocks_and_optional_warns(self): with tempfile.TemporaryDirectory() as td: