diff --git a/docs/dogfood/coding-tools-dogfood-transcript.json b/docs/dogfood/coding-tools-dogfood-transcript.json index 762c9bf..af8c661 100644 --- a/docs/dogfood/coding-tools-dogfood-transcript.json +++ b/docs/dogfood/coding-tools-dogfood-transcript.json @@ -2,10 +2,10 @@ "cases": [ { "checks": [ - "PASS search_text finds add: {\"matches\": [{\"after\": [], \"before\": [], \"column\": 1, \"line\": 1, \"path\": \"tiny-js-project/src/math.js\", \"preview\": \"function add(a, b) {\"}], \"ok\": true, \"query\": \"function add\",...", + "PASS search_text finds add: {\"engine\": \"rg\", \"matches\": [{\"after\": [], \"before\": [], \"column\": 1, \"line\": 1, \"path\": \"tiny-js-project/src/math.js\", \"preview\": \"function add(a, b) {\"}], \"ok\": true, \"query\":...", "PASS read_file returns buggy source: {\"bytes_read\": 66, \"content\": \"function add(a, b) {\\n return a - b;\\n}\\n\\nmodule.exports = { add };\\n\", \"encoding\": \"utf-8\", \"end_line\": 5, \"next_start_line\": null, \"ok\": true,...", "PASS apply_patch fixes add: {\"affected_files\": [{\"operation\": \"update\", \"path\": \"tiny-js-project/src/math.js\"}], \"clean\": true, \"dry_run\": false, \"ok\": true, \"summary\": \"M tiny-js-project/src/math.js\", \"wa...", - "PASS exec_command npm test passes: {\"elapsed_ms\": 713, \"exit_code\": 0, \"ok\": true, \"session_id\": \"etdyVu5lHMjRrpZrl8Rjl-Vq\", \"signal\": null, \"status\": \"exited\", \"stderr\": \"\", \"stderr_dropped_bytes\": 0, \"stderr_om...", + "PASS exec_command npm test passes: {\"elapsed_ms\": 424, \"exit_code\": 0, \"ok\": true, \"session_id\": \"eIwxXJOF1lUzP_OjOxSXY8qL\", \"signal\": null, \"status\": \"exited\", \"stderr\": \"\", \"stderr_dropped_bytes\": 0, \"stderr_om...", "PASS git_diff shows only math.js fix: {\"diff\": \"--- a/tiny-js-project/src/math.js\\n+++ b/tiny-js-project/src/math.js\\n@@ -1,5 +1,5 @@\\n function add(a, b) {\\n\\n- return a - b;\\n\\n+ return a + b;\\n\\n }\\n\\n \\n\\n mod..." ], "failures": [], @@ -16,7 +16,7 @@ "checks": [ "PASS read_file returns python source: {\"bytes_read\": 32, \"content\": \"def add(a, b):\\n return a + b\\n\", \"encoding\": \"utf-8\", \"end_line\": 2, \"next_start_line\": null, \"ok\": true, \"output_bytes\": 32, \"output_lines\": ...", "PASS apply_patch adds multiply: {\"affected_files\": [{\"operation\": \"update\", \"path\": \"tiny-python-project/src/math_utils.py\"}], \"clean\": true, \"dry_run\": false, \"ok\": true, \"summary\": \"M tiny-python-project/src...", - "PASS exec_command unittest passes: {\"elapsed_ms\": 136, \"exit_code\": 0, \"ok\": true, \"session_id\": \"Jbsd8Ibepw2fEvXSseToUcOi\", \"signal\": null, \"status\": \"exited\", \"stderr\": \"..\\n------------------------------------...", + "PASS exec_command unittest passes: {\"elapsed_ms\": 61, \"exit_code\": 0, \"ok\": true, \"session_id\": \"0hxF-ORjDM4rVSjpPJy0UkEp\", \"signal\": null, \"status\": \"exited\", \"stderr\": \"..\\n-------------------------------------...", "PASS git_diff shows multiply: {\"diff\": \"--- a/tiny-python-project/src/math_utils.py\\n+++ b/tiny-python-project/src/math_utils.py\\n@@ -1,2 +1,6 @@\\n def add(a, b):\\n\\n return a + b\\n\\n+\\n\\n+\\n\\n+def multi..." ], "failures": [], @@ -25,10 +25,10 @@ }, { "checks": [ - "PASS exec_command returns session_id: {\"elapsed_ms\": 94, \"exit_code\": null, \"ok\": true, \"session_id\": \"Y-zeUu11DSAZBpwWXa6QMhT0\", \"signal\": null, \"status\": \"running\", \"stderr\": \"\", \"stderr_dropped_bytes\": 0, \"stderr...", - "PASS write_stdin accepts hello: {\"exit_code\": null, \"ok\": true, \"session_id\": \"Y-zeUu11DSAZBpwWXa6QMhT0\", \"signal\": null, \"status\": \"running\", \"stderr\": \"\", \"stderr_dropped_bytes\": 0, \"stderr_omitted_bytes\": 0...", - "PASS write_stdin accepts exit: {\"exit_code\": 0, \"ok\": true, \"session_id\": \"Y-zeUu11DSAZBpwWXa6QMhT0\", \"signal\": null, \"status\": \"exited\", \"stderr\": \"\", \"stderr_dropped_bytes\": 0, \"stderr_omitted_bytes\": 0, \"s...", - "PASS kill_session terminates or reports already closed: {\"exit_code\": 0, \"killed\": false, \"ok\": true, \"session_id\": \"Y-zeUu11DSAZBpwWXa6QMhT0\", \"signal\": null, \"status\": \"exited\", \"stderr\": \"\", \"stderr_dropped_bytes\": 0, \"stderr_omit..." + "PASS exec_command returns session_id: {\"elapsed_ms\": 41, \"exit_code\": null, \"ok\": true, \"session_id\": \"r3j54TRIhYCbG_9_W0ngb4CW\", \"signal\": null, \"status\": \"running\", \"stderr\": \"\", \"stderr_dropped_bytes\": 0, \"stderr...", + "PASS write_stdin accepts hello: {\"exit_code\": null, \"ok\": true, \"session_id\": \"r3j54TRIhYCbG_9_W0ngb4CW\", \"signal\": null, \"status\": \"running\", \"stderr\": \"\", \"stderr_dropped_bytes\": 0, \"stderr_omitted_bytes\": 0...", + "PASS write_stdin accepts exit: {\"exit_code\": 0, \"ok\": true, \"session_id\": \"r3j54TRIhYCbG_9_W0ngb4CW\", \"signal\": null, \"status\": \"exited\", \"stderr\": \"\", \"stderr_dropped_bytes\": 0, \"stderr_omitted_bytes\": 0, \"s...", + "PASS kill_session terminates or reports already closed: {\"exit_code\": 0, \"killed\": false, \"ok\": true, \"session_id\": \"r3j54TRIhYCbG_9_W0ngb4CW\", \"signal\": null, \"status\": \"exited\", \"stderr\": \"\", \"stderr_dropped_bytes\": 0, \"stderr_omit..." ], "failures": [], "name": "long_running_stdin", @@ -55,7 +55,7 @@ }, "expected_rejection": false, "ok": true, - "summary": "{\"matches\": [{\"after\": [], \"before\": [], \"column\": 1, \"line\": 1, \"path\": \"tiny-js-project/src/math.js\", \"preview\": \"function add(a, b) {\"}], \"ok\": true, \"query\": \"function add\",...", + "summary": "{\"engine\": \"rg\", \"matches\": [{\"after\": [], \"before\": [], \"column\": 1, \"line\": 1, \"path\": \"tiny-js-project/src/math.js\", \"preview\": \"function add(a, b) {\"}], \"ok\": true, \"query\":...", "tool": "search_text" }, { @@ -87,7 +87,7 @@ }, "expected_rejection": false, "ok": true, - "summary": "{\"elapsed_ms\": 713, \"exit_code\": 0, \"ok\": true, \"session_id\": \"etdyVu5lHMjRrpZrl8Rjl-Vq\", \"signal\": null, \"status\": \"exited\", \"stderr\": \"\", \"stderr_dropped_bytes\": 0, \"stderr_om...", + "summary": "{\"elapsed_ms\": 424, \"exit_code\": 0, \"ok\": true, \"session_id\": \"eIwxXJOF1lUzP_OjOxSXY8qL\", \"signal\": null, \"status\": \"exited\", \"stderr\": \"\", \"stderr_dropped_bytes\": 0, \"stderr_om...", "tool": "exec_command" }, { @@ -122,7 +122,7 @@ }, { "arguments": { - "cmd": "/home/codespace/.python/current/bin/python3 -m unittest discover -s tests", + "cmd": "/usr/local/bin/python3 -m unittest discover -s tests", "max_output_bytes": 40000, "timeout_ms": 20000, "tty": false, @@ -131,7 +131,7 @@ }, "expected_rejection": false, "ok": true, - "summary": "{\"elapsed_ms\": 136, \"exit_code\": 0, \"ok\": true, \"session_id\": \"Jbsd8Ibepw2fEvXSseToUcOi\", \"signal\": null, \"status\": \"exited\", \"stderr\": \"..\\n------------------------------------...", + "summary": "{\"elapsed_ms\": 61, \"exit_code\": 0, \"ok\": true, \"session_id\": \"0hxF-ORjDM4rVSjpPJy0UkEp\", \"signal\": null, \"status\": \"exited\", \"stderr\": \"..\\n-------------------------------------...", "tool": "exec_command" }, { @@ -148,7 +148,7 @@ }, { "arguments": { - "cmd": "/home/codespace/.python/current/bin/python3 repl.py", + "cmd": "/usr/local/bin/python3 repl.py", "max_output_bytes": 40000, "timeout_ms": 30000, "tty": true, @@ -157,36 +157,36 @@ }, "expected_rejection": false, "ok": true, - "summary": "{\"elapsed_ms\": 94, \"exit_code\": null, \"ok\": true, \"session_id\": \"Y-zeUu11DSAZBpwWXa6QMhT0\", \"signal\": null, \"status\": \"running\", \"stderr\": \"\", \"stderr_dropped_bytes\": 0, \"stderr...", + "summary": "{\"elapsed_ms\": 41, \"exit_code\": null, \"ok\": true, \"session_id\": \"r3j54TRIhYCbG_9_W0ngb4CW\", \"signal\": null, \"status\": \"running\", \"stderr\": \"\", \"stderr_dropped_bytes\": 0, \"stderr...", "tool": "exec_command" }, { "arguments": { "chars": "hello\n", - "session_id": "Y-zeUu11DSAZBpwWXa6QMhT0" + "session_id": "r3j54TRIhYCbG_9_W0ngb4CW" }, "expected_rejection": false, "ok": true, - "summary": "{\"exit_code\": null, \"ok\": true, \"session_id\": \"Y-zeUu11DSAZBpwWXa6QMhT0\", \"signal\": null, \"status\": \"running\", \"stderr\": \"\", \"stderr_dropped_bytes\": 0, \"stderr_omitted_bytes\": 0...", + "summary": "{\"exit_code\": null, \"ok\": true, \"session_id\": \"r3j54TRIhYCbG_9_W0ngb4CW\", \"signal\": null, \"status\": \"running\", \"stderr\": \"\", \"stderr_dropped_bytes\": 0, \"stderr_omitted_bytes\": 0...", "tool": "write_stdin" }, { "arguments": { "chars": "exit\n", - "session_id": "Y-zeUu11DSAZBpwWXa6QMhT0" + "session_id": "r3j54TRIhYCbG_9_W0ngb4CW" }, "expected_rejection": false, "ok": true, - "summary": "{\"exit_code\": 0, \"ok\": true, \"session_id\": \"Y-zeUu11DSAZBpwWXa6QMhT0\", \"signal\": null, \"status\": \"exited\", \"stderr\": \"\", \"stderr_dropped_bytes\": 0, \"stderr_omitted_bytes\": 0, \"s...", + "summary": "{\"exit_code\": 0, \"ok\": true, \"session_id\": \"r3j54TRIhYCbG_9_W0ngb4CW\", \"signal\": null, \"status\": \"exited\", \"stderr\": \"\", \"stderr_dropped_bytes\": 0, \"stderr_omitted_bytes\": 0, \"s...", "tool": "write_stdin" }, { "arguments": { - "session_id": "Y-zeUu11DSAZBpwWXa6QMhT0" + "session_id": "r3j54TRIhYCbG_9_W0ngb4CW" }, "expected_rejection": true, "ok": true, - "summary": "{\"exit_code\": 0, \"killed\": false, \"ok\": true, \"session_id\": \"Y-zeUu11DSAZBpwWXa6QMhT0\", \"signal\": null, \"status\": \"exited\", \"stderr\": \"\", \"stderr_dropped_bytes\": 0, \"stderr_omit...", + "summary": "{\"exit_code\": 0, \"killed\": false, \"ok\": true, \"session_id\": \"r3j54TRIhYCbG_9_W0ngb4CW\", \"signal\": null, \"status\": \"exited\", \"stderr\": \"\", \"stderr_dropped_bytes\": 0, \"stderr_omit...", "tool": "kill_session" }, { @@ -228,5 +228,5 @@ "tool": "git_diff" } ], - "workspace": "/tmp/codex-mcp-dogfood-xf95irl4/workspace" + "workspace": "/tmp/codex-mcp-dogfood-yqbv5f87/workspace" } diff --git a/reports/benchmark/mcp-latency.json b/reports/benchmark/mcp-latency.json index b6500ef..df75dcb 100644 --- a/reports/benchmark/mcp-latency.json +++ b/reports/benchmark/mcp-latency.json @@ -1,78 +1,78 @@ { "comparisons": [ { - "mcp_p95_ms": 0.764, - "native_p95_ms": 0.06, + "mcp_p95_ms": 1.264, + "native_p95_ms": 0.015, "operation": "read_file", - "p95_ratio": 12.733 + "p95_ratio": 84.267 }, { - "mcp_p95_ms": 63.788, - "native_p95_ms": 4.167, + "mcp_p95_ms": 6.827, + "native_p95_ms": 4.531, "operation": "search_text", - "p95_ratio": 15.308 + "p95_ratio": 1.507 }, { - "mcp_p95_ms": 46.678, - "native_p95_ms": 23.871, + "mcp_p95_ms": 22.947, + "native_p95_ms": 2.162, "operation": "exec_command", - "p95_ratio": 1.955 + "p95_ratio": 10.614 } ], "conclusion": "PASS", - "endpoint": "http://127.0.0.1:54941/mcp", + "endpoint": "http://127.0.0.1:52187/mcp", "exec_iterations": 4, "failures": [], "iterations": 8, "max_p95_ms": 5000, "metrics": { "mcp.exec_command": { - "max_ms": 46.68, - "min_ms": 46.139, - "p50_ms": 46.632, - "p95_ms": 46.678, + "max_ms": 22.95, + "min_ms": 22.828, + "p50_ms": 22.893, + "p95_ms": 22.947, "samples": 4 }, "mcp.read_file": { - "max_ms": 0.769, - "min_ms": 0.651, - "p50_ms": 0.734, - "p95_ms": 0.764, + "max_ms": 1.266, + "min_ms": 1.066, + "p50_ms": 1.125, + "p95_ms": 1.264, "samples": 8 }, "mcp.search_text": { - "max_ms": 63.811, - "min_ms": 57.391, - "p50_ms": 60.186, - "p95_ms": 63.788, + "max_ms": 6.841, + "min_ms": 6.371, + "p50_ms": 6.579, + "p95_ms": 6.827, "samples": 8 }, "mcp.tools_list": { - "max_ms": 1.129, - "min_ms": 1.028, - "p50_ms": 1.074, - "p95_ms": 1.128, + "max_ms": 1.927, + "min_ms": 1.569, + "p50_ms": 1.757, + "p95_ms": 1.896, "samples": 8 }, - "native.exec_python": { - "max_ms": 23.898, - "min_ms": 23.657, - "p50_ms": 23.702, - "p95_ms": 23.871, + "native.exec_command": { + "max_ms": 2.162, + "min_ms": 1.046, + "p50_ms": 2.147, + "p95_ms": 2.162, "samples": 4 }, "native.read_text": { - "max_ms": 0.066, - "min_ms": 0.028, - "p50_ms": 0.043, - "p95_ms": 0.06, + "max_ms": 0.015, + "min_ms": 0.011, + "p50_ms": 0.013, + "p95_ms": 0.015, "samples": 8 }, "native.search": { - "max_ms": 4.17, - "min_ms": 4.034, - "p50_ms": 4.063, - "p95_ms": 4.167, + "max_ms": 4.593, + "min_ms": 4.217, + "p50_ms": 4.368, + "p95_ms": 4.531, "samples": 8 } }, @@ -81,5 +81,5 @@ "Latency thresholds are intentionally broad; this smoke benchmark catches transport regressions and records trend evidence." ], "warmup": 2, - "workspace": "/tmp/codex-mcp-latency-m468wc1p/workspace" + "workspace": "/tmp/codex-mcp-latency-c4oq7h2v/workspace" } diff --git a/reports/benchmark/mcp-latency.md b/reports/benchmark/mcp-latency.md index 420b013..10c2c70 100644 --- a/reports/benchmark/mcp-latency.md +++ b/reports/benchmark/mcp-latency.md @@ -1,7 +1,7 @@ # MCP Runtime Latency Benchmark - Conclusion: **PASS** -- Endpoint: `http://127.0.0.1:54941/mcp` +- Endpoint: `http://127.0.0.1:52187/mcp` - Iterations: `8` - Exec iterations: `4` - Warmup iterations: `2` @@ -11,21 +11,21 @@ | metric | samples | min ms | p50 ms | p95 ms | max ms | | --- | ---: | ---: | ---: | ---: | ---: | -| `mcp.tools_list` | 8 | 1.028 | 1.074 | 1.128 | 1.129 | -| `mcp.read_file` | 8 | 0.651 | 0.734 | 0.764 | 0.769 | -| `mcp.search_text` | 8 | 57.391 | 60.186 | 63.788 | 63.811 | -| `mcp.exec_command` | 4 | 46.139 | 46.632 | 46.678 | 46.68 | -| `native.read_text` | 8 | 0.028 | 0.043 | 0.06 | 0.066 | -| `native.search` | 8 | 4.034 | 4.063 | 4.167 | 4.17 | -| `native.exec_python` | 4 | 23.657 | 23.702 | 23.871 | 23.898 | +| `mcp.tools_list` | 8 | 1.569 | 1.757 | 1.896 | 1.927 | +| `mcp.read_file` | 8 | 1.066 | 1.125 | 1.264 | 1.266 | +| `mcp.search_text` | 8 | 6.371 | 6.579 | 6.827 | 6.841 | +| `mcp.exec_command` | 4 | 22.828 | 22.893 | 22.947 | 22.95 | +| `native.read_text` | 8 | 0.011 | 0.013 | 0.015 | 0.015 | +| `native.search` | 8 | 4.217 | 4.368 | 4.531 | 4.593 | +| `native.exec_command` | 4 | 1.046 | 2.147 | 2.162 | 2.162 | ## Native Baseline Comparison | operation | MCP p95 ms | native p95 ms | ratio | | --- | ---: | ---: | ---: | -| `read_file` | 0.764 | 0.06 | 12.733 | -| `search_text` | 63.788 | 4.167 | 15.308 | -| `exec_command` | 46.678 | 23.871 | 1.955 | +| `read_file` | 1.264 | 0.015 | 84.267 | +| `search_text` | 6.827 | 4.531 | 1.507 | +| `exec_command` | 22.947 | 2.162 | 10.614 | ## Failures diff --git a/reports/benchmark/swebench-regression.json b/reports/benchmark/swebench-regression.json index edc165e..b88b7ec 100644 --- a/reports/benchmark/swebench-regression.json +++ b/reports/benchmark/swebench-regression.json @@ -2,17 +2,17 @@ "baseline": { "artifacts": [], "command": [ - "/root/venv/bin/python3", + "/usr/local/bin/python3", "-m", "swebench.harness.run_evaluation", "--dataset_name", "princeton-nlp/SWE-bench_Lite", "--predictions_path", - "/root/codex-tool-runtime-mcp/benchmarks/swebench/predictions/baseline_native.jsonl", + "/home/user/coding-tools-mcp/benchmarks/swebench/predictions/baseline_native.jsonl", "--max_workers", "2", "--run_id", - "codex_tool_runtime_native_smoke", + "coding_tools_native_smoke", "--instance_ids", "astropy__astropy-12907", "django__django-11099", @@ -33,24 +33,24 @@ "model_names": [ "baseline_native_scaffold" ], - "path": "/root/codex-tool-runtime-mcp/benchmarks/swebench/predictions/baseline_native.jsonl", + "path": "/home/user/coding-tools-mcp/benchmarks/swebench/predictions/baseline_native.jsonl", "placeholder": true, "report_paths": [], "resolved": null, "resolved_ids": [], "run": { "command": [ - "/root/venv/bin/python3", + "/usr/local/bin/python3", "-m", "swebench.harness.run_evaluation", "--dataset_name", "princeton-nlp/SWE-bench_Lite", "--predictions_path", - "/root/codex-tool-runtime-mcp/benchmarks/swebench/predictions/baseline_native.jsonl", + "/home/user/coding-tools-mcp/benchmarks/swebench/predictions/baseline_native.jsonl", "--max_workers", "2", "--run_id", - "codex_tool_runtime_native_smoke", + "coding_tools_native_smoke", "--instance_ids", "astropy__astropy-12907", "django__django-11099", @@ -73,17 +73,17 @@ "candidate": { "artifacts": [], "command": [ - "/root/venv/bin/python3", + "/usr/local/bin/python3", "-m", "swebench.harness.run_evaluation", "--dataset_name", "princeton-nlp/SWE-bench_Lite", "--predictions_path", - "/root/codex-tool-runtime-mcp/benchmarks/swebench/predictions/candidate_mcp.jsonl", + "/home/user/coding-tools-mcp/benchmarks/swebench/predictions/candidate_mcp.jsonl", "--max_workers", "2", "--run_id", - "codex_tool_runtime_mcp_smoke", + "coding_tools_mcp_smoke", "--instance_ids", "astropy__astropy-12907", "django__django-11099", @@ -104,24 +104,24 @@ "model_names": [ "candidate_mcp_scaffold" ], - "path": "/root/codex-tool-runtime-mcp/benchmarks/swebench/predictions/candidate_mcp.jsonl", + "path": "/home/user/coding-tools-mcp/benchmarks/swebench/predictions/candidate_mcp.jsonl", "placeholder": true, "report_paths": [], "resolved": null, "resolved_ids": [], "run": { "command": [ - "/root/venv/bin/python3", + "/usr/local/bin/python3", "-m", "swebench.harness.run_evaluation", "--dataset_name", "princeton-nlp/SWE-bench_Lite", "--predictions_path", - "/root/codex-tool-runtime-mcp/benchmarks/swebench/predictions/candidate_mcp.jsonl", + "/home/user/coding-tools-mcp/benchmarks/swebench/predictions/candidate_mcp.jsonl", "--max_workers", "2", "--run_id", - "codex_tool_runtime_mcp_smoke", + "coding_tools_mcp_smoke", "--instance_ids", "astropy__astropy-12907", "django__django-11099", @@ -144,15 +144,19 @@ "conclusion": "PREFLIGHT_ONLY", "dataset_name": "princeton-nlp/SWE-bench_Lite", "docker": { - "ran": false, - "returncode": null, - "stderr": "", - "stdout": "" + "command": [ + "/usr/bin/docker", + "version" + ], + "ran": true, + "returncode": 1, + "stderr": "failed to connect to the docker API at unix:///var/run/docker.sock; check if the path is correct and if the daemon is running: dial unix /var/run/docker.sock: connect: no such file or directory\n", + "stdout": "Client: Docker Engine - Community\n Version: 29.3.1\n API version: 1.54\n Go version: go1.25.8\n Git commit: c2be9cc\n Built: Wed Mar 25 16:13:43 2026\n OS/Arch: linux/amd64\n Context: default\n" }, "environment": { - "cwd": "/root/codex-tool-runtime-mcp", + "cwd": "/home/user/coding-tools-mcp", "executables": { - "docker": null, + "docker": "/usr/bin/docker", "git": "/usr/bin/git" }, "github": { @@ -169,9 +173,9 @@ "docker": "7.1.0", "swebench": "4.1.0" }, - "platform": "Linux-6.18.24-talos-x86_64-with-glibc2.36", - "python": "3.11.14 (main, Feb 20 2026, 09:55:14) [GCC 12.2.0]", - "python_executable": "/root/venv/bin/python3" + "platform": "Linux-6.18.5-x86_64-with-glibc2.39", + "python": "3.11.15 (main, Mar 3 2026, 09:26:23) [GCC 13.3.0]", + "python_executable": "/usr/local/bin/python3" }, "instances": [ { @@ -227,29 +231,28 @@ ], "limitations": [ "Prediction files are schema-valid placeholders, not model-generated patches.", - "Official SWE-bench evaluation requires a working Docker daemon.", - "Official SWE-bench evaluation requires an importable swebench harness." + "Official SWE-bench evaluation requires a working Docker daemon." ], "preflight": [ - "docker: missing - docker executable not found", - "swebench package: missing - swebench harness help/import failed", + "docker: missing - docker daemon unavailable: failed to connect to the docker API at unix:///var/run/docker.sock; check if the path is correct and if the daemon is running: dial unix /var/run/docker.sock: connect: no such file or directory", + "swebench package: ok - swebench harness help succeeded", "baseline predictions: 10 rows, placeholder=True", "candidate predictions: 10 rows, placeholder=True" ], "raw_dir": "reports/benchmark/swebench-regression/raw", "split": "test", - "subset_path": "/root/codex-tool-runtime-mcp/benchmarks/swebench/subsets/smoke-lite-10.json", + "subset_path": "/home/user/coding-tools-mcp/benchmarks/swebench/subsets/smoke-lite-10.json", "swebench_help": { "command": [ - "/root/venv/bin/python3", + "/usr/local/bin/python3", "-m", "swebench.harness.run_evaluation", "--help" ], "ran": true, - "returncode": 1, - "stderr": "2026-05-17 23:27:49,067 - datasets - INFO - TensorFlow version 2.15.1 available.\nTraceback (most recent call last):\n File \"\", line 189, in _run_module_as_main\n File \"\", line 112, in _get_module_details\n File \"/root/venv/lib/python3.11/site-packages/swebench/__init__.py\", line 7, in \n from swebench.harness.constants import (\n File \"/root/venv/lib/python3.11/site-packages/swebench/harness/__init__.py\", line 1, in \n from swebench.harness import (\n File \"/root/venv/lib/python3.11/site-packages/swebench/harness/docker_build.py\", line 19, in \n from swebench.harness.test_spec.test_spec import (\n File \"/root/venv/lib/python3.11/site-packages/swebench/harness/test_spec/__init__.py\", line 1, in \n from swebench.harness.test_spec import (\n File \"/root/venv/lib/python3.11/site-packages/swebench/harness/test_spec/test_spec.py\", line 20, in \n from swebench.harness.test_spec.create_scripts import (\n File \"/root/venv/lib/python3.11/site-packages/swebench/harness/test_spec/create_scripts.py\", line 2, in \n from swebench.harness.test_spec.javascript import (\n File \"/root/venv/lib/python3.11/site-packages/swebench/harness/test_spec/javascript.py\", line 9, in \n from swebench.harness.test_spec.utils import make_eval_script_list_common\n File \"/root/venv/lib/python3.11/site-packages/swebench/harness/test_spec/utils.py\", line 6, in \n from swebench.harness.utils import get_modified_files\n File \"/root/venv/lib/python3.11/site-packages/swebench/harness/utils.py\", line 10, in \n from datasets import Dataset, load_dataset, load_from_disk\n File \"/root/venv/lib/python3.11/site-packages/datasets/__init__.py\", line 17, in \n from .arrow_dataset import Column, Dataset\n File \"/root/venv/lib/python3.11/site-packages/datasets/arrow_dataset.py\", line 80, in \n from .arrow_writer import ArrowWriter, OptimizedTypedSequence\n File \"/root/venv/lib/python3.11/site-packages/datasets/arrow_writer.py\", line 30, in \n from .features import Audio, Features, Image, Pdf, Value, Video\n File \"/root/venv/lib/python3.11/site-packages/datasets/features/__init__.py\", line 22, in \n from .features import Array2D, Array3D, Array4D, Array5D, ClassLabel, Features, Json, LargeList, List, Sequence, Value\n File \"/root/venv/lib/python3.11/site-packages/datasets/features/features.py\", line 1183, in \n class Json:\n File \"/root/venv/lib/python3.11/site-packages/datasets/features/features.py\", line 1221, in Json\n pa_type: ClassVar[Any] = pa.json_()\n ^^^^^^^^\nAttributeError: module 'pyarrow' has no attribute 'json_'. Did you mean: 'json'?\n", - "stdout": "" + "returncode": 0, + "stderr": ":128: RuntimeWarning: 'swebench.harness.run_evaluation' found in sys.modules after import of package 'swebench.harness', but prior to execution of 'swebench.harness.run_evaluation'; this may result in unpredictable behaviour\n", + "stdout": "usage: run_evaluation.py [-h] [-d DATASET_NAME] [-s SPLIT]\n [-i INSTANCE_IDS [INSTANCE_IDS ...]] -p\n PREDICTIONS_PATH [--max_workers MAX_WORKERS]\n [--open_file_limit OPEN_FILE_LIMIT] [-t TIMEOUT]\n [--force_rebuild FORCE_REBUILD]\n [--cache_level {none,base,env,instance}]\n [--clean CLEAN] -id RUN_ID [-n NAMESPACE]\n [--instance_image_tag INSTANCE_IMAGE_TAG]\n [--env_image_tag ENV_IMAGE_TAG]\n [--rewrite_reports REWRITE_REPORTS]\n [--report_dir REPORT_DIR] [--modal MODAL]\n\nRun evaluation harness for the given dataset and predictions.\n\noptions:\n -h, --help show this help message and exit\n -d DATASET_NAME, --dataset_name DATASET_NAME\n Name of dataset or path to JSON file. (default: SWE-\n bench/SWE-bench_Lite)\n -s SPLIT, --split SPLIT\n Split of the dataset (default: test)\n -i INSTANCE_IDS [INSTANCE_IDS ...], --instance_ids INSTANCE_IDS [INSTANCE_IDS ...]\n Instance IDs to run (space separated) (default: None)\n -p PREDICTIONS_PATH, --predictions_path PREDICTIONS_PATH\n Path to predictions file - if 'gold', uses gold\n predictions (default: None)\n --max_workers MAX_WORKERS\n Maximum number of workers (should be <= 75% of CPU\n cores) (default: 4)\n --open_file_limit OPEN_FILE_LIMIT\n Open file limit (default: 4096)\n -t TIMEOUT, --timeout TIMEOUT\n Timeout (in seconds) for running tests for each\n instance (default: 1800)\n --force_rebuild FORCE_REBUILD\n Force rebuild of all images (default: False)\n --cache_level {none,base,env,instance}\n Cache level - remove images above this level (default:\n env)\n --clean CLEAN Clean images above cache level (default: False)\n -id RUN_ID, --run_id RUN_ID\n Run ID - identifies the run (default: None)\n -n NAMESPACE, --namespace NAMESPACE\n Namespace for images. (use \"none\" to use no namespace)\n (default: swebench)\n --instance_image_tag INSTANCE_IMAGE_TAG\n Instance image tag (default: latest)\n --env_image_tag ENV_IMAGE_TAG\n Environment image tag (default: latest)\n --rewrite_reports REWRITE_REPORTS\n Doesn't run new instances, only writes reports for\n instances with existing test outputs (default: False)\n --report_dir REPORT_DIR\n Directory to write reports to (default: .)\n --modal MODAL Run on Modal (default: False)\n" }, "swebench_install": null } diff --git a/reports/benchmark/swebench-regression.md b/reports/benchmark/swebench-regression.md index 43bb919..0f2a488 100644 --- a/reports/benchmark/swebench-regression.md +++ b/reports/benchmark/swebench-regression.md @@ -2,10 +2,10 @@ - Conclusion: **PREFLIGHT_ONLY** - Dataset: `princeton-nlp/SWE-bench_Lite` split `test` -- Smoke subset: `/root/codex-tool-runtime-mcp/benchmarks/swebench/subsets/smoke-lite-10.json` +- Smoke subset: `/home/user/coding-tools-mcp/benchmarks/swebench/subsets/smoke-lite-10.json` - Raw log directory: `reports/benchmark/swebench-regression/raw` -- Baseline predictions: `/root/codex-tool-runtime-mcp/benchmarks/swebench/predictions/baseline_native.jsonl` -- Candidate predictions: `/root/codex-tool-runtime-mcp/benchmarks/swebench/predictions/candidate_mcp.jsonl` +- Baseline predictions: `/home/user/coding-tools-mcp/benchmarks/swebench/predictions/baseline_native.jsonl` +- Candidate predictions: `/home/user/coding-tools-mcp/benchmarks/swebench/predictions/candidate_mcp.jsonl` - Baseline resolved: `None` - Candidate resolved: `None` - Baseline completed: `None` / `None` @@ -13,8 +13,8 @@ ## Preflight -- docker: missing - docker executable not found -- swebench package: missing - swebench harness help/import failed +- docker: missing - docker daemon unavailable: failed to connect to the docker API at unix:///var/run/docker.sock; check if the path is correct and if the daemon is running: dial unix /var/run/docker.sock: connect: no such file or directory +- swebench package: ok - swebench harness help succeeded - baseline predictions: 10 rows, placeholder=True - candidate predictions: 10 rows, placeholder=True @@ -34,8 +34,8 @@ ## Evaluation Commands ```bash -/root/venv/bin/python3 -m swebench.harness.run_evaluation --dataset_name princeton-nlp/SWE-bench_Lite --predictions_path /root/codex-tool-runtime-mcp/benchmarks/swebench/predictions/baseline_native.jsonl --max_workers 2 --run_id codex_tool_runtime_native_smoke --instance_ids astropy__astropy-12907 django__django-11099 matplotlib__matplotlib-18869 pallets__flask-4992 psf__requests-2317 pydata__xarray-3364 pytest-dev__pytest-5221 scikit-learn__scikit-learn-10297 sphinx-doc__sphinx-10325 sympy__sympy-12419 -/root/venv/bin/python3 -m swebench.harness.run_evaluation --dataset_name princeton-nlp/SWE-bench_Lite --predictions_path /root/codex-tool-runtime-mcp/benchmarks/swebench/predictions/candidate_mcp.jsonl --max_workers 2 --run_id codex_tool_runtime_mcp_smoke --instance_ids astropy__astropy-12907 django__django-11099 matplotlib__matplotlib-18869 pallets__flask-4992 psf__requests-2317 pydata__xarray-3364 pytest-dev__pytest-5221 scikit-learn__scikit-learn-10297 sphinx-doc__sphinx-10325 sympy__sympy-12419 +/usr/local/bin/python3 -m swebench.harness.run_evaluation --dataset_name princeton-nlp/SWE-bench_Lite --predictions_path /home/user/coding-tools-mcp/benchmarks/swebench/predictions/baseline_native.jsonl --max_workers 2 --run_id coding_tools_native_smoke --instance_ids astropy__astropy-12907 django__django-11099 matplotlib__matplotlib-18869 pallets__flask-4992 psf__requests-2317 pydata__xarray-3364 pytest-dev__pytest-5221 scikit-learn__scikit-learn-10297 sphinx-doc__sphinx-10325 sympy__sympy-12419 +/usr/local/bin/python3 -m swebench.harness.run_evaluation --dataset_name princeton-nlp/SWE-bench_Lite --predictions_path /home/user/coding-tools-mcp/benchmarks/swebench/predictions/candidate_mcp.jsonl --max_workers 2 --run_id coding_tools_mcp_smoke --instance_ids astropy__astropy-12907 django__django-11099 matplotlib__matplotlib-18869 pallets__flask-4992 psf__requests-2317 pydata__xarray-3364 pytest-dev__pytest-5221 scikit-learn__scikit-learn-10297 sphinx-doc__sphinx-10325 sympy__sympy-12419 ``` ## Harness Reports @@ -49,4 +49,3 @@ - Prediction files are schema-valid placeholders, not model-generated patches. - Official SWE-bench evaluation requires a working Docker daemon. -- Official SWE-bench evaluation requires an importable swebench harness. diff --git a/reports/benchmark/swebench-regression/raw/docker-version.json b/reports/benchmark/swebench-regression/raw/docker-version.json new file mode 100644 index 0000000..fad8d70 --- /dev/null +++ b/reports/benchmark/swebench-regression/raw/docker-version.json @@ -0,0 +1,10 @@ +{ + "command": [ + "/usr/bin/docker", + "version" + ], + "ran": true, + "returncode": 1, + "stderr": "failed to connect to the docker API at unix:///var/run/docker.sock; check if the path is correct and if the daemon is running: dial unix /var/run/docker.sock: connect: no such file or directory\n", + "stdout": "Client: Docker Engine - Community\n Version: 29.3.1\n API version: 1.54\n Go version: go1.25.8\n Git commit: c2be9cc\n Built: Wed Mar 25 16:13:43 2026\n OS/Arch: linux/amd64\n Context: default\n" +} diff --git a/reports/benchmark/swebench-regression/raw/docker-version.stderr.txt b/reports/benchmark/swebench-regression/raw/docker-version.stderr.txt new file mode 100644 index 0000000..8170520 --- /dev/null +++ b/reports/benchmark/swebench-regression/raw/docker-version.stderr.txt @@ -0,0 +1 @@ +failed to connect to the docker API at unix:///var/run/docker.sock; check if the path is correct and if the daemon is running: dial unix /var/run/docker.sock: connect: no such file or directory diff --git a/reports/benchmark/swebench-regression/raw/docker-version.stdout.txt b/reports/benchmark/swebench-regression/raw/docker-version.stdout.txt new file mode 100644 index 0000000..ec00d67 --- /dev/null +++ b/reports/benchmark/swebench-regression/raw/docker-version.stdout.txt @@ -0,0 +1,8 @@ +Client: Docker Engine - Community + Version: 29.3.1 + API version: 1.54 + Go version: go1.25.8 + Git commit: c2be9cc + Built: Wed Mar 25 16:13:43 2026 + OS/Arch: linux/amd64 + Context: default diff --git a/reports/benchmark/swebench-regression/raw/environment.json b/reports/benchmark/swebench-regression/raw/environment.json index bf3df8f..a5468f7 100644 --- a/reports/benchmark/swebench-regression/raw/environment.json +++ b/reports/benchmark/swebench-regression/raw/environment.json @@ -1,7 +1,7 @@ { - "cwd": "/root/codex-tool-runtime-mcp", + "cwd": "/home/user/coding-tools-mcp", "executables": { - "docker": null, + "docker": "/usr/bin/docker", "git": "/usr/bin/git" }, "github": { @@ -18,7 +18,7 @@ "docker": "7.1.0", "swebench": "4.1.0" }, - "platform": "Linux-6.18.24-talos-x86_64-with-glibc2.36", - "python": "3.11.14 (main, Feb 20 2026, 09:55:14) [GCC 12.2.0]", - "python_executable": "/root/venv/bin/python3" + "platform": "Linux-6.18.5-x86_64-with-glibc2.39", + "python": "3.11.15 (main, Mar 3 2026, 09:26:23) [GCC 13.3.0]", + "python_executable": "/usr/local/bin/python3" } diff --git a/reports/benchmark/swebench-regression/raw/swebench-help.json b/reports/benchmark/swebench-regression/raw/swebench-help.json index 49f68c2..2c715fe 100644 --- a/reports/benchmark/swebench-regression/raw/swebench-help.json +++ b/reports/benchmark/swebench-regression/raw/swebench-help.json @@ -1,12 +1,12 @@ { "command": [ - "/root/venv/bin/python3", + "/usr/local/bin/python3", "-m", "swebench.harness.run_evaluation", "--help" ], "ran": true, - "returncode": 1, - "stderr": "2026-05-17 23:27:49,067 - datasets - INFO - TensorFlow version 2.15.1 available.\nTraceback (most recent call last):\n File \"\", line 189, in _run_module_as_main\n File \"\", line 112, in _get_module_details\n File \"/root/venv/lib/python3.11/site-packages/swebench/__init__.py\", line 7, in \n from swebench.harness.constants import (\n File \"/root/venv/lib/python3.11/site-packages/swebench/harness/__init__.py\", line 1, in \n from swebench.harness import (\n File \"/root/venv/lib/python3.11/site-packages/swebench/harness/docker_build.py\", line 19, in \n from swebench.harness.test_spec.test_spec import (\n File \"/root/venv/lib/python3.11/site-packages/swebench/harness/test_spec/__init__.py\", line 1, in \n from swebench.harness.test_spec import (\n File \"/root/venv/lib/python3.11/site-packages/swebench/harness/test_spec/test_spec.py\", line 20, in \n from swebench.harness.test_spec.create_scripts import (\n File \"/root/venv/lib/python3.11/site-packages/swebench/harness/test_spec/create_scripts.py\", line 2, in \n from swebench.harness.test_spec.javascript import (\n File \"/root/venv/lib/python3.11/site-packages/swebench/harness/test_spec/javascript.py\", line 9, in \n from swebench.harness.test_spec.utils import make_eval_script_list_common\n File \"/root/venv/lib/python3.11/site-packages/swebench/harness/test_spec/utils.py\", line 6, in \n from swebench.harness.utils import get_modified_files\n File \"/root/venv/lib/python3.11/site-packages/swebench/harness/utils.py\", line 10, in \n from datasets import Dataset, load_dataset, load_from_disk\n File \"/root/venv/lib/python3.11/site-packages/datasets/__init__.py\", line 17, in \n from .arrow_dataset import Column, Dataset\n File \"/root/venv/lib/python3.11/site-packages/datasets/arrow_dataset.py\", line 80, in \n from .arrow_writer import ArrowWriter, OptimizedTypedSequence\n File \"/root/venv/lib/python3.11/site-packages/datasets/arrow_writer.py\", line 30, in \n from .features import Audio, Features, Image, Pdf, Value, Video\n File \"/root/venv/lib/python3.11/site-packages/datasets/features/__init__.py\", line 22, in \n from .features import Array2D, Array3D, Array4D, Array5D, ClassLabel, Features, Json, LargeList, List, Sequence, Value\n File \"/root/venv/lib/python3.11/site-packages/datasets/features/features.py\", line 1183, in \n class Json:\n File \"/root/venv/lib/python3.11/site-packages/datasets/features/features.py\", line 1221, in Json\n pa_type: ClassVar[Any] = pa.json_()\n ^^^^^^^^\nAttributeError: module 'pyarrow' has no attribute 'json_'. Did you mean: 'json'?\n", - "stdout": "" + "returncode": 0, + "stderr": ":128: RuntimeWarning: 'swebench.harness.run_evaluation' found in sys.modules after import of package 'swebench.harness', but prior to execution of 'swebench.harness.run_evaluation'; this may result in unpredictable behaviour\n", + "stdout": "usage: run_evaluation.py [-h] [-d DATASET_NAME] [-s SPLIT]\n [-i INSTANCE_IDS [INSTANCE_IDS ...]] -p\n PREDICTIONS_PATH [--max_workers MAX_WORKERS]\n [--open_file_limit OPEN_FILE_LIMIT] [-t TIMEOUT]\n [--force_rebuild FORCE_REBUILD]\n [--cache_level {none,base,env,instance}]\n [--clean CLEAN] -id RUN_ID [-n NAMESPACE]\n [--instance_image_tag INSTANCE_IMAGE_TAG]\n [--env_image_tag ENV_IMAGE_TAG]\n [--rewrite_reports REWRITE_REPORTS]\n [--report_dir REPORT_DIR] [--modal MODAL]\n\nRun evaluation harness for the given dataset and predictions.\n\noptions:\n -h, --help show this help message and exit\n -d DATASET_NAME, --dataset_name DATASET_NAME\n Name of dataset or path to JSON file. (default: SWE-\n bench/SWE-bench_Lite)\n -s SPLIT, --split SPLIT\n Split of the dataset (default: test)\n -i INSTANCE_IDS [INSTANCE_IDS ...], --instance_ids INSTANCE_IDS [INSTANCE_IDS ...]\n Instance IDs to run (space separated) (default: None)\n -p PREDICTIONS_PATH, --predictions_path PREDICTIONS_PATH\n Path to predictions file - if 'gold', uses gold\n predictions (default: None)\n --max_workers MAX_WORKERS\n Maximum number of workers (should be <= 75% of CPU\n cores) (default: 4)\n --open_file_limit OPEN_FILE_LIMIT\n Open file limit (default: 4096)\n -t TIMEOUT, --timeout TIMEOUT\n Timeout (in seconds) for running tests for each\n instance (default: 1800)\n --force_rebuild FORCE_REBUILD\n Force rebuild of all images (default: False)\n --cache_level {none,base,env,instance}\n Cache level - remove images above this level (default:\n env)\n --clean CLEAN Clean images above cache level (default: False)\n -id RUN_ID, --run_id RUN_ID\n Run ID - identifies the run (default: None)\n -n NAMESPACE, --namespace NAMESPACE\n Namespace for images. (use \"none\" to use no namespace)\n (default: swebench)\n --instance_image_tag INSTANCE_IMAGE_TAG\n Instance image tag (default: latest)\n --env_image_tag ENV_IMAGE_TAG\n Environment image tag (default: latest)\n --rewrite_reports REWRITE_REPORTS\n Doesn't run new instances, only writes reports for\n instances with existing test outputs (default: False)\n --report_dir REPORT_DIR\n Directory to write reports to (default: .)\n --modal MODAL Run on Modal (default: False)\n" } diff --git a/reports/benchmark/swebench-regression/raw/swebench-help.stderr.txt b/reports/benchmark/swebench-regression/raw/swebench-help.stderr.txt index 7e62259..f2b3a6c 100644 --- a/reports/benchmark/swebench-regression/raw/swebench-help.stderr.txt +++ b/reports/benchmark/swebench-regression/raw/swebench-help.stderr.txt @@ -1,36 +1 @@ -2026-05-17 23:27:49,067 - datasets - INFO - TensorFlow version 2.15.1 available. -Traceback (most recent call last): - File "", line 189, in _run_module_as_main - File "", line 112, in _get_module_details - File "/root/venv/lib/python3.11/site-packages/swebench/__init__.py", line 7, in - from swebench.harness.constants import ( - File "/root/venv/lib/python3.11/site-packages/swebench/harness/__init__.py", line 1, in - from swebench.harness import ( - File "/root/venv/lib/python3.11/site-packages/swebench/harness/docker_build.py", line 19, in - from swebench.harness.test_spec.test_spec import ( - File "/root/venv/lib/python3.11/site-packages/swebench/harness/test_spec/__init__.py", line 1, in - from swebench.harness.test_spec import ( - File "/root/venv/lib/python3.11/site-packages/swebench/harness/test_spec/test_spec.py", line 20, in - from swebench.harness.test_spec.create_scripts import ( - File "/root/venv/lib/python3.11/site-packages/swebench/harness/test_spec/create_scripts.py", line 2, in - from swebench.harness.test_spec.javascript import ( - File "/root/venv/lib/python3.11/site-packages/swebench/harness/test_spec/javascript.py", line 9, in - from swebench.harness.test_spec.utils import make_eval_script_list_common - File "/root/venv/lib/python3.11/site-packages/swebench/harness/test_spec/utils.py", line 6, in - from swebench.harness.utils import get_modified_files - File "/root/venv/lib/python3.11/site-packages/swebench/harness/utils.py", line 10, in - from datasets import Dataset, load_dataset, load_from_disk - File "/root/venv/lib/python3.11/site-packages/datasets/__init__.py", line 17, in - from .arrow_dataset import Column, Dataset - File "/root/venv/lib/python3.11/site-packages/datasets/arrow_dataset.py", line 80, in - from .arrow_writer import ArrowWriter, OptimizedTypedSequence - File "/root/venv/lib/python3.11/site-packages/datasets/arrow_writer.py", line 30, in - from .features import Audio, Features, Image, Pdf, Value, Video - File "/root/venv/lib/python3.11/site-packages/datasets/features/__init__.py", line 22, in - from .features import Array2D, Array3D, Array4D, Array5D, ClassLabel, Features, Json, LargeList, List, Sequence, Value - File "/root/venv/lib/python3.11/site-packages/datasets/features/features.py", line 1183, in - class Json: - File "/root/venv/lib/python3.11/site-packages/datasets/features/features.py", line 1221, in Json - pa_type: ClassVar[Any] = pa.json_() - ^^^^^^^^ -AttributeError: module 'pyarrow' has no attribute 'json_'. Did you mean: 'json'? +:128: RuntimeWarning: 'swebench.harness.run_evaluation' found in sys.modules after import of package 'swebench.harness', but prior to execution of 'swebench.harness.run_evaluation'; this may result in unpredictable behaviour diff --git a/reports/benchmark/swebench-regression/raw/swebench-help.stdout.txt b/reports/benchmark/swebench-regression/raw/swebench-help.stdout.txt index e69de29..875752c 100644 --- a/reports/benchmark/swebench-regression/raw/swebench-help.stdout.txt +++ b/reports/benchmark/swebench-regression/raw/swebench-help.stdout.txt @@ -0,0 +1,55 @@ +usage: run_evaluation.py [-h] [-d DATASET_NAME] [-s SPLIT] + [-i INSTANCE_IDS [INSTANCE_IDS ...]] -p + PREDICTIONS_PATH [--max_workers MAX_WORKERS] + [--open_file_limit OPEN_FILE_LIMIT] [-t TIMEOUT] + [--force_rebuild FORCE_REBUILD] + [--cache_level {none,base,env,instance}] + [--clean CLEAN] -id RUN_ID [-n NAMESPACE] + [--instance_image_tag INSTANCE_IMAGE_TAG] + [--env_image_tag ENV_IMAGE_TAG] + [--rewrite_reports REWRITE_REPORTS] + [--report_dir REPORT_DIR] [--modal MODAL] + +Run evaluation harness for the given dataset and predictions. + +options: + -h, --help show this help message and exit + -d DATASET_NAME, --dataset_name DATASET_NAME + Name of dataset or path to JSON file. (default: SWE- + bench/SWE-bench_Lite) + -s SPLIT, --split SPLIT + Split of the dataset (default: test) + -i INSTANCE_IDS [INSTANCE_IDS ...], --instance_ids INSTANCE_IDS [INSTANCE_IDS ...] + Instance IDs to run (space separated) (default: None) + -p PREDICTIONS_PATH, --predictions_path PREDICTIONS_PATH + Path to predictions file - if 'gold', uses gold + predictions (default: None) + --max_workers MAX_WORKERS + Maximum number of workers (should be <= 75% of CPU + cores) (default: 4) + --open_file_limit OPEN_FILE_LIMIT + Open file limit (default: 4096) + -t TIMEOUT, --timeout TIMEOUT + Timeout (in seconds) for running tests for each + instance (default: 1800) + --force_rebuild FORCE_REBUILD + Force rebuild of all images (default: False) + --cache_level {none,base,env,instance} + Cache level - remove images above this level (default: + env) + --clean CLEAN Clean images above cache level (default: False) + -id RUN_ID, --run_id RUN_ID + Run ID - identifies the run (default: None) + -n NAMESPACE, --namespace NAMESPACE + Namespace for images. (use "none" to use no namespace) + (default: swebench) + --instance_image_tag INSTANCE_IMAGE_TAG + Instance image tag (default: latest) + --env_image_tag ENV_IMAGE_TAG + Environment image tag (default: latest) + --rewrite_reports REWRITE_REPORTS + Doesn't run new instances, only writes reports for + instances with existing test outputs (default: False) + --report_dir REPORT_DIR + Directory to write reports to (default: .) + --modal MODAL Run on Modal (default: False) diff --git a/reports/compliance/latest.json b/reports/compliance/latest.json index 20b72d9..214c784 100644 --- a/reports/compliance/latest.json +++ b/reports/compliance/latest.json @@ -1,28 +1,28 @@ { - "commit": "7c18d6875cf15795d6bc34d3ac9d770e29272893", + "commit": "af85abd4882d027a531a1d17dfede254e18aa778", "dogfood": "passed", - "e2e": "passed", - "elapsed_seconds": 51.042, + "e2e": "not_run", + "elapsed_seconds": 0.749, "failures": [], "passed": true, "profile": "coding-tools-mcp-v0.1", "required_tools": { - "apply_patch": "passed", - "exec_command": "passed", - "git_diff": "passed", - "git_status": "passed", - "kill_session": "passed", - "list_dir": "passed", - "list_files": "passed", - "read_file": "passed", - "request_permissions": "passed", - "search_text": "passed", - "view_image": "passed", - "write_stdin": "passed" + "apply_patch": "not_measured", + "exec_command": "not_measured", + "git_diff": "not_measured", + "git_status": "not_measured", + "kill_session": "not_measured", + "list_dir": "not_measured", + "list_files": "not_measured", + "read_file": "not_measured", + "request_permissions": "not_measured", + "search_text": "not_measured", + "view_image": "not_measured", + "write_stdin": "not_measured" }, - "security": "passed", + "security": "not_run", "skipped": [], - "suite": "all", - "tests_run": 71, + "suite": "dogfood", + "tests_run": 1, "write_only": false } diff --git a/reports/compliance/latest.md b/reports/compliance/latest.md index dff526d..cf5027e 100644 --- a/reports/compliance/latest.md +++ b/reports/compliance/latest.md @@ -1,26 +1,26 @@ # Compliance Report - profile: `coding-tools-mcp-v0.1` -- commit: `7c18d6875cf15795d6bc34d3ac9d770e29272893` -- suite: `all` +- commit: `af85abd4882d027a531a1d17dfede254e18aa778` +- suite: `dogfood` - passed: `true` -- tests_run: `71` -- elapsed_seconds: `51.042` +- tests_run: `1` +- elapsed_seconds: `0.749` ## Required Tools -- `read_file`: passed -- `list_dir`: passed -- `list_files`: passed -- `search_text`: passed -- `apply_patch`: passed -- `exec_command`: passed -- `write_stdin`: passed -- `kill_session`: passed -- `git_status`: passed -- `git_diff`: passed -- `request_permissions`: passed -- `view_image`: passed +- `read_file`: not_measured +- `list_dir`: not_measured +- `list_files`: not_measured +- `search_text`: not_measured +- `apply_patch`: not_measured +- `exec_command`: not_measured +- `write_stdin`: not_measured +- `kill_session`: not_measured +- `git_status`: not_measured +- `git_diff`: not_measured +- `request_permissions`: not_measured +- `view_image`: not_measured ## Failures diff --git a/reports/dogfood/coding-tools-dogfood.json b/reports/dogfood/coding-tools-dogfood.json index 3bead08..a3d85ce 100644 --- a/reports/dogfood/coding-tools-dogfood.json +++ b/reports/dogfood/coding-tools-dogfood.json @@ -2,10 +2,10 @@ "cases": [ { "checks": [ - "PASS search_text finds add: {\"matches\": [{\"after\": [], \"before\": [], \"column\": 1, \"line\": 1, \"path\": \"tiny-js-project/src/math.js\", \"preview\": \"function add(a, b) {\"}], \"ok\": true, \"query\": \"function add\",...", + "PASS search_text finds add: {\"engine\": \"rg\", \"matches\": [{\"after\": [], \"before\": [], \"column\": 1, \"line\": 1, \"path\": \"tiny-js-project/src/math.js\", \"preview\": \"function add(a, b) {\"}], \"ok\": true, \"query\":...", "PASS read_file returns buggy source: {\"bytes_read\": 66, \"content\": \"function add(a, b) {\\n return a - b;\\n}\\n\\nmodule.exports = { add };\\n\", \"encoding\": \"utf-8\", \"end_line\": 5, \"next_start_line\": null, \"ok\": true,...", "PASS apply_patch fixes add: {\"affected_files\": [{\"operation\": \"update\", \"path\": \"tiny-js-project/src/math.js\"}], \"clean\": true, \"dry_run\": false, \"ok\": true, \"summary\": \"M tiny-js-project/src/math.js\", \"wa...", - "PASS exec_command npm test passes: {\"elapsed_ms\": 713, \"exit_code\": 0, \"ok\": true, \"session_id\": \"etdyVu5lHMjRrpZrl8Rjl-Vq\", \"signal\": null, \"status\": \"exited\", \"stderr\": \"\", \"stderr_dropped_bytes\": 0, \"stderr_om...", + "PASS exec_command npm test passes: {\"elapsed_ms\": 424, \"exit_code\": 0, \"ok\": true, \"session_id\": \"eIwxXJOF1lUzP_OjOxSXY8qL\", \"signal\": null, \"status\": \"exited\", \"stderr\": \"\", \"stderr_dropped_bytes\": 0, \"stderr_om...", "PASS git_diff shows only math.js fix: {\"diff\": \"--- a/tiny-js-project/src/math.js\\n+++ b/tiny-js-project/src/math.js\\n@@ -1,5 +1,5 @@\\n function add(a, b) {\\n\\n- return a - b;\\n\\n+ return a + b;\\n\\n }\\n\\n \\n\\n mod..." ], "failures": [], @@ -16,7 +16,7 @@ "checks": [ "PASS read_file returns python source: {\"bytes_read\": 32, \"content\": \"def add(a, b):\\n return a + b\\n\", \"encoding\": \"utf-8\", \"end_line\": 2, \"next_start_line\": null, \"ok\": true, \"output_bytes\": 32, \"output_lines\": ...", "PASS apply_patch adds multiply: {\"affected_files\": [{\"operation\": \"update\", \"path\": \"tiny-python-project/src/math_utils.py\"}], \"clean\": true, \"dry_run\": false, \"ok\": true, \"summary\": \"M tiny-python-project/src...", - "PASS exec_command unittest passes: {\"elapsed_ms\": 136, \"exit_code\": 0, \"ok\": true, \"session_id\": \"Jbsd8Ibepw2fEvXSseToUcOi\", \"signal\": null, \"status\": \"exited\", \"stderr\": \"..\\n------------------------------------...", + "PASS exec_command unittest passes: {\"elapsed_ms\": 61, \"exit_code\": 0, \"ok\": true, \"session_id\": \"0hxF-ORjDM4rVSjpPJy0UkEp\", \"signal\": null, \"status\": \"exited\", \"stderr\": \"..\\n-------------------------------------...", "PASS git_diff shows multiply: {\"diff\": \"--- a/tiny-python-project/src/math_utils.py\\n+++ b/tiny-python-project/src/math_utils.py\\n@@ -1,2 +1,6 @@\\n def add(a, b):\\n\\n return a + b\\n\\n+\\n\\n+\\n\\n+def multi..." ], "failures": [], @@ -25,10 +25,10 @@ }, { "checks": [ - "PASS exec_command returns session_id: {\"elapsed_ms\": 94, \"exit_code\": null, \"ok\": true, \"session_id\": \"Y-zeUu11DSAZBpwWXa6QMhT0\", \"signal\": null, \"status\": \"running\", \"stderr\": \"\", \"stderr_dropped_bytes\": 0, \"stderr...", - "PASS write_stdin accepts hello: {\"exit_code\": null, \"ok\": true, \"session_id\": \"Y-zeUu11DSAZBpwWXa6QMhT0\", \"signal\": null, \"status\": \"running\", \"stderr\": \"\", \"stderr_dropped_bytes\": 0, \"stderr_omitted_bytes\": 0...", - "PASS write_stdin accepts exit: {\"exit_code\": 0, \"ok\": true, \"session_id\": \"Y-zeUu11DSAZBpwWXa6QMhT0\", \"signal\": null, \"status\": \"exited\", \"stderr\": \"\", \"stderr_dropped_bytes\": 0, \"stderr_omitted_bytes\": 0, \"s...", - "PASS kill_session terminates or reports already closed: {\"exit_code\": 0, \"killed\": false, \"ok\": true, \"session_id\": \"Y-zeUu11DSAZBpwWXa6QMhT0\", \"signal\": null, \"status\": \"exited\", \"stderr\": \"\", \"stderr_dropped_bytes\": 0, \"stderr_omit..." + "PASS exec_command returns session_id: {\"elapsed_ms\": 41, \"exit_code\": null, \"ok\": true, \"session_id\": \"r3j54TRIhYCbG_9_W0ngb4CW\", \"signal\": null, \"status\": \"running\", \"stderr\": \"\", \"stderr_dropped_bytes\": 0, \"stderr...", + "PASS write_stdin accepts hello: {\"exit_code\": null, \"ok\": true, \"session_id\": \"r3j54TRIhYCbG_9_W0ngb4CW\", \"signal\": null, \"status\": \"running\", \"stderr\": \"\", \"stderr_dropped_bytes\": 0, \"stderr_omitted_bytes\": 0...", + "PASS write_stdin accepts exit: {\"exit_code\": 0, \"ok\": true, \"session_id\": \"r3j54TRIhYCbG_9_W0ngb4CW\", \"signal\": null, \"status\": \"exited\", \"stderr\": \"\", \"stderr_dropped_bytes\": 0, \"stderr_omitted_bytes\": 0, \"s...", + "PASS kill_session terminates or reports already closed: {\"exit_code\": 0, \"killed\": false, \"ok\": true, \"session_id\": \"r3j54TRIhYCbG_9_W0ngb4CW\", \"signal\": null, \"status\": \"exited\", \"stderr\": \"\", \"stderr_dropped_bytes\": 0, \"stderr_omit..." ], "failures": [], "name": "long_running_stdin", @@ -50,7 +50,7 @@ "direct_bypass": false, "endpoint": "http://127.0.0.1:8765/mcp", "final_git_diff": "{\"diff\": \"--- a/tiny-js-project/src/math.js\\n+++ b/tiny-js-project/src/math.js\\n@@ -1,5 +1,5 @@\\n function add(a, b) {\\n\\n- return a - b;\\n\\n+ return a + b;\\n\\n }\\n\\n \\n\\n module.exports = { add };\\n\\n--- a/tiny-python-project/src/math_utils.py\\n+++ b/tiny-python-project/src/math_utils.py\\n@@ -1,2 +1,6 @@\\n def add(a, b):\\n\\n return a + b\\n\\n+\\n\\n+\\n\\n+def multiply(a, b):\\n\\n+ return a * b\\n\", \"files\": [{\"binary\": false, \"path\": \"tiny-js-project/src/math.js\", \"status\": \"modified\"}, {\"binary\": false, \"path\": \"tiny-python-project/src/math_utils.py\", \"status\": \"modified\"}], \"ok\": true, \"output_bytes\": 364, \"output_lines\": 30, \"truncated\": false, \"truncated_by\": null, \"warnings\": [\"non-git diff fallback\"]}", - "fixture_root": "/tmp/codex-mcp-dogfood-xf95irl4", + "fixture_root": "/tmp/codex-mcp-dogfood-yqbv5f87", "initialize": { "capabilities": { "logging": {}, @@ -63,12 +63,12 @@ "serverInfo": { "name": "coding-tools-mcp", "title": "Coding Tools MCP", - "version": "0.1.3" + "version": "0.1.4" } }, "known_limitations": [], "prompt": "Use only MCP tools to search/read, patch, test, exercise stdin, and inspect diff for deterministic fixtures.", - "server_command": "python3 -m coding_tools_mcp --workspace {workspace} --host 127.0.0.1 --port 8765", + "server_command": "coding-tools-mcp --workspace {workspace} --host 127.0.0.1 --port 8765", "tool_calls": [ { "arguments": { @@ -77,7 +77,7 @@ }, "expected_rejection": false, "ok": true, - "summary": "{\"matches\": [{\"after\": [], \"before\": [], \"column\": 1, \"line\": 1, \"path\": \"tiny-js-project/src/math.js\", \"preview\": \"function add(a, b) {\"}], \"ok\": true, \"query\": \"function add\",...", + "summary": "{\"engine\": \"rg\", \"matches\": [{\"after\": [], \"before\": [], \"column\": 1, \"line\": 1, \"path\": \"tiny-js-project/src/math.js\", \"preview\": \"function add(a, b) {\"}], \"ok\": true, \"query\":...", "tool": "search_text" }, { @@ -109,7 +109,7 @@ }, "expected_rejection": false, "ok": true, - "summary": "{\"elapsed_ms\": 713, \"exit_code\": 0, \"ok\": true, \"session_id\": \"etdyVu5lHMjRrpZrl8Rjl-Vq\", \"signal\": null, \"status\": \"exited\", \"stderr\": \"\", \"stderr_dropped_bytes\": 0, \"stderr_om...", + "summary": "{\"elapsed_ms\": 424, \"exit_code\": 0, \"ok\": true, \"session_id\": \"eIwxXJOF1lUzP_OjOxSXY8qL\", \"signal\": null, \"status\": \"exited\", \"stderr\": \"\", \"stderr_dropped_bytes\": 0, \"stderr_om...", "tool": "exec_command" }, { @@ -144,7 +144,7 @@ }, { "arguments": { - "cmd": "/home/codespace/.python/current/bin/python3 -m unittest discover -s tests", + "cmd": "/usr/local/bin/python3 -m unittest discover -s tests", "max_output_bytes": 40000, "timeout_ms": 20000, "tty": false, @@ -153,7 +153,7 @@ }, "expected_rejection": false, "ok": true, - "summary": "{\"elapsed_ms\": 136, \"exit_code\": 0, \"ok\": true, \"session_id\": \"Jbsd8Ibepw2fEvXSseToUcOi\", \"signal\": null, \"status\": \"exited\", \"stderr\": \"..\\n------------------------------------...", + "summary": "{\"elapsed_ms\": 61, \"exit_code\": 0, \"ok\": true, \"session_id\": \"0hxF-ORjDM4rVSjpPJy0UkEp\", \"signal\": null, \"status\": \"exited\", \"stderr\": \"..\\n-------------------------------------...", "tool": "exec_command" }, { @@ -170,7 +170,7 @@ }, { "arguments": { - "cmd": "/home/codespace/.python/current/bin/python3 repl.py", + "cmd": "/usr/local/bin/python3 repl.py", "max_output_bytes": 40000, "timeout_ms": 30000, "tty": true, @@ -179,36 +179,36 @@ }, "expected_rejection": false, "ok": true, - "summary": "{\"elapsed_ms\": 94, \"exit_code\": null, \"ok\": true, \"session_id\": \"Y-zeUu11DSAZBpwWXa6QMhT0\", \"signal\": null, \"status\": \"running\", \"stderr\": \"\", \"stderr_dropped_bytes\": 0, \"stderr...", + "summary": "{\"elapsed_ms\": 41, \"exit_code\": null, \"ok\": true, \"session_id\": \"r3j54TRIhYCbG_9_W0ngb4CW\", \"signal\": null, \"status\": \"running\", \"stderr\": \"\", \"stderr_dropped_bytes\": 0, \"stderr...", "tool": "exec_command" }, { "arguments": { "chars": "hello\n", - "session_id": "Y-zeUu11DSAZBpwWXa6QMhT0" + "session_id": "r3j54TRIhYCbG_9_W0ngb4CW" }, "expected_rejection": false, "ok": true, - "summary": "{\"exit_code\": null, \"ok\": true, \"session_id\": \"Y-zeUu11DSAZBpwWXa6QMhT0\", \"signal\": null, \"status\": \"running\", \"stderr\": \"\", \"stderr_dropped_bytes\": 0, \"stderr_omitted_bytes\": 0...", + "summary": "{\"exit_code\": null, \"ok\": true, \"session_id\": \"r3j54TRIhYCbG_9_W0ngb4CW\", \"signal\": null, \"status\": \"running\", \"stderr\": \"\", \"stderr_dropped_bytes\": 0, \"stderr_omitted_bytes\": 0...", "tool": "write_stdin" }, { "arguments": { "chars": "exit\n", - "session_id": "Y-zeUu11DSAZBpwWXa6QMhT0" + "session_id": "r3j54TRIhYCbG_9_W0ngb4CW" }, "expected_rejection": false, "ok": true, - "summary": "{\"exit_code\": 0, \"ok\": true, \"session_id\": \"Y-zeUu11DSAZBpwWXa6QMhT0\", \"signal\": null, \"status\": \"exited\", \"stderr\": \"\", \"stderr_dropped_bytes\": 0, \"stderr_omitted_bytes\": 0, \"s...", + "summary": "{\"exit_code\": 0, \"ok\": true, \"session_id\": \"r3j54TRIhYCbG_9_W0ngb4CW\", \"signal\": null, \"status\": \"exited\", \"stderr\": \"\", \"stderr_dropped_bytes\": 0, \"stderr_omitted_bytes\": 0, \"s...", "tool": "write_stdin" }, { "arguments": { - "session_id": "Y-zeUu11DSAZBpwWXa6QMhT0" + "session_id": "r3j54TRIhYCbG_9_W0ngb4CW" }, "expected_rejection": true, "ok": true, - "summary": "{\"exit_code\": 0, \"killed\": false, \"ok\": true, \"session_id\": \"Y-zeUu11DSAZBpwWXa6QMhT0\", \"signal\": null, \"status\": \"exited\", \"stderr\": \"\", \"stderr_dropped_bytes\": 0, \"stderr_omit...", + "summary": "{\"exit_code\": 0, \"killed\": false, \"ok\": true, \"session_id\": \"r3j54TRIhYCbG_9_W0ngb4CW\", \"signal\": null, \"status\": \"exited\", \"stderr\": \"\", \"stderr_dropped_bytes\": 0, \"stderr_omit...", "tool": "kill_session" }, { @@ -253,7 +253,11 @@ "tools": [ "apply_patch", "exec_command", + "get_default_cwd", + "git_blame", "git_diff", + "git_log", + "git_show", "git_status", "kill_session", "list_dir", @@ -261,8 +265,10 @@ "read_file", "request_permissions", "search_text", + "server_info", + "set_default_cwd", "view_image", "write_stdin" ], - "workspace": "/tmp/codex-mcp-dogfood-xf95irl4/workspace" + "workspace": "/tmp/codex-mcp-dogfood-yqbv5f87/workspace" } diff --git a/reports/dogfood/coding-tools-dogfood.md b/reports/dogfood/coding-tools-dogfood.md index bcc7de5..8541c9f 100644 --- a/reports/dogfood/coding-tools-dogfood.md +++ b/reports/dogfood/coding-tools-dogfood.md @@ -2,8 +2,8 @@ - Conclusion: **PASS** - Endpoint: `http://127.0.0.1:8765/mcp` -- Workspace: `/tmp/codex-mcp-dogfood-xf95irl4/workspace` -- Server command: `python3 -m coding_tools_mcp --workspace {workspace} --host 127.0.0.1 --port 8765` +- Workspace: `/tmp/codex-mcp-dogfood-yqbv5f87/workspace` +- Server command: `coding-tools-mcp --workspace {workspace} --host 127.0.0.1 --port 8765` - Codex version: `unknown` - Direct filesystem/shell bypass during task execution: `False` @@ -11,7 +11,11 @@ - `apply_patch` - `exec_command` +- `get_default_cwd` +- `git_blame` - `git_diff` +- `git_log` +- `git_show` - `git_status` - `kill_session` - `list_dir` @@ -19,6 +23,8 @@ - `read_file` - `request_permissions` - `search_text` +- `server_info` +- `set_default_cwd` - `view_image` - `write_stdin` @@ -29,23 +35,23 @@ Use only MCP tools to search/read, patch, test, exercise stdin, and inspect diff ## Case Results ### js_bugfix: PASS -- PASS search_text finds add: {"matches": [{"after": [], "before": [], "column": 1, "line": 1, "path": "tiny-js-project/src/math.js", "preview": "function add(a, b) {"}], "ok": true, "query": "function add",... +- PASS search_text finds add: {"engine": "rg", "matches": [{"after": [], "before": [], "column": 1, "line": 1, "path": "tiny-js-project/src/math.js", "preview": "function add(a, b) {"}], "ok": true, "query":... - PASS read_file returns buggy source: {"bytes_read": 66, "content": "function add(a, b) {\n return a - b;\n}\n\nmodule.exports = { add };\n", "encoding": "utf-8", "end_line": 5, "next_start_line": null, "ok": true,... - PASS apply_patch fixes add: {"affected_files": [{"operation": "update", "path": "tiny-js-project/src/math.js"}], "clean": true, "dry_run": false, "ok": true, "summary": "M tiny-js-project/src/math.js", "wa... -- PASS exec_command npm test passes: {"elapsed_ms": 713, "exit_code": 0, "ok": true, "session_id": "etdyVu5lHMjRrpZrl8Rjl-Vq", "signal": null, "status": "exited", "stderr": "", "stderr_dropped_bytes": 0, "stderr_om... +- PASS exec_command npm test passes: {"elapsed_ms": 424, "exit_code": 0, "ok": true, "session_id": "eIwxXJOF1lUzP_OjOxSXY8qL", "signal": null, "status": "exited", "stderr": "", "stderr_dropped_bytes": 0, "stderr_om... - PASS git_diff shows only math.js fix: {"diff": "--- a/tiny-js-project/src/math.js\n+++ b/tiny-js-project/src/math.js\n@@ -1,5 +1,5 @@\n function add(a, b) {\n\n- return a - b;\n\n+ return a + b;\n\n }\n\n \n\n mod... ### python_new_function: PASS - PASS read_file returns python source: {"bytes_read": 32, "content": "def add(a, b):\n return a + b\n", "encoding": "utf-8", "end_line": 2, "next_start_line": null, "ok": true, "output_bytes": 32, "output_lines": ... - PASS apply_patch adds multiply: {"affected_files": [{"operation": "update", "path": "tiny-python-project/src/math_utils.py"}], "clean": true, "dry_run": false, "ok": true, "summary": "M tiny-python-project/src... -- PASS exec_command unittest passes: {"elapsed_ms": 136, "exit_code": 0, "ok": true, "session_id": "Jbsd8Ibepw2fEvXSseToUcOi", "signal": null, "status": "exited", "stderr": "..\n------------------------------------... +- PASS exec_command unittest passes: {"elapsed_ms": 61, "exit_code": 0, "ok": true, "session_id": "0hxF-ORjDM4rVSjpPJy0UkEp", "signal": null, "status": "exited", "stderr": "..\n-------------------------------------... - PASS git_diff shows multiply: {"diff": "--- a/tiny-python-project/src/math_utils.py\n+++ b/tiny-python-project/src/math_utils.py\n@@ -1,2 +1,6 @@\n def add(a, b):\n\n return a + b\n\n+\n\n+\n\n+def multi... ### long_running_stdin: PASS -- PASS exec_command returns session_id: {"elapsed_ms": 94, "exit_code": null, "ok": true, "session_id": "Y-zeUu11DSAZBpwWXa6QMhT0", "signal": null, "status": "running", "stderr": "", "stderr_dropped_bytes": 0, "stderr... -- PASS write_stdin accepts hello: {"exit_code": null, "ok": true, "session_id": "Y-zeUu11DSAZBpwWXa6QMhT0", "signal": null, "status": "running", "stderr": "", "stderr_dropped_bytes": 0, "stderr_omitted_bytes": 0... -- PASS write_stdin accepts exit: {"exit_code": 0, "ok": true, "session_id": "Y-zeUu11DSAZBpwWXa6QMhT0", "signal": null, "status": "exited", "stderr": "", "stderr_dropped_bytes": 0, "stderr_omitted_bytes": 0, "s... -- PASS kill_session terminates or reports already closed: {"exit_code": 0, "killed": false, "ok": true, "session_id": "Y-zeUu11DSAZBpwWXa6QMhT0", "signal": null, "status": "exited", "stderr": "", "stderr_dropped_bytes": 0, "stderr_omit... +- PASS exec_command returns session_id: {"elapsed_ms": 41, "exit_code": null, "ok": true, "session_id": "r3j54TRIhYCbG_9_W0ngb4CW", "signal": null, "status": "running", "stderr": "", "stderr_dropped_bytes": 0, "stderr... +- PASS write_stdin accepts hello: {"exit_code": null, "ok": true, "session_id": "r3j54TRIhYCbG_9_W0ngb4CW", "signal": null, "status": "running", "stderr": "", "stderr_dropped_bytes": 0, "stderr_omitted_bytes": 0... +- PASS write_stdin accepts exit: {"exit_code": 0, "ok": true, "session_id": "r3j54TRIhYCbG_9_W0ngb4CW", "signal": null, "status": "exited", "stderr": "", "stderr_dropped_bytes": 0, "stderr_omitted_bytes": 0, "s... +- PASS kill_session terminates or reports already closed: {"exit_code": 0, "killed": false, "ok": true, "session_id": "r3j54TRIhYCbG_9_W0ngb4CW", "signal": null, "status": "exited", "stderr": "", "stderr_dropped_bytes": 0, "stderr_omit... ### workspace_escape: PASS - PASS read_file rejects ../ escape: {"error": {"category": "security", "code": "PATH_OUTSIDE_WORKSPACE", "details": {}, "message": "Path escapes the configured workspace.", "retryable": false}, "ok": false} @@ -61,12 +67,12 @@ Use only MCP tools to search/read, patch, test, exercise stdin, and inspect diff - `git_diff` ok=True args={"path": "tiny-js-project/src/math.js", "paths": ["tiny-js-project/src/math.js"]} - `read_file` ok=True args={"path": "tiny-python-project/src/math_utils.py"} - `apply_patch` ok=True args={"patch": "*** Begin Patch\n*** Update File: tiny-python-project/src/math_utils.py\n@@\n def add(a, b):\n return a + b\n+\n+\n+def multiply(a, b):\n+ return a * b\n*** End Patch\n"} -- `exec_command` ok=True args={"cmd": "/home/codespace/.python/current/bin/python3 -m unittest discover -s tests", "max_output_bytes": 40000, "timeout_ms": 20000, "tty": false, "workdir": "tiny-python-project", "yield_time_ms": 20000} +- `exec_command` ok=True args={"cmd": "/usr/local/bin/python3 -m unittest discover -s tests", "max_output_bytes": 40000, "timeout_ms": 20000, "tty": false, "workdir": "tiny-python-project", "yield_time_ms": 20000} - `git_diff` ok=True args={"path": "tiny-python-project/src/math_utils.py", "paths": ["tiny-python-project/src/math_utils.py"]} -- `exec_command` ok=True args={"cmd": "/home/codespace/.python/current/bin/python3 repl.py", "max_output_bytes": 40000, "timeout_ms": 30000, "tty": true, "workdir": "long-running-project", "yield_time_ms": 20000} -- `write_stdin` ok=True args={"chars": "hello\n", "session_id": "Y-zeUu11DSAZBpwWXa6QMhT0"} -- `write_stdin` ok=True args={"chars": "exit\n", "session_id": "Y-zeUu11DSAZBpwWXa6QMhT0"} -- `kill_session` ok=True expected_rejection args={"session_id": "Y-zeUu11DSAZBpwWXa6QMhT0"} +- `exec_command` ok=True args={"cmd": "/usr/local/bin/python3 repl.py", "max_output_bytes": 40000, "timeout_ms": 30000, "tty": true, "workdir": "long-running-project", "yield_time_ms": 20000} +- `write_stdin` ok=True args={"chars": "hello\n", "session_id": "r3j54TRIhYCbG_9_W0ngb4CW"} +- `write_stdin` ok=True args={"chars": "exit\n", "session_id": "r3j54TRIhYCbG_9_W0ngb4CW"} +- `kill_session` ok=True expected_rejection args={"session_id": "r3j54TRIhYCbG_9_W0ngb4CW"} - `read_file` ok=False expected_rejection args={"path": "../outside-secret.txt"} - `apply_patch` ok=False expected_rejection args={"patch": "*** Begin Patch\n*** Update File: ../outside-secret.txt\n@@\n-DOGFOOD-OUTSIDE-SECRET\n+MODIFIED\n*** End Patch\n"} - `exec_command` ok=False expected_rejection args={"cmd": "cat ../outside-secret.txt", "max_output_bytes": 40000, "timeout_ms": 10000, "tty": false, "yield_time_ms": 10000} diff --git a/tests/compliance/fixtures.py b/tests/compliance/fixtures.py index aa1fbe1..d6ea704 100644 --- a/tests/compliance/fixtures.py +++ b/tests/compliance/fixtures.py @@ -98,6 +98,8 @@ def init_git(root: Path) -> None: run(["git", "init", "-q"], root) run(["git", "config", "user.email", "compliance@example.invalid"], root) run(["git", "config", "user.name", "Compliance Runner"], root) + run(["git", "config", "commit.gpgsign", "false"], root) + run(["git", "config", "tag.gpgsign", "false"], root) run(["git", "add", "-A"], root) run(["git", "commit", "-q", "-m", "baseline fixture"], root) diff --git a/tests/compliance/test_runtime_helpers.py b/tests/compliance/test_runtime_helpers.py index 767c50c..1975b8a 100644 --- a/tests/compliance/test_runtime_helpers.py +++ b/tests/compliance/test_runtime_helpers.py @@ -253,6 +253,8 @@ def test_default_cwd_and_git_convenience_tools(self) -> None: ["git", "init", "-q"], ["git", "config", "user.email", "test@example.invalid"], ["git", "config", "user.name", "Runtime Test"], + ["git", "config", "commit.gpgsign", "false"], + ["git", "config", "tag.gpgsign", "false"], ["git", "add", "-A"], ["git", "commit", "-q", "-m", "initial commit"], ):