From 277a4c7a8cb6d46d7be2c69f1186052e5996e22d Mon Sep 17 00:00:00 2001 From: rounakbende10 Date: Tue, 21 Jul 2026 18:41:11 -0400 Subject: [PATCH 1/3] fix(resume): regenerate Pi models.json on URL change Pi agent stores the server URL in a bind-mounted models.json file that doesn't persist to MinIO. When resuming with a new server_url, the URL replacement walks all JSON files but misses this mount file. Added Pi models.json regeneration inline in the URL replacement script. --- src/coding_agent_bench/api.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/coding_agent_bench/api.py b/src/coding_agent_bench/api.py index 69e737e..81b0ae0 100644 --- a/src/coding_agent_bench/api.py +++ b/src/coding_agent_bench/api.py @@ -534,6 +534,13 @@ async def resume_job(job_id: str, req: ResumeJobRequest = ResumeJobRequest()): " with open(path, 'w') as f: f.write(content)", " replaced += 1", "print(f'Replaced URL in {replaced} files')", + "# Regenerate Pi mount file with new URL", + "agent = c.get('agents', [{}])[0]", + "mounts = c.get('environment', {}).get('mounts', [])", + "for m in mounts:", + " src, tgt = m.get('source',''), m.get('target','')", + f" if agent.get('name') == 'pi' and 'models.json' in tgt: json.dump({{'providers': {{'vllm': {{'baseUrl': 'https://' + new_host + '/v1', 'api': 'openai-completions', 'apiKey': 'NONE', 'models': [{{'id': agent.get('model_name',''), 'name': agent.get('model_name',''), 'contextWindow': 262000}}]}}}}}}, open(src, 'w')); print('Regenerated Pi models.json')", + f" if agent.get('name') == 'codex' and 'config.toml' in tgt: open(src, 'w').write('[api]\\nbase_url = \"https://' + new_host + '\"\\napi_key = \"sk-no-key\"\\n[model]\\nmodel_id = \"' + agent.get('model_name','') + '\"\\n'); print('Regenerated Codex config.toml')", ] replace_script = "\n".join(replace_lines) url_replace_step = f" && python3 -c {shlex.quote(replace_script)}" From 2073f5720eaa481756c369d595cdc2d6547bf525 Mon Sep 17 00:00:00 2001 From: rounakbende10 Date: Tue, 21 Jul 2026 23:26:41 -0400 Subject: [PATCH 2/3] fix(resume): don't exit early when no env var URLs found The URL replacement script had 'if not hosts: exit(0)' which killed the entire script before reaching Pi models.json / Codex config.toml mount file regeneration when the harness stores URLs in mount files instead of env vars. --- src/coding_agent_bench/api.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/coding_agent_bench/api.py b/src/coding_agent_bench/api.py index 81b0ae0..43af7e7 100644 --- a/src/coding_agent_bench/api.py +++ b/src/coding_agent_bench/api.py @@ -521,18 +521,18 @@ async def resume_job(job_id: str, req: ResumeJobRequest = ResumeJobRequest()): " for m in re.finditer('https?://([^\"\\\\s,}/]+)', v):", " hosts.add(m.group(1).split('/')[0])", "hosts = [h for h in hosts if h != new_host and h.endswith(new_domain)]", - "if not hosts: exit(0)", "replaced = 0", - "for root, dirs, files in os.walk(job_dir):", - " for file in files:", - " if not file.endswith('.json'): continue", - " path = os.path.join(root, file)", - " with open(path, 'r') as f: content = f.read()", - " orig = content", - " for h in hosts: content = content.replace(h, new_host)", - " if content != orig:", - " with open(path, 'w') as f: f.write(content)", - " replaced += 1", + "if hosts:", + " for root, dirs, files in os.walk(job_dir):", + " for file in files:", + " if not file.endswith('.json'): continue", + " path = os.path.join(root, file)", + " with open(path, 'r') as f: content = f.read()", + " orig = content", + " for h in hosts: content = content.replace(h, new_host)", + " if content != orig:", + " with open(path, 'w') as f: f.write(content)", + " replaced += 1", "print(f'Replaced URL in {replaced} files')", "# Regenerate Pi mount file with new URL", "agent = c.get('agents', [{}])[0]", From 408fd33b7118a15844b24e0de0ca11db20433b70 Mon Sep 17 00:00:00 2001 From: rounakbende10 Date: Wed, 22 Jul 2026 13:53:43 -0400 Subject: [PATCH 3/3] fix(resume): use mc rm + mc cp to prevent ghost trial dirs in MinIO mc cp --recursive only adds files, leaving deleted trial dirs as ghosts. mc rm + mc cp guarantees MinIO exactly matches local disk after resume. --- src/coding_agent_bench/api.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/coding_agent_bench/api.py b/src/coding_agent_bench/api.py index 43af7e7..5ce1b5d 100644 --- a/src/coding_agent_bench/api.py +++ b/src/coding_agent_bench/api.py @@ -534,13 +534,19 @@ async def resume_job(job_id: str, req: ResumeJobRequest = ResumeJobRequest()): " with open(path, 'w') as f: f.write(content)", " replaced += 1", "print(f'Replaced URL in {replaced} files')", - "# Regenerate Pi mount file with new URL", + "# Regenerate Pi/Codex mount files with new URL", + f"server_url = {json.dumps(req.server_url)}", "agent = c.get('agents', [{}])[0]", + "model_name = agent.get('model_name', '')", "mounts = c.get('environment', {}).get('mounts', [])", "for m in mounts:", " src, tgt = m.get('source',''), m.get('target','')", - f" if agent.get('name') == 'pi' and 'models.json' in tgt: json.dump({{'providers': {{'vllm': {{'baseUrl': 'https://' + new_host + '/v1', 'api': 'openai-completions', 'apiKey': 'NONE', 'models': [{{'id': agent.get('model_name',''), 'name': agent.get('model_name',''), 'contextWindow': 262000}}]}}}}}}, open(src, 'w')); print('Regenerated Pi models.json')", - f" if agent.get('name') == 'codex' and 'config.toml' in tgt: open(src, 'w').write('[api]\\nbase_url = \"https://' + new_host + '\"\\napi_key = \"sk-no-key\"\\n[model]\\nmodel_id = \"' + agent.get('model_name','') + '\"\\n'); print('Regenerated Codex config.toml')", + " if agent.get('name') == 'pi' and 'models.json' in tgt:", + " json.dump({'providers': {'vllm': {'baseUrl': server_url, 'api': 'openai-completions', 'apiKey': 'NONE', 'models': [{'id': model_name, 'name': model_name}]}}}, open(src, 'w'))", + " print('Regenerated Pi models.json')", + " if agent.get('name') == 'codex' and 'config.toml' in tgt:", + " open(src, 'w').write(f'[api]\\nbase_url = \"{server_url}\"\\napi_key = \"sk-no-key\"\\n[model]\\nmodel_id = \"{model_name}\"\\n')", + " print('Regenerated Codex config.toml')", ] replace_script = "\n".join(replace_lines) url_replace_step = f" && python3 -c {shlex.quote(replace_script)}" @@ -550,7 +556,8 @@ async def resume_job(job_id: str, req: ResumeJobRequest = ResumeJobRequest()): f" && mc cp --recursive minio/results/{shlex.quote(original_job_name)}/ {job_dir}/" f"{url_replace_step}" f" && uv run --no-sync --no-cache harbor jobs resume -p {job_dir}{filter_flags}" - f" ; mc cp --recursive {job_dir}/ minio/results/{shlex.quote(original_job_name)}/" + f" ; mc rm --recursive --force minio/results/{shlex.quote(original_job_name)}/" + f" && mc cp --recursive {job_dir}/ minio/results/{shlex.quote(original_job_name)}/" ) command = ["sh", "-c", shell_command]