From 31f867e345b3880ae18c12013015feaf900228d9 Mon Sep 17 00:00:00 2001 From: Onyeka Nwamba Date: Mon, 25 May 2026 01:37:28 +0100 Subject: [PATCH 01/18] feat: add AgentRail SWE lifecycle dry run benchmark --- .gitignore | 5 +- benchmarks/agentrail-swe-lifecycle/README.md | 47 + .../agentrail-swe-lifecycle/dataset.json | 8304 +++++++++++++++++ .../fixtures/sample-dataset.json | 75 + .../agentrail-swe-lifecycle/lib/dataset.mjs | 75 + .../lib/local-lifecycle.mjs | 121 + .../agentrail-swe-lifecycle/lib/results.mjs | 56 + .../agentrail-swe-lifecycle/lib/upstream.mjs | 87 + benchmarks/agentrail-swe-lifecycle/run.mjs | 101 + .../scenarios/local-mock-lifecycle.json | 63 + package.json | 3 + ...efresh-agentrail-swe-lifecycle-dataset.mjs | 199 + test/agentrail-swe-lifecycle-dataset.test.js | 72 + test/agentrail-swe-lifecycle-runner.test.js | 714 ++ 14 files changed, 9921 insertions(+), 1 deletion(-) create mode 100644 benchmarks/agentrail-swe-lifecycle/README.md create mode 100644 benchmarks/agentrail-swe-lifecycle/dataset.json create mode 100644 benchmarks/agentrail-swe-lifecycle/fixtures/sample-dataset.json create mode 100644 benchmarks/agentrail-swe-lifecycle/lib/dataset.mjs create mode 100644 benchmarks/agentrail-swe-lifecycle/lib/local-lifecycle.mjs create mode 100644 benchmarks/agentrail-swe-lifecycle/lib/results.mjs create mode 100644 benchmarks/agentrail-swe-lifecycle/lib/upstream.mjs create mode 100644 benchmarks/agentrail-swe-lifecycle/run.mjs create mode 100644 benchmarks/agentrail-swe-lifecycle/scenarios/local-mock-lifecycle.json create mode 100644 scripts/refresh-agentrail-swe-lifecycle-dataset.mjs create mode 100644 test/agentrail-swe-lifecycle-dataset.test.js create mode 100644 test/agentrail-swe-lifecycle-runner.test.js diff --git a/.gitignore b/.gitignore index da05d18..006475f 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ AGENTS.md !.env.example .agentrail/ .agentrail.legacy/ +.agentrail-bench/ .idea/ .vscode/ browser-artifacts/ @@ -49,7 +50,9 @@ verify-browser.mjs verify-waitlist.mjs verify-waitlist-api.mjs verify-landing-claims.mjs -benchmarks/ +benchmarks/* +!benchmarks/agentrail-swe-lifecycle/ +!benchmarks/agentrail-swe-lifecycle/** scripts/phase1-dogfood.mjs scripts/phase2-dogfood.mjs scripts/phase3-dogfood.mjs diff --git a/benchmarks/agentrail-swe-lifecycle/README.md b/benchmarks/agentrail-swe-lifecycle/README.md new file mode 100644 index 0000000..093d186 --- /dev/null +++ b/benchmarks/agentrail-swe-lifecycle/README.md @@ -0,0 +1,47 @@ +# AgentRail SWE Lifecycle Dataset + +This directory defines AgentRail's wrapper dataset for token-efficiency +benchmarking on SWE-bench-style coding tasks. + +The checked-in `dataset.json` does not vendor SWE-bench problem statements, +gold patches, test patches, or evaluator payloads. It stores only a stable +manifest of upstream task references plus AgentRail lifecycle metadata. + +## Sources + +- Task selection: Artificial Analysis Coding Agent Index methodology, + `SWE-Bench-Pro-Hard-AA`. +- Upstream task rows: `ScaleAI/SWE-bench_Pro`, `default/test` split on + Hugging Face. + +## Lifecycle Wrapper + +Each task is intended to run through an AgentRail lifecycle: + +1. create a synthetic GitHub or Linear issue from the upstream problem statement +2. ingest that issue into AgentRail +3. assign it to a managed AgentRail runner +4. produce a patch or pull-request artifact +5. run the SWE-bench evaluator as CI +6. convert deterministic review findings into AgentRail review feedback +7. retry through AgentRail when CI or review is blocking +8. mark final pass/fail from the evaluator and lifecycle state + +The first scenario is `local-mock-lifecycle`, which simulates GitHub/Linear, +PRs, CI, and code review locally. Live GitHub/Linear scenarios can be added +after the local lifecycle is repeatable. + +## Refresh + +Regenerate the manifest from upstream sources: + +```bash +npm run benchmark:agentrail-swe:refresh +``` + +Validate the checked-in manifest shape: + +```bash +npm run benchmark:agentrail-swe:validate +``` + diff --git a/benchmarks/agentrail-swe-lifecycle/dataset.json b/benchmarks/agentrail-swe-lifecycle/dataset.json new file mode 100644 index 0000000..bcb9cfc --- /dev/null +++ b/benchmarks/agentrail-swe-lifecycle/dataset.json @@ -0,0 +1,8304 @@ +{ + "version": 1, + "id": "agentrail-swe-lifecycle", + "name": "AgentRail SWE Lifecycle Token Efficiency Dataset", + "description": "AgentRail lifecycle wrapper around Artificial Analysis SWE-Bench-Pro-Hard-AA task selection and ScaleAI/SWE-bench_Pro upstream rows.", + "defaultScenario": "local-mock-lifecycle", + "upstream": { + "taskSelection": { + "name": "Artificial Analysis Coding Agent Index", + "benchmark": "SWE-Bench-Pro-Hard-AA", + "sourceUrl": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", + "evaluatedTaskCount": 150 + }, + "taskData": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + } + }, + "lifecycle": { + "issueProviders": [ + "github", + "linear" + ], + "defaultIssueProvider": "github", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record", + "finalOracle": "swebench_fail_to_pass" + }, + "metrics": { + "primary": "tokens_per_successful_task", + "guardrails": [ + "success_rate", + "cost_per_successful_task", + "wall_clock_time_per_successful_task", + "ci_retry_count", + "review_retry_count" + ] + }, + "tasks": [ + { + "id": "agentrail-swe-pro-hard-aa-001", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_ansible__ansible-1bd7dcf339dd8b6c50bc16670be2448a206f4fdb-vba6da65a0f3baefda7a058ebbd0a8dcafb8512f5", + "selectedInstanceId": "instance_ansible__ansible-1bd7dcf339dd8b6c50bc16670be2448a206f4fdb-vba6da65a0f3baefda7a058ebbd0a8dcafb8512f5", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "ansible/ansible", + "baseCommit": "20ef733ee02ba688757998404c1926381356b031", + "language": "python", + "tags": [ + "python", + "security_feat", + "performance_feat", + "security_knowledge", + "back_end_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "ansible.ansible-ansible__ansible-1bd7dcf339dd8b6c50bc16670be2448a206f4fdb-vba6da65a0f3baefda7a058ebbd0a8dcafb8512f5", + "beforeRepoSetCommand": "git reset --hard 20ef733ee02ba688757998404c1926381356b031\ngit clean -fd \ngit checkout 20ef733ee02ba688757998404c1926381356b031 \ngit checkout 1bd7dcf339dd8b6c50bc16670be2448a206f4fdb -- test/units/plugins/lookup/test_password.py test/units/utils/test_encrypt.py", + "selectedTestFiles": [ + "test/units/utils/test_encrypt.py", + "test/units/plugins/lookup/test_password.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-002", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_ansible__ansible-1c06c46cc14324df35ac4f39a45fb3ccd602195d-v0f01c69f1e2528b935359cfe578530722bca2c59", + "selectedInstanceId": "instance_ansible__ansible-1c06c46cc14324df35ac4f39a45fb3ccd602195d-v0f01c69f1e2528b935359cfe578530722bca2c59", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "ansible/ansible", + "baseCommit": "6198c7377f545207218fe8eb2e3cfa9673ff8f5e", + "language": "python", + "tags": [ + "python", + "major_bug", + "back_end_knowledge", + "security_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "ansible.ansible-ansible__ansible-1c06c46cc14324df35ac4f39a45fb3ccd602195d-v0f01c69f1e2528b935359cfe578530722bca2c59", + "beforeRepoSetCommand": "git reset --hard 6198c7377f545207218fe8eb2e3cfa9673ff8f5e\ngit clean -fd \ngit checkout 6198c7377f545207218fe8eb2e3cfa9673ff8f5e \ngit checkout 1c06c46cc14324df35ac4f39a45fb3ccd602195d -- test/integration/targets/templating/tasks/main.yml test/units/_internal/_yaml/__init__.py test/units/_internal/_yaml/test_dumper.py test/units/plugins/filter/test_core.py", + "selectedTestFiles": [ + "test/units/plugins/filter/test_core.py", + "test/units/_internal/_yaml/__init__.py", + "test/units/_internal/_yaml/test_dumper.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-003", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_ansible__ansible-34db57a47f875d11c4068567b9ec7ace174ec4cf-v1055803c3a812189a1133297f7f5468579283f86", + "selectedInstanceId": "instance_ansible__ansible-34db57a47f875d11c4068567b9ec7ace174ec4cf-v1055803c3a812189a1133297f7f5468579283f86", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "ansible/ansible", + "baseCommit": "d63a71e3f83fc23defb97393367859634881b8da", + "language": "python", + "tags": [ + "python", + "performance_bug", + "core_feat", + "performance_feat", + "customization_feat", + "infrastructure_knowledge", + "back_end_knowledge", + "devops_knowledge", + "performance_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "ansible.ansible-ansible__ansible-34db57a47f875d11c4068567b9ec7ace174ec4cf-v1055803c3a812189a1133297f7f5468579283f86", + "beforeRepoSetCommand": "git reset --hard d63a71e3f83fc23defb97393367859634881b8da\ngit clean -fd \ngit checkout d63a71e3f83fc23defb97393367859634881b8da \ngit checkout 34db57a47f875d11c4068567b9ec7ace174ec4cf -- test/units/module_utils/facts/hardware/linux_data.py test/units/module_utils/facts/hardware/test_linux_get_cpu_info.py", + "selectedTestFiles": [ + "test/units/module_utils/facts/hardware/test_linux_get_cpu_info.py", + "test/units/module_utils/facts/hardware/linux_data.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-004", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_ansible__ansible-40ade1f84b8bb10a63576b0ac320c13f57c87d34-v6382ea168a93d80a64aab1fbd8c4f02dc5ada5bf", + "selectedInstanceId": "instance_ansible__ansible-40ade1f84b8bb10a63576b0ac320c13f57c87d34-v6382ea168a93d80a64aab1fbd8c4f02dc5ada5bf", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "ansible/ansible", + "baseCommit": "9ab63986ad528a6ad5bf4c59fe104d5106d6ef9b", + "language": "python", + "tags": [ + "python", + "major_bug", + "data_bug", + "edge_case_bug", + "compatibility_bug", + "back_end_knowledge", + "infrastructure_knowledge", + "devops_knowledge", + "api_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "ansible.ansible-ansible__ansible-40ade1f84b8bb10a63576b0ac320c13f57c87d34-v6382ea168a93d80a64aab1fbd8c4f02dc5ada5bf", + "beforeRepoSetCommand": "git reset --hard 9ab63986ad528a6ad5bf4c59fe104d5106d6ef9b\ngit clean -fd \ngit checkout 9ab63986ad528a6ad5bf4c59fe104d5106d6ef9b \ngit checkout 40ade1f84b8bb10a63576b0ac320c13f57c87d34 -- test/integration/targets/mount_facts/aliases test/integration/targets/mount_facts/meta/main.yml test/integration/targets/mount_facts/tasks/main.yml test/units/modules/mount_facts_data.py test/units/modules/test_mount_facts.py", + "selectedTestFiles": [ + "test/units/modules/test_mount_facts.py::test_list_mounts[gen_fstab_entries-\\nUUID=6b8b920d-f334-426e-a440-1207d0d8725b", + "test/units/modules/test_mount_facts.py::test_handle_deduplication_warning[True-False]", + "test/units/modules/test_mount_facts.py::test_parse_mount_bin_stdout_unknown", + "test/units/modules/test_mount_facts.py::test_handle_deduplication", + "test/units/modules/test_mount_facts.py::test_gen_mounts_by_source_timeout[ignore-False-False]", + "test/units/modules/test_mount_facts.py::test_get_mount_size_timeout[error-False-True]", + "test/units/modules/test_mount_facts.py::test_parse_mount_bin_stdout[node", + "test/units/modules/test_mount_facts.py::test_etc_filesystems_linux", + "test/units/modules/test_mount_facts.py::test_gen_mounts_by_sources[sources8-file_data8-/dev/gpt/rootfs", + "test/units/modules/test_mount_facts.py::test_gen_mounts_by_source_timeout[warn-True-False]", + "test/units/modules/test_mount_facts.py::test_handle_deduplication_warning[None-True]", + "test/units/modules/test_mount_facts.py::test_gen_mounts_by_sources[sources3-file_data3-proc", + "test/units/modules/test_mount_facts.py::test_parse_mount_bin_stdout[proc", + "test/units/modules/test_mount_facts.py::test_list_mounts[gen_fstab_entries-#", + "test/units/modules/test_mount_facts.py::test_invocation", + "test/units/modules/test_mount_facts.py::test_invalid_mount_binary[invalid_binary2-argument", + "test/units/modules/test_mount_facts.py::test_gen_mounts_by_sources[sources7-file_data7-/dev/gpt/rootfs", + "test/units/modules/test_mount_facts.py::test_gen_mounts_by_sources[sources9-file_data9-/dev/gpt/rootfs", + "test/units/modules/test_mount_facts.py::test_invalid_timeout", + "test/units/modules/test_mount_facts.py::test_gen_mounts_by_sources[sources1-file_data1-proc", + "test/units/modules/test_mount_facts.py::test_handle_deduplication_warning[False-False]", + "test/units/modules/test_mount_facts.py::test_list_mounts[gen_fstab_entries-proc", + "test/units/modules/test_mount_facts.py::test_invalid_mount_binary[false-Failed", + "test/units/modules/test_mount_facts.py::test_get_mount_size_timeout[warn-True-False]", + "test/units/modules/test_mount_facts.py::test_get_mount_size_timeout[None-False-True]", + "test/units/modules/test_mount_facts.py::test_invalid_mount_binary[ansible_test_missing_binary-Failed", + "test/units/modules/test_mount_facts.py::test_gen_mounts_by_sources[sources2-file_data2-proc", + "test/units/modules/test_mount_facts.py::test_get_mount_size_timeout[ignore-False-False]", + "test/units/modules/test_mount_facts.py::test_list_mounts[gen_vfstab_entries-#device", + "test/units/modules/test_mount_facts.py::test_get_mount_facts", + "test/units/modules/test_mount_facts.py::test_list_mounts[gen_aix_filesystems_entries-*\\n*", + "test/units/modules/test_mount_facts.py::test_gen_mounts_by_source_timeout[None-False-True]", + "test/units/modules/test_mount_facts.py::test_invalid_source", + "test/units/modules/test_mount_facts.py::test_gen_mounts_by_sources[sources10-file_data10-node", + "test/units/modules/test_mount_facts.py::test_list_mounts[gen_mnttab_entries-rpool/ROOT/solaris", + "test/units/modules/test_mount_facts.py::test_gen_mounts_by_sources[sources12-file_data12--results12]", + "test/units/modules/test_mount_facts.py::test_get_mounts_facts_filtering[devices-proc-proc", + "test/units/modules/test_mount_facts.py::test_gen_mounts_by_sources[sources5-file_data5-proc", + "test/units/modules/test_mount_facts.py::test_parse_mount_bin_stdout[/dev/gpt/rootfs", + "test/units/modules/test_mount_facts.py::test_get_mounts_size", + "test/units/modules/test_mount_facts.py::test_get_mounts_facts_filtering[fstypes-proc-proc", + "test/units/modules/test_mount_facts.py::test_gen_mounts_by_sources[sources6-file_data6-/dev/gpt/rootfs", + "test/units/modules/test_mount_facts.py::test_gen_mounts_by_sources[sources4-file_data4-proc", + "test/units/modules/test_mount_facts.py::test_gen_mounts_by_sources[sources0-file_data0-proc", + "test/units/modules/test_mount_facts.py::test_gen_mounts_by_sources[sources11-file_data11-/dev/sd0a", + "test/units/modules/test_mount_facts.py::test_duplicate_source", + "test/units/modules/test_mount_facts.py::test_parse_mount_bin_stdout[/dev/sd0a", + "test/units/modules/test_mount_facts.py::test_get_partition_uuid" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-005", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_ansible__ansible-4c5ce5a1a9e79a845aff4978cfeb72a0d4ecf7d6-v1055803c3a812189a1133297f7f5468579283f86", + "selectedInstanceId": "instance_ansible__ansible-4c5ce5a1a9e79a845aff4978cfeb72a0d4ecf7d6-v1055803c3a812189a1133297f7f5468579283f86", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "ansible/ansible", + "baseCommit": "8a175f59c939ca29ad56f3fa9edbc37a8656879a", + "language": "python", + "tags": [ + "python", + "code_quality_enh", + "dev_ops_enh", + "back_end_knowledge", + "infrastructure_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "ansible.ansible-ansible__ansible-4c5ce5a1a9e79a845aff4978cfeb72a0d4ecf7d6-v1055803c3a812189a1133297f7f5468579283f86", + "beforeRepoSetCommand": "git reset --hard 8a175f59c939ca29ad56f3fa9edbc37a8656879a\ngit clean -fd \ngit checkout 8a175f59c939ca29ad56f3fa9edbc37a8656879a \ngit checkout 4c5ce5a1a9e79a845aff4978cfeb72a0d4ecf7d6 -- .azure-pipelines/templates/test.yml test/integration/targets/apt/tasks/apt.yml test/integration/targets/dnf/tasks/dnf.yml test/integration/targets/dnf/tasks/gpg.yml test/integration/targets/module_utils_respawn/aliases test/integration/targets/module_utils_respawn/library/respawnme.py test/integration/targets/module_utils_respawn/tasks/main.yml test/integration/targets/module_utils_selinux/aliases test/integration/targets/module_utils_selinux/tasks/main.yml test/integration/targets/module_utils_selinux/tasks/selinux.yml test/integration/targets/setup_rpm_repo/library/create_repo.py test/integration/targets/setup_rpm_repo/tasks/main.yml test/integration/targets/setup_rpm_repo/vars/RedHat-8.yml test/integration/targets/yum/tasks/yum.yml test/support/integration/plugins/modules/sefcontext.py test/units/executor/module_common/test_recursive_finder.py test/units/module_utils/basic/test_imports.py test/units/module_utils/basic/test_selinux.py test/utils/shippable/incidental/remote.sh test/utils/shippable/remote.sh", + "selectedTestFiles": [ + "test/integration/targets/module_utils_respawn/library/respawnme.py", + "test/units/module_utils/basic/test_selinux.py", + "test/integration/targets/setup_rpm_repo/library/create_repo.py", + "test/support/integration/plugins/modules/sefcontext.py", + "test/units/module_utils/basic/test_imports.py", + "test/units/executor/module_common/test_recursive_finder.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-006", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_ansible__ansible-5640093f1ca63fd6af231cc8a7fb7d40e1907b8c-vba6da65a0f3baefda7a058ebbd0a8dcafb8512f5", + "selectedInstanceId": "instance_ansible__ansible-5640093f1ca63fd6af231cc8a7fb7d40e1907b8c-vba6da65a0f3baefda7a058ebbd0a8dcafb8512f5", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "ansible/ansible", + "baseCommit": "a5a13246ce02fd4c4ce997627b4f42152e59d288", + "language": "python", + "tags": [ + "python", + "regression_bug", + "compatibility_bug", + "back_end_knowledge", + "infrastructure_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "ansible.ansible-ansible__ansible-5640093f1ca63fd6af231cc8a7fb7d40e1907b8c-vba6da65a0f3baefda7a058ebbd0a8dcafb8512f5", + "beforeRepoSetCommand": "git reset --hard a5a13246ce02fd4c4ce997627b4f42152e59d288\ngit clean -fd \ngit checkout a5a13246ce02fd4c4ce997627b4f42152e59d288 \ngit checkout 5640093f1ca63fd6af231cc8a7fb7d40e1907b8c -- test/integration/targets/gathering_facts/runme.sh test/integration/targets/gathering_facts/test_module_defaults.yml test/integration/targets/package/tasks/main.yml test/integration/targets/service/tasks/tests.yml test/units/plugins/action/test_gather_facts.py", + "selectedTestFiles": [ + "test/units/plugins/action/test_gather_facts.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-007", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_ansible__ansible-5e88cd9972f10b66dd97e1ee684c910c6a2dd25e-v906c969b551b346ef54a2c0b41e04f632b7b73c2", + "selectedInstanceId": "instance_ansible__ansible-5e88cd9972f10b66dd97e1ee684c910c6a2dd25e-v906c969b551b346ef54a2c0b41e04f632b7b73c2", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "ansible/ansible", + "baseCommit": "5fa2d29c9a06baa8cc5d271c41e23b1f99b3814a", + "language": "python", + "tags": [ + "python", + "core_feat", + "api_feat", + "integration_feat", + "security_feat", + "back_end_knowledge", + "infrastructure_knowledge", + "api_knowledge", + "security_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "ansible.ansible-ansible__ansible-5e88cd9972f10b66dd97e1ee684c910c6a2dd25e-v906c969b551b346ef54a2c0b41e04f632b7b73c2", + "beforeRepoSetCommand": "git reset --hard 5fa2d29c9a06baa8cc5d271c41e23b1f99b3814a\ngit clean -fd \ngit checkout 5fa2d29c9a06baa8cc5d271c41e23b1f99b3814a \ngit checkout 5e88cd9972f10b66dd97e1ee684c910c6a2dd25e -- test/units/modules/network/netvisor/test_pn_user.py", + "selectedTestFiles": [ + "test/units/modules/network/netvisor/test_pn_user.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-008", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_ansible__ansible-709484969c8a4ffd74b839a673431a8c5caa6457-vba6da65a0f3baefda7a058ebbd0a8dcafb8512f5", + "selectedInstanceId": "instance_ansible__ansible-709484969c8a4ffd74b839a673431a8c5caa6457-vba6da65a0f3baefda7a058ebbd0a8dcafb8512f5", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "ansible/ansible", + "baseCommit": "35809806d3ab5d66fbb9696dc6a0009383e50673", + "language": "python", + "tags": [ + "python", + "compatibility_bug", + "code_quality_enh", + "major_bug", + "back_end_knowledge", + "desktop_knowledge", + "infrastructure_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "ansible.ansible-ansible__ansible-709484969c8a4ffd74b839a673431a8c5caa6457-vba6da65a0f3baefda7a058ebbd0a8dcafb8512f5", + "beforeRepoSetCommand": "git reset --hard 35809806d3ab5d66fbb9696dc6a0009383e50673\ngit clean -fd \ngit checkout 35809806d3ab5d66fbb9696dc6a0009383e50673 \ngit checkout 709484969c8a4ffd74b839a673431a8c5caa6457 -- test/units/module_utils/facts/test_sysctl.py", + "selectedTestFiles": [ + "test/units/module_utils/facts/test_sysctl.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-009", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_ansible__ansible-7e1a347695c7987ae56ef1b6919156d9254010ad-v390e508d27db7a51eece36bb6d9698b63a5b638a", + "selectedInstanceId": "instance_ansible__ansible-7e1a347695c7987ae56ef1b6919156d9254010ad-v390e508d27db7a51eece36bb6d9698b63a5b638a", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "ansible/ansible", + "baseCommit": "20ec92728004bc94729ffc08cbc483b7496c0c1f", + "language": "python", + "tags": [ + "python", + "core_feat", + "integration_feat", + "infrastructure_knowledge", + "networking_knowledge", + "devops_knowledge", + "back_end_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "ansible.ansible-ansible__ansible-7e1a347695c7987ae56ef1b6919156d9254010ad-v390e508d27db7a51eece36bb6d9698b63a5b638a", + "beforeRepoSetCommand": "git reset --hard 20ec92728004bc94729ffc08cbc483b7496c0c1f\ngit clean -fd \ngit checkout 20ec92728004bc94729ffc08cbc483b7496c0c1f \ngit checkout 7e1a347695c7987ae56ef1b6919156d9254010ad -- test/units/modules/network/icx/fixtures/lag_running_config.txt test/units/modules/network/icx/test_icx_linkagg.py", + "selectedTestFiles": [ + "test/units/modules/network/icx/test_icx_linkagg.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-010", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_ansible__ansible-935528e22e5283ee3f63a8772830d3d01f55ed8c-vba6da65a0f3baefda7a058ebbd0a8dcafb8512f5", + "selectedInstanceId": "instance_ansible__ansible-935528e22e5283ee3f63a8772830d3d01f55ed8c-vba6da65a0f3baefda7a058ebbd0a8dcafb8512f5", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "ansible/ansible", + "baseCommit": "43300e22798e4c9bd8ec2e321d28c5e8d2018aeb", + "language": "python", + "tags": [ + "python", + "edge_case_bug", + "integration_bug", + "back_end_knowledge", + "devops_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "ansible.ansible-ansible__ansible-935528e22e5283ee3f63a8772830d3d01f55ed8c-vba6da65a0f3baefda7a058ebbd0a8dcafb8512f5", + "beforeRepoSetCommand": "git reset --hard 43300e22798e4c9bd8ec2e321d28c5e8d2018aeb\ngit clean -fd \ngit checkout 43300e22798e4c9bd8ec2e321d28c5e8d2018aeb \ngit checkout 935528e22e5283ee3f63a8772830d3d01f55ed8c -- test/integration/targets/connection_windows_ssh/runme.sh test/units/plugins/connection/test_ssh.py", + "selectedTestFiles": [ + "test/units/plugins/connection/test_ssh.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-011", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_ansible__ansible-942424e10b2095a173dbd78e7128f52f7995849b-v30a923fb5c164d6cd18280c02422f75e611e8fb2", + "selectedInstanceId": "instance_ansible__ansible-942424e10b2095a173dbd78e7128f52f7995849b-v30a923fb5c164d6cd18280c02422f75e611e8fb2", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "ansible/ansible", + "baseCommit": "dd44449b6ec454a46426a020608fe3763287101f", + "language": "python", + "tags": [ + "python", + "major_bug", + "integration_bug", + "edge_case_bug", + "back_end_knowledge", + "infrastructure_knowledge", + "devops_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "ansible.ansible-ansible__ansible-942424e10b2095a173dbd78e7128f52f7995849b-v30a923fb5c164d6cd18280c02422f75e611e8fb2", + "beforeRepoSetCommand": "git reset --hard dd44449b6ec454a46426a020608fe3763287101f\ngit clean -fd \ngit checkout dd44449b6ec454a46426a020608fe3763287101f \ngit checkout 942424e10b2095a173dbd78e7128f52f7995849b -- test/units/plugins/connection/test_winrm.py", + "selectedTestFiles": [ + "test/units/plugins/connection/test_winrm.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-012", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_ansible__ansible-949c503f2ef4b2c5d668af0492a5c0db1ab86140-v0f01c69f1e2528b935359cfe578530722bca2c59", + "selectedInstanceId": "instance_ansible__ansible-949c503f2ef4b2c5d668af0492a5c0db1ab86140-v0f01c69f1e2528b935359cfe578530722bca2c59", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "ansible/ansible", + "baseCommit": "375d3889de9f437bc120ade623c170198629375d", + "language": "python", + "tags": [ + "python", + "customization_feat", + "back_end_knowledge", + "infrastructure_knowledge", + "networking_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "ansible.ansible-ansible__ansible-949c503f2ef4b2c5d668af0492a5c0db1ab86140-v0f01c69f1e2528b935359cfe578530722bca2c59", + "beforeRepoSetCommand": "git reset --hard 375d3889de9f437bc120ade623c170198629375d\ngit clean -fd \ngit checkout 375d3889de9f437bc120ade623c170198629375d \ngit checkout 949c503f2ef4b2c5d668af0492a5c0db1ab86140 -- test/integration/targets/ansible-config/files/galaxy_server.ini test/integration/targets/ansible-config/tasks/main.yml test/units/galaxy/test_collection.py test/units/galaxy/test_token.py", + "selectedTestFiles": [ + "test/units/galaxy/test_collection.py", + "test/units/galaxy/test_token.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-013", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_ansible__ansible-9a21e247786ebd294dafafca1105fcd770ff46c6-v67cdaa49f89b34e42b69d5b7830b3c3ad3d8803f", + "selectedInstanceId": "instance_ansible__ansible-9a21e247786ebd294dafafca1105fcd770ff46c6-v67cdaa49f89b34e42b69d5b7830b3c3ad3d8803f", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "ansible/ansible", + "baseCommit": "4c8c40fd3d4a58defdc80e7d22aa8d26b731353e", + "language": "python", + "tags": [ + "python", + "compatibility_bug", + "edge_case_bug", + "minor_bug", + "integration_bug", + "back_end_knowledge", + "devops_knowledge", + "infrastructure_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "ansible.ansible-ansible__ansible-9a21e247786ebd294dafafca1105fcd770ff46c6-v67cdaa49f89b34e42b69d5b7830b3c3ad3d8803f", + "beforeRepoSetCommand": "git reset --hard 4c8c40fd3d4a58defdc80e7d22aa8d26b731353e\ngit clean -fd \ngit checkout 4c8c40fd3d4a58defdc80e7d22aa8d26b731353e \ngit checkout 9a21e247786ebd294dafafca1105fcd770ff46c6 -- test/units/module_utils/basic/test_platform_distribution.py test/units/module_utils/common/test_sys_info.py test/units/modules/test_service.py", + "selectedTestFiles": [ + "test/units/module_utils/common/test_sys_info.py::test_get_distribution_not_linux[SunOS-Solaris]", + "test/units/module_utils/common/test_sys_info.py::test_get_distribution_not_linux[Darwin-Darwin]", + "test/units/module_utils/common/test_sys_info.py::test_get_distribution_version_not_linux[SunOS-11.4]", + "test/units/module_utils/common/test_sys_info.py::test_get_distribution_version_not_linux[Darwin-19.6.0]", + "test/units/module_utils/common/test_sys_info.py::test_get_distribution_not_linux[FreeBSD-Freebsd]", + "test/units/module_utils/common/test_sys_info.py::test_get_distribution_version_not_linux[FreeBSD-12.1]" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-014", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_ansible__ansible-a1569ea4ca6af5480cf0b7b3135f5e12add28a44-v0f01c69f1e2528b935359cfe578530722bca2c59", + "selectedInstanceId": "instance_ansible__ansible-a1569ea4ca6af5480cf0b7b3135f5e12add28a44-v0f01c69f1e2528b935359cfe578530722bca2c59", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "ansible/ansible", + "baseCommit": "f10d11bcdc54c9b7edc0111eb38c59a88e396d0a", + "language": "python", + "tags": [ + "python", + "major_bug", + "integration_bug", + "performance_bug", + "back_end_knowledge", + "networking_knowledge", + "api_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "ansible.ansible-ansible__ansible-a1569ea4ca6af5480cf0b7b3135f5e12add28a44-v0f01c69f1e2528b935359cfe578530722bca2c59", + "beforeRepoSetCommand": "git reset --hard f10d11bcdc54c9b7edc0111eb38c59a88e396d0a\ngit clean -fd \ngit checkout f10d11bcdc54c9b7edc0111eb38c59a88e396d0a \ngit checkout a1569ea4ca6af5480cf0b7b3135f5e12add28a44 -- test/integration/targets/iptables/tasks/chain_management.yml test/units/modules/test_iptables.py", + "selectedTestFiles": [ + "test/units/modules/test_iptables.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-015", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_ansible__ansible-b6290e1d156af608bd79118d209a64a051c55001-v390e508d27db7a51eece36bb6d9698b63a5b638a", + "selectedInstanceId": "instance_ansible__ansible-b6290e1d156af608bd79118d209a64a051c55001-v390e508d27db7a51eece36bb6d9698b63a5b638a", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "ansible/ansible", + "baseCommit": "b6e71c5ffb8ba382b6f49fc9b25e6d8bc78a9a88", + "language": "python", + "tags": [ + "python", + "core_feat", + "integration_feat", + "back_end_knowledge", + "networking_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "ansible.ansible-ansible__ansible-b6290e1d156af608bd79118d209a64a051c55001-v390e508d27db7a51eece36bb6d9698b63a5b638a", + "beforeRepoSetCommand": "git reset --hard b6e71c5ffb8ba382b6f49fc9b25e6d8bc78a9a88\ngit clean -fd \ngit checkout b6e71c5ffb8ba382b6f49fc9b25e6d8bc78a9a88 \ngit checkout b6290e1d156af608bd79118d209a64a051c55001 -- test/units/modules/network/icx/fixtures/icx_logging_config.cfg test/units/modules/network/icx/test_icx_logging.py", + "selectedTestFiles": [ + "test/units/modules/network/icx/test_icx_logging.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-016", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_ansible__ansible-b8025ac160146319d2b875be3366b60c852dd35d-v0f01c69f1e2528b935359cfe578530722bca2c59", + "selectedInstanceId": "instance_ansible__ansible-b8025ac160146319d2b875be3366b60c852dd35d-v0f01c69f1e2528b935359cfe578530722bca2c59", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "ansible/ansible", + "baseCommit": "fa093d8adf03c88908caa38fe70e0db2711e801c", + "language": "python", + "tags": [ + "python", + "major_bug", + "integration_bug", + "compatibility_bug", + "back_end_knowledge", + "devops_knowledge", + "networking_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "ansible.ansible-ansible__ansible-b8025ac160146319d2b875be3366b60c852dd35d-v0f01c69f1e2528b935359cfe578530722bca2c59", + "beforeRepoSetCommand": "git reset --hard fa093d8adf03c88908caa38fe70e0db2711e801c\ngit clean -fd \ngit checkout fa093d8adf03c88908caa38fe70e0db2711e801c \ngit checkout b8025ac160146319d2b875be3366b60c852dd35d -- test/integration/targets/get_url/tasks/ciphers.yml test/integration/targets/get_url/tasks/main.yml test/integration/targets/lookup_url/tasks/main.yml test/integration/targets/uri/tasks/ciphers.yml test/integration/targets/uri/tasks/main.yml test/units/module_utils/urls/test_Request.py test/units/module_utils/urls/test_fetch_url.py", + "selectedTestFiles": [ + "test/units/module_utils/urls/test_fetch_url.py", + "test/units/module_utils/urls/test_Request.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-017", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_ansible__ansible-bf98f031f3f5af31a2d78dc2f0a58fe92ebae0bb-v1055803c3a812189a1133297f7f5468579283f86", + "selectedInstanceId": "instance_ansible__ansible-bf98f031f3f5af31a2d78dc2f0a58fe92ebae0bb-v1055803c3a812189a1133297f7f5468579283f86", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "ansible/ansible", + "baseCommit": "17332532973343248297e3d3c746738d1f3b2f56", + "language": "python", + "tags": [ + "python", + "major_bug", + "regression_bug", + "data_bug", + "back_end_knowledge", + "devops_knowledge", + "security_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "ansible.ansible-ansible__ansible-bf98f031f3f5af31a2d78dc2f0a58fe92ebae0bb-v1055803c3a812189a1133297f7f5468579283f86", + "beforeRepoSetCommand": "git reset --hard 17332532973343248297e3d3c746738d1f3b2f56\ngit clean -fd \ngit checkout 17332532973343248297e3d3c746738d1f3b2f56 \ngit checkout bf98f031f3f5af31a2d78dc2f0a58fe92ebae0bb -- test/units/module_utils/basic/test_no_log.py test/units/module_utils/basic/test_sanitize_keys.py", + "selectedTestFiles": [ + "test/units/module_utils/basic/test_sanitize_keys.py", + "test/units/module_utils/basic/test_no_log.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-018", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_ansible__ansible-cb94c0cc550df9e98f1247bc71d8c2b861c75049-v1055803c3a812189a1133297f7f5468579283f86", + "selectedInstanceId": "instance_ansible__ansible-cb94c0cc550df9e98f1247bc71d8c2b861c75049-v1055803c3a812189a1133297f7f5468579283f86", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "ansible/ansible", + "baseCommit": "96c19724394a32b9d3c596966be2f46e478681f8", + "language": "python", + "tags": [ + "python", + "performance_feat", + "customization_feat", + "back_end_knowledge", + "desktop_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "ansible.ansible-ansible__ansible-cb94c0cc550df9e98f1247bc71d8c2b861c75049-v1055803c3a812189a1133297f7f5468579283f86", + "beforeRepoSetCommand": "git reset --hard 96c19724394a32b9d3c596966be2f46e478681f8\ngit clean -fd \ngit checkout 96c19724394a32b9d3c596966be2f46e478681f8 \ngit checkout cb94c0cc550df9e98f1247bc71d8c2b861c75049 -- test/integration/targets/adhoc/aliases test/integration/targets/adhoc/runme.sh test/units/cli/test_adhoc.py", + "selectedTestFiles": [ + "test/units/cli/test_adhoc.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-019", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_ansible__ansible-cd473dfb2fdbc97acf3293c134b21cbbcfa89ec3-vba6da65a0f3baefda7a058ebbd0a8dcafb8512f5", + "selectedInstanceId": "instance_ansible__ansible-cd473dfb2fdbc97acf3293c134b21cbbcfa89ec3-vba6da65a0f3baefda7a058ebbd0a8dcafb8512f5", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "ansible/ansible", + "baseCommit": "e8ae7211dabbd07093f50d6dfb383da3bb14f13d", + "language": "python", + "tags": [ + "python", + "major_bug", + "edge_case_bug", + "data_bug", + "back_end_knowledge", + "devops_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "ansible.ansible-ansible__ansible-cd473dfb2fdbc97acf3293c134b21cbbcfa89ec3-vba6da65a0f3baefda7a058ebbd0a8dcafb8512f5", + "beforeRepoSetCommand": "git reset --hard e8ae7211dabbd07093f50d6dfb383da3bb14f13d\ngit clean -fd \ngit checkout e8ae7211dabbd07093f50d6dfb383da3bb14f13d \ngit checkout cd473dfb2fdbc97acf3293c134b21cbbcfa89ec3 -- test/integration/targets/playbook/runme.sh test/units/playbook/test_play.py", + "selectedTestFiles": [ + "test/units/playbook/test_play.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-020", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_ansible__ansible-cd9c4eb5a6b2bfaf4a6709f001ce3d0c92c1eed2-v0f01c69f1e2528b935359cfe578530722bca2c59", + "selectedInstanceId": "instance_ansible__ansible-cd9c4eb5a6b2bfaf4a6709f001ce3d0c92c1eed2-v0f01c69f1e2528b935359cfe578530722bca2c59", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "ansible/ansible", + "baseCommit": "585ef6c55e87c10c1ce7d59ebe9c33dd6dbe5afb", + "language": "python", + "tags": [ + "python", + "major_bug", + "compatibility_bug", + "back_end_knowledge", + "infrastructure_knowledge", + "devops_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "ansible.ansible-ansible__ansible-cd9c4eb5a6b2bfaf4a6709f001ce3d0c92c1eed2-v0f01c69f1e2528b935359cfe578530722bca2c59", + "beforeRepoSetCommand": "git reset --hard 585ef6c55e87c10c1ce7d59ebe9c33dd6dbe5afb\ngit clean -fd \ngit checkout 585ef6c55e87c10c1ce7d59ebe9c33dd6dbe5afb \ngit checkout cd9c4eb5a6b2bfaf4a6709f001ce3d0c92c1eed2 -- test/units/module_utils/facts/hardware/linux/fixtures/sysinfo test/units/module_utils/facts/hardware/linux/test_get_sysinfo_facts.py", + "selectedTestFiles": [ + "test/units/module_utils/facts/hardware/linux/test_get_sysinfo_facts.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-021", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_ansible__ansible-d30fc6c0b359f631130b0e979d9a78a7b3747d48-v1055803c3a812189a1133297f7f5468579283f86", + "selectedInstanceId": "instance_ansible__ansible-d30fc6c0b359f631130b0e979d9a78a7b3747d48-v1055803c3a812189a1133297f7f5468579283f86", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "ansible/ansible", + "baseCommit": "a58fcde3a0d7a93c363ae7af4e6ee03001b96d82", + "language": "python", + "tags": [ + "python", + "edge_case_bug", + "code_quality_enh", + "dev_ops_enh", + "back_end_knowledge", + "devops_knowledge", + "infrastructure_knowledge", + "security_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "ansible.ansible-ansible__ansible-d30fc6c0b359f631130b0e979d9a78a7b3747d48-v1055803c3a812189a1133297f7f5468579283f86", + "beforeRepoSetCommand": "git reset --hard a58fcde3a0d7a93c363ae7af4e6ee03001b96d82\ngit clean -fd \ngit checkout a58fcde3a0d7a93c363ae7af4e6ee03001b96d82 \ngit checkout d30fc6c0b359f631130b0e979d9a78a7b3747d48 -- test/integration/targets/ansible-galaxy-collection/library/setup_collections.py test/integration/targets/ansible-galaxy-collection/tasks/install.yml test/integration/targets/ansible-galaxy-collection/tasks/main.yml test/units/galaxy/test_collection.py", + "selectedTestFiles": [ + "test/units/galaxy/test_collection.py", + "test/integration/targets/ansible-galaxy-collection/library/setup_collections.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-022", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_ansible__ansible-d58e69c82d7edd0583dd8e78d76b075c33c3151e-v173091e2e36d38c978002990795f66cfc0af30ad", + "selectedInstanceId": "instance_ansible__ansible-d58e69c82d7edd0583dd8e78d76b075c33c3151e-v173091e2e36d38c978002990795f66cfc0af30ad", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "ansible/ansible", + "baseCommit": "a6e671db25381ed111bbad0ab3e7d97366395d05", + "language": "python", + "tags": [ + "python", + "regression_bug", + "major_bug", + "back_end_knowledge", + "api_knowledge", + "infrastructure_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "ansible.ansible-ansible__ansible-d58e69c82d7edd0583dd8e78d76b075c33c3151e-v173091e2e36d38c978002990795f66cfc0af30ad", + "beforeRepoSetCommand": "git reset --hard a6e671db25381ed111bbad0ab3e7d97366395d05\ngit clean -fd \ngit checkout a6e671db25381ed111bbad0ab3e7d97366395d05 \ngit checkout d58e69c82d7edd0583dd8e78d76b075c33c3151e -- test/integration/targets/get_url/tasks/main.yml test/integration/targets/uri/tasks/main.yml test/sanity/ignore.txt test/units/module_utils/urls/test_Request.py test/units/module_utils/urls/test_fetch_url.py test/units/module_utils/urls/test_gzip.py", + "selectedTestFiles": [ + "test/units/module_utils/urls/test_fetch_url.py", + "test/units/module_utils/urls/test_gzip.py", + "test/units/module_utils/urls/test_Request.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-023", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_ansible__ansible-d6d2251929c84c3aa883bad7db0f19cc9ff0339e-v30a923fb5c164d6cd18280c02422f75e611e8fb2", + "selectedInstanceId": "instance_ansible__ansible-d6d2251929c84c3aa883bad7db0f19cc9ff0339e-v30a923fb5c164d6cd18280c02422f75e611e8fb2", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "ansible/ansible", + "baseCommit": "02e00aba3fd7b646a4f6d6af72159c2b366536bf", + "language": "python", + "tags": [ + "python", + "performance_bug", + "edge_case_bug", + "back_end_knowledge", + "devops_knowledge", + "infrastructure_knowledge", + "performance_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "ansible.ansible-ansible__ansible-d6d2251929c84c3aa883bad7db0f19cc9ff0339e-v30a923fb5c164d6cd18280c02422f75e611e8fb2", + "beforeRepoSetCommand": "git reset --hard 02e00aba3fd7b646a4f6d6af72159c2b366536bf\ngit clean -fd \ngit checkout 02e00aba3fd7b646a4f6d6af72159c2b366536bf \ngit checkout d6d2251929c84c3aa883bad7db0f19cc9ff0339e -- test/integration/targets/ansible-playbook-callbacks/callbacks_list.expected test/integration/targets/handlers/handler_notify_earlier_handler.yml test/integration/targets/handlers/runme.sh test/integration/targets/old_style_vars_plugins/runme.sh test/units/executor/test_play_iterator.py test/units/plugins/strategy/test_linear.py", + "selectedTestFiles": [ + "test/units/executor/test_play_iterator.py", + "test/units/plugins/strategy/test_linear.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-024", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_ansible__ansible-de5858f48dc9e1ce9117034e0d7e76806f420ca8-v1055803c3a812189a1133297f7f5468579283f86", + "selectedInstanceId": "instance_ansible__ansible-de5858f48dc9e1ce9117034e0d7e76806f420ca8-v1055803c3a812189a1133297f7f5468579283f86", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "ansible/ansible", + "baseCommit": "a1730af91f94552e8aaff4bedfb8dcae00bd284d", + "language": "python", + "tags": [ + "python", + "core_feat", + "performance_feat", + "api_feat", + "back_end_knowledge", + "api_knowledge", + "performance_knowledge", + "devops_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "ansible.ansible-ansible__ansible-de5858f48dc9e1ce9117034e0d7e76806f420ca8-v1055803c3a812189a1133297f7f5468579283f86", + "beforeRepoSetCommand": "git reset --hard a1730af91f94552e8aaff4bedfb8dcae00bd284d\ngit clean -fd \ngit checkout a1730af91f94552e8aaff4bedfb8dcae00bd284d \ngit checkout de5858f48dc9e1ce9117034e0d7e76806f420ca8 -- test/integration/targets/ansible-galaxy-collection/library/setup_collections.py test/integration/targets/ansible-galaxy-collection/tasks/install.yml test/integration/targets/ansible-galaxy-collection/tasks/main.yml test/integration/targets/ansible-galaxy-collection/vars/main.yml test/units/galaxy/test_api.py", + "selectedTestFiles": [ + "test/integration/targets/ansible-galaxy-collection/library/setup_collections.py", + "test/units/galaxy/test_api.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-025", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_ansible__ansible-deb54e4c5b32a346f1f0b0a14f1c713d2cc2e961-vba6da65a0f3baefda7a058ebbd0a8dcafb8512f5", + "selectedInstanceId": "instance_ansible__ansible-deb54e4c5b32a346f1f0b0a14f1c713d2cc2e961-vba6da65a0f3baefda7a058ebbd0a8dcafb8512f5", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "ansible/ansible", + "baseCommit": "ac1ca40fb3ed0a0fce387ae70cb8937208a13e03", + "language": "python", + "tags": [ + "python", + "customization_feat", + "back_end_knowledge", + "devops_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "ansible.ansible-ansible__ansible-deb54e4c5b32a346f1f0b0a14f1c713d2cc2e961-vba6da65a0f3baefda7a058ebbd0a8dcafb8512f5", + "beforeRepoSetCommand": "git reset --hard ac1ca40fb3ed0a0fce387ae70cb8937208a13e03\ngit clean -fd \ngit checkout ac1ca40fb3ed0a0fce387ae70cb8937208a13e03 \ngit checkout deb54e4c5b32a346f1f0b0a14f1c713d2cc2e961 -- test/units/galaxy/test_collection.py", + "selectedTestFiles": [ + "test/units/galaxy/test_collection.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-026", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_ansible__ansible-e64c6c1ca50d7d26a8e7747d8eb87642e767cd74-v0f01c69f1e2528b935359cfe578530722bca2c59", + "selectedInstanceId": "instance_ansible__ansible-e64c6c1ca50d7d26a8e7747d8eb87642e767cd74-v0f01c69f1e2528b935359cfe578530722bca2c59", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "ansible/ansible", + "baseCommit": "a0aad17912da687a3b0b5a573ab6ed0394b569ad", + "language": "python", + "tags": [ + "python", + "major_bug", + "data_bug", + "compatibility_bug", + "edge_case_bug", + "back_end_knowledge", + "devops_knowledge", + "performance_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "ansible.ansible-ansible__ansible-e64c6c1ca50d7d26a8e7747d8eb87642e767cd74-v0f01c69f1e2528b935359cfe578530722bca2c59", + "beforeRepoSetCommand": "git reset --hard a0aad17912da687a3b0b5a573ab6ed0394b569ad\ngit clean -fd \ngit checkout a0aad17912da687a3b0b5a573ab6ed0394b569ad \ngit checkout e64c6c1ca50d7d26a8e7747d8eb87642e767cd74 -- test/units/modules/test_unarchive.py", + "selectedTestFiles": [ + "test/units/modules/test_unarchive.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-027", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_ansible__ansible-ecea15c508f0e081525be036cf76bbb56dbcdd9d-vba6da65a0f3baefda7a058ebbd0a8dcafb8512f5", + "selectedInstanceId": "instance_ansible__ansible-ecea15c508f0e081525be036cf76bbb56dbcdd9d-vba6da65a0f3baefda7a058ebbd0a8dcafb8512f5", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "ansible/ansible", + "baseCommit": "01e7915b0a9778a934a0f0e9e9d110dbef7e31ec", + "language": "python", + "tags": [ + "python", + "core_feat", + "ui_ux_enh", + "back_end_knowledge", + "devops_knowledge", + "api_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "ansible.ansible-ansible__ansible-ecea15c508f0e081525be036cf76bbb56dbcdd9d-vba6da65a0f3baefda7a058ebbd0a8dcafb8512f5", + "beforeRepoSetCommand": "git reset --hard 01e7915b0a9778a934a0f0e9e9d110dbef7e31ec\ngit clean -fd \ngit checkout 01e7915b0a9778a934a0f0e9e9d110dbef7e31ec \ngit checkout ecea15c508f0e081525be036cf76bbb56dbcdd9d -- test/integration/targets/ansible-galaxy-collection/tasks/download.yml test/integration/targets/ansible-galaxy-collection/tasks/install.yml test/units/cli/test_galaxy.py test/units/galaxy/test_collection.py", + "selectedTestFiles": [ + "test/units/galaxy/test_collection.py", + "test/units/cli/test_galaxy.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-028", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_ansible__ansible-f02a62db509dc7463fab642c9c3458b9bc3476cc-v390e508d27db7a51eece36bb6d9698b63a5b638a", + "selectedInstanceId": "instance_ansible__ansible-f02a62db509dc7463fab642c9c3458b9bc3476cc-v390e508d27db7a51eece36bb6d9698b63a5b638a", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "ansible/ansible", + "baseCommit": "73248bf27d4c6094799512b95993382ea2139e72", + "language": "python", + "tags": [ + "python", + "core_feat", + "api_feat", + "back_end_knowledge", + "infrastructure_knowledge", + "api_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "ansible.ansible-ansible__ansible-f02a62db509dc7463fab642c9c3458b9bc3476cc-v390e508d27db7a51eece36bb6d9698b63a5b638a", + "beforeRepoSetCommand": "git reset --hard 73248bf27d4c6094799512b95993382ea2139e72\ngit clean -fd \ngit checkout 73248bf27d4c6094799512b95993382ea2139e72 \ngit checkout f02a62db509dc7463fab642c9c3458b9bc3476cc -- test/integration/targets/netapp_eseries_drive_firmware/aliases test/integration/targets/netapp_eseries_drive_firmware/tasks/main.yml test/integration/targets/netapp_eseries_drive_firmware/tasks/run.yml test/units/modules/storage/netapp/test_netapp_e_drive_firmware.py", + "selectedTestFiles": [ + "test/units/modules/storage/netapp/test_netapp_e_drive_firmware.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-029", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_ansible__ansible-f327e65d11bb905ed9f15996024f857a95592629-vba6da65a0f3baefda7a058ebbd0a8dcafb8512f5", + "selectedInstanceId": "instance_ansible__ansible-f327e65d11bb905ed9f15996024f857a95592629-vba6da65a0f3baefda7a058ebbd0a8dcafb8512f5", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "ansible/ansible", + "baseCommit": "f533d46572113655a0a698beab4b38671744a458", + "language": "python", + "tags": [ + "python", + "technical_debt_enh", + "refactoring_enh", + "compatibility_bug", + "devops_knowledge", + "infrastructure_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "ansible.ansible-ansible__ansible-f327e65d11bb905ed9f15996024f857a95592629-vba6da65a0f3baefda7a058ebbd0a8dcafb8512f5", + "beforeRepoSetCommand": "git reset --hard f533d46572113655a0a698beab4b38671744a458\ngit clean -fd \ngit checkout f533d46572113655a0a698beab4b38671744a458 \ngit checkout f327e65d11bb905ed9f15996024f857a95592629 -- test/units/cli/test_galaxy.py test/units/utils/collection_loader/test_collection_loader.py", + "selectedTestFiles": [ + "test/units/utils/collection_loader/test_collection_loader.py", + "test/units/cli/test_galaxy.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-030", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_element-hq__element-web-1216285ed2e82e62f8780b6702aa0f9abdda0b34-vnan", + "selectedInstanceId": "instance_element-hq__element-web-1216285ed2e82e62f8780b6702aa0f9abdda0b34-vnan", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "element-hq/element-web", + "baseCommit": "d7a6e3ec65cf28d2454ccb357874828bc8147eb0", + "language": "js", + "tags": [ + "js", + "ui_ux_feat", + "accessibility_feat", + "front_end_knowledge", + "web_knowledge", + "ui_ux_knowledge", + "accessibility_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "element-hq.element-element-hq__element-web-1216285ed2e82e62f8780b6702aa0f9abdda0b34", + "beforeRepoSetCommand": "git reset --hard d7a6e3ec65cf28d2454ccb357874828bc8147eb0\ngit clean -fd \ngit checkout d7a6e3ec65cf28d2454ccb357874828bc8147eb0 \ngit checkout 1216285ed2e82e62f8780b6702aa0f9abdda0b34 -- test/components/views/elements/ExternalLink-test.tsx test/components/views/elements/__snapshots__/ExternalLink-test.tsx.snap", + "selectedTestFiles": [ + "test/components/views/elements/__snapshots__/ExternalLink-test.tsx.snap", + "test/components/views/elements/ExternalLink-test.tsx", + "test/components/views/elements/ExternalLink-test.ts" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-031", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_element-hq__element-web-2760bfc8369f1bee640d6d7a7e910783143d4c5f-vnan", + "selectedInstanceId": "instance_element-hq__element-web-2760bfc8369f1bee640d6d7a7e910783143d4c5f-vnan", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "element-hq/element-web", + "baseCommit": "cdffd1ca1f7b60334a8ca3bba64d0a4e6d2b68d0", + "language": "js", + "tags": [ + "js", + "minor_bug", + "ui_ux_bug", + "ui_ux_enh", + "front_end_knowledge", + "web_knowledge", + "ui_ux_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "element-hq.element-element-hq__element-web-2760bfc8369f1bee640d6d7a7e910783143d4c5f", + "beforeRepoSetCommand": "git reset --hard cdffd1ca1f7b60334a8ca3bba64d0a4e6d2b68d0\ngit clean -fd \ngit checkout cdffd1ca1f7b60334a8ca3bba64d0a4e6d2b68d0 \ngit checkout 2760bfc8369f1bee640d6d7a7e910783143d4c5f -- test/components/views/right_panel/UserInfo-test.tsx", + "selectedTestFiles": [ + "test/components/views/right_panel/UserInfo-test.ts", + "test/audio/VoiceMessageRecording-test.ts", + "test/voice-broadcast/components/molecules/VoiceBroadcastRecordingBody-test.ts", + "test/stores/room-list/algorithms/list-ordering/NaturalAlgorithm-test.ts", + "test/components/views/right_panel/UserInfo-test.tsx", + "test/components/views/dialogs/ManualDeviceKeyVerificationDialog-test.ts", + "test/components/views/rooms/wysiwyg_composer/components/FormattingButtons-test.ts", + "test/components/views/messages/MLocationBody-test.ts" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-032", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_element-hq__element-web-33299af5c9b7a7ec5a9c31d578d4ec5b18088fb7-vnan", + "selectedInstanceId": "instance_element-hq__element-web-33299af5c9b7a7ec5a9c31d578d4ec5b18088fb7-vnan", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "element-hq/element-web", + "baseCommit": "8166306e0f8951a9554bf1437f7ef6eef54a3267", + "language": "js", + "tags": [ + "js", + "core_feat", + "ui_ux_feat", + "front_end_knowledge", + "ui_ux_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "element-hq.element-element-hq__element-web-33299af5c9b7a7ec5a9c31d578d4ec5b18088fb7", + "beforeRepoSetCommand": "git reset --hard 8166306e0f8951a9554bf1437f7ef6eef54a3267\ngit clean -fd \ngit checkout 8166306e0f8951a9554bf1437f7ef6eef54a3267 \ngit checkout 33299af5c9b7a7ec5a9c31d578d4ec5b18088fb7 -- test/components/views/rooms/RoomHeader-test.tsx test/components/views/rooms/__snapshots__/RoomHeader-test.tsx.snap", + "selectedTestFiles": [ + "test/components/views/settings/Notifications-test.ts", + "test/stores/widgets/WidgetPermissionStore-test.ts", + "test/components/views/rooms/__snapshots__/RoomHeader-test.tsx.snap", + "test/components/views/polls/pollHistory/PollListItemEnded-test.ts", + "test/components/views/rooms/RoomHeader-test.tsx", + "test/hooks/useProfileInfo-test.ts", + "test/stores/room-list/algorithms/RecentAlgorithm-test.ts", + "test/components/views/elements/AppTile-test.ts", + "test/i18n-test/languageHandler-test.ts", + "test/components/views/settings/devices/deleteDevices-test.ts", + "test/utils/DateUtils-test.ts", + "test/components/views/dialogs/CreateRoomDialog-test.ts", + "test/editor/history-test.ts", + "test/components/views/rooms/wysiwyg_composer/components/WysiwygComposer-test.ts", + "test/voice-broadcast/components/molecules/VoiceBroadcastPlaybackBody-test.ts", + "test/components/views/rooms/RoomHeader-test.ts", + "test/components/structures/RightPanel-test.ts", + "test/stores/RoomViewStore-test.ts", + "test/voice-broadcast/stores/VoiceBroadcastRecordingsStore-test.ts", + "test/components/structures/auth/ForgotPassword-test.ts" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-033", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_element-hq__element-web-404c412bcb694f04ba0c4d5479541203d701bca0-vnan", + "selectedInstanceId": "instance_element-hq__element-web-404c412bcb694f04ba0c4d5479541203d701bca0-vnan", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "element-hq/element-web", + "baseCommit": "f152613f830ec32a3de3d7f442816a63a4c732c5", + "language": "js", + "tags": [ + "js", + "ui_ux_bug", + "major_bug", + "front_end_knowledge", + "web_knowledge", + "database_knowledge", + "ui_ux_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "element-hq.element-element-hq__element-web-404c412bcb694f04ba0c4d5479541203d701bca0", + "beforeRepoSetCommand": "git reset --hard f152613f830ec32a3de3d7f442816a63a4c732c5\ngit clean -fd \ngit checkout f152613f830ec32a3de3d7f442816a63a4c732c5 \ngit checkout 404c412bcb694f04ba0c4d5479541203d701bca0 -- test/MatrixClientPeg-test.ts", + "selectedTestFiles": [ + "test/components/views/settings/tabs/user/SecurityUserSettingsTab-test.ts", + "test/utils/pillify-test.ts", + "test/utils/notifications-test.ts", + "test/components/views/settings/AddPrivilegedUsers-test.ts", + "test/SlidingSyncManager-test.ts", + "test/utils/device/parseUserAgent-test.ts", + "test/MatrixClientPeg-test.ts" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-034", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_element-hq__element-web-41dfec20bfe9b62cddbbbf621bef2e9aa9685157-vnan", + "selectedInstanceId": "instance_element-hq__element-web-41dfec20bfe9b62cddbbbf621bef2e9aa9685157-vnan", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "element-hq/element-web", + "baseCommit": "d5d1ec775caf2d3c9132122c1243898e99fdb2da", + "language": "js", + "tags": [ + "js", + "integration_feat", + "api_feat", + "security_feat", + "authentication_authorization_knowledge", + "security_knowledge", + "api_knowledge", + "back_end_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "element-hq.element-element-hq__element-web-41dfec20bfe9b62cddbbbf621bef2e9aa9685157", + "beforeRepoSetCommand": "git reset --hard d5d1ec775caf2d3c9132122c1243898e99fdb2da\ngit clean -fd \ngit checkout d5d1ec775caf2d3c9132122c1243898e99fdb2da \ngit checkout 41dfec20bfe9b62cddbbbf621bef2e9aa9685157 -- test/utils/AutoDiscoveryUtils-test.tsx", + "selectedTestFiles": [ + "test/components/views/rooms/wysiwyg_composer/EditWysiwygComposer-test.ts", + "test/utils/AutoDiscoveryUtils-test.ts", + "test/components/views/beacon/BeaconMarker-test.ts", + "test/components/views/rooms/wysiwyg_composer/SendWysiwygComposer-test.ts", + "test/utils/LruCache-test.ts", + "test/utils/beacon/bounds-test.ts", + "test/components/views/dialogs/ChangelogDialog-test.ts", + "test/utils/AutoDiscoveryUtils-test.tsx", + "test/components/views/messages/MBeaconBody-test.ts", + "test/components/views/rooms/VoiceRecordComposerTile-test.ts", + "test/components/views/rooms/BasicMessageComposer-test.ts" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-035", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_element-hq__element-web-44b98896a79ede48f5ad7ff22619a39d5f6ff03c-vnan", + "selectedInstanceId": "instance_element-hq__element-web-44b98896a79ede48f5ad7ff22619a39d5f6ff03c-vnan", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "element-hq/element-web", + "baseCommit": "19f9f9856451a8e4cce6d313d19ca8aed4b5d6b4", + "language": "js", + "tags": [ + "js", + "code_quality_enh", + "refactoring_enh", + "front_end_knowledge", + "ui_ux_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "element-hq.element-element-hq__element-web-44b98896a79ede48f5ad7ff22619a39d5f6ff03c", + "beforeRepoSetCommand": "git reset --hard 19f9f9856451a8e4cce6d313d19ca8aed4b5d6b4\ngit clean -fd \ngit checkout 19f9f9856451a8e4cce6d313d19ca8aed4b5d6b4 \ngit checkout 44b98896a79ede48f5ad7ff22619a39d5f6ff03c -- test/components/views/settings/SetIntegrationManager-test.tsx test/components/views/settings/__snapshots__/SetIntegrationManager-test.tsx.snap test/components/views/settings/tabs/user/GeneralUserSettingsTab-test.tsx test/components/views/settings/tabs/user/__snapshots__/GeneralUserSettingsTab-test.tsx.snap test/components/views/settings/tabs/user/__snapshots__/SecurityUserSettingsTab-test.tsx.snap", + "selectedTestFiles": [ + "test/components/views/elements/FilterTabGroup-test.ts", + "test/components/views/settings/SetIntegrationManager-test.tsx", + "test/components/views/context_menus/MessageContextMenu-test.ts", + "test/components/views/settings/tabs/user/GeneralUserSettingsTab-test.tsx", + "test/stores/ReleaseAnnouncementStore-test.ts", + "test/components/views/elements/Pill-test.ts", + "test/utils/StorageManager-test.ts", + "test/components/views/right_panel/PinnedMessagesCard-test.ts", + "test/components/views/user-onboarding/UserOnboardingPage-test.ts", + "test/components/views/settings/tabs/user/__snapshots__/SecurityUserSettingsTab-test.tsx.snap", + "test/components/structures/auth/ForgotPassword-test.ts", + "test/modules/ProxiedModuleApi-test.ts", + "test/components/views/settings/SetIntegrationManager-test.ts", + "test/components/views/settings/tabs/user/VoiceUserSettingsTab-test.ts", + "test/voice-broadcast/components/molecules/VoiceBroadcastRecordingPip-test.ts", + "test/components/views/dialogs/ConfirmRedactDialog-test.ts", + "test/voice-broadcast/components/molecules/VoiceBroadcastPlaybackBody-test.ts", + "test/components/views/settings/tabs/user/SecurityUserSettingsTab-test.ts", + "test/components/views/settings/tabs/user/__snapshots__/GeneralUserSettingsTab-test.tsx.snap", + "test/components/views/settings/__snapshots__/SetIntegrationManager-test.tsx.snap", + "test/components/structures/SpaceHierarchy-test.ts", + "test/utils/sets-test.ts" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-036", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_element-hq__element-web-459df4583e01e4744a52d45446e34183385442d6-vnan", + "selectedInstanceId": "instance_element-hq__element-web-459df4583e01e4744a52d45446e34183385442d6-vnan", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "element-hq/element-web", + "baseCommit": "dd91250111dcf4f398e125e14c686803315ebf5d", + "language": "js", + "tags": [ + "js", + "major_bug", + "regression_bug", + "edge_case_bug", + "integration_bug", + "front_end_knowledge", + "ui_ux_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "element-hq.element-element-hq__element-web-459df4583e01e4744a52d45446e34183385442d6", + "beforeRepoSetCommand": "git reset --hard dd91250111dcf4f398e125e14c686803315ebf5d\ngit clean -fd \ngit checkout dd91250111dcf4f398e125e14c686803315ebf5d \ngit checkout 459df4583e01e4744a52d45446e34183385442d6 -- test/components/views/voip/PipView-test.tsx test/voice-broadcast/components/molecules/VoiceBroadcastPreRecordingPip-test.tsx test/voice-broadcast/models/VoiceBroadcastPreRecording-test.ts test/voice-broadcast/stores/VoiceBroadcastPreRecordingStore-test.ts test/voice-broadcast/utils/setUpVoiceBroadcastPreRecording-test.ts test/voice-broadcast/utils/startNewVoiceBroadcastRecording-test.ts", + "selectedTestFiles": [ + "test/components/views/messages/CallEvent-test.ts", + "test/components/views/settings/devices/DeviceExpandDetailsButton-test.ts", + "test/utils/localRoom/isRoomReady-test.ts", + "test/settings/watchers/FontWatcher-test.ts", + "test/utils/device/clientInformation-test.ts", + "test/voice-broadcast/utils/setUpVoiceBroadcastPreRecording-test.ts", + "test/voice-broadcast/models/VoiceBroadcastPreRecording-test.ts", + "test/voice-broadcast/stores/VoiceBroadcastPreRecordingStore-test.ts", + "test/components/views/rooms/RoomPreviewBar-test.ts", + "test/components/views/voip/PipView-test.tsx", + "test/components/views/right_panel/UserInfo-test.ts", + "test/utils/membership-test.ts", + "test/utils/location/locationEventGeoUri-test.ts", + "test/components/views/context_menus/ThreadListContextMenu-test.ts", + "test/components/views/beacon/BeaconListItem-test.ts", + "test/voice-broadcast/utils/startNewVoiceBroadcastRecording-test.ts", + "test/components/views/right_panel/PinnedMessagesCard-test.ts", + "test/voice-broadcast/components/molecules/VoiceBroadcastPreRecordingPip-test.tsx", + "test/components/views/location/Map-test.ts", + "test/voice-broadcast/utils/findRoomLiveVoiceBroadcastFromUserAndDevice-test.ts", + "test/components/views/elements/AppTile-test.ts" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-037", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_element-hq__element-web-4fec436883b601a3cac2d4a58067e597f737b817-vnan", + "selectedInstanceId": "instance_element-hq__element-web-4fec436883b601a3cac2d4a58067e597f737b817-vnan", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "element-hq/element-web", + "baseCommit": "b8bb8f163a89cb6f2af0ac1cfc97e89deb938368", + "language": "js", + "tags": [ + "js", + "ui_ux_feat", + "core_feat", + "customization_feat", + "front_end_knowledge", + "ui_ux_knowledge", + "api_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "element-hq.element-element-hq__element-web-4fec436883b601a3cac2d4a58067e597f737b817", + "beforeRepoSetCommand": "git reset --hard b8bb8f163a89cb6f2af0ac1cfc97e89deb938368\ngit clean -fd \ngit checkout b8bb8f163a89cb6f2af0ac1cfc97e89deb938368 \ngit checkout 4fec436883b601a3cac2d4a58067e597f737b817 -- test/components/views/settings/devices/CurrentDeviceSection-test.tsx test/components/views/settings/devices/DeviceDetailHeading-test.tsx test/components/views/settings/devices/DeviceDetails-test.tsx test/components/views/settings/devices/FilteredDeviceList-test.tsx test/components/views/settings/devices/__snapshots__/CurrentDeviceSection-test.tsx.snap test/components/views/settings/devices/__snapshots__/DeviceDetailHeading-test.tsx.snap test/components/views/settings/devices/__snapshots__/DeviceDetails-test.tsx.snap test/components/views/settings/tabs/user/SessionManagerTab-test.tsx test/components/views/settings/tabs/user/__snapshots__/SessionManagerTab-test.tsx.snap", + "selectedTestFiles": [ + "test/components/views/settings/devices/DeviceDetails-test.ts", + "test/components/views/settings/tabs/user/__snapshots__/SessionManagerTab-test.tsx.snap", + "test/components/views/settings/devices/__snapshots__/DeviceDetails-test.tsx.snap", + "test/components/views/settings/devices/CurrentDeviceSection-test.tsx", + "test/components/views/settings/tabs/user/SessionManagerTab-test.ts", + "test/components/views/settings/devices/__snapshots__/DeviceDetailHeading-test.tsx.snap", + "test/components/views/settings/devices/__snapshots__/CurrentDeviceSection-test.tsx.snap", + "test/components/views/rooms/SearchResultTile-test.ts", + "test/components/structures/RightPanel-test.ts", + "test/components/views/settings/devices/DeviceDetailHeading-test.tsx", + "test/stores/widgets/StopGapWidgetDriver-test.ts", + "test/components/views/settings/devices/FilteredDeviceList-test.tsx", + "test/components/views/settings/devices/DeviceDetails-test.tsx", + "test/components/views/elements/ExternalLink-test.ts", + "test/components/views/settings/devices/CurrentDeviceSection-test.ts", + "test/settings/controllers/ThemeController-test.ts", + "test/components/views/settings/tabs/user/SessionManagerTab-test.tsx", + "test/components/views/settings/devices/DeviceDetailHeading-test.ts" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-038", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_element-hq__element-web-56c7fc1948923b4b3f3507799e725ac16bcf8018-vnan", + "selectedInstanceId": "instance_element-hq__element-web-56c7fc1948923b4b3f3507799e725ac16bcf8018-vnan", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "element-hq/element-web", + "baseCommit": "9d8efacede71e3057383684446df3bde21e7bb1a", + "language": "js", + "tags": [ + "js", + "ui_ux_bug", + "minor_bug", + "front_end_knowledge", + "ui_ux_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "element-hq.element-element-hq__element-web-56c7fc1948923b4b3f3507799e725ac16bcf8018", + "beforeRepoSetCommand": "git reset --hard 9d8efacede71e3057383684446df3bde21e7bb1a\ngit clean -fd \ngit checkout 9d8efacede71e3057383684446df3bde21e7bb1a \ngit checkout 56c7fc1948923b4b3f3507799e725ac16bcf8018 -- test/unit-tests/components/views/settings/encryption/ResetIdentityPanel-test.tsx test/unit-tests/components/views/settings/encryption/__snapshots__/ResetIdentityPanel-test.tsx.snap test/unit-tests/components/views/settings/tabs/user/__snapshots__/EncryptionUserSettingsTab-test.tsx.snap", + "selectedTestFiles": [ + "test/unit-tests/utils/exportUtils/PlainTextExport-test.ts", + "test/unit-tests/components/views/dialogs/CreateRoomDialog-test.ts", + "test/unit-tests/components/views/dialogs/security/RestoreKeyBackupDialog-test.ts", + "test/unit-tests/components/views/elements/RoomTopic-test.ts", + "test/unit-tests/components/views/settings/tabs/user/__snapshots__/EncryptionUserSettingsTab-test.tsx.snap", + "test/unit-tests/components/views/settings/SecureBackupPanel-test.ts", + "test/unit-tests/utils/crypto/deviceInfo-test.ts", + "test/unit-tests/components/views/settings/tabs/user/EncryptionUserSettingsTab-test.ts", + "test/unit-tests/utils/device/clientInformation-test.ts", + "test/unit-tests/widgets/ManagedHybrid-test.ts", + "test/unit-tests/hooks/useNotificationSettings-test.ts", + "test/unit-tests/components/views/rooms/NotificationBadge/UnreadNotificationBadge-test.ts", + "test/unit-tests/utils/direct-messages-test.ts", + "test/unit-tests/components/views/spaces/SpacePanel-test.ts", + "test/unit-tests/hooks/useUnreadNotifications-test.ts", + "test/unit-tests/utils/dm/findDMForUser-test.ts", + "test/unit-tests/utils/oidc/authorize-test.ts", + "test/unit-tests/components/views/settings/encryption/__snapshots__/ResetIdentityPanel-test.tsx.snap", + "test/unit-tests/components/views/settings/devices/FilteredDeviceList-test.ts", + "test/unit-tests/components/views/messages/DecryptionFailureBody-test.ts", + "test/unit-tests/components/views/voip/CallView-test.ts", + "test/unit-tests/stores/ReleaseAnnouncementStore-test.ts", + "test/unit-tests/hooks/useDebouncedCallback-test.ts", + "test/unit-tests/components/views/elements/RoomFacePile-test.ts", + "test/unit-tests/components/views/settings/encryption/ResetIdentityPanel-test.ts", + "test/unit-tests/utils/enums-test.ts", + "test/unit-tests/utils/exportUtils/HTMLExport-test.ts", + "test/unit-tests/components/views/beacon/OwnBeaconStatus-test.ts", + "test/unit-tests/editor/range-test.ts", + "test/unit-tests/components/views/settings/tabs/user/SidebarUserSettingsTab-test.ts", + "test/unit-tests/components/structures/RoomStatusBar-test.ts", + "test/unit-tests/components/views/dialogs/InviteDialog-test.ts", + "test/unit-tests/SecurityManager-test.ts", + "test/unit-tests/components/views/settings/encryption/ResetIdentityPanel-test.tsx", + "test/unit-tests/utils/objects-test.ts" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-039", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_element-hq__element-web-72a8f8f03b1a01bb70ef8a5bb61759416991b32c-vnan", + "selectedInstanceId": "instance_element-hq__element-web-72a8f8f03b1a01bb70ef8a5bb61759416991b32c-vnan", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "element-hq/element-web", + "baseCommit": "650b9cb0cf9bb10674057e232f4792acf83f2e46", + "language": "js", + "tags": [ + "js", + "ui_ux_enh", + "front_end_knowledge", + "ui_ux_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "element-hq.element-element-hq__element-web-72a8f8f03b1a01bb70ef8a5bb61759416991b32c", + "beforeRepoSetCommand": "git reset --hard 650b9cb0cf9bb10674057e232f4792acf83f2e46\ngit clean -fd \ngit checkout 650b9cb0cf9bb10674057e232f4792acf83f2e46 \ngit checkout 72a8f8f03b1a01bb70ef8a5bb61759416991b32c -- test/hooks/useWindowWidth-test.ts", + "selectedTestFiles": [ + "test/audio/VoiceMessageRecording-test.ts", + "test/utils/exportUtils/HTMLExport-test.ts", + "test/components/views/messages/MessageEvent-test.ts", + "test/components/views/location/LocationPicker-test.ts", + "test/components/views/dialogs/ConfirmRedactDialog-test.ts", + "test/components/views/settings/AddPrivilegedUsers-test.ts", + "test/utils/DMRoomMap-test.ts", + "test/components/views/rooms/MessageComposer-test.ts", + "test/components/views/settings/CrossSigningPanel-test.ts", + "test/components/views/dialogs/AccessSecretStorageDialog-test.ts", + "test/stores/AutoRageshakeStore-test.ts", + "test/hooks/useWindowWidth-test.ts", + "test/components/views/context_menus/ContextMenu-test.ts", + "test/utils/SearchInput-test.ts", + "test/components/views/rooms/RoomTile-test.ts", + "test/stores/room-list/algorithms/list-ordering/NaturalAlgorithm-test.ts", + "test/components/views/messages/EncryptionEvent-test.ts", + "test/notifications/PushRuleVectorState-test.ts", + "test/components/views/rooms/wysiwyg_composer/components/WysiwygComposer-test.ts", + "test/components/views/messages/MStickerBody-test.ts", + "test/stores/room-list/SpaceWatcher-test.ts" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-040", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_element-hq__element-web-75c2c1a572fa45d1ea1d1a96e9e36e303332ecaa-vnan", + "selectedInstanceId": "instance_element-hq__element-web-75c2c1a572fa45d1ea1d1a96e9e36e303332ecaa-vnan", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "element-hq/element-web", + "baseCommit": "1f8fbc819795b9c19c567b4737109183acf86162", + "language": "js", + "tags": [ + "js", + "core_feat", + "front_end_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "element-hq.element-element-hq__element-web-75c2c1a572fa45d1ea1d1a96e9e36e303332ecaa", + "beforeRepoSetCommand": "git reset --hard 1f8fbc819795b9c19c567b4737109183acf86162\ngit clean -fd \ngit checkout 1f8fbc819795b9c19c567b4737109183acf86162 \ngit checkout 75c2c1a572fa45d1ea1d1a96e9e36e303332ecaa -- test/audio/VoiceRecording-test.ts", + "selectedTestFiles": [ + "test/components/views/beacon/OwnBeaconStatus-test.ts", + "test/modules/AppModule-test.ts", + "test/voice-broadcast/stores/VoiceBroadcastPlaybacksStore-test.ts", + "test/components/views/settings/devices/CurrentDeviceSection-test.ts", + "test/components/views/rooms/RoomPreviewBar-test.ts", + "test/components/views/context_menus/EmbeddedPage-test.ts", + "test/components/views/elements/QRCode-test.ts", + "test/audio/VoiceRecording-test.ts", + "test/voice-broadcast/stores/VoiceBroadcastPreRecordingStore-test.ts", + "test/components/views/dialogs/AccessSecretStorageDialog-test.ts", + "test/components/views/right_panel/UserInfo-test.ts", + "test/voice-broadcast/utils/hasRoomLiveVoiceBroadcast-test.ts", + "test/components/views/voip/CallView-test.ts", + "test/utils/numbers-test.ts", + "test/notifications/PushRuleVectorState-test.ts", + "test/components/views/settings/devices/SecurityRecommendations-test.ts", + "test/voice-broadcast/utils/VoiceBroadcastResumer-test.ts", + "test/createRoom-test.ts", + "test/utils/maps-test.ts" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-041", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_element-hq__element-web-776ffa47641c7ec6d142ab4a47691c30ebf83c2e", + "selectedInstanceId": "instance_element-hq__element-web-776ffa47641c7ec6d142ab4a47691c30ebf83c2e", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "element-hq/element-web", + "baseCommit": "8b54be6f48631083cb853cda5def60d438daa14f", + "language": "js", + "tags": [ + "js", + "ui_ux_enh", + "code_quality_enh", + "refactoring_enh", + "front_end_knowledge", + "web_knowledge", + "ui_ux_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "element-hq.element-element-hq__element-web-776ffa47641c7ec6d142ab4a47691c30ebf83c2e", + "beforeRepoSetCommand": "git reset --hard 8b54be6f48631083cb853cda5def60d438daa14f\ngit clean -fd \ngit checkout 8b54be6f48631083cb853cda5def60d438daa14f \ngit checkout 776ffa47641c7ec6d142ab4a47691c30ebf83c2e -- test/components/views/settings/devices/__snapshots__/CurrentDeviceSection-test.tsx.snap test/components/views/settings/shared/SettingsSubsectionHeading-test.tsx test/components/views/settings/shared/__snapshots__/SettingsSubsectionHeading-test.tsx.snap test/components/views/settings/tabs/user/SessionManagerTab-test.tsx test/components/views/settings/tabs/user/__snapshots__/SessionManagerTab-test.tsx.snap", + "selectedTestFiles": [ + "test/components/views/settings/tabs/user/SessionManagerTab-test.tsx", + "test/components/views/settings/shared/__snapshots__/SettingsSubsectionHeading-test.tsx.snap", + "/app/test/components/views/settings/devices/CurrentDeviceSection-test.ts", + "test/components/views/settings/devices/__snapshots__/CurrentDeviceSection-test.tsx.snap", + "test/components/views/settings/tabs/user/__snapshots__/SessionManagerTab-test.tsx.snap", + "/app/test/components/views/settings/tabs/user/SessionManagerTab-test.ts", + "test/components/views/settings/shared/SettingsSubsectionHeading-test.tsx" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-042", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_element-hq__element-web-9a31cd0fa849da810b4fac6c6c015145e850b282-vnan", + "selectedInstanceId": "instance_element-hq__element-web-9a31cd0fa849da810b4fac6c6c015145e850b282-vnan", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "element-hq/element-web", + "baseCommit": "b03433ef8b83a4c82b9d879946fb1ab5afaca522", + "language": "js", + "tags": [ + "js", + "core_feat", + "ui_ux_feat", + "localization_feat", + "front_end_knowledge", + "web_knowledge", + "ui_ux_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "element-hq.element-element-hq__element-web-9a31cd0fa849da810b4fac6c6c015145e850b282", + "beforeRepoSetCommand": "git reset --hard b03433ef8b83a4c82b9d879946fb1ab5afaca522\ngit clean -fd \ngit checkout b03433ef8b83a4c82b9d879946fb1ab5afaca522 \ngit checkout 9a31cd0fa849da810b4fac6c6c015145e850b282 -- test/components/views/settings/JoinRuleSettings-test.tsx", + "selectedTestFiles": [ + "test/components/views/messages/MessageActionBar-test.ts", + "test/components/views/settings/EventIndexPanel-test.ts", + "test/voice-broadcast/stores/VoiceBroadcastPreRecordingStore-test.ts", + "test/components/views/elements/LabelledCheckbox-test.ts", + "test/components/views/settings/tabs/user/SessionManagerTab-test.ts", + "test/components/structures/AutocompleteInput-test.ts", + "test/components/views/settings/JoinRuleSettings-test.tsx", + "test/components/views/settings/JoinRuleSettings-test.ts", + "test/stores/SetupEncryptionStore-test.ts", + "test/components/views/context_menus/SpaceContextMenu-test.ts" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-043", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_element-hq__element-web-aec454dd6feeb93000380523cbb0b3681c0275fd-vnan", + "selectedInstanceId": "instance_element-hq__element-web-aec454dd6feeb93000380523cbb0b3681c0275fd-vnan", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "element-hq/element-web", + "baseCommit": "1c039fcd3880ef4fefa58812d375104d2d70fe6c", + "language": "js", + "tags": [ + "js", + "core_feat", + "performance_enh", + "code_quality_enh", + "front_end_knowledge", + "back_end_knowledge", + "web_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "element-hq.element-element-hq__element-web-aec454dd6feeb93000380523cbb0b3681c0275fd", + "beforeRepoSetCommand": "git reset --hard 1c039fcd3880ef4fefa58812d375104d2d70fe6c\ngit clean -fd \ngit checkout 1c039fcd3880ef4fefa58812d375104d2d70fe6c \ngit checkout aec454dd6feeb93000380523cbb0b3681c0275fd -- test/components/views/elements/Pill-test.tsx test/components/views/elements/__snapshots__/Pill-test.tsx.snap test/contexts/SdkContext-test.ts test/stores/UserProfilesStore-test.ts test/utils/LruCache-test.ts", + "selectedTestFiles": [ + "test/components/views/elements/__snapshots__/Pill-test.tsx.snap", + "test/components/views/dialogs/InviteDialog-test.ts", + "test/i18n-test/languageHandler-test.ts", + "test/components/structures/AutocompleteInput-test.ts", + "test/voice-broadcast/audio/VoiceBroadcastRecorder-test.ts", + "test/stores/UserProfilesStore-test.ts", + "test/components/views/beacon/DialogSidebar-test.ts", + "test/components/views/elements/Pill-test.tsx", + "test/components/views/messages/MImageBody-test.ts", + "test/utils/LruCache-test.ts", + "test/components/views/dialogs/ExportDialog-test.ts", + "test/contexts/SdkContext-test.ts", + "test/components/views/elements/AccessibleButton-test.ts", + "test/components/views/rooms/wysiwyg_composer/components/WysiwygComposer-test.ts" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-044", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_element-hq__element-web-ca58617cee8aa91c93553449bfdf9b3465a5119b-vnan", + "selectedInstanceId": "instance_element-hq__element-web-ca58617cee8aa91c93553449bfdf9b3465a5119b-vnan", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "element-hq/element-web", + "baseCommit": "5583d07f25071ceb4f84462150717b68a244f166", + "language": "js", + "tags": [ + "js", + "code_quality_enh", + "minor_bug", + "refactoring_enh", + "front_end_knowledge", + "web_knowledge", + "performance_knowledge", + "ui_ux_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "element-hq.element-element-hq__element-web-ca58617cee8aa91c93553449bfdf9b3465a5119b", + "beforeRepoSetCommand": "git reset --hard 5583d07f25071ceb4f84462150717b68a244f166\ngit clean -fd \ngit checkout 5583d07f25071ceb4f84462150717b68a244f166 \ngit checkout ca58617cee8aa91c93553449bfdf9b3465a5119b -- test/LegacyCallHandler-test.ts", + "selectedTestFiles": [ + "test/components/views/rooms/NotificationBadge/NotificationBadge-test.ts", + "test/LegacyCallHandler-test.ts", + "test/components/views/elements/LabelledCheckbox-test.ts", + "test/modules/ModuleRunner-test.ts", + "test/utils/beacon/geolocation-test.ts", + "test/components/views/context_menus/ContextMenu-test.ts", + "test/components/views/settings/tabs/room/VoipRoomSettingsTab-test.ts", + "test/components/views/settings/devices/SecurityRecommendations-test.ts", + "test/components/views/audio_messages/RecordingPlayback-test.ts", + "test/components/structures/auth/ForgotPassword-test.ts", + "test/components/views/spaces/SpacePanel-test.ts", + "test/components/views/settings/devices/filter-test.ts", + "test/components/views/rooms/wysiwyg_composer/utils/createMessageContent-test.ts", + "test/audio/Playback-test.ts" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-045", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_element-hq__element-web-cf3c899dd1f221aa1a1f4c5a80dffc05b9c21c85-vnan", + "selectedInstanceId": "instance_element-hq__element-web-cf3c899dd1f221aa1a1f4c5a80dffc05b9c21c85-vnan", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "element-hq/element-web", + "baseCommit": "973513cc758030917cb339ba35d6436bc2c7d5dd", + "language": "js", + "tags": [ + "js", + "ui_ux_enh", + "web_knowledge", + "front_end_knowledge", + "ui_ux_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "element-hq.element-element-hq__element-web-cf3c899dd1f221aa1a1f4c5a80dffc05b9c21c85", + "beforeRepoSetCommand": "git reset --hard 973513cc758030917cb339ba35d6436bc2c7d5dd\ngit clean -fd \ngit checkout 973513cc758030917cb339ba35d6436bc2c7d5dd \ngit checkout cf3c899dd1f221aa1a1f4c5a80dffc05b9c21c85 -- test/voice-broadcast/components/atoms/LiveBadge-test.tsx test/voice-broadcast/components/atoms/VoiceBroadcastHeader-test.tsx test/voice-broadcast/components/atoms/__snapshots__/LiveBadge-test.tsx.snap test/voice-broadcast/components/atoms/__snapshots__/VoiceBroadcastHeader-test.tsx.snap test/voice-broadcast/components/molecules/VoiceBroadcastPlaybackBody-test.tsx test/voice-broadcast/components/molecules/VoiceBroadcastRecordingBody-test.tsx test/voice-broadcast/components/molecules/__snapshots__/VoiceBroadcastPlaybackBody-test.tsx.snap test/voice-broadcast/components/molecules/__snapshots__/VoiceBroadcastRecordingBody-test.tsx.snap test/voice-broadcast/components/molecules/__snapshots__/VoiceBroadcastRecordingPip-test.tsx.snap test/voice-broadcast/models/VoiceBroadcastPlayback-test.ts", + "selectedTestFiles": [ + "test/utils/device/clientInformation-test.ts", + "test/components/views/elements/ProgressBar-test.ts", + "test/voice-broadcast/components/molecules/VoiceBroadcastRecordingBody-test.tsx", + "test/voice-broadcast/components/molecules/VoiceBroadcastRecordingPip-test.ts", + "test/voice-broadcast/components/atoms/VoiceBroadcastHeader-test.tsx", + "test/voice-broadcast/components/molecules/VoiceBroadcastRecordingBody-test.ts", + "test/voice-broadcast/components/atoms/VoiceBroadcastHeader-test.ts", + "test/voice-broadcast/components/atoms/LiveBadge-test.tsx", + "test/voice-broadcast/components/atoms/__snapshots__/VoiceBroadcastHeader-test.tsx.snap", + "test/voice-broadcast/components/atoms/__snapshots__/LiveBadge-test.tsx.snap", + "test/voice-broadcast/components/atoms/LiveBadge-test.ts", + "test/voice-broadcast/components/molecules/VoiceBroadcastPlaybackBody-test.tsx", + "test/voice-broadcast/components/molecules/__snapshots__/VoiceBroadcastPlaybackBody-test.tsx.snap", + "test/audio/Playback-test.ts", + "test/voice-broadcast/models/VoiceBroadcastPlayback-test.ts", + "test/voice-broadcast/components/molecules/VoiceBroadcastPlaybackBody-test.ts", + "test/voice-broadcast/components/molecules/__snapshots__/VoiceBroadcastRecordingBody-test.tsx.snap", + "test/utils/local-room-test.ts", + "test/components/views/dialogs/ForwardDialog-test.ts", + "test/voice-broadcast/components/molecules/__snapshots__/VoiceBroadcastRecordingPip-test.tsx.snap", + "test/events/forward/getForwardableEvent-test.ts", + "test/utils/dm/createDmLocalRoom-test.ts" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-046", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_element-hq__element-web-dae13ac8522fc6d41e64d1ac6e3174486fdcce0c-vnan", + "selectedInstanceId": "instance_element-hq__element-web-dae13ac8522fc6d41e64d1ac6e3174486fdcce0c-vnan", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "element-hq/element-web", + "baseCommit": "526645c79160ab1ad4b4c3845de27d51263a405e", + "language": "js", + "tags": [ + "js", + "code_quality_enh", + "refactoring_enh", + "ui_ux_enh", + "front_end_knowledge", + "web_knowledge", + "ui_ux_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "element-hq.element-element-hq__element-web-dae13ac8522fc6d41e64d1ac6e3174486fdcce0c", + "beforeRepoSetCommand": "git reset --hard 526645c79160ab1ad4b4c3845de27d51263a405e\ngit clean -fd \ngit checkout 526645c79160ab1ad4b4c3845de27d51263a405e \ngit checkout dae13ac8522fc6d41e64d1ac6e3174486fdcce0c -- test/Unread-test.ts test/test-utils/threads.ts", + "selectedTestFiles": [ + "test/test-utils/threads.ts", + "test/editor/position-test.ts", + "test/components/views/typography/Caption-test.ts", + "test/components/structures/UserMenu-test.ts", + "test/components/views/settings/devices/SelectableDeviceTile-test.ts", + "test/utils/localRoom/isRoomReady-test.ts", + "test/Unread-test.ts", + "test/voice-broadcast/utils/shouldDisplayAsVoiceBroadcastRecordingTile-test.ts" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-047", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_element-hq__element-web-ecfd1736e5dd9808e87911fc264e6c816653e1a9-vnan", + "selectedInstanceId": "instance_element-hq__element-web-ecfd1736e5dd9808e87911fc264e6c816653e1a9-vnan", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "element-hq/element-web", + "baseCommit": "f34c1609c3c42f095b59bc068620f342894f94ed", + "language": "js", + "tags": [ + "js", + "core_feat", + "minor_bug", + "ui_ux_bug", + "front_end_knowledge", + "ui_ux_knowledge", + "performance_knowledge", + "web_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "element-hq.element-element-hq__element-web-ecfd1736e5dd9808e87911fc264e6c816653e1a9", + "beforeRepoSetCommand": "git reset --hard f34c1609c3c42f095b59bc068620f342894f94ed\ngit clean -fd \ngit checkout f34c1609c3c42f095b59bc068620f342894f94ed \ngit checkout ecfd1736e5dd9808e87911fc264e6c816653e1a9 -- test/components/structures/RoomSearchView-test.tsx test/components/views/rooms/SearchResultTile-test.tsx", + "selectedTestFiles": [ + "test/components/views/rooms/SearchResultTile-test.ts", + "test/utils/sets-test.ts", + "test/components/structures/RoomSearchView-test.tsx", + "test/components/structures/RoomSearchView-test.ts", + "test/utils/direct-messages-test.ts", + "test/components/views/settings/devices/DeviceExpandDetailsButton-test.ts", + "test/components/views/context_menus/ThreadListContextMenu-test.ts", + "test/components/views/rooms/SearchResultTile-test.tsx" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-048", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_flipt-io__flipt-05d7234fa582df632f70a7cd10194d61bd7043b9", + "selectedInstanceId": "instance_flipt-io__flipt-05d7234fa582df632f70a7cd10194d61bd7043b9", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "flipt-io/flipt", + "baseCommit": "b64891e57df74861e89ebcfa81394e4bc096f8c7", + "language": "go", + "tags": [ + "go", + "api_feat", + "code_quality_enh", + "scalability_enh", + "back_end_knowledge", + "database_knowledge", + "api_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "flipt-io.flipt-flipt-io__flipt-05d7234fa582df632f70a7cd10194d61bd7043b9", + "beforeRepoSetCommand": "git reset --hard b64891e57df74861e89ebcfa81394e4bc096f8c7\ngit clean -fd \ngit checkout b64891e57df74861e89ebcfa81394e4bc096f8c7 \ngit checkout 05d7234fa582df632f70a7cd10194d61bd7043b9 -- internal/storage/fs/object/file_test.go internal/storage/fs/object/fileinfo_test.go internal/storage/fs/snapshot_test.go internal/storage/fs/store_test.go", + "selectedTestFiles": [ + "TestNewFile", + "TestRemapScheme", + "TestFSWithoutIndex", + "TestSupportedSchemes", + "TestFileInfoIsDir", + "TestFileInfo", + "TestGetVersion" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-049", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_flipt-io__flipt-1737085488ecdcd3299c8e61af45a8976d457b7e", + "selectedInstanceId": "instance_flipt-io__flipt-1737085488ecdcd3299c8e61af45a8976d457b7e", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "flipt-io/flipt", + "baseCommit": "1f6255dda6648ecafd94826cda3fac2486af4b0f", + "language": "go", + "tags": [ + "go", + "major_bug", + "data_bug", + "compatibility_bug", + "back_end_knowledge", + "devops_knowledge", + "cloud_knowledge", + "api_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "flipt-io.flipt-flipt-io__flipt-1737085488ecdcd3299c8e61af45a8976d457b7e", + "beforeRepoSetCommand": "git reset --hard 1f6255dda6648ecafd94826cda3fac2486af4b0f\ngit clean -fd \ngit checkout 1f6255dda6648ecafd94826cda3fac2486af4b0f \ngit checkout 1737085488ecdcd3299c8e61af45a8976d457b7e -- internal/ext/importer_test.go", + "selectedTestFiles": [ + "TestImport" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-050", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_flipt-io__flipt-21a935ad7886cc50c46852be21b37f363a926af0", + "selectedInstanceId": "instance_flipt-io__flipt-21a935ad7886cc50c46852be21b37f363a926af0", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "flipt-io/flipt", + "baseCommit": "4e1cd36398ee73acf7d9235b517f05178651c464", + "language": "go", + "tags": [ + "go", + "integration_feat", + "customization_feat", + "back_end_knowledge", + "devops_knowledge", + "infrastructure_knowledge", + "performance_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "flipt-io.flipt-flipt-io__flipt-21a935ad7886cc50c46852be21b37f363a926af0", + "beforeRepoSetCommand": "git reset --hard 4e1cd36398ee73acf7d9235b517f05178651c464\ngit clean -fd \ngit checkout 4e1cd36398ee73acf7d9235b517f05178651c464 \ngit checkout 21a935ad7886cc50c46852be21b37f363a926af0 -- config/config_test.go", + "selectedTestFiles": [ + "TestScheme", + "TestValidate", + "TestDatabaseProtocol", + "TestLoad", + "TestServeHTTP", + "TestCacheBackend", + "TestLogEncoding" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-051", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_flipt-io__flipt-2ce8a0331e8a8f63f2c1b555db8277ffe5aa2e63", + "selectedInstanceId": "instance_flipt-io__flipt-2ce8a0331e8a8f63f2c1b555db8277ffe5aa2e63", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "flipt-io/flipt", + "baseCommit": "f3421c143953d2a2e3f4373f8ec366e0904f9bdd", + "language": "go", + "tags": [ + "go", + "compatibility_bug", + "major_bug", + "back_end_knowledge", + "api_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "flipt-io.flipt-flipt-io__flipt-2ce8a0331e8a8f63f2c1b555db8277ffe5aa2e63", + "beforeRepoSetCommand": "git reset --hard f3421c143953d2a2e3f4373f8ec366e0904f9bdd\ngit clean -fd \ngit checkout f3421c143953d2a2e3f4373f8ec366e0904f9bdd \ngit checkout 2ce8a0331e8a8f63f2c1b555db8277ffe5aa2e63 -- internal/server/middleware/grpc/middleware_test.go", + "selectedTestFiles": [ + "TestCacheUnaryInterceptor_UpdateVariant", + "TestAuditUnaryInterceptor_OrderRollout", + "TestAuditUnaryInterceptor_CreateSegment", + "TestAuthMetadataAuditUnaryInterceptor", + "TestAuditUnaryInterceptor_CreateDistribution", + "TestFliptAcceptServerVersionUnaryInterceptor", + "TestAuditUnaryInterceptor_UpdateFlag", + "TestCacheUnaryInterceptor_UpdateFlag", + "TestAuditUnaryInterceptor_DeleteDistribution", + "TestAuditUnaryInterceptor_UpdateVariant", + "TestAuditUnaryInterceptor_OrderRule", + "TestAuditUnaryInterceptor_DeleteVariant", + "TestCacheUnaryInterceptor_CreateVariant", + "TestCacheUnaryInterceptor_Evaluation_Variant", + "TestAuditUnaryInterceptor_UpdateSegment", + "TestAuditUnaryInterceptor_CreateToken", + "TestAuditUnaryInterceptor_CreateRule", + "TestCacheUnaryInterceptor_DeleteVariant", + "TestEvaluationUnaryInterceptor_Noop", + "TestAuditUnaryInterceptor_CreateRollout", + "TestAuditUnaryInterceptor_DeleteRollout", + "TestAuditUnaryInterceptor_CreateConstraint", + "TestAuditUnaryInterceptor_DeleteNamespace", + "TestAuditUnaryInterceptor_CreateFlag", + "TestAuditUnaryInterceptor_CreateVariant", + "TestAuditUnaryInterceptor_UpdateRule", + "TestCacheUnaryInterceptor_DeleteFlag", + "TestAuditUnaryInterceptor_DeleteSegment", + "TestAuditUnaryInterceptor_DeleteConstraint", + "TestCacheUnaryInterceptor_Evaluate", + "TestErrorUnaryInterceptor", + "TestAuditUnaryInterceptor_UpdateConstraint", + "TestCacheUnaryInterceptor_GetFlag", + "TestValidationUnaryInterceptor", + "TestAuditUnaryInterceptor_DeleteFlag", + "TestAuditUnaryInterceptor_UpdateRollout", + "TestEvaluationUnaryInterceptor_BatchEvaluation", + "TestCacheUnaryInterceptor_Evaluation_Boolean", + "TestAuditUnaryInterceptor_CreateNamespace", + "TestAuditUnaryInterceptor_DeleteRule", + "TestAuditUnaryInterceptor_UpdateNamespace", + "TestEvaluationUnaryInterceptor_Evaluation", + "TestAuditUnaryInterceptor_UpdateDistribution" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-052", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_flipt-io__flipt-381b90f718435c4694380b5fcd0d5cf8e3b5a25a", + "selectedInstanceId": "instance_flipt-io__flipt-381b90f718435c4694380b5fcd0d5cf8e3b5a25a", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "flipt-io/flipt", + "baseCommit": "0ed96dc5d33768c4b145d68d52e80e7bce3790d0", + "language": "go", + "tags": [ + "go", + "integration_feat", + "code_quality_enh", + "back_end_knowledge", + "api_knowledge", + "devops_knowledge", + "cloud_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "flipt-io.flipt-flipt-io__flipt-381b90f718435c4694380b5fcd0d5cf8e3b5a25a", + "beforeRepoSetCommand": "git reset --hard 0ed96dc5d33768c4b145d68d52e80e7bce3790d0\ngit clean -fd \ngit checkout 0ed96dc5d33768c4b145d68d52e80e7bce3790d0 \ngit checkout 381b90f718435c4694380b5fcd0d5cf8e3b5a25a -- internal/config/config_test.go", + "selectedTestFiles": [ + "TestServeHTTP", + "TestTracingExporter", + "Test_mustBindEnv", + "TestCacheBackend", + "TestDefaultDatabaseRoot", + "TestLogEncoding", + "TestLoad", + "TestJSONSchema", + "TestScheme", + "TestMarshalYAML", + "TestDatabaseProtocol" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-053", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_flipt-io__flipt-3b2c25ee8a3ac247c3fad13ad8d64ace34ec8ee7", + "selectedInstanceId": "instance_flipt-io__flipt-3b2c25ee8a3ac247c3fad13ad8d64ace34ec8ee7", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "flipt-io/flipt", + "baseCommit": "8d72418bf67cec833da7f59beeecb5abfd48cb05", + "language": "go", + "tags": [ + "go", + "major_bug", + "integration_bug", + "compatibility_bug", + "back_end_knowledge", + "api_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "flipt-io.flipt-flipt-io__flipt-3b2c25ee8a3ac247c3fad13ad8d64ace34ec8ee7", + "beforeRepoSetCommand": "git reset --hard 8d72418bf67cec833da7f59beeecb5abfd48cb05\ngit clean -fd \ngit checkout 8d72418bf67cec833da7f59beeecb5abfd48cb05 \ngit checkout 3b2c25ee8a3ac247c3fad13ad8d64ace34ec8ee7 -- internal/server/ofrep/evaluation_test.go internal/server/ofrep/extensions_test.go internal/server/ofrep/middleware_test.go", + "selectedTestFiles": [ + "TestEvaluateFlag_Failure", + "TestEvaluateFlag_Success", + "TestGetProviderConfiguration", + "TestEvaluateBulkSuccess" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-054", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_flipt-io__flipt-3d5a345f94c2adc8a0eaa102c189c08ad4c0f8e8", + "selectedInstanceId": "instance_flipt-io__flipt-3d5a345f94c2adc8a0eaa102c189c08ad4c0f8e8", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "flipt-io/flipt", + "baseCommit": "91cc1b9fc38280a53a36e1e9543d87d7306144b2", + "language": "go", + "tags": [ + "go", + "performance_enh", + "dev_ops_enh", + "back_end_knowledge", + "devops_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "flipt-io.flipt-flipt-io__flipt-3d5a345f94c2adc8a0eaa102c189c08ad4c0f8e8", + "beforeRepoSetCommand": "git reset --hard 91cc1b9fc38280a53a36e1e9543d87d7306144b2\ngit clean -fd \ngit checkout 91cc1b9fc38280a53a36e1e9543d87d7306144b2 \ngit checkout 3d5a345f94c2adc8a0eaa102c189c08ad4c0f8e8 -- internal/config/config_test.go internal/tracing/tracing_test.go", + "selectedTestFiles": [ + "TestLoad", + "TestCacheBackend", + "TestLogEncoding", + "TestMarshalYAML", + "Test_mustBindEnv", + "TestDefaultDatabaseRoot", + "TestAnalyticsClickhouseConfiguration", + "TestJSONSchema", + "TestDatabaseProtocol", + "TestGetConfigFile", + "TestServeHTTP", + "TestScheme", + "TestTracingExporter" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-055", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_flipt-io__flipt-40007b9d97e3862bcef8c20ae6c87b22ea0627f0", + "selectedInstanceId": "instance_flipt-io__flipt-40007b9d97e3862bcef8c20ae6c87b22ea0627f0", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "flipt-io/flipt", + "baseCommit": "bbf0a917fbdf4c92017f760b63727b921eb9fc98", + "language": "go", + "tags": [ + "go", + "api_feat", + "security_feat", + "integration_feat", + "back_end_knowledge", + "api_knowledge", + "security_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "flipt-io.flipt-flipt-io__flipt-40007b9d97e3862bcef8c20ae6c87b22ea0627f0", + "beforeRepoSetCommand": "git reset --hard bbf0a917fbdf4c92017f760b63727b921eb9fc98\ngit clean -fd \ngit checkout bbf0a917fbdf4c92017f760b63727b921eb9fc98 \ngit checkout 40007b9d97e3862bcef8c20ae6c87b22ea0627f0 -- internal/config/config_test.go internal/server/authn/method/github/server_test.go", + "selectedTestFiles": [ + "TestLoad", + "TestGithubSimpleOrganizationDecode" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-056", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_flipt-io__flipt-518ec324b66a07fdd95464a5e9ca5fe7681ad8f9", + "selectedInstanceId": "instance_flipt-io__flipt-518ec324b66a07fdd95464a5e9ca5fe7681ad8f9", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "flipt-io/flipt", + "baseCommit": "0018c5df774444117b107dfe3fe503d4c7126d73", + "language": "go", + "tags": [ + "go", + "regression_bug", + "ui_ux_bug", + "data_bug", + "back_end_knowledge", + "web_knowledge", + "api_knowledge", + "devops_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "flipt-io.flipt-flipt-io__flipt-518ec324b66a07fdd95464a5e9ca5fe7681ad8f9", + "beforeRepoSetCommand": "git reset --hard 0018c5df774444117b107dfe3fe503d4c7126d73\ngit clean -fd \ngit checkout 0018c5df774444117b107dfe3fe503d4c7126d73 \ngit checkout 518ec324b66a07fdd95464a5e9ca5fe7681ad8f9 -- internal/config/config_test.go", + "selectedTestFiles": [ + "TestLoad" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-057", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_flipt-io__flipt-690672523398c2b6f6e4562f0bf9868664ab894f", + "selectedInstanceId": "instance_flipt-io__flipt-690672523398c2b6f6e4562f0bf9868664ab894f", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "flipt-io/flipt", + "baseCommit": "6da20eb7afb693a1cbee2482272e3aee2fbd43ee", + "language": "go", + "tags": [ + "go", + "refactoring_enh", + "code_quality_enh", + "back_end_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "flipt-io.flipt-flipt-io__flipt-690672523398c2b6f6e4562f0bf9868664ab894f", + "beforeRepoSetCommand": "git reset --hard 6da20eb7afb693a1cbee2482272e3aee2fbd43ee\ngit clean -fd \ngit checkout 6da20eb7afb693a1cbee2482272e3aee2fbd43ee \ngit checkout 690672523398c2b6f6e4562f0bf9868664ab894f -- internal/cmd/grpc_test.go internal/tracing/tracing_test.go", + "selectedTestFiles": [ + "TestGetTraceExporter", + "TestNewResourceDefault" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-058", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_flipt-io__flipt-6fd0f9e2587f14ac1fdd1c229f0bcae0468c8daa", + "selectedInstanceId": "instance_flipt-io__flipt-6fd0f9e2587f14ac1fdd1c229f0bcae0468c8daa", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "flipt-io/flipt", + "baseCommit": "563a8c4593610e431f0c3db55e88a679c4386991", + "language": "go", + "tags": [ + "go", + "core_feat", + "back_end_knowledge", + "infrastructure_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "flipt-io.flipt-flipt-io__flipt-6fd0f9e2587f14ac1fdd1c229f0bcae0468c8daa", + "beforeRepoSetCommand": "git reset --hard 563a8c4593610e431f0c3db55e88a679c4386991\ngit clean -fd \ngit checkout 563a8c4593610e431f0c3db55e88a679c4386991 \ngit checkout 6fd0f9e2587f14ac1fdd1c229f0bcae0468c8daa -- internal/config/config_test.go internal/oci/file_test.go", + "selectedTestFiles": [ + "TestLogEncoding", + "TestMarshalYAML", + "TestNewStore", + "TestStore_Fetch", + "TestLoad", + "TestServeHTTP", + "TestTracingExporter", + "Test_mustBindEnv", + "TestDatabaseProtocol", + "TestScheme", + "TestJSONSchema", + "TestStore_Fetch_InvalidMediaType", + "TestCacheBackend" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-059", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_flipt-io__flipt-756f00f79ba8abf9fe53f3c6c818123b42eb7355", + "selectedInstanceId": "instance_flipt-io__flipt-756f00f79ba8abf9fe53f3c6c818123b42eb7355", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "flipt-io/flipt", + "baseCommit": "266e5e143e87519047b9be3202a0aba273b83de3", + "language": "go", + "tags": [ + "go", + "refactoring_enh", + "code_quality_enh", + "scalability_enh", + "technical_debt_enh", + "back_end_knowledge", + "api_knowledge", + "infrastructure_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "flipt-io.flipt-flipt-io__flipt-756f00f79ba8abf9fe53f3c6c818123b42eb7355", + "beforeRepoSetCommand": "git reset --hard 266e5e143e87519047b9be3202a0aba273b83de3\ngit clean -fd \ngit checkout 266e5e143e87519047b9be3202a0aba273b83de3 \ngit checkout 756f00f79ba8abf9fe53f3c6c818123b42eb7355 -- internal/config/config_test.go", + "selectedTestFiles": [ + "TestLogEncoding", + "TestServeHTTP", + "TestLoad", + "TestDatabaseProtocol", + "TestScheme", + "TestJSONSchema", + "TestCacheBackend" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-060", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_flipt-io__flipt-8bd3604dc54b681f1f0f7dd52cbc70b3024184b6", + "selectedInstanceId": "instance_flipt-io__flipt-8bd3604dc54b681f1f0f7dd52cbc70b3024184b6", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "flipt-io/flipt", + "baseCommit": "25a5f278e1116ca22f86d86b4a5259ca05ef2623", + "language": "go", + "tags": [ + "go", + "critical_bug", + "back_end_knowledge", + "devops_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "flipt-io.flipt-flipt-io__flipt-8bd3604dc54b681f1f0f7dd52cbc70b3024184b6", + "beforeRepoSetCommand": "git reset --hard 25a5f278e1116ca22f86d86b4a5259ca05ef2623\ngit clean -fd \ngit checkout 25a5f278e1116ca22f86d86b4a5259ca05ef2623 \ngit checkout 8bd3604dc54b681f1f0f7dd52cbc70b3024184b6 -- internal/server/audit/template/executer_test.go internal/server/audit/template/leveled_logger_test.go internal/server/audit/webhook/client_test.go", + "selectedTestFiles": [ + "TestExecuter_JSON_Failure", + "TestExecuter_Execute", + "TestConstructorWebhookTemplate", + "TestWebhookClient", + "TestExecuter_Execute_toJson_valid_Json", + "TestLeveledLogger", + "TestConstructorWebhookClient" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-061", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_flipt-io__flipt-96820c3ad10b0b2305e8877b6b303f7fafdf815f", + "selectedInstanceId": "instance_flipt-io__flipt-96820c3ad10b0b2305e8877b6b303f7fafdf815f", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "flipt-io/flipt", + "baseCommit": "8dd44097778951eaa6976631d35bc418590d1555", + "language": "go", + "tags": [ + "go", + "major_bug", + "data_bug", + "compatibility_bug", + "back_end_knowledge", + "api_knowledge", + "authentication_authorization_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "flipt-io.flipt-flipt-io__flipt-96820c3ad10b0b2305e8877b6b303f7fafdf815f", + "beforeRepoSetCommand": "git reset --hard 8dd44097778951eaa6976631d35bc418590d1555\ngit clean -fd \ngit checkout 8dd44097778951eaa6976631d35bc418590d1555 \ngit checkout 96820c3ad10b0b2305e8877b6b303f7fafdf815f -- internal/oci/ecr/credentials_store_test.go internal/oci/ecr/ecr_test.go internal/oci/ecr/mock_Client_test.go internal/oci/ecr/mock_PrivateClient_test.go internal/oci/ecr/mock_PublicClient_test.go internal/oci/file_test.go", + "selectedTestFiles": [ + "TestStore_List", + "TestFile", + "TestWithCredentials", + "TestStore_Fetch_InvalidMediaType", + "TestStore_FetchWithECR", + "TestAuthenicationTypeIsValid", + "TestParseReference", + "TestStore_Fetch", + "TestECRCredential", + "TestStore_Copy", + "TestPrivateClient", + "TestDefaultClientFunc", + "TestWithManifestVersion", + "TestCredential", + "TestPublicClient", + "TestStore_Build" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-062", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_flipt-io__flipt-9d25c18b79bc7829a6fb08ec9e8793d5d17e2868", + "selectedInstanceId": "instance_flipt-io__flipt-9d25c18b79bc7829a6fb08ec9e8793d5d17e2868", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "flipt-io/flipt", + "baseCommit": "fa8f302adcde48b4a94bf45726fe4da73de5e5ab", + "language": "go", + "tags": [ + "go", + "core_feat", + "api_feat", + "integration_feat", + "back_end_knowledge", + "api_knowledge", + "devops_knowledge", + "infrastructure_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "flipt-io.flipt-flipt-io__flipt-9d25c18b79bc7829a6fb08ec9e8793d5d17e2868", + "beforeRepoSetCommand": "git reset --hard fa8f302adcde48b4a94bf45726fe4da73de5e5ab\ngit clean -fd \ngit checkout fa8f302adcde48b4a94bf45726fe4da73de5e5ab \ngit checkout 9d25c18b79bc7829a6fb08ec9e8793d5d17e2868 -- internal/server/evaluation/ofrep_bridge_test.go internal/server/ofrep/evaluation_test.go internal/server/ofrep/extensions_test.go", + "selectedTestFiles": [ + "TestBoolean_SegmentMatch_Constraint_EntityId", + "TestEvaluator_ErrorParsingDateTime", + "TestVariant_FlagNotFound", + "TestEvaluator_DistributionNotMatched", + "TestEvaluator_MatchAll_RolloutDistribution", + "TestEvaluator_MatchAny_NoConstraints", + "TestBoolean_DefaultRule_NoRollouts", + "TestEvaluator_MatchAll_RolloutDistribution_MultiRule", + "TestEvaluateFlag_Failure", + "TestBoolean_DefaultRuleFallthrough_WithPercentageRollout", + "TestBoolean_PercentageRuleMatch", + "TestEvaluator_RulesOutOfOrder", + "TestEvaluator_ErrorGettingRules", + "TestEvaluator_FlagDisabled_DefaultVariant", + "TestVariant_Success", + "TestOFREPEvaluationBridge_Boolean", + "TestEvaluateFlag_Success", + "Test_Server_AllowsNamespaceScopedAuthentication", + "TestBoolean_NonBooleanFlagError", + "TestVariant_EvaluateFailure_OnGetEvaluationRules", + "TestBoolean_SegmentMatch_MultipleSegments_WithAnd", + "Test_matchesNumber", + "TestEvaluator_MatchAll_NoDistributions_DefaultVariant", + "TestEvaluator_DistributionNotMatched_DefaultVariant", + "TestEvaluator_MatchAny_NoDistributions_DefaultVariant", + "TestBoolean_FlagNotFoundError", + "TestEvaluator_ErrorGettingDistributions", + "TestBatch_Success", + "TestGetProviderConfiguration", + "TestBatch_UnknownFlagType", + "Test_matchesDateTime", + "TestEvaluator_MatchAny_SingleVariantDistribution", + "TestEvaluator_FlagNoRules_DefaultVariant", + "TestBatch_InternalError_GetFlag", + "TestEvaluator_MultipleZeroRolloutDistributions", + "TestEvaluator_MatchEntityId", + "Test_matchesBool", + "TestEvaluator_NonVariantFlag", + "TestOFREPEvaluationBridge_Variant", + "TestEvaluator_MatchAll_MultipleSegments", + "TestEvaluator_MatchAny_RolloutDistribution", + "TestBoolean_SegmentMatch_MultipleConstraints", + "TestEvaluator_MatchAny_RolloutDistribution_MultiRule", + "TestVariant_FlagDisabled", + "TestEvaluator_FlagNoRules", + "TestVariant_NonVariantFlag", + "TestBoolean_RulesOutOfOrder", + "TestBoolean_PercentageRuleFallthrough_SegmentMatch", + "Test_matchesString", + "TestEvaluator_FlagDisabled", + "TestEvaluator_MatchAll_NoConstraints", + "TestEvaluator_FirstRolloutRuleIsZero", + "TestEvaluator_MatchAll_NoVariants_NoDistributions", + "TestEvaluator_MatchAll_SingleVariantDistribution", + "TestEvaluator_MatchAny_NoVariants_NoDistributions", + "TestEvaluator_ErrorParsingNumber" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-063", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_flipt-io__flipt-9f8127f225a86245fa35dca4885c2daef824ee55", + "selectedInstanceId": "instance_flipt-io__flipt-9f8127f225a86245fa35dca4885c2daef824ee55", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "flipt-io/flipt", + "baseCommit": "2d0ff0c91a63a1165f5ca528faa1f0785b1f730c", + "language": "go", + "tags": [ + "go", + "core_feat", + "integration_feat", + "back_end_knowledge", + "database_knowledge", + "infrastructure_knowledge", + "api_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "flipt-io.flipt-flipt-io__flipt-9f8127f225a86245fa35dca4885c2daef824ee55", + "beforeRepoSetCommand": "git reset --hard 2d0ff0c91a63a1165f5ca528faa1f0785b1f730c\ngit clean -fd \ngit checkout 2d0ff0c91a63a1165f5ca528faa1f0785b1f730c \ngit checkout 9f8127f225a86245fa35dca4885c2daef824ee55 -- .github/workflows/test.yml internal/storage/sql/db_test.go", + "selectedTestFiles": [ + "TestMigratorExpectedVersions", + "TestOpen", + "TestMigratorRun", + "TestDBTestSuite", + "TestParse", + "TestMigratorRun_NoChange" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-064", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_flipt-io__flipt-a0cbc0cb65ae601270bdbe3f5313e2dfd49c80e4", + "selectedInstanceId": "instance_flipt-io__flipt-a0cbc0cb65ae601270bdbe3f5313e2dfd49c80e4", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "flipt-io/flipt", + "baseCommit": "fee220d0a20adfb21686685bef2a1d6c2ff6fc17", + "language": "go", + "tags": [ + "go", + "core_feat", + "customization_feat", + "back_end_knowledge", + "devops_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "flipt-io.flipt-flipt-io__flipt-a0cbc0cb65ae601270bdbe3f5313e2dfd49c80e4", + "beforeRepoSetCommand": "git reset --hard fee220d0a20adfb21686685bef2a1d6c2ff6fc17\ngit clean -fd \ngit checkout fee220d0a20adfb21686685bef2a1d6c2ff6fc17 \ngit checkout a0cbc0cb65ae601270bdbe3f5313e2dfd49c80e4 -- internal/config/config_test.go", + "selectedTestFiles": [ + "TestLoad" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-065", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_flipt-io__flipt-a42d38a1bb1df267c53d9d4a706cf34825ae3da9", + "selectedInstanceId": "instance_flipt-io__flipt-a42d38a1bb1df267c53d9d4a706cf34825ae3da9", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "flipt-io/flipt", + "baseCommit": "ee02b164f6728d3227c42671028c67a4afd36918", + "language": "go", + "tags": [ + "go", + "api_feat", + "security_feat", + "security_enh", + "authentication_authorization_knowledge", + "security_knowledge", + "web_knowledge", + "full_stack_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "flipt-io.flipt-flipt-io__flipt-a42d38a1bb1df267c53d9d4a706cf34825ae3da9", + "beforeRepoSetCommand": "git reset --hard ee02b164f6728d3227c42671028c67a4afd36918\ngit clean -fd \ngit checkout ee02b164f6728d3227c42671028c67a4afd36918 \ngit checkout a42d38a1bb1df267c53d9d4a706cf34825ae3da9 -- internal/config/config_test.go test/api.sh test/config/test-with-auth.yml", + "selectedTestFiles": [ + "TestLoad", + "TestLogEncoding", + "TestServeHTTP", + "Test_mustBindEnv", + "TestDatabaseProtocol", + "TestScheme", + "TestCacheBackend", + "TestJSONSchema" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-066", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_flipt-io__flipt-af7a0be46d15f0b63f16a868d13f3b48a838e7ce", + "selectedInstanceId": "instance_flipt-io__flipt-af7a0be46d15f0b63f16a868d13f3b48a838e7ce", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "flipt-io/flipt", + "baseCommit": "165ba79a44732208147f516fa6fa4d1dc72b7008", + "language": "go", + "tags": [ + "go", + "dev_ops_enh", + "infrastructure_knowledge", + "devops_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "flipt-io.flipt-flipt-io__flipt-af7a0be46d15f0b63f16a868d13f3b48a838e7ce", + "beforeRepoSetCommand": "git reset --hard 165ba79a44732208147f516fa6fa4d1dc72b7008\ngit clean -fd \ngit checkout 165ba79a44732208147f516fa6fa4d1dc72b7008 \ngit checkout af7a0be46d15f0b63f16a868d13f3b48a838e7ce -- internal/config/config_test.go", + "selectedTestFiles": [ + "Test_mustBindEnv", + "TestJSONSchema", + "TestServeHTTP", + "TestLoad", + "TestScheme", + "TestCacheBackend", + "TestDatabaseProtocol", + "TestLogEncoding" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-067", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_flipt-io__flipt-b3cd920bbb25e01fdb2dab66a5a913363bc62f6c", + "selectedInstanceId": "instance_flipt-io__flipt-b3cd920bbb25e01fdb2dab66a5a913363bc62f6c", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "flipt-io/flipt", + "baseCommit": "490cc12996575ec353e9755ca1c21e222a4e5191", + "language": "go", + "tags": [ + "go", + "core_feat", + "performance_feat", + "customization_feat", + "back_end_knowledge", + "api_knowledge", + "devops_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "flipt-io.flipt-flipt-io__flipt-b3cd920bbb25e01fdb2dab66a5a913363bc62f6c", + "beforeRepoSetCommand": "git reset --hard 490cc12996575ec353e9755ca1c21e222a4e5191\ngit clean -fd \ngit checkout 490cc12996575ec353e9755ca1c21e222a4e5191 \ngit checkout b3cd920bbb25e01fdb2dab66a5a913363bc62f6c -- internal/ext/exporter_test.go", + "selectedTestFiles": [ + "TestImport", + "TestImport_Namespaces_Mix_And_Match", + "TestExport", + "TestImport_InvalidVersion", + "TestImport_Export", + "TestImport_FlagType_LTVersion1_1", + "FuzzImport", + "TestImport_Rollouts_LTVersion1_1" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-068", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_flipt-io__flipt-c1fd7a81ef9f23e742501bfb26d914eb683262aa", + "selectedInstanceId": "instance_flipt-io__flipt-c1fd7a81ef9f23e742501bfb26d914eb683262aa", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "flipt-io/flipt", + "baseCommit": "dbe263961b187e1c5d7fe34c65b000985a2da5a0", + "language": "go", + "tags": [ + "go", + "major_bug", + "security_bug", + "data_bug", + "back_end_knowledge", + "authentication_authorization_knowledge", + "security_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "flipt-io.flipt-flipt-io__flipt-c1fd7a81ef9f23e742501bfb26d914eb683262aa", + "beforeRepoSetCommand": "git reset --hard dbe263961b187e1c5d7fe34c65b000985a2da5a0\ngit clean -fd \ngit checkout dbe263961b187e1c5d7fe34c65b000985a2da5a0 \ngit checkout c1fd7a81ef9f23e742501bfb26d914eb683262aa -- internal/config/config_test.go", + "selectedTestFiles": [ + "TestLoad" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-069", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_flipt-io__flipt-c8d71ad7ea98d97546f01cce4ccb451dbcf37d3b", + "selectedInstanceId": "instance_flipt-io__flipt-c8d71ad7ea98d97546f01cce4ccb451dbcf37d3b", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "flipt-io/flipt", + "baseCommit": "29d3f9db40c83434d0e3cc082af8baec64c391a9", + "language": "go", + "tags": [ + "go", + "major_bug", + "data_bug", + "dev_ops_enh", + "back_end_knowledge", + "database_knowledge", + "api_knowledge", + "devops_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "flipt-io.flipt-flipt-io__flipt-c8d71ad7ea98d97546f01cce4ccb451dbcf37d3b", + "beforeRepoSetCommand": "git reset --hard 29d3f9db40c83434d0e3cc082af8baec64c391a9\ngit clean -fd \ngit checkout 29d3f9db40c83434d0e3cc082af8baec64c391a9 \ngit checkout c8d71ad7ea98d97546f01cce4ccb451dbcf37d3b -- internal/cue/validate_fuzz_test.go internal/cue/validate_test.go internal/storage/fs/snapshot_test.go", + "selectedTestFiles": [ + "FuzzValidate", + "TestFSWithIndex", + "TestFS_Invalid_VariantFlag_Distribution", + "TestFSWithoutIndex", + "TestValidate_Failure", + "TestFS_Invalid_VariantFlag_Segment", + "Test_Store", + "TestFS_Invalid_BooleanFlag_Segment", + "TestValidate_Latest_Segments_V2", + "TestValidate_V1_Success", + "TestValidate_Latest_Success" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-070", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_flipt-io__flipt-cd18e54a0371fa222304742c6312e9ac37ea86c1", + "selectedInstanceId": "instance_flipt-io__flipt-cd18e54a0371fa222304742c6312e9ac37ea86c1", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "flipt-io/flipt", + "baseCommit": "9e469bf851c6519616c2b220f946138b71fab047", + "language": "go", + "tags": [ + "go", + "major_bug", + "data_bug", + "back_end_knowledge", + "devops_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "flipt-io.flipt-flipt-io__flipt-cd18e54a0371fa222304742c6312e9ac37ea86c1", + "beforeRepoSetCommand": "git reset --hard 9e469bf851c6519616c2b220f946138b71fab047\ngit clean -fd \ngit checkout 9e469bf851c6519616c2b220f946138b71fab047 \ngit checkout cd18e54a0371fa222304742c6312e9ac37ea86c1 -- config/schema_test.go internal/config/config_test.go", + "selectedTestFiles": [ + "TestScheme", + "TestJSONSchema", + "Test_mustBindEnv", + "TestLoad", + "TestCacheBackend", + "TestTracingExporter", + "TestLogEncoding", + "Test_CUE", + "TestServeHTTP", + "TestDatabaseProtocol" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-071", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_flipt-io__flipt-cd2f3b0a9d4d8b8a6d3d56afab65851ecdc408e8", + "selectedInstanceId": "instance_flipt-io__flipt-cd2f3b0a9d4d8b8a6d3d56afab65851ecdc408e8", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "flipt-io/flipt", + "baseCommit": "a91a0258e72c0f0aac3d33ae5c226a85c80ecdf8", + "language": "go", + "tags": [ + "go", + "core_feat", + "back_end_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "flipt-io.flipt-flipt-io__flipt-cd2f3b0a9d4d8b8a6d3d56afab65851ecdc408e8", + "beforeRepoSetCommand": "git reset --hard a91a0258e72c0f0aac3d33ae5c226a85c80ecdf8\ngit clean -fd \ngit checkout a91a0258e72c0f0aac3d33ae5c226a85c80ecdf8 \ngit checkout cd2f3b0a9d4d8b8a6d3d56afab65851ecdc408e8 -- internal/server/evaluation/legacy_evaluator_test.go rpc/flipt/validation_fuzz_test.go rpc/flipt/validation_test.go", + "selectedTestFiles": [ + "Test_matchesNumber", + "Test_matchesString" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-072", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_flipt-io__flipt-e2bd19dafa7166c96b082fb2a59eb54b4be0d778", + "selectedInstanceId": "instance_flipt-io__flipt-e2bd19dafa7166c96b082fb2a59eb54b4be0d778", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "flipt-io/flipt", + "baseCommit": "0eaf98f050d86247bfe9bd7e41178865cb585060", + "language": "go", + "tags": [ + "go", + "major_bug", + "performance_feat", + "performance_enh", + "back_end_knowledge", + "api_knowledge", + "database_knowledge", + "devops_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "flipt-io.flipt-flipt-io__flipt-e2bd19dafa7166c96b082fb2a59eb54b4be0d778", + "beforeRepoSetCommand": "git reset --hard 0eaf98f050d86247bfe9bd7e41178865cb585060\ngit clean -fd \ngit checkout 0eaf98f050d86247bfe9bd7e41178865cb585060 \ngit checkout e2bd19dafa7166c96b082fb2a59eb54b4be0d778 -- internal/cache/cache_test.go internal/server/middleware/grpc/middleware_test.go internal/storage/cache/cache_test.go", + "selectedTestFiles": [ + "TestAuditUnaryInterceptor_UpdateVariant", + "TestAuditUnaryInterceptor_CreateSegment", + "TestGetJSONHandleGetError", + "TestAuditUnaryInterceptor_CreateToken", + "TestAuditUnaryInterceptor_UpdateRollout", + "TestEvaluationCacheUnaryInterceptor_Evaluate", + "TestCacheControlUnaryInterceptor", + "TestAuditUnaryInterceptor_DeleteVariant", + "TestGetFlagCached", + "TestGetEvaluationRulesCached", + "TestAuditUnaryInterceptor_CreateDistribution", + "TestEvaluationUnaryInterceptor_Noop", + "TestEvaluationUnaryInterceptor_Evaluation", + "TestAuditUnaryInterceptor_UpdateRule", + "TestAuditUnaryInterceptor_CreateNamespace", + "TestGetEvaluationRules", + "TestAuditUnaryInterceptor_DeleteDistribution", + "TestValidationUnaryInterceptor", + "TestAuditUnaryInterceptor_DeleteNamespace", + "TestWithDoNotStore", + "TestAuditUnaryInterceptor_UpdateConstraint", + "TestIsDoNotStore", + "TestAuditUnaryInterceptor_CreateVariant", + "TestEvaluationCacheUnaryInterceptor_Evaluation_Variant", + "TestAuditUnaryInterceptor_DeleteRollout", + "TestEvaluationUnaryInterceptor_BatchEvaluation", + "TestAuditUnaryInterceptor_DeleteSegment", + "TestAuditUnaryInterceptor_DeleteConstraint", + "TestAuditUnaryInterceptor_DeleteFlag", + "TestAuditUnaryInterceptor_UpdateNamespace", + "TestAuditUnaryInterceptor_CreateRule", + "TestAuthMetadataAuditUnaryInterceptor", + "TestAuditUnaryInterceptor_UpdateFlag", + "TestAuditUnaryInterceptor_CreateConstraint", + "TestErrorUnaryInterceptor", + "TestEvaluationCacheUnaryInterceptor_Evaluation_Boolean", + "TestAuditUnaryInterceptor_UpdateSegment", + "TestAuditUnaryInterceptor_DeleteRule", + "TestGetJSONHandleUnmarshalError", + "TestSetJSONHandleMarshalError", + "TestAuditUnaryInterceptor_CreateFlag", + "TestAuditUnaryInterceptor_UpdateDistribution", + "TestAuditUnaryInterceptor_CreateRollout" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-073", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_flipt-io__flipt-e50808c03e4b9d25a6a78af9c61a3b1616ea356b", + "selectedInstanceId": "instance_flipt-io__flipt-e50808c03e4b9d25a6a78af9c61a3b1616ea356b", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "flipt-io/flipt", + "baseCommit": "5069ba6fa22fbbf208352ff341ea7a85d6eca29f", + "language": "go", + "tags": [ + "go", + "api_feat", + "integration_feat", + "core_feat", + "back_end_knowledge", + "api_knowledge", + "infrastructure_knowledge", + "devops_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "flipt-io.flipt-flipt-io__flipt-e50808c03e4b9d25a6a78af9c61a3b1616ea356b", + "beforeRepoSetCommand": "git reset --hard 5069ba6fa22fbbf208352ff341ea7a85d6eca29f\ngit clean -fd \ngit checkout 5069ba6fa22fbbf208352ff341ea7a85d6eca29f \ngit checkout e50808c03e4b9d25a6a78af9c61a3b1616ea356b -- internal/config/config_test.go internal/server/audit/audit_test.go internal/server/auth/server_test.go internal/server/middleware/grpc/middleware_test.go internal/server/middleware/grpc/support_test.go", + "selectedTestFiles": [ + "TestAuditUnaryInterceptor_CreateFlag", + "TestCacheUnaryInterceptor_UpdateVariant", + "TestValidationUnaryInterceptor", + "TestTracingExporter", + "TestAuditUnaryInterceptor_DeleteConstraint", + "TestJSONSchema", + "TestAuditUnaryInterceptor_UpdateConstraint", + "TestAuditUnaryInterceptor_UpdateRule", + "TestAuditUnaryInterceptor_UpdateNamespace", + "TestAuditUnaryInterceptor_CreateNamespace", + "TestAuditUnaryInterceptor_DeleteVariant", + "TestAuditUnaryInterceptor_UpdateSegment", + "TestCacheUnaryInterceptor_GetFlag", + "TestAuditUnaryInterceptor_CreateVariant", + "TestAuditUnaryInterceptor_CreateConstraint", + "TestErrorUnaryInterceptor", + "TestSinkSpanExporter", + "TestServeHTTP", + "Test_mustBindEnv", + "TestEvaluationUnaryInterceptor_BatchEvaluation", + "TestScheme", + "TestAuditUnaryInterceptor_DeleteFlag", + "TestAuditUnaryInterceptor_DeleteNamespace", + "TestLoad", + "TestCacheUnaryInterceptor_DeleteFlag", + "TestAuditUnaryInterceptor_DeleteDistribution", + "TestAuditUnaryInterceptor_CreateSegment", + "TestEvaluationUnaryInterceptor_Noop", + "TestAuditUnaryInterceptor_DeleteRule", + "TestCacheUnaryInterceptor_DeleteVariant", + "TestAuditUnaryInterceptor_DeleteSegment", + "TestEvaluationUnaryInterceptor_Evaluation", + "TestAuditUnaryInterceptor_CreateRule", + "TestAuditUnaryInterceptor_UpdateDistribution", + "TestLogEncoding", + "TestAuditUnaryInterceptor_CreateDistribution", + "TestCacheBackend", + "TestCacheUnaryInterceptor_UpdateFlag", + "TestCacheUnaryInterceptor_Evaluate", + "TestAuditUnaryInterceptor_UpdateVariant", + "TestDatabaseProtocol", + "TestCacheUnaryInterceptor_CreateVariant", + "TestAuditUnaryInterceptor_UpdateFlag" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-074", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_flipt-io__flipt-ea9a2663b176da329b3f574da2ce2a664fc5b4a1", + "selectedInstanceId": "instance_flipt-io__flipt-ea9a2663b176da329b3f574da2ce2a664fc5b4a1", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "flipt-io/flipt", + "baseCommit": "866ba43dd49c238c97831362cdab50630b0b9aa7", + "language": "go", + "tags": [ + "go", + "major_bug", + "authentication_authorization_knowledge", + "ui_ux_knowledge", + "devops_knowledge", + "api_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "flipt-io.flipt-flipt-io__flipt-ea9a2663b176da329b3f574da2ce2a664fc5b4a1", + "beforeRepoSetCommand": "git reset --hard 866ba43dd49c238c97831362cdab50630b0b9aa7\ngit clean -fd \ngit checkout 866ba43dd49c238c97831362cdab50630b0b9aa7 \ngit checkout ea9a2663b176da329b3f574da2ce2a664fc5b4a1 -- internal/server/authz/engine/bundle/engine_test.go internal/server/authz/engine/rego/engine_test.go internal/server/authz/middleware/grpc/middleware_test.go internal/server/namespace_test.go", + "selectedTestFiles": [ + "TestEngine_IsAuthMethod", + "TestUpdateRule", + "TestUpdateConstraint", + "TestListRollouts_PaginationPageToken", + "TestOrderRules", + "TestListSegments_PaginationOffset", + "TestDeleteSegment", + "TestBatchEvaluate_FlagNotFound", + "TestDeleteDistribution", + "TestAuthorizationRequiredInterceptor", + "TestListNamespaces_WithAuthz", + "TestCreateSegment", + "TestUpdateDistribution", + "TestDeleteRollout", + "TestDeleteNamespace_HasFlagsWithForce", + "TestDeleteNamespace", + "TestDeleteRule", + "TestBatchEvaluate_FlagNotFoundExcluded", + "TestBatchEvaluate", + "TestCreateRule_MultipleSegments", + "TestEngine_IsAllowed", + "TestListSegments_PaginationPageToken", + "TestCreateRollout", + "TestDeleteConstraint", + "TestListFlags_PaginationOffset", + "TestDeleteNamespace_NonExistent", + "TestListNamespaces_PaginationOffset", + "TestListRules_PaginationOffset", + "TestEngine_NewEngine", + "TestCreateDistribution", + "TestListRules_PaginationPageToken", + "TestDeleteNamespace_HasFlags", + "TestUpdateRollout", + "TestOrderRollouts", + "TestUpdateNamespace", + "TestDeleteNamespace_ProtectedWithForce", + "TestListFlags_PaginationPageToken", + "TestUpdateSegment", + "TestCreateRule", + "TestListNamespaces_PaginationPageToken", + "TestCreateNamespace", + "TestCreateConstraint", + "TestViewableNamespaces", + "TestBatchEvaluate_NamespaceMismatch", + "TestDeleteNamespace_Protected" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-075", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_flipt-io__flipt-ee02b164f6728d3227c42671028c67a4afd36918", + "selectedInstanceId": "instance_flipt-io__flipt-ee02b164f6728d3227c42671028c67a4afd36918", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "flipt-io/flipt", + "baseCommit": "e38e41543f08b762904ed8a08969d0b6aba67166", + "language": "go", + "tags": [ + "go", + "code_quality_enh", + "refactoring_enh", + "technical_debt_enh", + "dev_ops_enh", + "back_end_knowledge", + "devops_knowledge", + "api_knowledge", + "security_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "flipt-io.flipt-flipt-io__flipt-ee02b164f6728d3227c42671028c67a4afd36918", + "beforeRepoSetCommand": "git reset --hard e38e41543f08b762904ed8a08969d0b6aba67166\ngit clean -fd \ngit checkout e38e41543f08b762904ed8a08969d0b6aba67166 \ngit checkout ee02b164f6728d3227c42671028c67a4afd36918 -- internal/release/check_test.go", + "selectedTestFiles": [ + "TestIs", + "TestCheck" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-076", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_flipt-io__flipt-f1bc91a1b999656dbdb2495ccb57bf2105b84920", + "selectedInstanceId": "instance_flipt-io__flipt-f1bc91a1b999656dbdb2495ccb57bf2105b84920", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "flipt-io/flipt", + "baseCommit": "56d261e7c28e47faf537061e6ad3b306459a3e93", + "language": "go", + "tags": [ + "go", + "code_quality_enh", + "scalability_enh", + "refactoring_enh", + "back_end_knowledge", + "api_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "flipt-io.flipt-flipt-io__flipt-f1bc91a1b999656dbdb2495ccb57bf2105b84920", + "beforeRepoSetCommand": "git reset --hard 56d261e7c28e47faf537061e6ad3b306459a3e93\ngit clean -fd \ngit checkout 56d261e7c28e47faf537061e6ad3b306459a3e93 \ngit checkout f1bc91a1b999656dbdb2495ccb57bf2105b84920 -- server/evaluator_test.go server/rule_test.go storage/db_test.go storage/evaluator_test.go storage/rule_test.go", + "selectedTestFiles": [ + "TestFlagsPagination", + "TestUpdateFlag_NotFound", + "TestUpdateRule", + "TestListSegments", + "TestCreateVariant_FlagNotFound", + "TestUpdateVariant_NotFound", + "TestUpdateVariant_DuplicateName", + "TestUpdateRuleAndDistribution", + "TestNew", + "TestEvaluate_RolloutDistribution", + "Test_matchesString", + "TestListRulesPagination", + "TestCreateRule", + "TestUpdateConstraint_NotFound", + "TestCreateConstraint", + "TestCreateDistribution", + "TestCreateVariant_DuplicateName", + "TestCreateSegment", + "TestEvaluate_NoConstraints", + "TestUpdateSegment_NotFound", + "TestCreateConstraint_ErrInvalid", + "TestCreateDistribution_NoRule", + "TestDeleteConstraint_NotFound", + "TestParse", + "TestGetRule", + "TestUpdateConstraint_ErrInvalid", + "TestCreateConstraint_SegmentNotFound", + "TestDeleteConstraint", + "Test_evaluate", + "TestGetSegment", + "TestCreateVariant_DuplicateName_DifferentFlag", + "TestUpdateSegment", + "TestCreateRule_FlagNotFound", + "TestErrorUnaryInterceptor", + "Test_matchesNumber", + "TestCreateRuleAndDistribution", + "TestEvaluate_FlagDisabled", + "TestDeleteSegment_NotFound", + "TestDeleteDistribution", + "Test_validate", + "TestDeleteSegment", + "TestCreateSegment_DuplicateKey", + "TestDeleteRule", + "TestWithCache", + "TestCreateRule_SegmentNotFound", + "TestDeleteRule_NotFound", + "TestGetSegmentNotFound", + "TestUpdateConstraint", + "TestEvaluate_NoVariants_NoDistributions", + "TestEvaluate_SingleVariantDistribution", + "TestDeleteVariant_NotFound", + "TestListSegmentsPagination", + "TestListRules", + "TestEvaluate_FlagNotFound", + "Test_matchesBool", + "TestDeleteFlag_NotFound", + "TestEvaluate", + "TestUpdateRule_NotFound", + "TestCreateFlag_DuplicateKey", + "TestOrderRules", + "TestEvaluate_FlagNoRules", + "TestUpdateDistribution" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-077", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_flipt-io__flipt-f36bd61fb1cee4669de1f00e59da462bfeae8765", + "selectedInstanceId": "instance_flipt-io__flipt-f36bd61fb1cee4669de1f00e59da462bfeae8765", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "flipt-io/flipt", + "baseCommit": "54e188b64f0dda5a1ab9caf8425f94dac3d08f40", + "language": "go", + "tags": [ + "go", + "major_bug", + "ui_ux_bug", + "back_end_knowledge", + "ui_ux_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "flipt-io.flipt-flipt-io__flipt-f36bd61fb1cee4669de1f00e59da462bfeae8765", + "beforeRepoSetCommand": "git reset --hard 54e188b64f0dda5a1ab9caf8425f94dac3d08f40\ngit clean -fd \ngit checkout 54e188b64f0dda5a1ab9caf8425f94dac3d08f40 \ngit checkout f36bd61fb1cee4669de1f00e59da462bfeae8765 -- internal/cue/validate_test.go", + "selectedTestFiles": [ + "TestValidate_Success", + "TestValidate_Failure" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-078", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_future-architect__vuls-0ec945d0510cdebf92cdd8999f94610772689f14", + "selectedInstanceId": "instance_future-architect__vuls-0ec945d0510cdebf92cdd8999f94610772689f14", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "future-architect/vuls", + "baseCommit": "d3bf2a6f26e8e549c0732c26fdcc82725d3c6633", + "language": "go", + "tags": [ + "go", + "data_bug", + "back_end_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "future-architect.vuls-future-architect__vuls-0ec945d0510cdebf92cdd8999f94610772689f14", + "beforeRepoSetCommand": "git reset --hard d3bf2a6f26e8e549c0732c26fdcc82725d3c6633\ngit clean -fd \ngit checkout d3bf2a6f26e8e549c0732c26fdcc82725d3c6633 \ngit checkout 0ec945d0510cdebf92cdd8999f94610772689f14 -- scanner/redhatbase_test.go", + "selectedTestFiles": [ + "Test_redhatBase_parseInstalledPackagesLine/not_standard_rpm_style_source_package", + "Test_redhatBase_parseInstalledPackagesLine/release_is_empty", + "Test_redhatBase_parseInstalledPackagesLine/not_standard_rpm_style_source_package_2", + "Test_redhatBase_parseInstalledPackagesLine", + "Test_redhatBase_parseInstalledPackagesLine/not_standard_rpm_style_source_package_3", + "Test_redhatBase_parseInstalledPackagesLine/release_is_empty_2" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-079", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_future-architect__vuls-1832b4ee3a20177ad313d806983127cb6e53f5cf", + "selectedInstanceId": "instance_future-architect__vuls-1832b4ee3a20177ad313d806983127cb6e53f5cf", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "future-architect/vuls", + "baseCommit": "78b52d6a7f480bd610b692de9bf0c86f57332f23", + "language": "go", + "tags": [ + "go", + "core_feat", + "integration_feat", + "back_end_knowledge", + "infrastructure_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "future-architect.vuls-future-architect__vuls-1832b4ee3a20177ad313d806983127cb6e53f5cf", + "beforeRepoSetCommand": "git reset --hard 78b52d6a7f480bd610b692de9bf0c86f57332f23\ngit clean -fd \ngit checkout 78b52d6a7f480bd610b692de9bf0c86f57332f23 \ngit checkout 1832b4ee3a20177ad313d806983127cb6e53f5cf -- config/os_test.go scanner/base_test.go scanner/freebsd_test.go scanner/macos_test.go", + "selectedTestFiles": [ + "Test_getAmazonLinuxVersion/1", + "TestParseDockerPs", + "Test_updatePortStatus/update_match_multi_address", + "TestEOL_IsStandardSupportEnded/Debian_9_supported", + "TestEOL_IsStandardSupportEnded/macOS_13.4.1_supported", + "TestEOL_IsStandardSupportEnded/Alpine_3.15_supported", + "TestPortScanConf_IsZero/not_zero", + "TestParseOSRelease", + "Test_findPortScanSuccessOn/asterisk_match", + "TestEOL_IsStandardSupportEnded/Alma_Linux_10_Not_Found", + "Test_parseSWVers/MacOS", + "Test_debian_parseGetPkgName/success", + "Test_windows_detectKBsFromKernelVersion/10.0.19045.2129", + "TestParseNeedsRestarting", + "Test_redhatBase_parseRpmQfLine/permission_denied_will_be_ignored", + "TestEOL_IsStandardSupportEnded/Ubuntu_14.04_eol", + "Test_redhatBase_parseRpmQfLine/err", + "TestEOL_IsStandardSupportEnded/Rocky_Linux_8_supported", + "TestScanModule_validate/err", + "TestEOL_IsStandardSupportEnded/Oracle_Linux_6_eol", + "TestEOL_IsStandardSupportEnded/Ubuntu_14.10_eol", + "TestPortScanConf_getScanTechniques/unknown", + "TestEOL_IsStandardSupportEnded/Alma_Linux_8_supported", + "TestEOL_IsStandardSupportEnded/Fedora_37_eol_since_2023-12-16", + "Test_detectScanDest/asterisk", + "Test_windows_detectKBsFromKernelVersion", + "TestSplitIntoBlocks", + "Test_base_parseLsProcExe", + "TestParseSSHConfiguration", + "TestEOL_IsStandardSupportEnded/amazon_linux_1_supported", + "Test_findPortScanSuccessOn/no_match_port", + "TestEOL_IsStandardSupportEnded/Ubuntu_22.04_supported", + "TestEOL_IsStandardSupportEnded/Fedora_38_eol_since_2024-05-15", + "Test_getAmazonLinuxVersion/2025", + "TestParseIp", + "TestParseChangelog", + "Test_getAmazonLinuxVersion/2", + "Test_detectScanDest/dup-addr-port", + "TestParseYumCheckUpdateLinesAmazon", + "TestParseAptCachePolicy", + "Test_findPortScanSuccessOn/no_match_address", + "TestParseApkInfo", + "Test_windows_detectKBsFromKernelVersion/10.0.19045.2130", + "TestEOL_IsStandardSupportEnded/Ubuntu_18.04_ext_supported", + "TestEOL_IsStandardSupportEnded/Ubuntu_20.04_supported", + "TestEOL_IsStandardSupportEnded/Debian_10_supported", + "TestEOL_IsStandardSupportEnded/Fedora_37_supported", + "Test_base_parseGrepProcMap/systemd", + "TestEOL_IsStandardSupportEnded/Ubuntu_20.10_supported", + "TestEOL_IsStandardSupportEnded/Fedora_39_not_found", + "TestParseInstalledPackagesLinesRedhat", + "Test_parseSystemInfo", + "Test_detectOSName/Windows_10_Version_21H2_for_x64-based_Systems", + "Test_detectOSName/err", + "Test_formatKernelVersion/major.minor.build.revision", + "TestEOL_IsStandardSupportEnded/Ubuntu_22.10_supported", + "Test_getAmazonLinuxVersion/2029", + "Test_redhatBase_parseDnfModuleList", + "TestHosts", + "TestDistro_MajorVersion", + "Test_parseWmiObject/happy", + "Test_macos_parseInstalledPackages/happy", + "TestEOL_IsStandardSupportEnded/Alpine_3.9_eol", + "Test_base_parseLsProcExe/systemd", + "Test_detectScanDest/single-addr", + "TestEOL_IsStandardSupportEnded/Debian_11_supported", + "TestEOL_IsStandardSupportEnded/Mac_OS_X_10.15_EOL", + "Test_updatePortStatus/update_multi_packages", + "Test_redhatBase_parseRpmQfLine/is_not_owned_by_any_package", + "Test_parseGetPackageMSU/happy", + "Test_detectScanDest", + "Test_parseSWVers/Mac_OS_X_Server", + "TestEOL_IsStandardSupportEnded/CentOS_stream10_Not_Found", + "Test_majorDotMinor", + "TestEOL_IsStandardSupportEnded/Alpine_3.18_not_found", + "Test_majorDotMinor/major_dot_minor", + "TestParseYumCheckUpdateLine", + "TestScanModule_validate", + "Test_parseInstalledPackages", + "TestScanModule_IsZero/zero", + "TestEOL_IsStandardSupportEnded/freebsd_13_supported", + "TestParseApkVersion", + "TestScanModule_IsZero", + "TestEOL_IsStandardSupportEnded/Debian_13_is_not_supported_yet", + "TestEOL_IsStandardSupportEnded/CentOS_6_eol", + "Test_detectOSName/Windows_Server_2022", + "Test_parseWindowsUpdaterSearch", + "TestParseChangelog/vlc", + "Test_parseSystemInfo/Workstation", + "Test_parseGetHotfix/happy", + "TestEOL_IsStandardSupportEnded/CentOS_7_supported", + "TestEOL_IsStandardSupportEnded/Rocky_Linux_9_supported", + "TestEOL_IsStandardSupportEnded/RHEL10_not_found", + "Test_detectScanDest/empty", + "TestEOL_IsStandardSupportEnded/Ubuntu_5.10_not_found", + "TestEOL_IsStandardSupportEnded/Fedora_32_supported", + "Test_formatKernelVersion/major.minor.build", + "Test_parseInstalledPackages/happy", + "Test_getAmazonLinuxVersion", + "TestEOL_IsStandardSupportEnded/freebsd_12_supported", + "Test_base_parseLsOf/lsof", + "TestEOL_IsStandardSupportEnded/Alpine_3.11_supported", + "TestEOL_IsStandardSupportEnded/Rocky_Linux_10_Not_Found", + "Test_parseSWVers", + "TestParseCheckRestart", + "TestEOL_IsStandardSupportEnded/Fedora_33_supported", + "Test_getAmazonLinuxVersion/2022", + "TestViaHTTP", + "TestScanUpdatablePackage", + "Test_parseWmiObject", + "TestEOL_IsStandardSupportEnded/Ubuntu_18.04_supported", + "TestEOL_IsStandardSupportEnded/alpine_3.10_supported", + "TestEOL_IsStandardSupportEnded/Ubuntu_21.04_supported", + "Test_getAmazonLinuxVersion/2023", + "TestIsRunningKernelRedHatLikeLinux", + "TestEOL_IsStandardSupportEnded/CentOS_8_supported", + "TestEOL_IsStandardSupportEnded/Windows_10_EOL", + "TestEOL_IsStandardSupportEnded/Oracle_Linux_10_not_found", + "TestParseIfconfig", + "TestEOL_IsStandardSupportEnded/Alpine_3.17_supported", + "Test_base_parseLsOf/lsof-duplicate-port", + "Test_macos_parseInstalledPackages", + "Test_redhatBase_rebootRequired/kerne_needs-reboot", + "TestIsRunningKernelSUSE", + "TestEOL_IsStandardSupportEnded/Debian_8_supported", + "TestEOL_IsStandardSupportEnded/Alpine_3.16_supported", + "TestEOL_IsStandardSupportEnded/freebsd_11_supported", + "Test_majorDotMinor/major", + "TestDecorateCmd", + "Test_parseSystemInfo/Domain_Controller", + "TestEOL_IsStandardSupportEnded", + "Test_updatePortStatus/nil_listen_ports", + "TestEOL_IsStandardSupportEnded/Alma_Linux_9_supported", + "TestEOL_IsStandardSupportEnded/Oracle_Linux_8_supported", + "TestEOL_IsStandardSupportEnded/Alpine_3.12_supported", + "TestParseInstalledPackagesLineFromRepoquery", + "TestEOL_IsStandardSupportEnded/Fedora_36_supported", + "Test_redhatBase_parseRpmQfLine/valid_line", + "TestEOL_IsStandardSupportEnded/Ubuntu_23.04_supported", + "TestSyslogConfValidate", + "Test_updatePortStatus/update_match_single_address", + "TestParseChangelog/realvnc-vnc-server", + "Test_windows_detectKBsFromKernelVersion/10.0.20348.9999", + "TestEOL_IsStandardSupportEnded/RHEL6_eol", + "Test_parseWindowsUpdateHistory", + "TestPortScanConf_IsZero", + "Test_redhatBase_parseRpmQfLine", + "Test_findPortScanSuccessOn", + "TestEOL_IsStandardSupportEnded/amazon_linux_2022_supported", + "TestGetUpdatablePackNames", + "Test_findPortScanSuccessOn/open_empty", + "TestEOL_IsStandardSupportEnded/Fedora_38_supported", + "Test_redhatBase_rebootRequired/uek_kernel_no-reboot", + "TestPortScanConf_getScanTechniques/nil", + "TestParseSystemctlStatus", + "Test_redhatBase_rebootRequired/uek_kernel_needs-reboot", + "TestPortScanConf_getScanTechniques/single", + "Test_detectScanDest/multi-addr", + "Test_updatePortStatus/update_match_asterisk", + "Test_detectOSName", + "Test_redhatBase_rebootRequired", + "Test_detectOSName/Windows_Server_2019", + "TestScanUpdatablePackages", + "TestPortScanConf_getScanTechniques", + "TestScanModule_IsZero/not_zero", + "TestParseSSHScan", + "Test_parseWindowsUpdaterSearch/happy", + "Test_redhatBase_rebootRequired/kerne_no-reboot", + "TestEOL_IsStandardSupportEnded/freebsd_10_eol", + "Test_parseRegistry/happy", + "Test_getAmazonLinuxVersion/2031", + "Test_parseWindowsUpdateHistory/happy", + "TestEOL_IsStandardSupportEnded/amazon_linux_2_supported", + "Test_detectOSName/Windows_10_for_x64-based_Systems", + "TestParseBlock", + "Test_parseRegistry", + "Test_redhatBase_parseRpmQfLine/No_such_file_or_directory_will_be_ignored", + "TestEOL_IsStandardSupportEnded/Ubuntu_20.04_ext_supported", + "Test_parseSystemInfo/Server", + "TestEOL_IsStandardSupportEnded/Ubuntu_21.10_supported", + "Test_updatePortStatus/nil_affected_procs", + "Test_majorDotMinor/empty", + "Test_findPortScanSuccessOn/port_empty", + "Test_parseSWVers/MacOS_Server", + "TestEOL_IsStandardSupportEnded/Fedora_35_supported", + "TestEOL_IsStandardSupportEnded/RHEL7_supported", + "Test_windows_parseIP/en", + "TestEOL_IsStandardSupportEnded/Windows_10_Version_22H2_supported", + "Test_base_parseGrepProcMap", + "Test_redhatBase_parseDnfModuleList/Success", + "TestEOL_IsStandardSupportEnded/amazon_linux_1_eol_on_2023-6-30", + "TestEOL_IsStandardSupportEnded/RHEL9_supported", + "Test_windows_parseIP", + "TestScanModule_validate/valid", + "TestEOL_IsStandardSupportEnded/Fedora_32_eol_since_2021-5-25", + "TestParsePkgInfo", + "Test_parseGetPackageMSU", + "TestPortScanConf_IsZero/zero", + "Test_parseSWVers/ProductName_error", + "TestToCpeURI", + "Test_findPortScanSuccessOn/single_match", + "TestEOL_IsStandardSupportEnded/Fedora_34_eol_since_2022-6-7", + "TestParseYumCheckUpdateLines", + "TestEOL_IsStandardSupportEnded/RHEL8_supported", + "TestEOL_IsStandardSupportEnded/Fedora_36_eol_since_2023-05-17", + "Test_windows_parseIP/ja", + "Test_parseGetComputerInfo", + "TestEOL_IsStandardSupportEnded/Fedora_34_supported", + "TestEOL_IsStandardSupportEnded/Oracle_Linux_9_supported", + "TestEOL_IsStandardSupportEnded/Debian_12_supported", + "TestNormalizedForWindows", + "TestEOL_IsStandardSupportEnded/amazon_linux_2023_supported", + "Test_parseGetHotfix", + "Test_getAmazonLinuxVersion/2018.03", + "TestEOL_IsStandardSupportEnded/CentOS_stream9_supported", + "TestEOL_IsStandardSupportEnded/Alpine_3.14_supported", + "TestEOL_IsStandardSupportEnded/Fedora_33_eol_since_2021-11-30", + "Test_updatePortStatus", + "TestParseInstalledPackagesLine", + "Test_getAmazonLinuxVersion/2017.09", + "TestParseLxdPs", + "TestEOL_IsStandardSupportEnded/CentOS_stream8_supported", + "TestEOL_IsStandardSupportEnded/Fedora_35_eol_since_2022-12-13", + "Test_debian_parseGetPkgName", + "TestParsePkgVersion", + "Test_parseSWVers/ProductVersion_error", + "Test_formatKernelVersion", + "Test_base_parseLsOf", + "TestIsAwsInstanceID", + "Test_windows_detectKBsFromKernelVersion/err", + "Test_majorDotMinor/major_dot_minor_dot_release", + "Test_windows_detectKBsFromKernelVersion/10.0.22621.1105", + "Test_getAmazonLinuxVersion/2027", + "Test_parseGetComputerInfo/happy", + "TestGetChangelogCache", + "TestPortScanConf_getScanTechniques/multiple", + "TestSplitAptCachePolicy", + "TestParseSSHKeygen", + "Test_parseSWVers/Mac_OS_X", + "TestEOL_IsStandardSupportEnded/Oracle_Linux_7_supported", + "TestEOL_IsStandardSupportEnded/freebsd_11_eol_on_2021-9-30", + "TestEOL_IsStandardSupportEnded/Ubuntu_16.04_supported", + "TestGetCveIDsFromChangelog", + "Test_windows_detectKBsFromKernelVersion/10.0.20348.1547", + "TestEOL_IsStandardSupportEnded/amazon_linux_2031_not_found" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-080", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_future-architect__vuls-50580f6e98eeb36f53f27222f7f4fdfea0b21e8d", + "selectedInstanceId": "instance_future-architect__vuls-50580f6e98eeb36f53f27222f7f4fdfea0b21e8d", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "future-architect/vuls", + "baseCommit": "472df0e1b6ab9b50f8605af957ad054c7a732938", + "language": "go", + "tags": [ + "go", + "integration_feat", + "core_feat", + "code_quality_enh", + "back_end_knowledge", + "security_knowledge", + "api_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "future-architect.vuls-future-architect__vuls-50580f6e98eeb36f53f27222f7f4fdfea0b21e8d", + "beforeRepoSetCommand": "git reset --hard 472df0e1b6ab9b50f8605af957ad054c7a732938\ngit clean -fd \ngit checkout 472df0e1b6ab9b50f8605af957ad054c7a732938 \ngit checkout 50580f6e98eeb36f53f27222f7f4fdfea0b21e8d -- detector/wordpress_test.go", + "selectedTestFiles": [ + "Test_convertToVinfos", + "Test_convertToVinfos/WordPress_vulnerabilities_Enterprise", + "Test_convertToVinfos/WordPress_vulnerabilities_Researcher" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-081", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_future-architect__vuls-5af1a227339e46c7abf3f2815e4c636a0c01098e", + "selectedInstanceId": "instance_future-architect__vuls-5af1a227339e46c7abf3f2815e4c636a0c01098e", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "future-architect/vuls", + "baseCommit": "053306944695e6616f320393f865f667e984481d", + "language": "go", + "tags": [ + "go", + "compatibility_bug", + "edge_case_bug", + "back_end_knowledge", + "infrastructure_knowledge", + "devops_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "future-architect.vuls-future-architect__vuls-5af1a227339e46c7abf3f2815e4c636a0c01098e", + "beforeRepoSetCommand": "git reset --hard 053306944695e6616f320393f865f667e984481d\ngit clean -fd \ngit checkout 053306944695e6616f320393f865f667e984481d \ngit checkout 5af1a227339e46c7abf3f2815e4c636a0c01098e -- scanner/redhatbase_test.go scanner/utils_test.go", + "selectedTestFiles": [ + "Test_base_parseGrepProcMap/systemd", + "Test_isRunningKernel/SUES_not_kernel", + "Test_parseGetComputerInfo", + "Test_detectOSName/Windows_10_Version_21H2_for_x64-based_Systems", + "Test_parseWmiObject", + "Test_parseSWVers/ProductName_error", + "Test_windows_parseIP/ja", + "TestNormalizedForWindows", + "Test_detectScanDest/single-addr", + "Test_isRunningKernel/SUSE_kernel_and_running", + "TestParseYumCheckUpdateLines", + "Test_windows_parseIP/en", + "Test_parseRegistry/happy", + "Test_redhatBase_parseRpmQfLine/is_not_owned_by_any_package", + "Test_detectScanDest/asterisk", + "Test_updatePortStatus/update_match_multi_address", + "TestParseAptCachePolicy", + "TestSplitIntoBlocks", + "TestParseInstalledPackagesLinesRedhat", + "TestDecorateCmd", + "TestParseSSHConfiguration", + "Test_detectOSName/Windows_10_for_x64-based_Systems", + "TestParseInstalledPackagesLineFromRepoquery", + "TestParseChangelog/vlc", + "Test_findPortScanSuccessOn/asterisk_match", + "TestParseYumCheckUpdateLine", + "Test_isRunningKernel/Amazon_kernel_and_running", + "TestParseYumCheckUpdateLinesAmazon", + "Test_base_parseLsOf", + "Test_redhatBase_rebootRequired/kerne_no-reboot", + "Test_formatKernelVersion/major.minor.build", + "Test_windows_detectKBsFromKernelVersion/10.0.22621.1105", + "Test_detectScanDest/multi-addr", + "TestParseSSHKeygen", + "TestGetChangelogCache", + "Test_updatePortStatus/update_match_asterisk", + "TestParseApkInfo", + "TestIsAwsInstanceID", + "Test_parseSWVers/Mac_OS_X_Server", + "Test_redhatBase_parseRpmQfLine/valid_line", + "Test_updatePortStatus/nil_affected_procs", + "Test_redhatBase_rebootRequired/kerne_needs-reboot", + "Test_macos_parseInstalledPackages/happy", + "TestScanUpdatablePackages", + "Test_parseRegistry", + "Test_windows_detectKBsFromKernelVersion/err", + "TestScanUpdatablePackage", + "Test_redhatBase_parseRpmQfLine", + "Test_parseGetPackageMSU/happy", + "Test_isRunningKernel", + "Test_isRunningKernel/Amazon_not_kernel", + "Test_macos_parseInstalledPackages", + "TestParseBlock", + "Test_parseGetHotfix", + "Test_isRunningKernel/old_redhat_kernel_release_style", + "Test_windows_detectKBsFromKernelVersion/10.0.20348.1547", + "Test_windows_parseIP", + "Test_detectScanDest", + "Test_parseInstalledPackages", + "Test_debian_parseGetPkgName/success", + "Test_redhatBase_parseRpmQfLine/No_such_file_or_directory_will_be_ignored", + "Test_isRunningKernel/Amazon_kernel_but_not_running", + "Test_base_parseLsProcExe/systemd", + "TestParseChangelog", + "Test_debian_parseGetPkgName", + "Test_updatePortStatus", + "Test_parseWmiObject/happy", + "Test_updatePortStatus/nil_listen_ports", + "TestGetCveIDsFromChangelog", + "Test_formatKernelVersion/major.minor.build.revision", + "Test_parseSystemInfo/Server", + "Test_detectOSName/err", + "Test_updatePortStatus/update_multi_packages", + "Test_parseWindowsUpdateHistory/happy", + "Test_windows_detectKBsFromKernelVersion/10.0.20348.9999", + "TestParseOSRelease", + "Test_isRunningKernel/SUES_kernel_but_not_running", + "Test_detectScanDest/empty", + "Test_parseInstalledPackages/happy", + "TestParsePkgInfo", + "Test_detectScanDest/dup-addr-port", + "Test_parseGetPackageMSU", + "Test_base_parseLsProcExe", + "Test_parseWindowsUpdateHistory", + "Test_findPortScanSuccessOn/open_empty", + "TestParseLxdPs", + "Test_findPortScanSuccessOn/no_match_port", + "TestParseSystemctlStatus", + "Test_findPortScanSuccessOn/port_empty", + "Test_parseWindowsUpdaterSearch", + "Test_findPortScanSuccessOn/no_match_address", + "Test_parseSWVers/MacOS_Server", + "Test_updatePortStatus/update_match_single_address", + "Test_redhatBase_parseRpmQfLine/permission_denied_will_be_ignored", + "TestParseChangelog/realvnc-vnc-server", + "Test_formatKernelVersion", + "Test_redhatBase_parseRpmQfLine/err", + "Test_windows_detectKBsFromKernelVersion", + "Test_base_parseLsOf/lsof-duplicate-port", + "Test_redhatBase_rebootRequired", + "Test_base_parseLsOf/lsof", + "Test_parseSystemInfo", + "Test_parseSWVers/Mac_OS_X", + "Test_detectOSName", + "Test_redhatBase_rebootRequired/uek_kernel_needs-reboot", + "Test_parseSWVers/ProductVersion_error", + "TestParsePkgVersion", + "Test_parseSystemInfo/Domain_Controller", + "Test_detectOSName/Windows_Server_2019", + "TestParseIp", + "TestGetUpdatablePackNames", + "TestSplitAptCachePolicy", + "Test_parseSystemInfo/Workstation", + "Test_windows_detectKBsFromKernelVersion/10.0.19045.2130", + "TestParseInstalledPackagesLine", + "Test_parseGetHotfix/happy", + "TestParseIfconfig", + "TestParseApkVersion", + "Test_isRunningKernel/kernel_is_kernel-debug,_but_pack_is_kernel", + "TestViaHTTP", + "Test_detectOSName/Windows_Server_2022", + "TestParseSSHScan", + "Test_parseWindowsUpdaterSearch/happy", + "TestParseCheckRestart", + "Test_parseSWVers/MacOS", + "TestParseDockerPs", + "Test_findPortScanSuccessOn/single_match", + "Test_windows_detectKBsFromKernelVersion/10.0.19045.2129", + "Test_base_parseGrepProcMap", + "TestParseNeedsRestarting", + "Test_parseGetComputerInfo/happy", + "Test_findPortScanSuccessOn", + "Test_parseSWVers", + "Test_redhatBase_rebootRequired/uek_kernel_no-reboot" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-082", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_future-architect__vuls-83bcca6e669ba2e4102f26c4a2b52f78c7861f1a", + "selectedInstanceId": "instance_future-architect__vuls-83bcca6e669ba2e4102f26c4a2b52f78c7861f1a", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "future-architect/vuls", + "baseCommit": "a124518d78779cd9daefd92bb66b25da37516363", + "language": "go", + "tags": [ + "go", + "core_feat", + "security_feat", + "integration_feat", + "performance_feat", + "security_knowledge", + "infrastructure_knowledge", + "networking_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "future-architect.vuls-future-architect__vuls-83bcca6e669ba2e4102f26c4a2b52f78c7861f1a", + "beforeRepoSetCommand": "git reset --hard a124518d78779cd9daefd92bb66b25da37516363\ngit clean -fd \ngit checkout a124518d78779cd9daefd92bb66b25da37516363 \ngit checkout 83bcca6e669ba2e4102f26c4a2b52f78c7861f1a -- scan/base_test.go", + "selectedTestFiles": [ + "TestSplitIntoBlocks", + "Test_detectScanDest/asterisk", + "Test_base_parseListenPorts/empty", + "Test_base_parseLsProcExe/systemd", + "Test_base_parseLsOf/lsof", + "Test_updatePortStatus/nil_affected_procs", + "Test_matchListenPorts", + "Test_detectScanDest/dup-addr", + "TestGetUpdatablePackNames", + "Test_updatePortStatus/nil_listen_ports", + "TestParseYumCheckUpdateLinesAmazon", + "TestParseSystemctlStatus", + "Test_detectScanDest", + "Test_base_parseGrepProcMap", + "Test_detectScanDest/single-addr", + "TestParseChangelog/vlc", + "TestParseDockerPs", + "TestParseYumCheckUpdateLine", + "Test_base_parseListenPorts/normal", + "Test_debian_parseGetPkgName", + "Test_debian_parseGetPkgName/success", + "Test_base_parseLsProcExe", + "TestParseIp", + "Test_detectScanDest/empty", + "Test_matchListenPorts/open_empty", + "TestGetChangelogCache", + "TestParseYumCheckUpdateLines", + "TestDecorateCmd", + "TestParseBlock", + "Test_matchListenPorts/single_match", + "Test_matchListenPorts/asterisk_match", + "TestParsePkgInfo", + "TestScanUpdatablePackages", + "Test_updatePortStatus/update_multi_packages", + "TestIsRunningKernelSUSE", + "TestParseIfconfig", + "TestParseInstalledPackagesLinesRedhat", + "Test_updatePortStatus/update_match_single_address", + "Test_updatePortStatus/update_match_asterisk", + "Test_base_parseListenPorts", + "TestParseChangelog/realvnc-vnc-server", + "Test_updatePortStatus", + "TestScanUpdatablePackage", + "Test_base_parseLsOf", + "Test_updatePortStatus/update_match_multi_address", + "TestParseAptCachePolicy", + "TestIsRunningKernelRedHatLikeLinux", + "TestGetCveIDsFromChangelog", + "TestSplitAptCachePolicy", + "Test_base_parseListenPorts/ipv6_loopback", + "TestIsAwsInstanceID", + "TestParseLxdPs", + "Test_detectScanDest/multi-addr", + "Test_matchListenPorts/no_match_address", + "Test_matchListenPorts/no_match_port", + "TestParseScanedPackagesLineRedhat", + "Test_matchListenPorts/port_empty", + "TestParseChangelog", + "TestParseNeedsRestarting", + "Test_base_parseListenPorts/asterisk", + "TestViaHTTP", + "Test_base_parseGrepProcMap/systemd", + "TestParseApkVersion", + "TestParseOSRelease", + "TestParseApkInfo", + "TestParsePkgVersion", + "TestParseCheckRestart" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-083", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_future-architect__vuls-86b60e1478e44d28b1aff6b9ac7e95ceb05bc5fc", + "selectedInstanceId": "instance_future-architect__vuls-86b60e1478e44d28b1aff6b9ac7e95ceb05bc5fc", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "future-architect/vuls", + "baseCommit": "42fdc08933d2b60adc555972b32ae599386b0b9c", + "language": "go", + "tags": [ + "go", + "core_feat", + "back_end_knowledge", + "infrastructure_knowledge", + "devops_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "future-architect.vuls-future-architect__vuls-86b60e1478e44d28b1aff6b9ac7e95ceb05bc5fc", + "beforeRepoSetCommand": "git reset --hard 42fdc08933d2b60adc555972b32ae599386b0b9c\ngit clean -fd \ngit checkout 42fdc08933d2b60adc555972b32ae599386b0b9c \ngit checkout 86b60e1478e44d28b1aff6b9ac7e95ceb05bc5fc -- config/tomlloader_test.go", + "selectedTestFiles": [ + "TestPortScanConf_getScanTechniques/multiple", + "TestEOL_IsStandardSupportEnded/freebsd_11_supported", + "TestEOL_IsStandardSupportEnded/Oracle_Linux_8_supported", + "TestEOL_IsStandardSupportEnded/Fedora_35_supported", + "TestEOL_IsStandardSupportEnded/Ubuntu_21.10_supported", + "TestScanModule_IsZero/not_zero", + "TestEOL_IsStandardSupportEnded/Oracle_Linux_7_supported", + "TestEOL_IsStandardSupportEnded/Fedora_32_eol_on_2022-5-17", + "TestPortScanConf_IsZero/not_zero", + "TestEOL_IsStandardSupportEnded/Oracle_Linux_9_not_found", + "TestEOL_IsStandardSupportEnded/RHEL6_eol", + "TestEOL_IsStandardSupportEnded/Fedora_35_eol_on_2022-12-7", + "TestEOL_IsStandardSupportEnded/CentOS_stream8_supported", + "TestEOL_IsStandardSupportEnded/Fedora_33_supported", + "TestEOL_IsStandardSupportEnded/Ubuntu_18.04_ext_supported", + "Test_majorDotMinor/major_dot_minor_dot_release", + "TestEOL_IsStandardSupportEnded/Alpine_3.15_supported", + "TestEOL_IsStandardSupportEnded/Ubuntu_12.10_not_found", + "TestEOL_IsStandardSupportEnded/amazon_linux_2_supported", + "TestEOL_IsStandardSupportEnded/Fedora_32_supported", + "TestEOL_IsStandardSupportEnded/Ubuntu_18.04_supported", + "Test_majorDotMinor/empty", + "TestScanModule_validate", + "TestPortScanConf_getScanTechniques", + "TestToCpeURI", + "TestScanModule_validate/err", + "TestEOL_IsStandardSupportEnded/freebsd_12_supported", + "TestEOL_IsStandardSupportEnded/Ubuntu_22.04_supported", + "Test_majorDotMinor/major", + "TestEOL_IsStandardSupportEnded/Rocky_Linux_9_Not_Found", + "TestEOL_IsStandardSupportEnded/Alpine_3.9_eol", + "TestEOL_IsStandardSupportEnded/Ubuntu_20.04_ext_supported", + "TestPortScanConf_getScanTechniques/unknown", + "Test_majorDotMinor", + "TestEOL_IsStandardSupportEnded/Ubuntu_20.10_supported", + "TestEOL_IsStandardSupportEnded/Alpine_3.14_supported", + "TestEOL_IsStandardSupportEnded/Debian_10_supported", + "TestPortScanConf_IsZero/zero", + "TestEOL_IsStandardSupportEnded", + "TestEOL_IsStandardSupportEnded/Alpine_3.12_supported", + "TestEOL_IsStandardSupportEnded/Ubuntu_16.04_supported", + "TestEOL_IsStandardSupportEnded/Alpine_3.11_supported", + "TestEOL_IsStandardSupportEnded/amazon_linux_2022_supported", + "TestEOL_IsStandardSupportEnded/Ubuntu_20.04_supported", + "TestEOL_IsStandardSupportEnded/CentOS_6_eol", + "TestEOL_IsStandardSupportEnded/Debian_11_supported", + "TestEOL_IsStandardSupportEnded/Rocky_Linux_8_supported", + "TestScanModule_IsZero/zero", + "TestHosts", + "TestEOL_IsStandardSupportEnded/CentOS_8_supported", + "TestEOL_IsStandardSupportEnded/Debian_9_supported", + "TestEOL_IsStandardSupportEnded/Alma_Linux_8_supported", + "TestEOL_IsStandardSupportEnded/Alpine_3.16_not_found", + "TestPortScanConf_IsZero", + "TestEOL_IsStandardSupportEnded/Alma_Linux_10_Not_Found", + "TestPortScanConf_getScanTechniques/nil", + "TestDistro_MajorVersion", + "TestEOL_IsStandardSupportEnded/freebsd_10_eol", + "TestEOL_IsStandardSupportEnded/RHEL8_supported", + "TestEOL_IsStandardSupportEnded/CentOS_stream10_Not_Found", + "TestEOL_IsStandardSupportEnded/Ubuntu_14.04_eol", + "TestEOL_IsStandardSupportEnded/Debian_8_supported", + "TestEOL_IsStandardSupportEnded/RHEL9_supported", + "TestEOL_IsStandardSupportEnded/Ubuntu_14.10_eol", + "TestEOL_IsStandardSupportEnded/alpine_3.10_supported", + "TestEOL_IsStandardSupportEnded/amazon_linux_1_eol_on_2023-6-30", + "TestEOL_IsStandardSupportEnded/Alma_Linux_9_supported", + "TestEOL_IsStandardSupportEnded/Ubuntu_21.04_supported", + "TestScanModule_validate/valid", + "TestEOL_IsStandardSupportEnded/RHEL7_supported", + "Test_majorDotMinor/major_dot_minor", + "TestEOL_IsStandardSupportEnded/Fedora_32_eol_on_2021-5-25", + "TestPortScanConf_getScanTechniques/single", + "TestEOL_IsStandardSupportEnded/RHEL10_not_found", + "TestSyslogConfValidate", + "TestEOL_IsStandardSupportEnded/Fedora_33_eol_on_2021-5-26", + "TestEOL_IsStandardSupportEnded/Oracle_Linux_6_eol", + "TestEOL_IsStandardSupportEnded/freebsd_13_supported", + "TestEOL_IsStandardSupportEnded/Fedora_34_supported", + "TestScanModule_IsZero", + "TestEOL_IsStandardSupportEnded/freebsd_11_eol_on_2021-9-30", + "TestEOL_IsStandardSupportEnded/Debian_12_is_not_supported_yet", + "TestEOL_IsStandardSupportEnded/amazon_linux_1_supported", + "TestEOL_IsStandardSupportEnded/CentOS_stream9_supported", + "TestEOL_IsStandardSupportEnded/CentOS_7_supported" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-084", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_future-architect__vuls-c11ba27509f733d7d280bdf661cbbe2e7a99df4c", + "selectedInstanceId": "instance_future-architect__vuls-c11ba27509f733d7d280bdf661cbbe2e7a99df4c", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "future-architect/vuls", + "baseCommit": "8a611f9ba6d470cbaa3b781c92c96f5fc9f0b3ba", + "language": "go", + "tags": [ + "go", + "data_bug", + "ui_ux_bug", + "back_end_knowledge", + "security_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "future-architect.vuls-future-architect__vuls-c11ba27509f733d7d280bdf661cbbe2e7a99df4c", + "beforeRepoSetCommand": "git reset --hard 8a611f9ba6d470cbaa3b781c92c96f5fc9f0b3ba\ngit clean -fd \ngit checkout 8a611f9ba6d470cbaa3b781c92c96f5fc9f0b3ba \ngit checkout c11ba27509f733d7d280bdf661cbbe2e7a99df4c -- models/library_test.go", + "selectedTestFiles": [ + "TestFormatMaxCvssScore", + "TestSummaries", + "TestExcept", + "TestFilterIgnorePkgsContainer", + "TestLibraryScanners_Find/miss", + "TestCvss3Scores", + "TestDistroAdvisories_AppendIfMissing/duplicate_no_append", + "TestStorePackageStatueses", + "TestVulnInfo_AttackVector/3.1:N", + "TestPackage_FormatVersionFromTo/nfy2", + "TestToSortedSlice", + "TestCvss2Scores", + "TestFilterByCvssOver", + "TestVulnInfo_AttackVector/2.0:L", + "TestPackage_FormatVersionFromTo", + "TestMerge", + "TestDistroAdvisories_AppendIfMissing/append", + "TestVendorLink", + "TestPackage_FormatVersionFromTo/nfy3", + "TestTitles", + "TestPackage_FormatVersionFromTo/nfy", + "TestCountGroupBySeverity", + "TestPackage_FormatVersionFromTo/fixed", + "TestVulnInfo_AttackVector/3.0:N", + "TestLibraryScanners_Find", + "TestVulnInfo_AttackVector", + "TestSortPackageStatues", + "TestPackage_FormatVersionFromTo/nfy#01", + "TestFilterIgnoreCveIDsContainer", + "TestFilterIgnoreCveIDs", + "TestVulnInfo_AttackVector/2.0:N", + "TestAppendIfMissing", + "TestMergeNewVersion", + "TestFilterUnfixed", + "TestIsDisplayUpdatableNum", + "TestSourceLinks", + "TestLibraryScanners_Find/multi_file", + "TestAddBinaryName", + "TestMaxCvssScores", + "TestFindByBinName", + "TestSortByConfiden", + "TestFilterIgnorePkgs", + "TestMaxCvss2Scores", + "TestDistroAdvisories_AppendIfMissing", + "TestVulnInfo_AttackVector/2.0:A", + "TestMaxCvss3Scores", + "TestLibraryScanners_Find/single_file" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-085", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_gravitational__teleport-0ac7334939981cf85b9591ac295c3816954e287e", + "selectedInstanceId": "instance_gravitational__teleport-0ac7334939981cf85b9591ac295c3816954e287e", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "gravitational/teleport", + "baseCommit": "42beccb27e0e5797a10db05bf126e529273d2d95", + "language": "go", + "tags": [ + "go", + "performance_enh", + "code_quality_enh", + "back_end_knowledge", + "database_knowledge", + "performance_knowledge", + "authentication_authorization_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "gravitational.teleport-gravitational__teleport-0ac7334939981cf85b9591ac295c3816954e287e", + "beforeRepoSetCommand": "git reset --hard 42beccb27e0e5797a10db05bf126e529273d2d95\ngit clean -fd \ngit checkout 42beccb27e0e5797a10db05bf126e529273d2d95 \ngit checkout 0ac7334939981cf85b9591ac295c3816954e287e -- lib/srv/db/access_test.go lib/srv/db/ha_test.go", + "selectedTestFiles": [ + "TestAccessMySQL/has_access_to_nothing", + "TestProxyProtocolPostgres", + "TestAccessPostgres/has_access_to_nothing", + "TestAuthTokens/correct_Postgres_Cloud_SQL_IAM_auth_token", + "TestAuthTokens/correct_Postgres_Redshift_IAM_auth_token", + "TestAccessPostgres", + "TestAuthTokens/incorrect_MySQL_RDS_IAM_auth_token", + "TestAccessMySQL/access_allowed_to_specific_user", + "TestHA", + "TestAuthTokens/incorrect_Postgres_Redshift_IAM_auth_token", + "TestAuditMySQL", + "TestAccessDisabled", + "TestAccessMySQL", + "TestAccessMySQL/has_access_to_all_database_users", + "TestAccessPostgres/access_allowed_to_specific_user/database", + "TestAuthTokens/incorrect_Postgres_RDS_IAM_auth_token", + "TestAuthTokens/correct_MySQL_RDS_IAM_auth_token", + "TestAccessPostgres/no_access_to_users", + "TestProxyClientDisconnectDueToIdleConnection", + "TestAuthTokens/incorrect_Postgres_Cloud_SQL_IAM_auth_token", + "TestDatabaseServerStart", + "TestProxyClientDisconnectDueToCertExpiration", + "TestAccessMySQL/access_denied_to_specific_user", + "TestAccessPostgres/has_access_to_all_database_names_and_users", + "TestAuthTokens", + "TestAccessPostgres/no_access_to_databases", + "TestAuditPostgres", + "TestAccessPostgres/access_denied_to_specific_user/database", + "TestProxyProtocolMySQL", + "TestAuthTokens/correct_Postgres_RDS_IAM_auth_token" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-086", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_gravitational__teleport-3fa6904377c006497169945428e8197158667910-v626ec2a48416b10a88641359a169d99e935ff037", + "selectedInstanceId": "instance_gravitational__teleport-3fa6904377c006497169945428e8197158667910-v626ec2a48416b10a88641359a169d99e935ff037", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "gravitational/teleport", + "baseCommit": "481158d6310e36e3c1115e25ab3fdf1c1ed45e60", + "language": "go", + "tags": [ + "go", + "major_bug", + "code_quality_enh", + "back_end_knowledge", + "infrastructure_knowledge", + "devops_knowledge", + "security_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "gravitational.teleport-gravitational__teleport-3fa6904377c006497169945428e8197158667910-v626ec2a48416b10a88641359a169d99e935ff03", + "beforeRepoSetCommand": "git reset --hard 481158d6310e36e3c1115e25ab3fdf1c1ed45e60\ngit clean -fd \ngit checkout 481158d6310e36e3c1115e25ab3fdf1c1ed45e60 \ngit checkout 3fa6904377c006497169945428e8197158667910 -- lib/kube/proxy/forwarder_test.go", + "selectedTestFiles": [ + "TestParseResourcePath//api/v1/watch/namespaces/kube-system/pods/foo", + "TestParseResourcePath", + "TestParseResourcePath//apis", + "TestParseResourcePath//api/v1/pods", + "TestParseResourcePath/#00", + "TestParseResourcePath//api/v1/namespaces/kube-system/pods", + "TestAuthenticate/unsupported_user_type", + "TestParseResourcePath//apis/apps/v1/", + "TestAuthenticate/local_user_and_cluster,_no_tunnel", + "TestParseResourcePath//apis/apps", + "TestParseResourcePath//api/v1/", + "TestParseResourcePath//api/v1/watch/pods", + "TestParseResourcePath//api/v1/nodes/foo/proxy/bar", + "TestParseResourcePath//apis/apiregistration.k8s.io/v1/apiservices/foo/status", + "TestAuthenticate/remote_user_and_local_cluster", + "TestParseResourcePath//apis/", + "TestAuthenticate/unknown_kubernetes_cluster_in_local_cluster", + "TestAuthenticate/custom_kubernetes_cluster_in_remote_cluster", + "TestGetKubeCreds/proxy_service,_no_kube_creds", + "TestParseResourcePath//apis/apps/v1", + "TestGetKubeCreds/kubernetes_service,_no_kube_creds", + "TestGetKubeCreds/proxy_service,_with_kube_creds", + "TestAuthenticate/local_user_and_cluster,_no_kubeconfig", + "TestParseResourcePath//api/v1/namespaces/kube-system/pods/foo/exec", + "TestParseResourcePath//", + "TestAuthenticate/local_user_and_remote_cluster", + "TestParseResourcePath//apis/apps/", + "TestParseResourcePath//api/v1/namespaces/kube-system", + "TestAuthenticate/remote_user_and_remote_cluster", + "TestParseResourcePath//api/", + "TestParseResourcePath//api", + "TestAuthenticate/authorization_failure", + "TestAuthenticate/custom_kubernetes_cluster_in_local_cluster", + "TestAuthenticate/local_user_and_remote_cluster,_no_tunnel", + "TestParseResourcePath//apis/rbac.authorization.k8s.io/v1/watch/clusterroles", + "TestAuthenticate/local_user_and_cluster", + "TestParseResourcePath//apis/rbac.authorization.k8s.io/v1/clusterroles/foo", + "TestParseResourcePath//api/v1/watch/namespaces/kube-system/pods", + "TestAuthenticate/kube_users_passed_in_request", + "TestParseResourcePath//api/v1", + "TestAuthenticate/local_user_and_remote_cluster,_no_kubeconfig", + "TestParseResourcePath//apis/rbac.authorization.k8s.io/v1/watch/clusterroles/foo", + "TestAuthenticate", + "TestGetKubeCreds", + "TestParseResourcePath//api/v1/namespaces/kube-system/pods/foo", + "TestGetKubeCreds/kubernetes_service,_with_kube_creds", + "TestParseResourcePath//apis/rbac.authorization.k8s.io/v1/clusterroles", + "TestParseResourcePath//api/v1/watch/namespaces/kube-system" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-087", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_gravitational__teleport-4d0117b50dc8cdb91c94b537a4844776b224cd3d", + "selectedInstanceId": "instance_gravitational__teleport-4d0117b50dc8cdb91c94b537a4844776b224cd3d", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "gravitational/teleport", + "baseCommit": "e2412a7c37314c9482d856f407f737c9e5b34bce", + "language": "go", + "tags": [ + "go", + "integration_feat", + "security_feat", + "performance_feat", + "back_end_knowledge", + "database_knowledge", + "infrastructure_knowledge", + "performance_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "gravitational.teleport-gravitational__teleport-4d0117b50dc8cdb91c94b537a4844776b224cd3d", + "beforeRepoSetCommand": "git reset --hard e2412a7c37314c9482d856f407f737c9e5b34bce\ngit clean -fd \ngit checkout e2412a7c37314c9482d856f407f737c9e5b34bce \ngit checkout 4d0117b50dc8cdb91c94b537a4844776b224cd3d -- lib/events/dynamoevents/dynamoevents_test.go", + "selectedTestFiles": [ + "TestDateRangeGenerator", + "TestDynamoevents" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-088", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_gravitational__teleport-5dca072bb4301f4579a15364fcf37cc0c39f7f6c", + "selectedInstanceId": "instance_gravitational__teleport-5dca072bb4301f4579a15364fcf37cc0c39f7f6c", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "gravitational/teleport", + "baseCommit": "d45e26cec6dc799afbb9eac4381d70f95c21c41f", + "language": "go", + "tags": [ + "go", + "performance_bug", + "edge_case_bug", + "integration_bug", + "security_bug", + "security_knowledge", + "authentication_authorization_knowledge", + "back_end_knowledge", + "networking_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "gravitational.teleport-gravitational__teleport-5dca072bb4301f4579a15364fcf37cc0c39f7f6c", + "beforeRepoSetCommand": "git reset --hard d45e26cec6dc799afbb9eac4381d70f95c21c41f\ngit clean -fd \ngit checkout d45e26cec6dc799afbb9eac4381d70f95c21c41f \ngit checkout 5dca072bb4301f4579a15364fcf37cc0c39f7f6c -- lib/kube/proxy/forwarder_test.go lib/kube/proxy/server_test.go", + "selectedTestFiles": [ + "TestParseResourcePath//api/v1/watch/namespaces/kube-system/pods/foo", + "TestAuthenticate/local_user_and_remote_cluster,_no_local_kube_users_or_groups", + "TestAuthenticate/unsupported_user_type", + "TestParseResourcePath//api/v1/watch/pods", + "TestParseResourcePath//apis/rbac.authorization.k8s.io/v1/watch/clusterroles/foo", + "TestParseResourcePath//api/v1/namespaces/kube-system/pods/foo", + "TestParseResourcePath//api/", + "TestAuthenticate/local_user_and_remote_cluster,_no_tunnel", + "TestParseResourcePath//apis/apps/v1", + "TestAuthenticate/custom_kubernetes_cluster_in_remote_cluster", + "TestAuthenticate/authorization_failure", + "TestParseResourcePath//api/v1/namespaces/kube-system", + "TestParseResourcePath//apis", + "TestParseResourcePath//api/v1/watch/namespaces/kube-system/pods", + "TestParseResourcePath//apis/apps", + "TestParseResourcePath//apis/apps/v1/", + "TestMTLSClientCAs/1000_CAs", + "TestParseResourcePath//api/v1/namespaces/kube-system/pods", + "TestParseResourcePath//api", + "TestParseResourcePath//apis/rbac.authorization.k8s.io/v1/clusterroles", + "TestParseResourcePath//apis/apiregistration.k8s.io/v1/apiservices/foo/status", + "TestParseResourcePath/#00", + "TestMTLSClientCAs/100_CAs", + "TestParseResourcePath//", + "TestAuthenticate/remote_user_and_local_cluster", + "TestAuthenticate/local_user_and_cluster,_no_kubeconfig", + "TestAuthenticate/local_user_and_cluster", + "TestAuthenticate", + "TestParseResourcePath//api/v1/pods", + "TestAuthenticate/local_user_and_remote_cluster,_no_kubeconfig", + "TestAuthenticate/remote_user_and_remote_cluster", + "TestAuthenticate/custom_kubernetes_cluster_in_local_cluster", + "TestParseResourcePath//api/v1/watch/namespaces/kube-system", + "TestParseResourcePath//apis/rbac.authorization.k8s.io/v1/clusterroles/foo", + "TestParseResourcePath", + "TestParseResourcePath//apis/rbac.authorization.k8s.io/v1/watch/clusterroles", + "TestParseResourcePath//apis/", + "TestMTLSClientCAs/1_CA", + "TestParseResourcePath//api/v1/namespaces/kube-system/pods/foo/exec", + "TestParseResourcePath//api/v1", + "TestParseResourcePath//api/v1/nodes/foo/proxy/bar", + "TestAuthenticate/unknown_kubernetes_cluster_in_local_cluster", + "TestAuthenticate/kube_users_passed_in_request", + "TestParseResourcePath//apis/apps/", + "TestAuthenticate/local_user_and_cluster,_no_tunnel", + "TestMTLSClientCAs", + "TestAuthenticate/local_user_and_remote_cluster", + "TestParseResourcePath//api/v1/" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-089", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_gravitational__teleport-629dc432eb191ca479588a8c49205debb83e80e2", + "selectedInstanceId": "instance_gravitational__teleport-629dc432eb191ca479588a8c49205debb83e80e2", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "gravitational/teleport", + "baseCommit": "c12ce90636425daef82da2a7fe2a495e9064d1f8", + "language": "go", + "tags": [ + "go", + "core_feat", + "code_quality_enh", + "back_end_knowledge", + "infrastructure_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "gravitational.teleport-gravitational__teleport-629dc432eb191ca479588a8c49205debb83e80e2", + "beforeRepoSetCommand": "git reset --hard c12ce90636425daef82da2a7fe2a495e9064d1f8\ngit clean -fd \ngit checkout c12ce90636425daef82da2a7fe2a495e9064d1f8 \ngit checkout 629dc432eb191ca479588a8c49205debb83e80e2 -- lib/utils/concurrentqueue/queue_test.go", + "selectedTestFiles": [ + "TestBackpressure", + "TestOrdering" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-090", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_gravitational__teleport-6a14edcf1ff010172fdbac622d0a474ed6af46de", + "selectedInstanceId": "instance_gravitational__teleport-6a14edcf1ff010172fdbac622d0a474ed6af46de", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "gravitational/teleport", + "baseCommit": "9c041361f9ff3270d252609d5aa92eb1251cd8d7", + "language": "go", + "tags": [ + "go", + "major_bug", + "regression_bug", + "back_end_knowledge", + "networking_knowledge", + "authentication_authorization_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "gravitational.teleport-gravitational__teleport-6a14edcf1ff010172fdbac622d0a474ed6af46de", + "beforeRepoSetCommand": "git reset --hard 9c041361f9ff3270d252609d5aa92eb1251cd8d7\ngit clean -fd \ngit checkout 9c041361f9ff3270d252609d5aa92eb1251cd8d7 \ngit checkout 6a14edcf1ff010172fdbac622d0a474ed6af46de -- lib/auth/trustedcluster_test.go", + "selectedTestFiles": [ + "TestRemoteClusterStatus" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-091", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_gravitational__teleport-6eaaf3a27e64f4ef4ef855bd35d7ec338cf17460-v626ec2a48416b10a88641359a169d99e935ff037", + "selectedInstanceId": "instance_gravitational__teleport-6eaaf3a27e64f4ef4ef855bd35d7ec338cf17460-v626ec2a48416b10a88641359a169d99e935ff037", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "gravitational/teleport", + "baseCommit": "a51596d8d779935e1dfa8d0fabce39d9edd91457", + "language": "go", + "tags": [ + "go", + "core_feat", + "performance_feat", + "api_feat", + "back_end_knowledge", + "api_knowledge", + "performance_knowledge", + "devops_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "gravitational.teleport-gravitational__teleport-6eaaf3a27e64f4ef4ef855bd35d7ec338cf17460-v626ec2a48416b10a88641359a169d99e935ff03", + "beforeRepoSetCommand": "git reset --hard a51596d8d779935e1dfa8d0fabce39d9edd91457\ngit clean -fd \ngit checkout a51596d8d779935e1dfa8d0fabce39d9edd91457 \ngit checkout 6eaaf3a27e64f4ef4ef855bd35d7ec338cf17460 -- lib/benchmark/linear_test.go", + "selectedTestFiles": [ + "TestGetBenchmarkNotEvenMultiple", + "TestGetBenchmark", + "TestValidateConfig" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-092", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_gravitational__teleport-769b4b5eec7286b7b14e179f2cc52e6b15d2d9f3-v626ec2a48416b10a88641359a169d99e935ff037", + "selectedInstanceId": "instance_gravitational__teleport-769b4b5eec7286b7b14e179f2cc52e6b15d2d9f3-v626ec2a48416b10a88641359a169d99e935ff037", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "gravitational/teleport", + "baseCommit": "1879807d3c30626ec02a730133ac5592cece310c", + "language": "go", + "tags": [ + "go", + "core_feat", + "integration_feat", + "customization_feat", + "security_feat", + "back_end_knowledge", + "security_knowledge", + "authentication_authorization_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "gravitational.teleport-gravitational__teleport-769b4b5eec7286b7b14e179f2cc52e6b15d2d9f3-v626ec2a48416b10a88641359a169d99e935ff03", + "beforeRepoSetCommand": "git reset --hard 1879807d3c30626ec02a730133ac5592cece310c\ngit clean -fd \ngit checkout 1879807d3c30626ec02a730133ac5592cece310c \ngit checkout 769b4b5eec7286b7b14e179f2cc52e6b15d2d9f3 -- tool/tsh/tsh_test.go", + "selectedTestFiles": [ + "TestMakeClient", + "TestReadClusterFlag", + "TestOptions/Incomplete_option", + "TestReadClusterFlag/nothing_set", + "TestFormatConnectCommand/no_default_user/database_are_specified", + "TestOptions/Forward_Agent_Local", + "TestFormatConnectCommand/unsupported_database_protocol", + "TestReadClusterFlag/TELEPORT_SITE_and_TELEPORT_CLUSTER_and_CLI_flag_is_set,_prefer_CLI", + "TestFetchDatabaseCreds", + "TestFormatConnectCommand/default_user_is_specified", + "TestOptions/Forward_Agent_No", + "TestOptions/Forward_Agent_InvalidValue", + "TestFormatConnectCommand/default_database_is_specified", + "TestReadClusterFlag/TELEPORT_SITE_set", + "TestOIDCLogin", + "TestRelogin", + "TestOptions/AddKeysToAgent_Invalid_Value", + "TestFormatConnectCommand/default_user/database_are_specified", + "TestOptions/Forward_Agent_Yes", + "TestFormatConnectCommand", + "TestOptions/Equals_Sign_Delimited", + "TestReadClusterFlag/TELEPORT_SITE_and_TELEPORT_CLUSTER_set,_prefer_TELEPORT_CLUSTER", + "TestOptions/Invalid_key", + "TestIdentityRead", + "TestOptions/Space_Delimited", + "TestReadClusterFlag/TELEPORT_CLUSTER_set", + "TestFailedLogin", + "TestOptions" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-093", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_gravitational__teleport-78b0d8c72637df1129fb6ff84fc49ef4b5ab1288", + "selectedInstanceId": "instance_gravitational__teleport-78b0d8c72637df1129fb6ff84fc49ef4b5ab1288", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "gravitational/teleport", + "baseCommit": "19c5768873a50f1f84f0906add418c29c68f4bc2", + "language": "go", + "tags": [ + "go", + "performance_feat", + "performance_enh", + "code_quality_enh", + "back_end_knowledge", + "infrastructure_knowledge", + "performance_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "gravitational.teleport-gravitational__teleport-78b0d8c72637df1129fb6ff84fc49ef4b5ab1288", + "beforeRepoSetCommand": "git reset --hard 19c5768873a50f1f84f0906add418c29c68f4bc2\ngit clean -fd \ngit checkout 19c5768873a50f1f84f0906add418c29c68f4bc2 \ngit checkout 78b0d8c72637df1129fb6ff84fc49ef4b5ab1288 -- lib/cache/fncache_test.go", + "selectedTestFiles": [ + "TestFnCacheCancellation", + "TestApplicationServers", + "TestDatabases", + "TestDatabaseServers", + "TestFnCacheSanity", + "TestState" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-094", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_gravitational__teleport-87a593518b6ce94624f6c28516ce38cc30cbea5a", + "selectedInstanceId": "instance_gravitational__teleport-87a593518b6ce94624f6c28516ce38cc30cbea5a", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "gravitational/teleport", + "baseCommit": "49ab2a7bfd935317818b20a5bd0b59ac4d5289c9", + "language": "go", + "tags": [ + "go", + "core_feat", + "api_feat", + "integration_feat", + "database_knowledge", + "cloud_knowledge", + "infrastructure_knowledge", + "authentication_authorization_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "gravitational.teleport-gravitational__teleport-87a593518b6ce94624f6c28516ce38cc30cbea5a", + "beforeRepoSetCommand": "git reset --hard 49ab2a7bfd935317818b20a5bd0b59ac4d5289c9\ngit clean -fd \ngit checkout 49ab2a7bfd935317818b20a5bd0b59ac4d5289c9 \ngit checkout 87a593518b6ce94624f6c28516ce38cc30cbea5a -- lib/client/conntest/database/sqlserver_test.go", + "selectedTestFiles": [ + "TestMySQLErrors/invalid_database_user", + "TestSQLServerErrors", + "TestPostgresErrors", + "TestPostgresErrors/invalid_database_error", + "TestMySQLErrors/invalid_database_user_access_denied", + "TestMySQLPing", + "TestMySQLErrors/connection_refused_string", + "TestMySQLErrors/connection_refused_host_not_allowed", + "TestMySQLErrors", + "TestSQLServerErrors/ConnectionRefusedError", + "TestPostgresErrors/invalid_user_error", + "TestSQLServerErrors/InvalidDatabaseNameError", + "TestMySQLErrors/invalid_database_name_access_denied", + "TestMySQLErrors/invalid_database_name", + "TestMySQLErrors/connection_refused_host_blocked", + "TestPostgresErrors/connection_refused_error", + "TestPostgresPing", + "TestSQLServerPing", + "TestSQLServerErrors/InvalidDatabaseUserError" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-095", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_gravitational__teleport-96019ce0be7a2c8e36363f359eb7c943b41dde70", + "selectedInstanceId": "instance_gravitational__teleport-96019ce0be7a2c8e36363f359eb7c943b41dde70", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "gravitational/teleport", + "baseCommit": "d05df372ce37abd7c190f9fbb68192a773330e63", + "language": "go", + "tags": [ + "go", + "major_bug", + "compatibility_bug", + "back_end_knowledge", + "api_knowledge", + "infrastructure_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "gravitational.teleport-gravitational__teleport-96019ce0be7a2c8e36363f359eb7c943b41dde70", + "beforeRepoSetCommand": "git reset --hard d05df372ce37abd7c190f9fbb68192a773330e63\ngit clean -fd \ngit checkout d05df372ce37abd7c190f9fbb68192a773330e63 \ngit checkout 96019ce0be7a2c8e36363f359eb7c943b41dde70 -- lib/kube/proxy/forwarder_test.go", + "selectedTestFiles": [ + "TestAuthenticate/local_user_and_remote_cluster,_no_tunnel", + "TestAuthenticate/unknown_kubernetes_cluster_in_local_cluster", + "TestAuthenticate" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-096", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_gravitational__teleport-b4e7cd3a5e246736d3fe8d6886af55030b232277", + "selectedInstanceId": "instance_gravitational__teleport-b4e7cd3a5e246736d3fe8d6886af55030b232277", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "gravitational/teleport", + "baseCommit": "85addfbd36943a4b655e1a4241979789e8b4ff22", + "language": "go", + "tags": [ + "go", + "critical_bug", + "security_bug", + "back_end_knowledge", + "security_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "gravitational.teleport-gravitational__teleport-b4e7cd3a5e246736d3fe8d6886af55030b232277", + "beforeRepoSetCommand": "git reset --hard 85addfbd36943a4b655e1a4241979789e8b4ff22\ngit clean -fd \ngit checkout 85addfbd36943a4b655e1a4241979789e8b4ff22 \ngit checkout b4e7cd3a5e246736d3fe8d6886af55030b232277 -- lib/backend/report_test.go", + "selectedTestFiles": [ + "TestReporterTopRequestsLimit", + "TestBuildKeyLabel", + "TestInit", + "TestParams" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-097", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_gravitational__teleport-b5d8169fc0a5e43fee2616c905c6d32164654dc6", + "selectedInstanceId": "instance_gravitational__teleport-b5d8169fc0a5e43fee2616c905c6d32164654dc6", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "gravitational/teleport", + "baseCommit": "103f3de22f3986d4f73d931b1e433e011b26a488", + "language": "go", + "tags": [ + "go", + "code_quality_enh", + "refactoring_enh", + "security_enh", + "back_end_knowledge", + "security_knowledge", + "api_knowledge", + "authentication_authorization_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "gravitational.teleport-gravitational__teleport-b5d8169fc0a5e43fee2616c905c6d32164654dc6", + "beforeRepoSetCommand": "git reset --hard 103f3de22f3986d4f73d931b1e433e011b26a488\ngit clean -fd \ngit checkout 103f3de22f3986d4f73d931b1e433e011b26a488 \ngit checkout b5d8169fc0a5e43fee2616c905c6d32164654dc6 -- lib/auth/init_test.go", + "selectedTestFiles": [ + "TestMigrateOSS/User", + "TestMigrateOSS", + "TestMigrateOSS/EmptyCluster", + "TestMigrateOSS/TrustedCluster" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-098", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_gravitational__teleport-ba6c4a135412c4296dd5551bd94042f0dc024504-v626ec2a48416b10a88641359a169d99e935ff037", + "selectedInstanceId": "instance_gravitational__teleport-ba6c4a135412c4296dd5551bd94042f0dc024504-v626ec2a48416b10a88641359a169d99e935ff037", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "gravitational/teleport", + "baseCommit": "396812cebff3ebfd075bbed04acefc65e787a537", + "language": "go", + "tags": [ + "go", + "dev_ops_enh", + "performance_enh", + "back_end_knowledge", + "infrastructure_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "gravitational.teleport-gravitational__teleport-ba6c4a135412c4296dd5551bd94042f0dc024504-v626ec2a48416b10a88641359a169d99e935ff03", + "beforeRepoSetCommand": "git reset --hard 396812cebff3ebfd075bbed04acefc65e787a537\ngit clean -fd \ngit checkout 396812cebff3ebfd075bbed04acefc65e787a537 \ngit checkout ba6c4a135412c4296dd5551bd94042f0dc024504 -- lib/service/service_test.go lib/service/state_test.go", + "selectedTestFiles": [ + "TestMonitor/ok_event_remains_in_recovering_state_because_not_enough_time_passed", + "TestProcessStateGetState/multiple_components,_one_is_recovering", + "TestProcessStateGetState/multiple_components,_one_is_degraded", + "TestMonitor/ok_event_in_new_component_causes_overall_OK_state", + "TestMonitor/ok_event_after_enough_time_causes_OK_state", + "TestMonitor", + "TestProcessStateGetState/one_component_in_stateOK", + "TestMonitor/ok_event_in_one_component_keeps_overall_status_degraded_due_to_other_component", + "TestMonitor/degraded_event_in_a_new_component_causes_degraded_state", + "TestMonitor/degraded_event_causes_degraded_state", + "TestProcessStateGetState/multiple_components,_one_is_starting", + "TestProcessStateGetState/no_components", + "TestMonitor/ok_event_in_new_component_causes_overall_recovering_state", + "TestProcessStateGetState/multiple_components_in_stateOK", + "TestProcessStateGetState", + "TestMonitor/ok_event_causes_recovering_state" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-099", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_gravitational__teleport-e6681abe6a7113cfd2da507f05581b7bdf398540-v626ec2a48416b10a88641359a169d99e935ff037", + "selectedInstanceId": "instance_gravitational__teleport-e6681abe6a7113cfd2da507f05581b7bdf398540-v626ec2a48416b10a88641359a169d99e935ff037", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "gravitational/teleport", + "baseCommit": "be52cd325af38f53fa6b6f869bc10b88160e06e2", + "language": "go", + "tags": [ + "go", + "core_feat", + "back_end_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "gravitational.teleport-gravitational__teleport-e6681abe6a7113cfd2da507f05581b7bdf398540-v626ec2a48416b10a88641359a169d99e935ff03", + "beforeRepoSetCommand": "git reset --hard be52cd325af38f53fa6b6f869bc10b88160e06e2\ngit clean -fd \ngit checkout be52cd325af38f53fa6b6f869bc10b88160e06e2 \ngit checkout e6681abe6a7113cfd2da507f05581b7bdf398540 -- lib/events/auditwriter_test.go lib/events/emitter_test.go lib/events/stream_test.go", + "selectedTestFiles": [ + "TestStreamerCompleteEmpty", + "TestAsyncEmitter/Receive", + "TestProtoStreamer/small_load_test_with_some_uneven_numbers", + "TestProtoStreamer/one_event_using_the_whole_part", + "TestWriterEmitter", + "TestAsyncEmitter/Slow", + "TestProtoStreamer/no_events", + "TestExport", + "TestAuditWriter", + "TestAsyncEmitter/Close", + "TestProtoStreamer/5MB_similar_to_S3_min_size_in_bytes", + "TestProtoStreamer", + "TestAuditWriter/ResumeMiddle", + "TestAsyncEmitter", + "TestAuditWriter/Backoff", + "TestAuditWriter/Session", + "TestAuditLog", + "TestAuditWriter/ResumeStart", + "TestProtoStreamer/get_a_part_per_message" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-100", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_gravitational__teleport-eda668c30d9d3b56d9c69197b120b01013611186", + "selectedInstanceId": "instance_gravitational__teleport-eda668c30d9d3b56d9c69197b120b01013611186", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "gravitational/teleport", + "baseCommit": "4f6f52f86d65f506d1884a9f56bcd919a8744734", + "language": "go", + "tags": [ + "go", + "major_bug", + "regression_bug", + "integration_bug", + "back_end_knowledge", + "cloud_knowledge", + "devops_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "gravitational.teleport-gravitational__teleport-eda668c30d9d3b56d9c69197b120b01013611186", + "beforeRepoSetCommand": "git reset --hard 4f6f52f86d65f506d1884a9f56bcd919a8744734\ngit clean -fd \ngit checkout 4f6f52f86d65f506d1884a9f56bcd919a8744734 \ngit checkout eda668c30d9d3b56d9c69197b120b01013611186 -- lib/kube/proxy/forwarder_test.go", + "selectedTestFiles": [ + "TestAuthenticate/local_user_and_cluster,_no_tunnel", + "TestParseResourcePath//", + "TestGetKubeCreds/proxy_service,_no_kube_creds", + "TestAuthenticate/local_user_and_remote_cluster,_no_local_kube_users_or_groups", + "TestGetServerInfo/GetServerInfo_gets_correct_public_addr_with_PublicAddr_set", + "TestParseResourcePath//apis/", + "TestMTLSClientCAs", + "TestAuthenticate/custom_kubernetes_cluster_in_local_cluster", + "TestParseResourcePath", + "TestParseResourcePath//api/v1/namespaces/kube-system/pods/foo/exec", + "TestParseResourcePath//apis", + "TestGetKubeCreds/kubernetes_service,_with_kube_creds", + "TestParseResourcePath//api/v1/namespaces/kube-system", + "TestAuthenticate/remote_user_and_local_cluster", + "TestParseResourcePath//apis/apps/v1", + "TestParseResourcePath//apis/rbac.authorization.k8s.io/v1/clusterroles/foo", + "TestAuthenticate/local_user_and_remote_cluster,_no_kubeconfig", + "TestGetKubeCreds/legacy_proxy_service,_with_kube_creds", + "TestParseResourcePath//api/v1/watch/namespaces/kube-system/pods/foo", + "TestAuthenticate/unsupported_user_type", + "TestNewClusterSessionLocal", + "TestClusterSessionDial", + "TestParseResourcePath//api/v1", + "TestParseResourcePath//apis/rbac.authorization.k8s.io/v1/clusterroles", + "TestParseResourcePath//apis/rbac.authorization.k8s.io/v1/watch/clusterroles/foo", + "TestParseResourcePath//apis/apps/", + "TestGetServerInfo/GetServerInfo_gets_listener_addr_with_PublicAddr_unset", + "TestParseResourcePath//api/v1/namespaces/kube-system/pods", + "TestParseResourcePath//apis/rbac.authorization.k8s.io/v1/watch/clusterroles", + "TestAuthenticate/remote_user_and_remote_cluster", + "TestNewClusterSessionRemote", + "TestParseResourcePath//api/v1/watch/namespaces/kube-system", + "TestParseResourcePath//api/v1/", + "TestParseResourcePath//api/v1/namespaces/kube-system/pods/foo", + "TestParseResourcePath//api/", + "TestParseResourcePath//api/v1/pods", + "TestGetKubeCreds/legacy_proxy_service,_no_kube_creds", + "TestParseResourcePath//api/v1/watch/namespaces/kube-system/pods", + "TestAuthenticate/local_user_and_remote_cluster", + "TestParseResourcePath//apis/apiregistration.k8s.io/v1/apiservices/foo/status", + "TestGetKubeCreds/Missing_cluster_does_not_fail_operation", + "TestParseResourcePath//apis/apps/v1/", + "TestParseResourcePath//apis/apps", + "TestGetKubeCreds/proxy_service,_with_kube_creds", + "TestGetKubeCreds", + "TestAuthenticate/authorization_failure", + "TestParseResourcePath/#00", + "TestMTLSClientCAs/1_CA", + "TestNewClusterSessionDirect", + "TestParseResourcePath//api", + "TestGetKubeCreds/kubernetes_service,_no_kube_creds", + "TestAuthenticate/local_user_and_cluster", + "TestAuthenticate/kube_users_passed_in_request", + "TestAuthenticate", + "TestParseResourcePath//api/v1/watch/pods", + "TestAuthenticate/local_user_and_cluster,_no_kubeconfig", + "TestAuthenticate/local_user_and_remote_cluster,_no_tunnel", + "TestGetServerInfo", + "TestMTLSClientCAs/100_CAs", + "TestParseResourcePath//api/v1/nodes/foo/proxy/bar", + "TestAuthenticate/custom_kubernetes_cluster_in_remote_cluster", + "TestMTLSClientCAs/1000_CAs", + "TestAuthenticate/unknown_kubernetes_cluster_in_local_cluster" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-101", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_gravitational__teleport-f432a71a13e698b6e1c4672a2e9e9c1f32d35c12", + "selectedInstanceId": "instance_gravitational__teleport-f432a71a13e698b6e1c4672a2e9e9c1f32d35c12", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "gravitational/teleport", + "baseCommit": "a4a6a3e42d90918341224dd7f2ba45856b1b6c70", + "language": "go", + "tags": [ + "go", + "code_quality_enh", + "refactoring_enh", + "dev_ops_enh", + "devops_knowledge", + "back_end_knowledge", + "infrastructure_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "gravitational.teleport-gravitational__teleport-f432a71a13e698b6e1c4672a2e9e9c1f32d35c12", + "beforeRepoSetCommand": "git reset --hard a4a6a3e42d90918341224dd7f2ba45856b1b6c70\ngit clean -fd \ngit checkout a4a6a3e42d90918341224dd7f2ba45856b1b6c70 \ngit checkout f432a71a13e698b6e1c4672a2e9e9c1f32d35c12 -- lib/auth/keystore/keystore_test.go", + "selectedTestFiles": [ + "TestKeyStore/raw_keystore", + "TestKeyStore" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-102", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_gravitational__teleport-fb0ab2b9b771377a689fd0d0374777c251e58bbf", + "selectedInstanceId": "instance_gravitational__teleport-fb0ab2b9b771377a689fd0d0374777c251e58bbf", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "gravitational/teleport", + "baseCommit": "b58ad484649e51b439ba11df387e25e23e8296d1", + "language": "go", + "tags": [ + "go", + "performance_feat", + "analytics_feat", + "code_quality_enh", + "back_end_knowledge", + "devops_knowledge", + "infrastructure_knowledge", + "performance_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "gravitational.teleport-gravitational__teleport-fb0ab2b9b771377a689fd0d0374777c251e58bbf", + "beforeRepoSetCommand": "git reset --hard b58ad484649e51b439ba11df387e25e23e8296d1\ngit clean -fd \ngit checkout b58ad484649e51b439ba11df387e25e23e8296d1 \ngit checkout fb0ab2b9b771377a689fd0d0374777c251e58bbf -- lib/utils/circular_buffer_test.go", + "selectedTestFiles": [ + "TestChConn", + "TestReadAtMost", + "TestNewCircularBuffer", + "TestEscapeControl", + "TestSlice", + "TestCircularBuffer_Data", + "TestUtils", + "TestFilterAWSRoles", + "TestConsolefLongComponent", + "TestAllowNewlines" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-103", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_gravitational__teleport-fd2959260ef56463ad8afa4c973f47a50306edd4", + "selectedInstanceId": "instance_gravitational__teleport-fd2959260ef56463ad8afa4c973f47a50306edd4", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "gravitational/teleport", + "baseCommit": "025143d85654c604656571c363d0c7b9a6579f62", + "language": "go", + "tags": [ + "go", + "api_feat", + "customization_feat", + "ui_ux_feat", + "cloud_knowledge", + "infrastructure_knowledge", + "devops_knowledge", + "api_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "gravitational.teleport-gravitational__teleport-fd2959260ef56463ad8afa4c973f47a50306edd4", + "beforeRepoSetCommand": "git reset --hard 025143d85654c604656571c363d0c7b9a6579f62\ngit clean -fd \ngit checkout 025143d85654c604656571c363d0c7b9a6579f62 \ngit checkout fd2959260ef56463ad8afa4c973f47a50306edd4 -- lib/config/configuration_test.go", + "selectedTestFiles": [ + "TestProxyKube/new_and_old_formats", + "TestProxyKube/legacy_format,_no_local_cluster", + "TestProxyKube/not_configured", + "TestProxyKube/legacy_format,_with_local_cluster", + "TestProxyKube", + "TestProxyKube/new_format_and_old_explicitly_disabled", + "TestProxyKube/new_format" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-104", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_internetarchive__openlibrary-08ac40d050a64e1d2646ece4959af0c42bf6b7b5-v0f5aece3601a5b4419f7ccec1dbda2071be28ee4", + "selectedInstanceId": "instance_internetarchive__openlibrary-08ac40d050a64e1d2646ece4959af0c42bf6b7b5-v0f5aece3601a5b4419f7ccec1dbda2071be28ee4", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "internetarchive/openlibrary", + "baseCommit": "1e32ae3ec085dc3ab8611b88b56baf6354fa0762", + "language": "python", + "tags": [ + "python", + "core_feat", + "code_quality_enh", + "integration_feat", + "back_end_knowledge", + "database_knowledge", + "api_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "internetarchive.openlibrary-internetarchive__openlibrary-08ac40d050a64e1d2646ece4959af0c42bf6b7b5-v0f5aece3601a5b4419f7ccec1dbda", + "beforeRepoSetCommand": "git reset --hard 1e32ae3ec085dc3ab8611b88b56baf6354fa0762\ngit clean -fd \ngit checkout 1e32ae3ec085dc3ab8611b88b56baf6354fa0762 \ngit checkout 08ac40d050a64e1d2646ece4959af0c42bf6b7b5 -- openlibrary/catalog/add_book/tests/test_add_book.py openlibrary/catalog/marc/tests/test_data/bin_expect/ithaca_college_75002321.json openlibrary/catalog/marc/tests/test_data/bin_expect/lesnoirsetlesrou0000garl_meta.json openlibrary/catalog/marc/tests/test_data/bin_expect/memoirsofjosephf00fouc_meta.json openlibrary/catalog/marc/tests/test_data/bin_expect/warofrebellionco1473unit_meta.json openlibrary/catalog/marc/tests/test_data/xml_expect/00schlgoog.json openlibrary/catalog/marc/tests/test_data/xml_expect/warofrebellionco1473unit.json", + "selectedTestFiles": [ + "openlibrary/catalog/marc/tests/test_parse.py", + "openlibrary/catalog/add_book/tests/test_add_book.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-105", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_internetarchive__openlibrary-0a90f9f0256e4f933523e9842799e39f95ae29ce-v76304ecdb3a5954fcf13feb710e8c40fcf24b73c", + "selectedInstanceId": "instance_internetarchive__openlibrary-0a90f9f0256e4f933523e9842799e39f95ae29ce-v76304ecdb3a5954fcf13feb710e8c40fcf24b73c", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "internetarchive/openlibrary", + "baseCommit": "5c6c22f3d2edf2f1b10f5dc335e32cb6a5f40341", + "language": "python", + "tags": [ + "python", + "major_bug", + "data_bug", + "integration_bug", + "edge_case_bug", + "back_end_knowledge", + "api_knowledge", + "database_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "internetarchive.openlibrary-internetarchive__openlibrary-0a90f9f0256e4f933523e9842799e39f95ae29ce-v76304ecdb3a5954fcf13feb710e8c", + "beforeRepoSetCommand": "git reset --hard 5c6c22f3d2edf2f1b10f5dc335e32cb6a5f40341\ngit clean -fd \ngit checkout 5c6c22f3d2edf2f1b10f5dc335e32cb6a5f40341 \ngit checkout 0a90f9f0256e4f933523e9842799e39f95ae29ce -- openlibrary/plugins/importapi/tests/test_code.py openlibrary/plugins/upstream/tests/test_utils.py openlibrary/utils/tests/test_isbn.py", + "selectedTestFiles": [ + "openlibrary/plugins/importapi/tests/test_code.py", + "openlibrary/utils/tests/test_isbn.py", + "openlibrary/plugins/upstream/tests/test_utils.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-106", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_internetarchive__openlibrary-0dc5b20fa186f9714f8a838178597e69f549d026-v2d9a6c849c60ed19fd0858ce9e40b7cc8e097e59", + "selectedInstanceId": "instance_internetarchive__openlibrary-0dc5b20fa186f9714f8a838178597e69f549d026-v2d9a6c849c60ed19fd0858ce9e40b7cc8e097e59", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "internetarchive/openlibrary", + "baseCommit": "bd9d2a04efbbaec1575faa02a02eea995badf7f0", + "language": "python", + "tags": [ + "python", + "core_feat", + "refactoring_enh", + "back_end_knowledge", + "ml_ai_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "internetarchive.openlibrary-internetarchive__openlibrary-0dc5b20fa186f9714f8a838178597e69f549d026-v2d9a6c849c60ed19fd0858ce9e40b", + "beforeRepoSetCommand": "git reset --hard bd9d2a04efbbaec1575faa02a02eea995badf7f0\ngit clean -fd \ngit checkout bd9d2a04efbbaec1575faa02a02eea995badf7f0 \ngit checkout 0dc5b20fa186f9714f8a838178597e69f549d026 -- openlibrary/catalog/marc/tests/test_data/bin_expect/880_Nihon_no_chasho.json openlibrary/catalog/marc/tests/test_data/bin_expect/880_arabic_french_many_linkages.json", + "selectedTestFiles": [ + "openlibrary/catalog/marc/tests/test_parse.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-107", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_internetarchive__openlibrary-30bc73a1395fba2300087c7f307e54bb5372b60a-v76304ecdb3a5954fcf13feb710e8c40fcf24b73c", + "selectedInstanceId": "instance_internetarchive__openlibrary-30bc73a1395fba2300087c7f307e54bb5372b60a-v76304ecdb3a5954fcf13feb710e8c40fcf24b73c", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "internetarchive/openlibrary", + "baseCommit": "540853735859789920caf9ee78a762ebe14f6103", + "language": "python", + "tags": [ + "python", + "performance_feat", + "performance_enh", + "code_quality_enh", + "back_end_knowledge", + "api_knowledge", + "database_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "internetarchive.openlibrary-internetarchive__openlibrary-30bc73a1395fba2300087c7f307e54bb5372b60a-v76304ecdb3a5954fcf13feb710e8c", + "beforeRepoSetCommand": "git reset --hard 540853735859789920caf9ee78a762ebe14f6103\ngit clean -fd \ngit checkout 540853735859789920caf9ee78a762ebe14f6103 \ngit checkout 30bc73a1395fba2300087c7f307e54bb5372b60a -- openlibrary/coverstore/tests/test_archive.py", + "selectedTestFiles": [ + "openlibrary/coverstore/tests/test_archive.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-108", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_internetarchive__openlibrary-4b7ea2977be2747496ba792a678940baa985f7ea-v0f5aece3601a5b4419f7ccec1dbda2071be28ee4", + "selectedInstanceId": "instance_internetarchive__openlibrary-4b7ea2977be2747496ba792a678940baa985f7ea-v0f5aece3601a5b4419f7ccec1dbda2071be28ee4", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "internetarchive/openlibrary", + "baseCommit": "86d9b8a203cab5bc446614f3cbffd77b4751084a", + "language": "python", + "tags": [ + "python", + "api_feat", + "core_feat", + "integration_feat", + "api_knowledge", + "back_end_knowledge", + "database_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "internetarchive.openlibrary-internetarchive__openlibrary-4b7ea2977be2747496ba792a678940baa985f7ea-v0f5aece3601a5b4419f7ccec1dbda", + "beforeRepoSetCommand": "git reset --hard 86d9b8a203cab5bc446614f3cbffd77b4751084a\ngit clean -fd \ngit checkout 86d9b8a203cab5bc446614f3cbffd77b4751084a \ngit checkout 4b7ea2977be2747496ba792a678940baa985f7ea -- openlibrary/catalog/add_book/tests/test_load_book.py", + "selectedTestFiles": [ + "openlibrary/catalog/add_book/tests/test_load_book.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-109", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_internetarchive__openlibrary-53e02a22972e9253aeded0e1981e6845e1e521fe-vfa6ff903cb27f336e17654595dd900fa943dcd91", + "selectedInstanceId": "instance_internetarchive__openlibrary-53e02a22972e9253aeded0e1981e6845e1e521fe-vfa6ff903cb27f336e17654595dd900fa943dcd91", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "internetarchive/openlibrary", + "baseCommit": "2e2140e23dde91a233a91e9a7d04648450387721", + "language": "python", + "tags": [ + "python", + "refactoring_enh", + "back_end_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "internetarchive.openlibrary-internetarchive__openlibrary-53e02a22972e9253aeded0e1981e6845e1e521fe-vfa6ff903cb27f336e17654595dd90", + "beforeRepoSetCommand": "git reset --hard 2e2140e23dde91a233a91e9a7d04648450387721\ngit clean -fd \ngit checkout 2e2140e23dde91a233a91e9a7d04648450387721 \ngit checkout 53e02a22972e9253aeded0e1981e6845e1e521fe -- openlibrary/tests/solr/test_update_work.py", + "selectedTestFiles": [ + "openlibrary/tests/solr/test_update_work.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-110", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_internetarchive__openlibrary-5c6c22f3d2edf2f1b10f5dc335e32cb6a5f40341-v76304ecdb3a5954fcf13feb710e8c40fcf24b73c", + "selectedInstanceId": "instance_internetarchive__openlibrary-5c6c22f3d2edf2f1b10f5dc335e32cb6a5f40341-v76304ecdb3a5954fcf13feb710e8c40fcf24b73c", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "internetarchive/openlibrary", + "baseCommit": "31d6ecf3c04c8e365e557f2a9a6a65df2b8d01a3", + "language": "python", + "tags": [ + "python", + "code_quality_enh", + "back_end_knowledge", + "devops_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "internetarchive.openlibrary-internetarchive__openlibrary-5c6c22f3d2edf2f1b10f5dc335e32cb6a5f40341-v76304ecdb3a5954fcf13feb710e8c", + "beforeRepoSetCommand": "git reset --hard 31d6ecf3c04c8e365e557f2a9a6a65df2b8d01a3\ngit clean -fd \ngit checkout 31d6ecf3c04c8e365e557f2a9a6a65df2b8d01a3 \ngit checkout 5c6c22f3d2edf2f1b10f5dc335e32cb6a5f40341 -- openlibrary/plugins/importapi/tests/test_code.py openlibrary/plugins/upstream/tests/test_utils.py", + "selectedTestFiles": [ + "openlibrary/plugins/importapi/tests/test_code.py", + "openlibrary/plugins/upstream/tests/test_utils.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-111", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_internetarchive__openlibrary-5de7de19211e71b29b2f2ba3b1dff2fe065d660f-v08d8e8889ec945ab821fb156c04c7d2e2810debb", + "selectedInstanceId": "instance_internetarchive__openlibrary-5de7de19211e71b29b2f2ba3b1dff2fe065d660f-v08d8e8889ec945ab821fb156c04c7d2e2810debb", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "internetarchive/openlibrary", + "baseCommit": "5f7d8d190e2f0d837545e582fd5db99aae51a979", + "language": "python", + "tags": [ + "python", + "refactoring_enh", + "back_end_knowledge", + "api_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "internetarchive.openlibrary-internetarchive__openlibrary-5de7de19211e71b29b2f2ba3b1dff2fe065d660f-v08d8e8889ec945ab821fb156c04c7", + "beforeRepoSetCommand": "git reset --hard 5f7d8d190e2f0d837545e582fd5db99aae51a979\ngit clean -fd \ngit checkout 5f7d8d190e2f0d837545e582fd5db99aae51a979 \ngit checkout 5de7de19211e71b29b2f2ba3b1dff2fe065d660f -- openlibrary/tests/core/test_models.py", + "selectedTestFiles": [ + "openlibrary/tests/core/test_models.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-112", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_internetarchive__openlibrary-9bdfd29fac883e77dcbc4208cab28c06fd963ab2-v76304ecdb3a5954fcf13feb710e8c40fcf24b73c", + "selectedInstanceId": "instance_internetarchive__openlibrary-9bdfd29fac883e77dcbc4208cab28c06fd963ab2-v76304ecdb3a5954fcf13feb710e8c40fcf24b73c", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "internetarchive/openlibrary", + "baseCommit": "b2086f9bf54a3a8289e558a8f542673e3d01b376", + "language": "python", + "tags": [ + "python", + "edge_case_bug", + "back_end_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "internetarchive.openlibrary-internetarchive__openlibrary-9bdfd29fac883e77dcbc4208cab28c06fd963ab2-v76304ecdb3a5954fcf13feb710e8c", + "beforeRepoSetCommand": "git reset --hard b2086f9bf54a3a8289e558a8f542673e3d01b376\ngit clean -fd \ngit checkout b2086f9bf54a3a8289e558a8f542673e3d01b376 \ngit checkout 9bdfd29fac883e77dcbc4208cab28c06fd963ab2 -- openlibrary/plugins/worksearch/tests/test_worksearch.py", + "selectedTestFiles": [ + "openlibrary/plugins/worksearch/tests/test_worksearch.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-113", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_internetarchive__openlibrary-9cd47f4dc21e273320d9e30d889c864f8cb20ccf-v0f5aece3601a5b4419f7ccec1dbda2071be28ee4", + "selectedInstanceId": "instance_internetarchive__openlibrary-9cd47f4dc21e273320d9e30d889c864f8cb20ccf-v0f5aece3601a5b4419f7ccec1dbda2071be28ee4", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "internetarchive/openlibrary", + "baseCommit": "7fab050b6a99923d9d2efcc2f79311580b88f8a9", + "language": "python", + "tags": [ + "python", + "major_bug", + "performance_bug", + "integration_bug", + "back_end_knowledge", + "api_knowledge", + "infrastructure_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "internetarchive.openlibrary-internetarchive__openlibrary-9cd47f4dc21e273320d9e30d889c864f8cb20ccf-v0f5aece3601a5b4419f7ccec1dbda", + "beforeRepoSetCommand": "git reset --hard 7fab050b6a99923d9d2efcc2f79311580b88f8a9\ngit clean -fd \ngit checkout 7fab050b6a99923d9d2efcc2f79311580b88f8a9 \ngit checkout 9cd47f4dc21e273320d9e30d889c864f8cb20ccf -- openlibrary/catalog/add_book/tests/test_add_book.py", + "selectedTestFiles": [ + "openlibrary/catalog/add_book/tests/test_add_book.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-114", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_internetarchive__openlibrary-c8996ecc40803b9155935fd7ff3b8e7be6c1437c-ve8fc82d8aae8463b752a211156c5b7b59f349237", + "selectedInstanceId": "instance_internetarchive__openlibrary-c8996ecc40803b9155935fd7ff3b8e7be6c1437c-ve8fc82d8aae8463b752a211156c5b7b59f349237", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "internetarchive/openlibrary", + "baseCommit": "f0341c0ba81c790241b782f5103ce5c9a6edf8e3", + "language": "python", + "tags": [ + "python", + "data_bug", + "back_end_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "internetarchive.openlibrary-internetarchive__openlibrary-c8996ecc40803b9155935fd7ff3b8e7be6c1437c-ve8fc82d8aae8463b752a211156c5b", + "beforeRepoSetCommand": "git reset --hard f0341c0ba81c790241b782f5103ce5c9a6edf8e3\ngit clean -fd \ngit checkout f0341c0ba81c790241b782f5103ce5c9a6edf8e3 \ngit checkout c8996ecc40803b9155935fd7ff3b8e7be6c1437c -- openlibrary/catalog/add_book/tests/test_add_book.py openlibrary/tests/catalog/test_utils.py", + "selectedTestFiles": [ + "openlibrary/catalog/add_book/tests/test_add_book.py", + "openlibrary/tests/catalog/test_utils.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-115", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_internetarchive__openlibrary-e1e502986a3b003899a8347ac8a7ff7b08cbfc39-v08d8e8889ec945ab821fb156c04c7d2e2810debb", + "selectedInstanceId": "instance_internetarchive__openlibrary-e1e502986a3b003899a8347ac8a7ff7b08cbfc39-v08d8e8889ec945ab821fb156c04c7d2e2810debb", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "internetarchive/openlibrary", + "baseCommit": "77c16d530b4d5c0f33d68bead2c6b329aee9b996", + "language": "python", + "tags": [ + "python", + "ui_ux_feat", + "core_feat", + "code_quality_enh", + "back_end_knowledge", + "web_knowledge", + "ui_ux_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "internetarchive.openlibrary-internetarchive__openlibrary-e1e502986a3b003899a8347ac8a7ff7b08cbfc39-v08d8e8889ec945ab821fb156c04c7", + "beforeRepoSetCommand": "git reset --hard 77c16d530b4d5c0f33d68bead2c6b329aee9b996\ngit clean -fd \ngit checkout 77c16d530b4d5c0f33d68bead2c6b329aee9b996 \ngit checkout e1e502986a3b003899a8347ac8a7ff7b08cbfc39 -- openlibrary/plugins/upstream/tests/test_table_of_contents.py", + "selectedTestFiles": [ + "openlibrary/plugins/upstream/tests/test_table_of_contents.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-116", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_internetarchive__openlibrary-f343c08f89c772f7ba6c0246f384b9e6c3dc0add-v08d8e8889ec945ab821fb156c04c7d2e2810debb", + "selectedInstanceId": "instance_internetarchive__openlibrary-f343c08f89c772f7ba6c0246f384b9e6c3dc0add-v08d8e8889ec945ab821fb156c04c7d2e2810debb", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "internetarchive/openlibrary", + "baseCommit": "53d376b148897466bb86d5accb51912bbbe9a8ed", + "language": "python", + "tags": [ + "python", + "core_feat", + "customization_feat", + "back_end_knowledge", + "database_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "internetarchive.openlibrary-internetarchive__openlibrary-f343c08f89c772f7ba6c0246f384b9e6c3dc0add-v08d8e8889ec945ab821fb156c04c7", + "beforeRepoSetCommand": "git reset --hard 53d376b148897466bb86d5accb51912bbbe9a8ed\ngit clean -fd \ngit checkout 53d376b148897466bb86d5accb51912bbbe9a8ed \ngit checkout f343c08f89c772f7ba6c0246f384b9e6c3dc0add -- openlibrary/catalog/add_book/tests/test_load_book.py", + "selectedTestFiles": [ + "openlibrary/catalog/add_book/tests/test_load_book.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-117", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_navidrome__navidrome-0488fb92cb02a82924fb1181bf1642f2e87096db", + "selectedInstanceId": "instance_navidrome__navidrome-0488fb92cb02a82924fb1181bf1642f2e87096db", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "navidrome/navidrome", + "baseCommit": "61903facdf5d56277bf57c7aa83bce7fb35b597a", + "language": "go", + "tags": [ + "go", + "performance_bug", + "ui_ux_bug", + "minor_bug", + "web_knowledge", + "api_knowledge", + "ui_ux_knowledge", + "full_stack_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "navidrome.navidrome-navidrome__navidrome-0488fb92cb02a82924fb1181bf1642f2e87096db", + "beforeRepoSetCommand": "git reset --hard 61903facdf5d56277bf57c7aa83bce7fb35b597a\ngit clean -fd \ngit checkout 61903facdf5d56277bf57c7aa83bce7fb35b597a \ngit checkout 0488fb92cb02a82924fb1181bf1642f2e87096db -- core/artwork/artwork_internal_test.go core/artwork/artwork_test.go server/subsonic/media_retrieval_test.go", + "selectedTestFiles": [ + "TestArtwork", + "TestSubsonicApi" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-118", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_navidrome__navidrome-10108c63c9b5bdf2966ffb3239bbfd89683e37b7", + "selectedInstanceId": "instance_navidrome__navidrome-10108c63c9b5bdf2966ffb3239bbfd89683e37b7", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "navidrome/navidrome", + "baseCommit": "aac6e2cb0774aa256c00098b2d88bf8af288da79", + "language": "go", + "tags": [ + "go", + "minor_bug", + "customization_feat", + "back_end_knowledge", + "web_knowledge", + "api_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "navidrome.navidrome-navidrome__navidrome-10108c63c9b5bdf2966ffb3239bbfd89683e37b7", + "beforeRepoSetCommand": "git reset --hard aac6e2cb0774aa256c00098b2d88bf8af288da79\ngit clean -fd \ngit checkout aac6e2cb0774aa256c00098b2d88bf8af288da79 \ngit checkout 10108c63c9b5bdf2966ffb3239bbfd89683e37b7 -- server/serve_index_test.go server/serve_test.go", + "selectedTestFiles": [ + "TestServer" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-119", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_navidrome__navidrome-27875ba2dd1673ddf8affca526b0664c12c3b98b", + "selectedInstanceId": "instance_navidrome__navidrome-27875ba2dd1673ddf8affca526b0664c12c3b98b", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "navidrome/navidrome", + "baseCommit": "28f7ef43c1083fab4f7eabbade6a5bab83615e73", + "language": "go", + "tags": [ + "go", + "core_feat", + "customization_feat", + "integration_feat", + "back_end_knowledge", + "infrastructure_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "navidrome.navidrome-navidrome__navidrome-27875ba2dd1673ddf8affca526b0664c12c3b98b", + "beforeRepoSetCommand": "git reset --hard 28f7ef43c1083fab4f7eabbade6a5bab83615e73\ngit clean -fd \ngit checkout 28f7ef43c1083fab4f7eabbade6a5bab83615e73 \ngit checkout 27875ba2dd1673ddf8affca526b0664c12c3b98b -- server/serve_index_test.go", + "selectedTestFiles": [ + "TestServer" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-120", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_navidrome__navidrome-28389fb05e1523564dfc61fa43ed8eb8a10f938c", + "selectedInstanceId": "instance_navidrome__navidrome-28389fb05e1523564dfc61fa43ed8eb8a10f938c", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "navidrome/navidrome", + "baseCommit": "5d8318f7b362c5b044f892f775d264a5d40e4e24", + "language": "go", + "tags": [ + "go", + "core_feat", + "api_feat", + "code_quality_enh", + "refactoring_enh", + "back_end_knowledge", + "database_knowledge", + "api_knowledge", + "authentication_authorization_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "navidrome.navidrome-navidrome__navidrome-28389fb05e1523564dfc61fa43ed8eb8a10f938c", + "beforeRepoSetCommand": "git reset --hard 5d8318f7b362c5b044f892f775d264a5d40e4e24\ngit clean -fd \ngit checkout 5d8318f7b362c5b044f892f775d264a5d40e4e24 \ngit checkout 28389fb05e1523564dfc61fa43ed8eb8a10f938c -- core/playlists_test.go model/playlists_test.go", + "selectedTestFiles": [ + "TestModel" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-121", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_navidrome__navidrome-3972616585e82305eaf26aa25697b3f5f3082288", + "selectedInstanceId": "instance_navidrome__navidrome-3972616585e82305eaf26aa25697b3f5f3082288", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "navidrome/navidrome", + "baseCommit": "d0ce0303864d6859ee683214baab9c647f7467fe", + "language": "go", + "tags": [ + "go", + "api_feat", + "core_feat", + "back_end_knowledge", + "api_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "navidrome.navidrome-navidrome__navidrome-3972616585e82305eaf26aa25697b3f5f3082288", + "beforeRepoSetCommand": "git reset --hard d0ce0303864d6859ee683214baab9c647f7467fe\ngit clean -fd \ngit checkout d0ce0303864d6859ee683214baab9c647f7467fe \ngit checkout 3972616585e82305eaf26aa25697b3f5f3082288 -- model/criteria/criteria_suite_test.go model/criteria/criteria_test.go model/criteria/operators_test.go", + "selectedTestFiles": [ + "TestCriteria" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-122", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_navidrome__navidrome-7073d18b54da7e53274d11c9e2baef1242e8769e", + "selectedInstanceId": "instance_navidrome__navidrome-7073d18b54da7e53274d11c9e2baef1242e8769e", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "navidrome/navidrome", + "baseCommit": "7fc964aec5d432480d76bb90eee14a09aca5125f", + "language": "go", + "tags": [ + "go", + "code_quality_enh", + "refactoring_enh", + "back_end_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "navidrome.navidrome-navidrome__navidrome-7073d18b54da7e53274d11c9e2baef1242e8769e", + "beforeRepoSetCommand": "git reset --hard 7fc964aec5d432480d76bb90eee14a09aca5125f\ngit clean -fd \ngit checkout 7fc964aec5d432480d76bb90eee14a09aca5125f \ngit checkout 7073d18b54da7e53274d11c9e2baef1242e8769e -- core/agents/lastfm/agent_test.go core/agents/lastfm/client_test.go core/agents/listenbrainz/agent_test.go core/agents/listenbrainz/auth_router_test.go core/agents/listenbrainz/client_test.go core/agents/spotify/client_test.go", + "selectedTestFiles": [ + "TestListenBrainz", + "TestSpotify", + "TestLastFM" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-123", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_navidrome__navidrome-87d4db7638b37eeb754b217440ab7a372f669205", + "selectedInstanceId": "instance_navidrome__navidrome-87d4db7638b37eeb754b217440ab7a372f669205", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "navidrome/navidrome", + "baseCommit": "213ceeca7893d3c85eb688e6e99c09dd6cd7e453", + "language": "go", + "tags": [ + "go", + "core_feat", + "minor_bug", + "back_end_knowledge", + "ui_ux_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "navidrome.navidrome-navidrome__navidrome-87d4db7638b37eeb754b217440ab7a372f669205", + "beforeRepoSetCommand": "git reset --hard 213ceeca7893d3c85eb688e6e99c09dd6cd7e453\ngit clean -fd \ngit checkout 213ceeca7893d3c85eb688e6e99c09dd6cd7e453 \ngit checkout 87d4db7638b37eeb754b217440ab7a372f669205 -- core/artwork_internal_test.go", + "selectedTestFiles": [ + "TestCore" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-124", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_navidrome__navidrome-89b12b34bea5687c70e4de2109fd1e7330bb2ba2", + "selectedInstanceId": "instance_navidrome__navidrome-89b12b34bea5687c70e4de2109fd1e7330bb2ba2", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "navidrome/navidrome", + "baseCommit": "4e0177ee5340b888126092d1146c1c7b6a92fed8", + "language": "go", + "tags": [ + "go", + "edge_case_bug", + "back_end_knowledge", + "api_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "navidrome.navidrome-navidrome__navidrome-89b12b34bea5687c70e4de2109fd1e7330bb2ba2", + "beforeRepoSetCommand": "git reset --hard 4e0177ee5340b888126092d1146c1c7b6a92fed8\ngit clean -fd \ngit checkout 4e0177ee5340b888126092d1146c1c7b6a92fed8 \ngit checkout 89b12b34bea5687c70e4de2109fd1e7330bb2ba2 -- core/agents/lastfm_test.go tests/fake_http_client.go tests/fixtures/lastfm.artist.error3.json tests/fixtures/lastfm.artist.error6.json tests/fixtures/lastfm.artist.getinfo.unknown.json tests/fixtures/lastfm.artist.getsimilar.unknown.json tests/fixtures/lastfm.artist.gettoptracks.unknown.json utils/lastfm/client_test.go utils/lastfm/responses_test.go", + "selectedTestFiles": [ + "TestAgents", + "TestLastFM" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-125", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_navidrome__navidrome-8d56ec898e776e7e53e352cb9b25677975787ffc", + "selectedInstanceId": "instance_navidrome__navidrome-8d56ec898e776e7e53e352cb9b25677975787ffc", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "navidrome/navidrome", + "baseCommit": "5064cb2a46e4b7ca6a834b3c48a409eec8ca1830", + "language": "go", + "tags": [ + "go", + "minor_bug", + "back_end_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "navidrome.navidrome-navidrome__navidrome-8d56ec898e776e7e53e352cb9b25677975787ffc", + "beforeRepoSetCommand": "git reset --hard 5064cb2a46e4b7ca6a834b3c48a409eec8ca1830\ngit clean -fd \ngit checkout 5064cb2a46e4b7ca6a834b3c48a409eec8ca1830 \ngit checkout 8d56ec898e776e7e53e352cb9b25677975787ffc -- persistence/album_repository_test.go", + "selectedTestFiles": [ + "TestPersistence" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-126", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_navidrome__navidrome-97434c1789a6444b30aae5ff5aa124a96a88f504", + "selectedInstanceId": "instance_navidrome__navidrome-97434c1789a6444b30aae5ff5aa124a96a88f504", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "navidrome/navidrome", + "baseCommit": "f8ee6db72a015f33b41879d1ba253eeff8884bfd", + "language": "go", + "tags": [ + "go", + "core_feat", + "api_feat", + "code_quality_enh", + "back_end_knowledge", + "database_knowledge", + "api_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "navidrome.navidrome-navidrome__navidrome-97434c1789a6444b30aae5ff5aa124a96a88f504", + "beforeRepoSetCommand": "git reset --hard f8ee6db72a015f33b41879d1ba253eeff8884bfd\ngit clean -fd \ngit checkout f8ee6db72a015f33b41879d1ba253eeff8884bfd \ngit checkout 97434c1789a6444b30aae5ff5aa124a96a88f504 -- core/players_test.go", + "selectedTestFiles": [ + "TestCore" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-127", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_navidrome__navidrome-9c3b4561652a15846993d477003e111f0df0c585", + "selectedInstanceId": "instance_navidrome__navidrome-9c3b4561652a15846993d477003e111f0df0c585", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "navidrome/navidrome", + "baseCommit": "23bebe4e06124becf1000e88472ae71a6ca7de4c", + "language": "go", + "tags": [ + "go", + "core_feat", + "dev_ops_enh", + "back_end_knowledge", + "devops_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "navidrome.navidrome-navidrome__navidrome-9c3b4561652a15846993d477003e111f0df0c585", + "beforeRepoSetCommand": "git reset --hard 23bebe4e06124becf1000e88472ae71a6ca7de4c\ngit clean -fd \ngit checkout 23bebe4e06124becf1000e88472ae71a6ca7de4c \ngit checkout 9c3b4561652a15846993d477003e111f0df0c585 -- log/formatters_test.go", + "selectedTestFiles": [ + "TestLevelThreshold/errorLogLevel", + "TestEntryMessage", + "TestLevelThreshold", + "TestLevels", + "TestEntryDataValues", + "TestLevels/undefinedAcceptedLevels", + "TestLevelThreshold/unknownLogLevel", + "TestEntryDataValues/map_value", + "TestLevels/definedAcceptedLevels", + "TestInvalidRegex", + "TestEntryDataValues/match_on_key", + "TestEntryDataValues/string_value", + "TestLog" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-128", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_navidrome__navidrome-b65e76293a917ee2dfc5d4b373b1c62e054d0dca", + "selectedInstanceId": "instance_navidrome__navidrome-b65e76293a917ee2dfc5d4b373b1c62e054d0dca", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "navidrome/navidrome", + "baseCommit": "5f6f74ff2dde934aae2f2def911506d8c4ba66e1", + "language": "go", + "tags": [ + "go", + "core_feat", + "code_quality_enh", + "api_feat", + "web_knowledge", + "api_knowledge", + "full_stack_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "navidrome.navidrome-navidrome__navidrome-b65e76293a917ee2dfc5d4b373b1c62e054d0dca", + "beforeRepoSetCommand": "git reset --hard 5f6f74ff2dde934aae2f2def911506d8c4ba66e1\ngit clean -fd \ngit checkout 5f6f74ff2dde934aae2f2def911506d8c4ba66e1 \ngit checkout b65e76293a917ee2dfc5d4b373b1c62e054d0dca -- server/events/diode_test.go server/events/sse_test.go server/subsonic/middlewares_test.go", + "selectedTestFiles": [ + "TestSubsonicApi", + "TestEvents" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-129", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_navidrome__navidrome-d21932bd1b2379b0ebca2d19e5d8bae91040268a", + "selectedInstanceId": "instance_navidrome__navidrome-d21932bd1b2379b0ebca2d19e5d8bae91040268a", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "navidrome/navidrome", + "baseCommit": "c72add516a0f260e83a289c2355b2e74071311e0", + "language": "go", + "tags": [ + "go", + "core_feat", + "code_quality_enh", + "refactoring_enh", + "database_knowledge", + "back_end_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "navidrome.navidrome-navidrome__navidrome-d21932bd1b2379b0ebca2d19e5d8bae91040268a", + "beforeRepoSetCommand": "git reset --hard c72add516a0f260e83a289c2355b2e74071311e0\ngit clean -fd \ngit checkout c72add516a0f260e83a289c2355b2e74071311e0 \ngit checkout d21932bd1b2379b0ebca2d19e5d8bae91040268a -- persistence/sql_smartplaylist_test.go", + "selectedTestFiles": [ + "TestPersistence" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-130", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_navidrome__navidrome-d5df102f9f97c21715c756069c9e141da2a422dc", + "selectedInstanceId": "instance_navidrome__navidrome-d5df102f9f97c21715c756069c9e141da2a422dc", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "navidrome/navidrome", + "baseCommit": "20271df4fb0b94e201ed5e4b6501d591aa8cd813", + "language": "go", + "tags": [ + "go", + "api_feat", + "core_feat", + "ui_ux_feat", + "back_end_knowledge", + "api_knowledge", + "security_knowledge", + "ui_ux_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "navidrome.navidrome-navidrome__navidrome-d5df102f9f97c21715c756069c9e141da2a422dc", + "beforeRepoSetCommand": "git reset --hard 20271df4fb0b94e201ed5e4b6501d591aa8cd813\ngit clean -fd \ngit checkout 20271df4fb0b94e201ed5e4b6501d591aa8cd813 \ngit checkout d5df102f9f97c21715c756069c9e141da2a422dc -- core/share_test.go", + "selectedTestFiles": [ + "TestCore" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-131", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_navidrome__navidrome-e12a14a87d392ac70ee4cc8079e3c3e0103dbcb2", + "selectedInstanceId": "instance_navidrome__navidrome-e12a14a87d392ac70ee4cc8079e3c3e0103dbcb2", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "navidrome/navidrome", + "baseCommit": "0079a9b93873621fa0bed5439a0f753f48e081b5", + "language": "go", + "tags": [ + "go", + "core_feat", + "performance_enh", + "ui_ux_enh", + "ui_ux_knowledge", + "full_stack_knowledge", + "database_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "navidrome.navidrome-navidrome__navidrome-e12a14a87d392ac70ee4cc8079e3c3e0103dbcb2", + "beforeRepoSetCommand": "git reset --hard 0079a9b93873621fa0bed5439a0f753f48e081b5\ngit clean -fd \ngit checkout 0079a9b93873621fa0bed5439a0f753f48e081b5 \ngit checkout e12a14a87d392ac70ee4cc8079e3c3e0103dbcb2 -- scanner/metadata/ffmpeg/ffmpeg_test.go scanner/metadata/metadata_test.go scanner/metadata/taglib/taglib_test.go", + "selectedTestFiles": [ + "TestFFMpeg", + "TestMetadata", + "TestTagLib" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-132", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_navidrome__navidrome-f78257235ec3429ef42af6687738cd327ec77ce8", + "selectedInstanceId": "instance_navidrome__navidrome-f78257235ec3429ef42af6687738cd327ec77ce8", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "navidrome/navidrome", + "baseCommit": "1a6a284bc124d579c44053a6b0435cd20ead715c", + "language": "go", + "tags": [ + "go", + "core_feat", + "customization_feat", + "back_end_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "navidrome.navidrome-navidrome__navidrome-f78257235ec3429ef42af6687738cd327ec77ce8", + "beforeRepoSetCommand": "git reset --hard 1a6a284bc124d579c44053a6b0435cd20ead715c\ngit clean -fd \ngit checkout 1a6a284bc124d579c44053a6b0435cd20ead715c \ngit checkout f78257235ec3429ef42af6687738cd327ec77ce8 -- log/log_test.go", + "selectedTestFiles": [ + "TestLevelThreshold", + "TestInvalidRegex", + "TestLevelThreshold/errorLogLevel", + "TestEntryDataValues/map_value", + "TestEntryMessage", + "TestLevels/undefinedAcceptedLevels", + "TestEntryDataValues/match_on_key", + "TestLevels", + "TestEntryDataValues", + "TestEntryDataValues/string_value", + "TestLevels/definedAcceptedLevels", + "TestLog", + "TestLevelThreshold/unknownLogLevel" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-133", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_navidrome__navidrome-fa85e2a7816a6fe3829a4c0d8e893e982b0985da", + "selectedInstanceId": "instance_navidrome__navidrome-fa85e2a7816a6fe3829a4c0d8e893e982b0985da", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "navidrome/navidrome", + "baseCommit": "5360283bb0368c5226b301f99b7095a54407b053", + "language": "go", + "tags": [ + "go", + "data_bug", + "edge_case_bug", + "integration_bug", + "back_end_knowledge", + "database_knowledge", + "authentication_authorization_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "navidrome.navidrome-navidrome__navidrome-fa85e2a7816a6fe3829a4c0d8e893e982b0985da", + "beforeRepoSetCommand": "git reset --hard 5360283bb0368c5226b301f99b7095a54407b053\ngit clean -fd \ngit checkout 5360283bb0368c5226b301f99b7095a54407b053 \ngit checkout fa85e2a7816a6fe3829a4c0d8e893e982b0985da -- core/players_test.go persistence/persistence_suite_test.go persistence/persistence_test.go persistence/player_repository_test.go", + "selectedTestFiles": [ + "TestCore", + "TestPersistence" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-134", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_NodeBB__NodeBB-6489e9fd9ed16ea743cc5627f4d86c72fbdb3a8a-v2c59007b1005cd5cd14cbb523ca5229db1fd2dd8", + "selectedInstanceId": "instance_nodebb__nodebb-6489e9fd9ed16ea743cc5627f4d86c72fbdb3a8a-v2c59007b1005cd5cd14cbb523ca5229db1fd2dd8", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "NodeBB/NodeBB", + "baseCommit": "84dfda59e6a0e8a77240f939a7cb8757e6eaf945", + "language": "js", + "tags": [ + "js", + "refactoring_enh", + "major_bug", + "data_bug", + "database_knowledge", + "back_end_knowledge", + "infrastructure_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "nodebb.nodebb-NodeBB__NodeBB-6489e9fd9ed16ea743cc5627f4d86c72fbdb3a8a-v2c59007b1005cd5cd14cbb523ca5229db1fd2dd8", + "beforeRepoSetCommand": "git reset --hard 84dfda59e6a0e8a77240f939a7cb8757e6eaf945\ngit clean -fd \ngit checkout 84dfda59e6a0e8a77240f939a7cb8757e6eaf945 \ngit checkout 6489e9fd9ed16ea743cc5627f4d86c72fbdb3a8a -- test/posts/uploads.js", + "selectedTestFiles": [ + "test/posts/uploads.js" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-135", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_NodeBB__NodeBB-9c576a0758690f45a6ca03b5884c601e473bf2c1-vd59a5728dfc977f44533186ace531248c2917516", + "selectedInstanceId": "instance_nodebb__nodebb-9c576a0758690f45a6ca03b5884c601e473bf2c1-vd59a5728dfc977f44533186ace531248c2917516", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "NodeBB/NodeBB", + "baseCommit": "09f3ac6574b3192497df0403306aff8d6f20448b", + "language": "js", + "tags": [ + "js", + "code_quality_enh", + "security_enh", + "ui_ux_enh", + "refactoring_enh", + "back_end_knowledge", + "authentication_authorization_knowledge", + "security_knowledge", + "ui_ux_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "nodebb.nodebb-NodeBB__NodeBB-9c576a0758690f45a6ca03b5884c601e473bf2c1-vd59a5728dfc977f44533186ace531248c2917516", + "beforeRepoSetCommand": "git reset --hard 09f3ac6574b3192497df0403306aff8d6f20448b\ngit clean -fd \ngit checkout 09f3ac6574b3192497df0403306aff8d6f20448b \ngit checkout 9c576a0758690f45a6ca03b5884c601e473bf2c1 -- test/user.js test/user/emails.js", + "selectedTestFiles": [ + "test/user.js", + "test/user/emails.js" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-136", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_NodeBB__NodeBB-a5afad27e52fd336163063ba40dcadc80233ae10-vd59a5728dfc977f44533186ace531248c2917516", + "selectedInstanceId": "instance_nodebb__nodebb-a5afad27e52fd336163063ba40dcadc80233ae10-vd59a5728dfc977f44533186ace531248c2917516", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "NodeBB/NodeBB", + "baseCommit": "7800016f2f1b89d2d3cfea6a7da7c77096b7b927", + "language": "js", + "tags": [ + "js", + "api_feat", + "code_quality_enh", + "back_end_knowledge", + "web_knowledge", + "api_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "nodebb.nodebb-NodeBB__NodeBB-a5afad27e52fd336163063ba40dcadc80233ae10-vd59a5728dfc977f44533186ace531248c2917516", + "beforeRepoSetCommand": "git reset --hard 7800016f2f1b89d2d3cfea6a7da7c77096b7b927\ngit clean -fd \ngit checkout 7800016f2f1b89d2d3cfea6a7da7c77096b7b927 \ngit checkout a5afad27e52fd336163063ba40dcadc80233ae10 -- test/messaging.js test/user.js", + "selectedTestFiles": [ + "test/i18n.js", + "test/user.js", + "test/messaging.js" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-137", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_NodeBB__NodeBB-be43cd25974681c9743d424238b7536c357dc8d3-vf2cf3cbd463b7ad942381f1c6d077626485a1e9e", + "selectedInstanceId": "instance_nodebb__nodebb-be43cd25974681c9743d424238b7536c357dc8d3-vf2cf3cbd463b7ad942381f1c6d077626485a1e9e", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "NodeBB/NodeBB", + "baseCommit": "f24b630e1afcb8135144be66d67a09a61b21753e", + "language": "js", + "tags": [ + "js", + "core_feat", + "ui_ux_feat", + "localization_feat", + "customization_feat", + "back_end_knowledge", + "ui_ux_knowledge", + "database_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "nodebb.nodebb-NodeBB__NodeBB-be43cd25974681c9743d424238b7536c357dc8d3-vf2cf3cbd463b7ad942381f1c6d077626485a1e9e", + "beforeRepoSetCommand": "git reset --hard f24b630e1afcb8135144be66d67a09a61b21753e\ngit clean -fd \ngit checkout f24b630e1afcb8135144be66d67a09a61b21753e \ngit checkout be43cd25974681c9743d424238b7536c357dc8d3 -- test/posts.js", + "selectedTestFiles": [ + "test/posts.js" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-138", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_NodeBB__NodeBB-f2082d7de85eb62a70819f4f3396dd85626a0c0a-vd59a5728dfc977f44533186ace531248c2917516", + "selectedInstanceId": "instance_nodebb__nodebb-f2082d7de85eb62a70819f4f3396dd85626a0c0a-vd59a5728dfc977f44533186ace531248c2917516", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "NodeBB/NodeBB", + "baseCommit": "f0d989e4ba5b0dccff6e56022fc6d378d05ab404", + "language": "js", + "tags": [ + "js", + "api_feat", + "core_feat", + "integration_feat", + "back_end_knowledge", + "api_knowledge", + "web_knowledge", + "authentication_authorization_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "nodebb.nodebb-NodeBB__NodeBB-f2082d7de85eb62a70819f4f3396dd85626a0c0a-vd59a5728dfc977f44533186ace531248c2917516", + "beforeRepoSetCommand": "git reset --hard f0d989e4ba5b0dccff6e56022fc6d378d05ab404\ngit clean -fd \ngit checkout f0d989e4ba5b0dccff6e56022fc6d378d05ab404 \ngit checkout f2082d7de85eb62a70819f4f3396dd85626a0c0a -- test/posts.js", + "selectedTestFiles": [ + "test/posts.js" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-139", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_protonmail__webclients-2f2f6c311c6128fe86976950d3c0c2db07b03921", + "selectedInstanceId": "instance_protonmail__webclients-2f2f6c311c6128fe86976950d3c0c2db07b03921", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "protonmail/webclients", + "baseCommit": "4d0ef1ed136e80692723f148dc0390dcf28ba9dc", + "language": "js", + "tags": [ + "js", + "compatibility_bug", + "back_end_knowledge", + "api_knowledge", + "security_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "protonmail.webclients-protonmail__webclients-2f2f6c311c6128fe86976950d3c0c2db07b03921", + "beforeRepoSetCommand": "git reset --hard 4d0ef1ed136e80692723f148dc0390dcf28ba9dc\ngit clean -fd \ngit checkout 4d0ef1ed136e80692723f148dc0390dcf28ba9dc \ngit checkout 2f2f6c311c6128fe86976950d3c0c2db07b03921 -- applications/drive/src/app/store/_shares/useShareActions.test.ts", + "selectedTestFiles": [ + "applications/drive/src/app/store/_shares/useShareActions.test.ts", + "src/app/store/_shares/useShareActions.test.ts" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-140", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_protonmail__webclients-863d524b5717b9d33ce08a0f0535e3fd8e8d1ed8", + "selectedInstanceId": "instance_protonmail__webclients-863d524b5717b9d33ce08a0f0535e3fd8e8d1ed8", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "protonmail/webclients", + "baseCommit": "464a02f3da87d165d1bfc330e5310c7c6e5e9734", + "language": "js", + "tags": [ + "js", + "performance_enh", + "ui_ux_enh", + "front_end_knowledge", + "web_knowledge", + "ui_ux_knowledge", + "performance_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "protonmail.webclients-protonmail__webclients-863d524b5717b9d33ce08a0f0535e3fd8e8d1ed8", + "beforeRepoSetCommand": "git reset --hard 464a02f3da87d165d1bfc330e5310c7c6e5e9734\ngit clean -fd \ngit checkout 464a02f3da87d165d1bfc330e5310c7c6e5e9734 \ngit checkout 863d524b5717b9d33ce08a0f0535e3fd8e8d1ed8 -- packages/components/payments/client-extensions/usePollEvents.test.ts", + "selectedTestFiles": [ + "packages/components/payments/client-extensions/usePollEvents.test.ts", + "payments/client-extensions/usePollEvents.test.ts" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-141", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_protonmail__webclients-b530a3db50cb33e5064464addbcbef1465856ce6", + "selectedInstanceId": "instance_protonmail__webclients-b530a3db50cb33e5064464addbcbef1465856ce6", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "protonmail/webclients", + "baseCommit": "32394eda944448efcd48d2487c2878cd402e612c", + "language": "js", + "tags": [ + "js", + "code_quality_enh", + "refactoring_enh", + "front_end_knowledge", + "web_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "protonmail.webclients-protonmail__webclients-b530a3db50cb33e5064464addbcbef1465856ce6", + "beforeRepoSetCommand": "git reset --hard 32394eda944448efcd48d2487c2878cd402e612c\ngit clean -fd \ngit checkout 32394eda944448efcd48d2487c2878cd402e612c \ngit checkout b530a3db50cb33e5064464addbcbef1465856ce6 -- applications/mail/src/app/containers/onboardingChecklist/hooks/useCanCheckItem.test.ts", + "selectedTestFiles": [ + "applications/mail/src/app/containers/onboardingChecklist/hooks/useCanCheckItem.test.ts", + "src/app/containers/onboardingChecklist/hooks/useCanCheckItem.test.ts" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-142", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_qutebrowser__qutebrowser-36ade4bba504eb96f05d32ceab9972df7eb17bcc-v2ef375ac784985212b1805e1d0431dc8f1b3c171", + "selectedInstanceId": "instance_qutebrowser__qutebrowser-36ade4bba504eb96f05d32ceab9972df7eb17bcc-v2ef375ac784985212b1805e1d0431dc8f1b3c171", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "qutebrowser/qutebrowser", + "baseCommit": "73f93008f6c8104dcb317b10bae2c6d156674b33", + "language": "python", + "tags": [ + "python", + "customization_feat", + "code_quality_enh", + "back_end_knowledge", + "desktop_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "qutebrowser.qutebrowser-qutebrowser__qutebrowser-36ade4bba504eb96f05d32ceab9972df7eb17bcc-v2ef375ac784985212b1805e1d0431dc8f1b3c", + "beforeRepoSetCommand": "git reset --hard 73f93008f6c8104dcb317b10bae2c6d156674b33\ngit clean -fd \ngit checkout 73f93008f6c8104dcb317b10bae2c6d156674b33 \ngit checkout 36ade4bba504eb96f05d32ceab9972df7eb17bcc -- tests/unit/config/test_qtargs.py", + "selectedTestFiles": [ + "tests/unit/config/test_qtargs.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-143", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_qutebrowser__qutebrowser-394bfaed6544c952c6b3463751abab3176ad4997-vafb3e8e01b31319c66c4e666b8a3b1d8ba55db24", + "selectedInstanceId": "instance_qutebrowser__qutebrowser-394bfaed6544c952c6b3463751abab3176ad4997-vafb3e8e01b31319c66c4e666b8a3b1d8ba55db24", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "qutebrowser/qutebrowser", + "baseCommit": "d1164925c55f2417f1c3130b0196830bc2a3d25d", + "language": "python", + "tags": [ + "python", + "refactoring_enh", + "code_quality_enh", + "technical_debt_enh", + "desktop_knowledge", + "back_end_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "qutebrowser.qutebrowser-qutebrowser__qutebrowser-394bfaed6544c952c6b3463751abab3176ad4997-vafb3e8e01b31319c66c4e666b8a3b1d8ba55d", + "beforeRepoSetCommand": "git reset --hard d1164925c55f2417f1c3130b0196830bc2a3d25d\ngit clean -fd \ngit checkout d1164925c55f2417f1c3130b0196830bc2a3d25d \ngit checkout 394bfaed6544c952c6b3463751abab3176ad4997 -- tests/unit/misc/test_elf.py tests/unit/utils/test_version.py", + "selectedTestFiles": [ + "tests/unit/utils/test_version.py", + "tests/unit/misc/test_elf.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-144", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_qutebrowser__qutebrowser-3e21c8214a998cb1058defd15aabb24617a76402-v5fc38aaf22415ab0b70567368332beee7955b367", + "selectedInstanceId": "instance_qutebrowser__qutebrowser-3e21c8214a998cb1058defd15aabb24617a76402-v5fc38aaf22415ab0b70567368332beee7955b367", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "qutebrowser/qutebrowser", + "baseCommit": "fce306d5f184f185b660f35ec2808b6745d57520", + "language": "python", + "tags": [ + "python", + "code_quality_enh", + "refactoring_enh", + "desktop_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "qutebrowser.qutebrowser-qutebrowser__qutebrowser-3e21c8214a998cb1058defd15aabb24617a76402-v5fc38aaf22415ab0b70567368332beee7955b", + "beforeRepoSetCommand": "git reset --hard fce306d5f184f185b660f35ec2808b6745d57520\ngit clean -fd \ngit checkout fce306d5f184f185b660f35ec2808b6745d57520 \ngit checkout 3e21c8214a998cb1058defd15aabb24617a76402 -- tests/unit/keyinput/test_keyutils.py", + "selectedTestFiles": [ + "tests/unit/keyinput/test_bindingtrie.py", + "tests/unit/keyinput/test_keyutils.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-145", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_qutebrowser__qutebrowser-473a15f7908f2bb6d670b0e908ab34a28d8cf7e2-v363c8a7e5ccdf6968fc7ab84a2053ac78036691d", + "selectedInstanceId": "instance_qutebrowser__qutebrowser-473a15f7908f2bb6d670b0e908ab34a28d8cf7e2-v363c8a7e5ccdf6968fc7ab84a2053ac78036691d", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "qutebrowser/qutebrowser", + "baseCommit": "8e08f046aeffb38445cc659050f7a083b4a9649c", + "language": "python", + "tags": [ + "python", + "regression_bug", + "localization_feat", + "critical_bug", + "desktop_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "qutebrowser.qutebrowser-qutebrowser__qutebrowser-473a15f7908f2bb6d670b0e908ab34a28d8cf7e2-v363c8a7e5ccdf6968fc7ab84a2053ac780366", + "beforeRepoSetCommand": "git reset --hard 8e08f046aeffb38445cc659050f7a083b4a9649c\ngit clean -fd \ngit checkout 8e08f046aeffb38445cc659050f7a083b4a9649c \ngit checkout 473a15f7908f2bb6d670b0e908ab34a28d8cf7e2 -- tests/unit/config/test_qtargs_locale_workaround.py", + "selectedTestFiles": [ + "tests/unit/config/test_qtargs_locale_workaround.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-146", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_qutebrowser__qutebrowser-e34dfc68647d087ca3175d9ad3f023c30d8c9746-v363c8a7e5ccdf6968fc7ab84a2053ac78036691d", + "selectedInstanceId": "instance_qutebrowser__qutebrowser-e34dfc68647d087ca3175d9ad3f023c30d8c9746-v363c8a7e5ccdf6968fc7ab84a2053ac78036691d", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "qutebrowser/qutebrowser", + "baseCommit": "c984983bc4cf6f9148e16ea17369597f67774ff9", + "language": "python", + "tags": [ + "python", + "edge_case_bug", + "data_bug", + "web_knowledge", + "desktop_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "qutebrowser.qutebrowser-qutebrowser__qutebrowser-e34dfc68647d087ca3175d9ad3f023c30d8c9746-v363c8a7e5ccdf6968fc7ab84a2053ac780366", + "beforeRepoSetCommand": "git reset --hard c984983bc4cf6f9148e16ea17369597f67774ff9\ngit clean -fd \ngit checkout c984983bc4cf6f9148e16ea17369597f67774ff9 \ngit checkout e34dfc68647d087ca3175d9ad3f023c30d8c9746 -- tests/unit/utils/test_urlutils.py", + "selectedTestFiles": [ + "tests/unit/utils/test_urlutils.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-147", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_qutebrowser__qutebrowser-fcfa069a06ade76d91bac38127f3235c13d78eb1-v5fc38aaf22415ab0b70567368332beee7955b367", + "selectedInstanceId": "instance_qutebrowser__qutebrowser-fcfa069a06ade76d91bac38127f3235c13d78eb1-v5fc38aaf22415ab0b70567368332beee7955b367", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "qutebrowser/qutebrowser", + "baseCommit": "74671c167f6f18a5494ffe44809e6a0e1c6ea8e9", + "language": "python", + "tags": [ + "python", + "core_feat", + "technical_debt_enh", + "back_end_knowledge", + "database_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "qutebrowser.qutebrowser-qutebrowser__qutebrowser-fcfa069a06ade76d91bac38127f3235c13d78eb1-v5fc38aaf22415ab0b70567368332beee7955b", + "beforeRepoSetCommand": "git reset --hard 74671c167f6f18a5494ffe44809e6a0e1c6ea8e9\ngit clean -fd \ngit checkout 74671c167f6f18a5494ffe44809e6a0e1c6ea8e9 \ngit checkout fcfa069a06ade76d91bac38127f3235c13d78eb1 -- tests/unit/misc/test_sql.py", + "selectedTestFiles": [ + "tests/unit/misc/test_sql.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-148", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_qutebrowser__qutebrowser-ff1c025ad3210506fc76e1f604d8c8c27637d88e-v363c8a7e5ccdf6968fc7ab84a2053ac78036691d", + "selectedInstanceId": "instance_qutebrowser__qutebrowser-ff1c025ad3210506fc76e1f604d8c8c27637d88e-v363c8a7e5ccdf6968fc7ab84a2053ac78036691d", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "qutebrowser/qutebrowser", + "baseCommit": "e545faaf7b18d451c082f697675f0ab0e7599ed1", + "language": "python", + "tags": [ + "python", + "ui_ux_feat", + "customization_feat", + "code_quality_enh", + "full_stack_knowledge", + "desktop_knowledge", + "ui_ux_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "qutebrowser.qutebrowser-qutebrowser__qutebrowser-ff1c025ad3210506fc76e1f604d8c8c27637d88e-v363c8a7e5ccdf6968fc7ab84a2053ac780366", + "beforeRepoSetCommand": "git reset --hard e545faaf7b18d451c082f697675f0ab0e7599ed1\ngit clean -fd \ngit checkout e545faaf7b18d451c082f697675f0ab0e7599ed1 \ngit checkout ff1c025ad3210506fc76e1f604d8c8c27637d88e -- tests/helpers/fixtures.py tests/unit/config/test_configinit.py tests/unit/config/test_configtypes.py", + "selectedTestFiles": [ + "tests/unit/config/test_configfiles.py", + "tests/unit/config/test_configcache.py", + "tests/unit/config/test_configtypes.py", + "tests/unit/config/test_config.py", + "tests/helpers/fixtures.py", + "tests/unit/config/test_configinit.py", + "tests/unit/config/test_stylesheet.py", + "tests/unit/config/test_configcommands.py", + "tests/unit/config/test_configdata.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-149", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_tutao__tutanota-1ff82aa365763cee2d609c9d19360ad87fdf2ec7-vc4e41fd0029957297843cb9dec4a25c7c756f029", + "selectedInstanceId": "instance_tutao__tutanota-1ff82aa365763cee2d609c9d19360ad87fdf2ec7-vc4e41fd0029957297843cb9dec4a25c7c756f029", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "tutao/tutanota", + "baseCommit": "2e5d877afe8a1b8fc9ad3ac0a0d579bbad364224", + "language": "ts", + "tags": [ + "ts", + "data_bug", + "integration_bug", + "back_end_knowledge", + "security_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "tutao.tutanota-tutao__tutanota-1ff82aa365763cee2d609c9d19360ad87fdf2ec7-vc4e41fd0029957297843cb9dec4a25c7c756f029", + "beforeRepoSetCommand": "git reset --hard 2e5d877afe8a1b8fc9ad3ac0a0d579bbad364224\ngit clean -fd \ngit checkout 2e5d877afe8a1b8fc9ad3ac0a0d579bbad364224 \ngit checkout 1ff82aa365763cee2d609c9d19360ad87fdf2ec7 -- test/tests/api/worker/rest/EntityRestCacheTest.ts", + "selectedTestFiles": [ + "test/tests/api/worker/crypto/OwnerEncSessionKeysUpdateQueueTest.js", + "test/tests/api/worker/search/EventQueueTest.js", + "test/tests/misc/parsing/MailAddressParserTest.js", + "test/tests/api/worker/search/IndexUtilsTest.js", + "test/tests/gui/GuiUtilsTest.js", + "test/tests/api/worker/rest/EntityRestCacheTest.js", + "test/tests/api/worker/SuspensionHandlerTest.js", + "test/tests/misc/credentials/CredentialsProviderTest.js", + "test/tests/misc/credentials/CredentialsKeyProviderTest.js", + "test/tests/contacts/VCardImporterTest.js", + "test/tests/misc/SchedulerTest.js", + "test/tests/mail/export/ExporterTest.js", + "test/tests/misc/webauthn/WebauthnClientTest.js", + "test/tests/mail/TemplateSearchFilterTest.js", + "test/tests/calendar/eventeditor/CalendarEventModelTest.js", + "test/tests/api/worker/crypto/CompatibilityTest.js", + "test/tests/api/worker/rest/CustomCacheHandlerTest.js", + "test/tests/api/main/EntropyCollectorTest.js", + "test/tests/misc/ParserTest.js", + "test/tests/misc/UsageTestModelTest.js", + "test/tests/translations/TranslationKeysTest.js", + "test/tests/api/worker/facades/BlobAccessTokenFacadeTest.js", + "test/tests/gui/base/WizardDialogNTest.js", + "test/tests/api/common/utils/FileUtilsTest.js", + "test/tests/api/worker/search/TokenizerTest.js", + "test/tests/mail/KnowledgeBaseSearchFilterTest.js", + "test/tests/misc/HtmlSanitizerTest.js", + "test/tests/api/worker/rest/EntityRestCacheTest.ts", + "test/tests/gui/ColorTest.js", + "test/tests/misc/PasswordUtilsTest.js", + "test/tests/api/worker/facades/MailFacadeTest.js", + "test/tests/settings/login/secondfactor/SecondFactorEditModelTest.js", + "test/tests/api/common/error/RestErrorTest.js", + "test/tests/misc/FormatValidatorTest.js", + "test/tests/api/worker/CompressionTest.js", + "test/tests/api/worker/EventBusClientTest.js", + "test/tests/api/common/error/TutanotaErrorTest.js", + "test/tests/api/worker/utils/SleepDetectorTest.js", + "test/tests/misc/FormatterTest.js", + "test/tests/api/worker/search/SuggestionFacadeTest.js", + "test/tests/calendar/AlarmSchedulerTest.js", + "test/tests/api/worker/rest/ServiceExecutorTest.js", + "test/tests/contacts/ContactUtilsTest.js", + "test/tests/api/worker/rest/CborDateEncoderTest.js", + "test/tests/calendar/CalendarImporterTest.js", + "test/tests/login/LoginViewModelTest.js", + "test/tests/api/worker/search/GroupInfoIndexerTest.js", + "test/tests/api/worker/facades/BlobFacadeTest.js", + "test/tests/mail/SendMailModelTest.js", + "test/tests/api/worker/search/MailIndexerTest.js", + "test/tests/mail/export/BundlerTest.js", + "test/tests/contacts/ContactMergeUtilsTest.js", + "test/tests/mail/model/FolderSystemTest.js", + "test/tests/calendar/CalendarViewModelTest.js", + "test/tests/api/common/utils/LoggerTest.js", + "test/tests/misc/NewsModelTest.js", + "test/tests/misc/OutOfOfficeNotificationTest.js", + "test/tests/serviceworker/SwTest.js", + "test/tests/api/worker/UrlifierTest.js", + "test/tests/api/common/utils/CommonFormatterTest.js", + "test/tests/api/worker/facades/LoginFacadeTest.js", + "test/tests/calendar/CalendarGuiUtilsTest.js", + "test/tests/misc/LanguageViewModelTest.js", + "test/tests/subscription/CreditCardViewModelTest.js", + "test/tests/calendar/CalendarUtilsTest.js", + "test/tests/api/worker/rest/EntityRestClientTest.js", + "test/tests/calendar/CalendarModelTest.js", + "test/tests/support/FaqModelTest.js", + "test/tests/api/worker/rest/EphemeralCacheStorageTest.js", + "test/tests/calendar/EventDragHandlerTest.js", + "test/tests/gui/animation/AnimationsTest.js", + "test/tests/calendar/eventeditor/CalendarEventWhoModelTest.js", + "test/tests/mail/InboxRuleHandlerTest.js", + "test/tests/api/common/utils/PlainTextSearchTest.js", + "test/tests/file/FileControllerTest.js", + "test/tests/calendar/eventeditor/CalendarEventAlarmModelTest.js", + "test/tests/mail/MailModelTest.js", + "test/tests/api/common/utils/BirthdayUtilsTest.js", + "test/tests/api/worker/search/SearchFacadeTest.js", + "test/tests/calendar/CalendarParserTest.js", + "test/tests/misc/ListModelTest.js", + "test/tests/api/worker/crypto/CryptoFacadeTest.js", + "test/tests/api/worker/facades/CalendarFacadeTest.js", + "test/tests/api/worker/search/ContactIndexerTest.js", + "test/tests/misc/news/items/ReferralLinkNewsTest.js", + "test/tests/api/worker/search/SearchIndexEncodingTest.js", + "test/tests/api/worker/search/IndexerCoreTest.js", + "test/tests/api/worker/facades/UserFacadeTest.js", + "test/tests/api/worker/rest/CacheStorageProxyTest.js", + "test/tests/subscription/PriceUtilsTest.js", + "test/tests/misc/credentials/NativeCredentialsEncryptionTest.js", + "test/tests/misc/RecipientsModelTest.js", + "test/tests/contacts/VCardExporterTest.js", + "test/tests/settings/whitelabel/CustomColorEditorTest.js", + "test/tests/api/worker/facades/ConfigurationDbTest.js", + "test/tests/mail/MailUtilsSignatureTest.js", + "test/tests/api/worker/facades/MailAddressFacadeTest.js", + "test/tests/subscription/SubscriptionUtilsTest.js", + "test/tests/settings/TemplateEditorModelTest.js", + "test/tests/misc/DeviceConfigTest.js", + "test/tests/calendar/eventeditor/CalendarEventWhenModelTest.js", + "test/tests/settings/mailaddress/MailAddressTableModelTest.js", + "test/tests/misc/ClientDetectorTest.js", + "test/tests/api/common/utils/EntityUtilsTest.js", + "test/tests/api/worker/rest/RestClientTest.js", + "test/tests/api/worker/search/IndexerTest.js", + "test/tests/settings/UserDataExportTest.js", + "test/tests/gui/ThemeControllerTest.js" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-150", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_tutao__tutanota-51818218c6ae33de00cbea3a4d30daac8c34142e-vc4e41fd0029957297843cb9dec4a25c7c756f029", + "selectedInstanceId": "instance_tutao__tutanota-51818218c6ae33de00cbea3a4d30daac8c34142e-vc4e41fd0029957297843cb9dec4a25c7c756f029", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "tutao/tutanota", + "baseCommit": "dac77208814de95c4018bcf13137324153cc9a3a", + "language": "ts", + "tags": [ + "ts", + "regression_bug", + "ui_ux_bug", + "compatibility_bug", + "desktop_knowledge", + "ui_ux_knowledge", + "full_stack_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "tutao.tutanota-tutao__tutanota-51818218c6ae33de00cbea3a4d30daac8c34142e-vc4e41fd0029957297843cb9dec4a25c7c756f029", + "beforeRepoSetCommand": "git reset --hard dac77208814de95c4018bcf13137324153cc9a3a\ngit clean -fd \ngit checkout dac77208814de95c4018bcf13137324153cc9a3a \ngit checkout 51818218c6ae33de00cbea3a4d30daac8c34142e -- test/client/desktop/DesktopDownloadManagerTest.ts test/client/nodemocker.ts", + "selectedTestFiles": [ + "test/client/desktop/DesktopDownloadManagerTest.ts", + "test/api/Suite.ts", + "test/client/nodemocker.ts" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + } + ] +} diff --git a/benchmarks/agentrail-swe-lifecycle/fixtures/sample-dataset.json b/benchmarks/agentrail-swe-lifecycle/fixtures/sample-dataset.json new file mode 100644 index 0000000..942715b --- /dev/null +++ b/benchmarks/agentrail-swe-lifecycle/fixtures/sample-dataset.json @@ -0,0 +1,75 @@ +{ + "version": 1, + "id": "agentrail-swe-lifecycle-sample", + "defaultScenario": "local-mock-lifecycle", + "tasks": [ + { + "id": "agentrail-swe-pro-hard-aa-001", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_owner__repo-aaaa", + "selectedInstanceId": "instance_owner__repo-aaaa", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "owner/repo", + "baseCommit": "1111111111111111111111111111111111111111", + "language": "python", + "tags": ["python", "bugfix"], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "owner.repo-test", + "beforeRepoSetCommand": "git reset --hard 1111111111111111111111111111111111111111", + "selectedTestFiles": ["tests/test_bug.py"], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-002", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "INSTANCE_OWNER__REPO-BBBB", + "selectedInstanceId": "instance_owner__repo-selected-bbbb", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "owner/ui-repo", + "baseCommit": "2222222222222222222222222222222222222222", + "language": "ts", + "tags": ["ts", "ui"], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "owner.repo-test", + "beforeRepoSetCommand": "git reset --hard 2222222222222222222222222222222222222222", + "selectedTestFiles": ["tests/test_ui.ts"], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + } + ] +} diff --git a/benchmarks/agentrail-swe-lifecycle/lib/dataset.mjs b/benchmarks/agentrail-swe-lifecycle/lib/dataset.mjs new file mode 100644 index 0000000..e165c62 --- /dev/null +++ b/benchmarks/agentrail-swe-lifecycle/lib/dataset.mjs @@ -0,0 +1,75 @@ +import { readFile } from "node:fs/promises"; + +export async function loadLifecycleDataset(filePath) { + const dataset = JSON.parse(await readFile(filePath, "utf8")); + if (dataset.version !== 1 || !Array.isArray(dataset.tasks) || !dataset.tasks.every(isValidTask)) { + throw new Error(`Invalid AgentRail SWE lifecycle dataset at ${filePath}.`); + } + return dataset; +} + +function isValidTask(task) { + return isRecord(task) + && isNonEmptyString(task.id) + && isNonEmptyString(task.repo) + && (task.language === null || typeof task.language === "string") + && (task.tags === undefined || Array.isArray(task.tags)) + && isRecord(task.upstream) + && isNonEmptyString(task.upstream.instanceId) + && ( + task.upstream.selectedInstanceId === undefined + || task.upstream.selectedInstanceId === null + || typeof task.upstream.selectedInstanceId === "string" + ); +} + +function isRecord(value) { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +function isNonEmptyString(value) { + return typeof value === "string" && value.length > 0; +} + +export function selectLifecycleTasks(dataset, options = {}) { + let tasks = [...dataset.tasks].sort((left, right) => left.id.localeCompare(right.id)); + if (options.taskId) tasks = tasks.filter((task) => matchesTaskId(task, options.taskId)); + if (options.repo) tasks = tasks.filter((task) => task.repo === options.repo); + if (options.language) tasks = tasks.filter((task) => task.language === options.language); + if (options.tag) tasks = tasks.filter((task) => Array.isArray(task.tags) && task.tags.includes(options.tag)); + if (options.seed !== undefined) tasks = seededShuffle(tasks, String(options.seed)); + const count = normalizeCount(options.count, tasks.length); + return tasks.slice(0, count); +} + +function matchesTaskId(task, taskId) { + if (task.id === taskId) return true; + const normalizedTaskId = String(taskId).toLowerCase(); + return [task.upstream?.instanceId, task.upstream?.selectedInstanceId] + .filter((upstreamId) => typeof upstreamId === "string") + .some((upstreamId) => upstreamId.toLowerCase() === normalizedTaskId); +} + +function normalizeCount(value, fallback) { + if (value === undefined || value === null) return fallback; + if (!/^[1-9]\d*$/u.test(String(value))) { + throw new Error(`--count must be a positive integer, received ${value}.`); + } + const parsed = Number(value); + if (!Number.isSafeInteger(parsed)) { + throw new Error(`--count must be a positive integer, received ${value}.`); + } + return parsed; +} + +function seededShuffle(items, seed) { + const shuffled = [...items]; + let state = 0; + for (const char of seed) state = (state * 31 + char.charCodeAt(0)) >>> 0; + for (let index = shuffled.length - 1; index > 0; index -= 1) { + state = (1664525 * state + 1013904223) >>> 0; + const swapIndex = state % (index + 1); + [shuffled[index], shuffled[swapIndex]] = [shuffled[swapIndex], shuffled[index]]; + } + return shuffled; +} diff --git a/benchmarks/agentrail-swe-lifecycle/lib/local-lifecycle.mjs b/benchmarks/agentrail-swe-lifecycle/lib/local-lifecycle.mjs new file mode 100644 index 0000000..45602f2 --- /dev/null +++ b/benchmarks/agentrail-swe-lifecycle/lib/local-lifecycle.mjs @@ -0,0 +1,121 @@ +import { createHash } from "node:crypto"; + +const DETERMINISTIC_CREATED_AT = "1970-01-01T00:00:00.000Z"; +const RUN_ID_PATTERN = /^run_[A-Za-z0-9_-]+$/u; +const TASK_ID_PATTERN = /^[a-z0-9]+(?:-[a-z0-9]+)*$/u; +const REPO_PATTERN = /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u; +const BASE_COMMIT_PATTERN = /^[a-f0-9]{40}$/u; +const SUPPORTED_ISSUE_PROVIDERS = new Set(["github", "linear"]); + +export function buildLocalLifecycleArtifacts({ task, upstream, runId }) { + const issueProvider = validateArtifactInput({ task, upstream, runId }); + return { + issue: { + provider: issueProvider, + mode: "local_mock", + externalId: `${runId}:${task.id}:issue`, + title: issueTitleForTask(task, upstream), + body: upstream.problemStatement, + repo: task.repo, + baseCommit: task.baseCommit, + createdAt: DETERMINISTIC_CREATED_AT + }, + agentrailTask: { + id: syntheticTaskId({ runId, taskId: task.id }), + identifier: `${issueProvider}:${task.repo}:issues/${runId}-${task.id}`, + status: "todo", + sourceInstanceId: upstream.instanceId, + availableActions: ["start"], + createdAt: DETERMINISTIC_CREATED_AT + }, + pullRequest: { + provider: issueProvider, + mode: "synthetic_pr_record", + status: "not_created", + number: null, + headSha: null, + branch: null + }, + ci: { + provider: "swebench_evaluator", + overallStatus: "not_run", + failureSummaries: [] + }, + review: { + provider: "deterministic_review", + outcome: "not_reviewed", + blockingFindings: [] + }, + final: { + status: "pending", + success: false, + reason: null + } + }; +} + +function issueTitleForTask(task, upstream) { + return `[${task.upstream?.instanceId ?? upstream.instanceId}] ${task.repo}`; +} + +function syntheticTaskId({ runId, taskId }) { + const digest = createHash("sha256") + .update(runId) + .update("\0") + .update(taskId) + .digest("hex") + .slice(0, 24); + return `tsk_${digest}`; +} + +function validateArtifactInput({ task, upstream, runId }) { + if (!isRecord(task)) { + throw new Error("task must be a non-null object."); + } + if (!isRecord(upstream)) { + throw new Error("upstream must be a non-null object."); + } + if (task.lifecycle !== undefined && !isRecord(task.lifecycle)) { + throw new Error("task.lifecycle must be a non-null object when present."); + } + if (typeof runId !== "string" || !RUN_ID_PATTERN.test(runId)) { + throw new Error("runId must match /^run_[A-Za-z0-9_-]+$/."); + } + if (typeof task.id !== "string" || !TASK_ID_PATTERN.test(task.id)) { + throw new Error("task.id must match /^[a-z0-9]+(?:-[a-z0-9]+)*$/."); + } + if (typeof task.repo !== "string" || !REPO_PATTERN.test(task.repo)) { + throw new Error("task.repo must be an owner/name repository."); + } + if (typeof task.baseCommit !== "string" || !BASE_COMMIT_PATTERN.test(task.baseCommit)) { + throw new Error("task.baseCommit must be a 40-character lowercase hex SHA."); + } + if (typeof upstream.instanceId !== "string" || upstream.instanceId.length === 0) { + throw new Error("upstream.instanceId must be a non-empty string."); + } + if (typeof upstream.problemStatement !== "string" || upstream.problemStatement.length === 0) { + throw new Error("upstream.problemStatement must be a non-empty string."); + } + if ( + task.upstream !== undefined + && !isRecord(task.upstream) + ) { + throw new Error("task.upstream must be a non-null object when present."); + } + if ( + isRecord(task.upstream) + && task.upstream.instanceId !== undefined + && (typeof task.upstream.instanceId !== "string" || task.upstream.instanceId.length === 0) + ) { + throw new Error("task.upstream.instanceId must be a non-empty string."); + } + const issueProvider = task.lifecycle?.issueProvider ?? "github"; + if (!SUPPORTED_ISSUE_PROVIDERS.has(issueProvider)) { + throw new Error("issueProvider must be github or linear."); + } + return issueProvider; +} + +function isRecord(value) { + return typeof value === "object" && value !== null && !Array.isArray(value); +} diff --git a/benchmarks/agentrail-swe-lifecycle/lib/results.mjs b/benchmarks/agentrail-swe-lifecycle/lib/results.mjs new file mode 100644 index 0000000..637db17 --- /dev/null +++ b/benchmarks/agentrail-swe-lifecycle/lib/results.mjs @@ -0,0 +1,56 @@ +import { mkdir, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; + +export function aggregateLifecycleResults(results) { + const totalTasks = results.length; + const successfulTasks = results.filter((result) => result.final?.success === true).length; + const normalizedTokenTotals = results.map((result) => normalizeTokenTotal(result.tokenUsage)); + const totalTokens = normalizedTokenTotals.reduce((sum, total) => sum + (total ?? 0), 0); + const tokenUsageUnavailableTasks = normalizedTokenTotals.filter((total) => total === null).length; + + return { + totalTasks, + successfulTasks, + successRate: totalTasks > 0 ? successfulTasks / totalTasks : 0, + totalTokens, + tokenUsageUnavailableTasks, + tokensPerSuccessfulTask: successfulTasks > 0 ? totalTokens / successfulTasks : null + }; +} + +export async function writeLifecycleResultFiles({ outputDir, run }) { + await mkdir(outputDir, { recursive: true }); + const completeMarkerPath = path.join(outputDir, "complete.marker"); + await rm(completeMarkerPath, { force: true }); + await writeFile(path.join(outputDir, "results.json"), `${JSON.stringify(run, null, 2)}\n`, "utf8"); + await writeFile(path.join(outputDir, "summary.md"), renderSummary(run), "utf8"); + await writeFile(completeMarkerPath, "complete\n", "utf8"); +} + +function normalizeTokenTotal(tokenUsage) { + if (!tokenUsage || tokenUsage.totalTokens === undefined) return null; + if (typeof tokenUsage.totalTokens === "number") { + return Number.isSafeInteger(tokenUsage.totalTokens) && tokenUsage.totalTokens >= 0 ? tokenUsage.totalTokens : null; + } + if (typeof tokenUsage.totalTokens === "string" && /^(0|[1-9]\d*)$/u.test(tokenUsage.totalTokens)) { + const totalTokens = Number(tokenUsage.totalTokens); + return Number.isSafeInteger(totalTokens) ? totalTokens : null; + } + return null; +} + +function renderSummary(run) { + const aggregate = run.aggregate; + return [ + "# AgentRail SWE Lifecycle Benchmark", + "", + `- Run: ${run.runId}`, + `- Tasks: ${aggregate.totalTasks}`, + `- Successful tasks: ${aggregate.successfulTasks}`, + `- Success rate: ${(aggregate.successRate * 100).toFixed(2)}%`, + `- Total tokens: ${aggregate.totalTokens ?? "unavailable"}`, + `- Token usage unavailable tasks: ${aggregate.tokenUsageUnavailableTasks}`, + `- Tokens per successful task: ${aggregate.tokensPerSuccessfulTask ?? "unavailable"}`, + "" + ].join("\n"); +} diff --git a/benchmarks/agentrail-swe-lifecycle/lib/upstream.mjs b/benchmarks/agentrail-swe-lifecycle/lib/upstream.mjs new file mode 100644 index 0000000..c4c2c65 --- /dev/null +++ b/benchmarks/agentrail-swe-lifecycle/lib/upstream.mjs @@ -0,0 +1,87 @@ +const HF_ROWS_URL = "https://datasets-server.huggingface.co/rows"; +const HF_DATASET = "ScaleAI/SWE-bench_Pro"; +const HF_CONFIG = "default"; +const HF_SPLIT = "test"; +const PAGE_LENGTH = 100; + +export function buildHuggingFaceRowsUrl({ offset = 0, length = PAGE_LENGTH } = {}) { + const url = new URL(HF_ROWS_URL); + url.searchParams.set("dataset", HF_DATASET); + url.searchParams.set("config", HF_CONFIG); + url.searchParams.set("split", HF_SPLIT); + url.searchParams.set("offset", String(offset)); + url.searchParams.set("length", String(length)); + return url; +} + +export async function resolveUpstreamRowsForTasks(tasks, { fetchImpl = globalThis.fetch } = {}) { + if (typeof fetchImpl !== "function") { + throw new Error("A fetch implementation is required to resolve upstream rows."); + } + + const wanted = new Set(tasks.map((task) => upstreamInstanceIdForTask(task).toLowerCase())); + const rowsByInstanceId = new Map(); + + for (let offset = 0; rowsByInstanceId.size < wanted.size; offset += PAGE_LENGTH) { + const response = await fetchImpl(buildHuggingFaceRowsUrl({ offset, length: PAGE_LENGTH })); + const text = await response.text(); + if (!response.ok) { + throw new Error(`Failed to fetch SWE-bench Pro rows: HTTP ${response.status}: ${text.slice(0, 200)}`); + } + + const body = JSON.parse(text); + const pageRows = rowsFromPageBody(body); + if (pageRows.length === 0) break; + + for (const row of pageRows) { + const instanceId = String(row?.instance_id ?? "").toLowerCase(); + if (wanted.has(instanceId)) rowsByInstanceId.set(instanceId, row); + } + + if (isFinalPage({ body, offset, pageLength: PAGE_LENGTH, rowCount: pageRows.length })) break; + } + + const missing = [...wanted].filter((instanceId) => !rowsByInstanceId.has(instanceId)); + if (missing.length > 0) { + throw new Error(`Missing upstream rows: ${missing.join(", ")}`); + } + + return tasks.map((task) => redactUpstreamRowForLifecycle(rowsByInstanceId.get(upstreamInstanceIdForTask(task).toLowerCase()))); +} + +export function redactUpstreamRowForLifecycle(row) { + return { + repo: row.repo, + instanceId: row.instance_id, + baseCommit: row.base_commit, + problemStatement: row.problem_statement, + requirements: nullableString(row.requirements), + interface: nullableString(row.interface), + repoLanguage: nullableString(row.repo_language) + }; +} + +function rowsFromPageBody(body) { + if (!Array.isArray(body?.rows)) return []; + return body.rows.map((entry) => entry?.row).filter((row) => row && typeof row === "object"); +} + +function isFinalPage({ body, offset, pageLength, rowCount }) { + const totalRows = Number(body?.num_rows_total); + if (Number.isFinite(totalRows) && offset + rowCount >= totalRows) return true; + return rowCount < pageLength; +} + +function upstreamInstanceIdForTask(task) { + const instanceId = task?.upstream?.instanceId; + if (typeof instanceId !== "string" || instanceId.length === 0) { + throw new Error("Each task must include upstream.instanceId to resolve upstream rows."); + } + return instanceId; +} + +function nullableString(value) { + if (typeof value !== "string") return null; + const trimmed = value.trim(); + return trimmed.length > 0 && trimmed.toLowerCase() !== "nan" ? trimmed : null; +} diff --git a/benchmarks/agentrail-swe-lifecycle/run.mjs b/benchmarks/agentrail-swe-lifecycle/run.mjs new file mode 100644 index 0000000..4cd0db5 --- /dev/null +++ b/benchmarks/agentrail-swe-lifecycle/run.mjs @@ -0,0 +1,101 @@ +#!/usr/bin/env node +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { loadLifecycleDataset, selectLifecycleTasks } from "./lib/dataset.mjs"; +import { buildLocalLifecycleArtifacts } from "./lib/local-lifecycle.mjs"; +import { aggregateLifecycleResults, writeLifecycleResultFiles } from "./lib/results.mjs"; + +const RUN_ID_PATTERN = /^run_[A-Za-z0-9_-]+$/u; +const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", ".."); +const defaultDatasetPath = path.join(repoRoot, "benchmarks", "agentrail-swe-lifecycle", "dataset.json"); + +const flags = parseArgs(process.argv.slice(2)); + +if (!flags.dryRun) { + throw new Error("Only --dry-run is implemented in this phase. Model execution is planned for the next phase."); +} + +const dataset = await loadLifecycleDataset(flags.dataset ?? defaultDatasetPath); +const tasks = selectLifecycleTasks(dataset, flags); +if (tasks.length === 0) { + throw new Error("No lifecycle tasks matched the provided filters."); +} +const runId = flags.runId ?? `run_${new Date().toISOString().replaceAll(/[:.]/gu, "-")}`; +validateRunId(runId); +const outputDir = flags.outputDir ?? path.join(repoRoot, ".agentrail-bench", "agentrail-swe-lifecycle", runId); +const results = tasks.map((task) => buildDryRunTaskResult({ task, runId })); +const run = { + version: 1, + runId, + mode: "dry_run", + dataset: dataset.id, + scenario: dataset.defaultScenario, + aggregate: aggregateLifecycleResults(results), + tasks: results +}; + +await writeLifecycleResultFiles({ outputDir, run }); + +console.log(JSON.stringify({ status: "ok", outputDir, tasks: tasks.length }, null, 2)); + +function buildDryRunTaskResult({ task, runId }) { + const upstream = { + instanceId: task.upstream.instanceId, + problemStatement: `Dry-run placeholder for ${task.upstream.instanceId}. Use upstream resolver in execution mode.` + }; + + return { + taskId: task.id, + upstreamInstanceId: task.upstream.instanceId, + repo: task.repo, + lifecycle: buildLocalLifecycleArtifacts({ task, upstream, runId }), + final: { success: false, status: "dry_run" }, + tokenUsage: null + }; +} + +function parseArgs(argv) { + const flags = {}; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === "--dry-run") { + flags.dryRun = true; + } else if (arg === "--dataset") { + flags.dataset = readFlagValue({ argv, index: ++index, flag: arg }); + } else if (arg === "--count") { + flags.count = readFlagValue({ argv, index: ++index, flag: arg }); + } else if (arg === "--repo") { + flags.repo = readFlagValue({ argv, index: ++index, flag: arg }); + } else if (arg === "--language") { + flags.language = readFlagValue({ argv, index: ++index, flag: arg }); + } else if (arg === "--tag") { + flags.tag = readFlagValue({ argv, index: ++index, flag: arg }); + } else if (arg === "--task") { + flags.taskId = readFlagValue({ argv, index: ++index, flag: arg }); + } else if (arg === "--seed") { + flags.seed = readFlagValue({ argv, index: ++index, flag: arg }); + } else if (arg === "--output-dir") { + flags.outputDir = readFlagValue({ argv, index: ++index, flag: arg }); + } else if (arg === "--run-id") { + flags.runId = readFlagValue({ argv, index: ++index, flag: arg }); + } else { + throw new Error(`Unknown argument: ${arg}`); + } + } + return flags; +} + +function readFlagValue({ argv, index, flag }) { + const value = argv[index]; + if (value === undefined || value.startsWith("--")) { + throw new Error(`${flag} requires a value.`); + } + return value; +} + +function validateRunId(runId) { + if (!RUN_ID_PATTERN.test(runId)) { + throw new Error("runId must match /^run_[A-Za-z0-9_-]+$/."); + } +} diff --git a/benchmarks/agentrail-swe-lifecycle/scenarios/local-mock-lifecycle.json b/benchmarks/agentrail-swe-lifecycle/scenarios/local-mock-lifecycle.json new file mode 100644 index 0000000..a81b563 --- /dev/null +++ b/benchmarks/agentrail-swe-lifecycle/scenarios/local-mock-lifecycle.json @@ -0,0 +1,63 @@ +{ + "version": 1, + "id": "local-mock-lifecycle", + "description": "Run SWE-bench Pro tasks through an AgentRail lifecycle using local deterministic provider doubles instead of live GitHub, Linear, hosted CI, or human review.", + "providers": { + "issue": { + "default": "github", + "supported": ["github", "linear"], + "mode": "local_mock", + "bodyFrom": "upstream.problem_statement" + }, + "intake": { + "mode": "agentrail_local", + "required": true + }, + "pullRequest": { + "mode": "synthetic_pr_record", + "required": true + }, + "ci": { + "provider": "swebench_evaluator", + "mode": "local", + "required": true + }, + "review": { + "provider": "deterministic_review", + "mode": "local", + "required": true, + "blockingRules": [ + "evaluator_failed", + "forbidden_file_changed", + "gold_patch_or_test_patch_referenced", + "oversized_generated_or_vendor_churn" + ] + }, + "ship": { + "mode": "merge_ready_record", + "mergeRequired": false + } + }, + "retryPolicy": { + "maxCiRetries": 2, + "maxReviewRetries": 1 + }, + "successCriteria": [ + "upstream_evaluator_passed", + "no_blocking_review_findings", + "agentrail_task_reached_merge_ready_or_done" + ], + "tokenAccounting": { + "sourceOfTruth": "provider_reported_usage", + "include": [ + "input_tokens", + "output_tokens", + "reasoning_tokens_when_reported", + "cached_tokens_when_reported", + "tool_result_tokens_inserted_into_model_context", + "all_retry_ci_review_and_ship_turns" + ], + "doNotEstimateMissingProviderFields": true + } +} + diff --git a/package.json b/package.json index 2bca1fc..84e31d5 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,9 @@ "smoke:sdk:python": "node scripts/smoke-test-sdk-packages.mjs --python", "smoke:sdk": "node scripts/smoke-test-sdk-packages.mjs --all", "smoke:sandbox:live": "npm --prefix sdk/typescript ci && npm --prefix sdk/typescript run build && node scripts/agentrail-sandbox-smoke.mjs --mode live", + "benchmark:agentrail-swe:refresh": "node scripts/refresh-agentrail-swe-lifecycle-dataset.mjs", + "benchmark:agentrail-swe:run": "node benchmarks/agentrail-swe-lifecycle/run.mjs --dry-run", + "benchmark:agentrail-swe:validate": "node --test test/agentrail-swe-lifecycle-dataset.test.js test/agentrail-swe-lifecycle-runner.test.js", "lint:openapi": "spectral lint docs/api/task-lifecycle.openapi.yaml docs/api/intake-routing-admin.openapi.yaml -r .spectral.yaml", "test": "npm --prefix sdk/typescript ci && npm --prefix sdk/typescript run build && npm run test:service", "test:service": "node --test 'test/*.test.js' 'test/*.test.ts'", diff --git a/scripts/refresh-agentrail-swe-lifecycle-dataset.mjs b/scripts/refresh-agentrail-swe-lifecycle-dataset.mjs new file mode 100644 index 0000000..9175b2d --- /dev/null +++ b/scripts/refresh-agentrail-swe-lifecycle-dataset.mjs @@ -0,0 +1,199 @@ +#!/usr/bin/env node +import { mkdir, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const outputPath = path.join(repoRoot, "benchmarks", "agentrail-swe-lifecycle", "dataset.json"); + +const AA_METHODOLOGY_URL = "https://artificialanalysis.ai/methodology/coding-agents-benchmarking"; +const HF_DATASET = "ScaleAI/SWE-bench_Pro"; +const HF_CONFIG = "default"; +const HF_SPLIT = "test"; +const HF_ROWS_URL = "https://datasets-server.huggingface.co/rows"; + +const html = await fetchText(AA_METHODOLOGY_URL); +const selectedIds = extractSweBenchProHardAaIds(html); +if (selectedIds.length !== 150) { + throw new Error(`Expected 150 SWE-Bench-Pro-Hard-AA task ids from Artificial Analysis, found ${selectedIds.length}.`); +} + +const upstreamRows = await fetchAllHuggingFaceRows(); +const rowsByLowerId = new Map(upstreamRows.map((row) => [row.instance_id.toLowerCase(), row])); +const missing = selectedIds.filter((id) => !rowsByLowerId.has(id.toLowerCase())); +if (missing.length > 0) { + throw new Error(`Selected ids missing from ${HF_DATASET}: ${missing.slice(0, 10).join(", ")}`); +} + +const dataset = { + version: 1, + id: "agentrail-swe-lifecycle", + name: "AgentRail SWE Lifecycle Token Efficiency Dataset", + description: "AgentRail lifecycle wrapper around Artificial Analysis SWE-Bench-Pro-Hard-AA task selection and ScaleAI/SWE-bench_Pro upstream rows.", + defaultScenario: "local-mock-lifecycle", + upstream: { + taskSelection: { + name: "Artificial Analysis Coding Agent Index", + benchmark: "SWE-Bench-Pro-Hard-AA", + sourceUrl: AA_METHODOLOGY_URL, + evaluatedTaskCount: selectedIds.length + }, + taskData: { + dataset: HF_DATASET, + config: HF_CONFIG, + split: HF_SPLIT, + sourceUrl: "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + } + }, + lifecycle: { + issueProviders: ["github", "linear"], + defaultIssueProvider: "github", + intake: "agentrail_task_ingestion", + assignment: "agentrail_managed_agent", + submission: "synthetic_pull_request_or_patch_record", + ci: "swebench_evaluator", + review: "deterministic_review", + ship: "merge_ready_record", + finalOracle: "swebench_fail_to_pass" + }, + metrics: { + primary: "tokens_per_successful_task", + guardrails: [ + "success_rate", + "cost_per_successful_task", + "wall_clock_time_per_successful_task", + "ci_retry_count", + "review_retry_count" + ] + }, + tasks: selectedIds.map((selectedId, index) => { + const row = rowsByLowerId.get(selectedId.toLowerCase()); + return buildTaskRecord({ row, selectedId, index }); + }) +}; + +await mkdir(path.dirname(outputPath), { recursive: true }); +await writeFile(outputPath, `${JSON.stringify(dataset, null, 2)}\n`, "utf8"); +console.log(`Wrote ${dataset.tasks.length} AgentRail SWE lifecycle task references to ${path.relative(repoRoot, outputPath)}.`); + +function extractSweBenchProHardAaIds(html) { + return [...html.matchAll(/
  • (instance_[^<]+)<\/li>/g)] + .map((match) => decodeHtml(match[1])) + .filter((id) => /^instance_[A-Za-z0-9_.-]+__[A-Za-z0-9_.-]+-[A-Za-z0-9_.-]+(?:-v[A-Za-z0-9_.-]+)?$/u.test(id)); +} + +async function fetchAllHuggingFaceRows() { + const rows = []; + for (let offset = 0; ; offset += 100) { + const url = new URL(HF_ROWS_URL); + url.searchParams.set("dataset", HF_DATASET); + url.searchParams.set("config", HF_CONFIG); + url.searchParams.set("split", HF_SPLIT); + url.searchParams.set("offset", String(offset)); + url.searchParams.set("length", "100"); + + const body = await fetchJson(url); + const pageRows = Array.isArray(body.rows) ? body.rows.map((entry) => entry.row) : []; + if (pageRows.length === 0) break; + rows.push(...pageRows); + if (pageRows.length < 100) break; + } + return rows; +} + +function buildTaskRecord({ row, selectedId, index }) { + const issueCategories = parseJsonishList(row.issue_categories); + const issueSpecificity = parseJsonishList(row.issue_specificity); + const selectedTestFiles = parseJsonishList(row.selected_test_files_to_run); + return { + id: `agentrail-swe-pro-hard-aa-${String(index + 1).padStart(3, "0")}`, + upstream: { + dataset: HF_DATASET, + config: HF_CONFIG, + split: HF_SPLIT, + instanceId: row.instance_id, + selectedInstanceId: selectedId, + sourceUrl: "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + repo: normalizeRepo(row.repo), + baseCommit: row.base_commit, + language: normalizeNullableString(row.repo_language), + tags: uniqueStrings([ + normalizeNullableString(row.repo_language), + ...issueSpecificity, + ...issueCategories + ]), + lifecycle: { + scenario: "local-mock-lifecycle", + issueProvider: "github", + issueBodyFrom: "upstream.problem_statement", + intake: "agentrail_task_ingestion", + assignment: "agentrail_managed_agent", + submission: "synthetic_pull_request_or_patch_record", + ci: "swebench_evaluator", + review: "deterministic_review", + ship: "merge_ready_record" + }, + evaluation: { + oracle: "swebench_fail_to_pass", + dockerhubTag: normalizeNullableString(row.dockerhub_tag), + beforeRepoSetCommand: normalizeNullableString(row.before_repo_set_cmd), + selectedTestFiles, + maxCiRetries: 2, + maxReviewRetries: 1 + } + }; +} + +async function fetchText(url) { + const response = await fetch(url); + if (!response.ok) { + throw new Error(`GET ${url} failed with HTTP ${response.status}.`); + } + return await response.text(); +} + +async function fetchJson(url) { + const response = await fetch(url); + const text = await response.text(); + if (!response.ok) { + throw new Error(`GET ${url} failed with HTTP ${response.status}: ${text.slice(0, 300)}`); + } + return JSON.parse(text); +} + +function decodeHtml(value) { + return value + .replaceAll("&", "&") + .replaceAll("<", "<") + .replaceAll(">", ">") + .replaceAll(""", "\"") + .replaceAll("'", "'"); +} + +function normalizeRepo(value) { + return String(value ?? "").trim(); +} + +function normalizeNullableString(value) { + if (typeof value !== "string") return null; + const trimmed = value.trim(); + if (!trimmed || trimmed.toLowerCase() === "nan" || trimmed.toLowerCase() === "null") return null; + return trimmed; +} + +function parseJsonishList(value) { + const normalized = normalizeNullableString(value); + if (!normalized) return []; + try { + const parsed = JSON.parse(normalized.replaceAll("'", "\"")); + if (Array.isArray(parsed)) return uniqueStrings(parsed.map((entry) => String(entry))); + } catch { + // Fall through to comma parsing. + } + return uniqueStrings(normalized.split(/[,;\n]/u).map((entry) => entry.trim()).filter(Boolean)); +} + +function uniqueStrings(values) { + return [...new Set(values.filter((value) => typeof value === "string" && value.trim().length > 0).map((value) => value.trim()))]; +} diff --git a/test/agentrail-swe-lifecycle-dataset.test.js b/test/agentrail-swe-lifecycle-dataset.test.js new file mode 100644 index 0000000..46d8f90 --- /dev/null +++ b/test/agentrail-swe-lifecycle-dataset.test.js @@ -0,0 +1,72 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import path from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; + +const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const datasetPath = path.join(repoRoot, "benchmarks", "agentrail-swe-lifecycle", "dataset.json"); +const scenarioPath = path.join(repoRoot, "benchmarks", "agentrail-swe-lifecycle", "scenarios", "local-mock-lifecycle.json"); + +test("AgentRail SWE lifecycle dataset wraps AA SWE-Bench-Pro-Hard-AA task references", async () => { + const dataset = JSON.parse(await readFile(datasetPath, "utf8")); + assert.equal(dataset.version, 1); + assert.equal(dataset.id, "agentrail-swe-lifecycle"); + assert.equal(dataset.defaultScenario, "local-mock-lifecycle"); + assert.equal(dataset.upstream.taskSelection.benchmark, "SWE-Bench-Pro-Hard-AA"); + assert.equal(dataset.upstream.taskSelection.evaluatedTaskCount, 150); + assert.equal(dataset.upstream.taskData.dataset, "ScaleAI/SWE-bench_Pro"); + assert.equal(dataset.tasks.length, 150); + + const ids = new Set(); + const upstreamIds = new Set(); + for (const task of dataset.tasks) { + assert.match(task.id, /^agentrail-swe-pro-hard-aa-\d{3}$/u); + assert.ok(!ids.has(task.id), `duplicate task id ${task.id}`); + ids.add(task.id); + + assert.equal(task.upstream.dataset, "ScaleAI/SWE-bench_Pro"); + assert.equal(task.upstream.config, "default"); + assert.equal(task.upstream.split, "test"); + assert.match(task.upstream.instanceId, /^instance_.+__.+-/u); + assert.ok(!upstreamIds.has(task.upstream.instanceId.toLowerCase()), `duplicate upstream id ${task.upstream.instanceId}`); + upstreamIds.add(task.upstream.instanceId.toLowerCase()); + + assert.equal(typeof task.repo, "string"); + assert.ok(task.repo.includes("/"), `repo should be owner/name: ${task.repo}`); + assert.match(task.baseCommit, /^[a-f0-9]{40}$/u); + assert.equal(task.lifecycle.scenario, "local-mock-lifecycle"); + assert.equal(task.lifecycle.intake, "agentrail_task_ingestion"); + assert.equal(task.lifecycle.ci, "swebench_evaluator"); + assert.equal(task.lifecycle.review, "deterministic_review"); + assert.equal(task.evaluation.oracle, "swebench_fail_to_pass"); + assert.equal(task.evaluation.maxCiRetries, 2); + assert.equal(task.evaluation.maxReviewRetries, 1); + + assert.equal(task.problem_statement, undefined); + assert.equal(task.problemStatement, undefined); + assert.equal(task.patch, undefined); + assert.equal(task.test_patch, undefined); + assert.equal(task.testPatch, undefined); + assert.equal(task.fail_to_pass, undefined); + assert.equal(task.pass_to_pass, undefined); + } +}); + +test("local mock lifecycle scenario defines the AgentRail product path", async () => { + const scenario = JSON.parse(await readFile(scenarioPath, "utf8")); + assert.equal(scenario.version, 1); + assert.equal(scenario.id, "local-mock-lifecycle"); + assert.equal(scenario.providers.issue.mode, "local_mock"); + assert.deepEqual(scenario.providers.issue.supported, ["github", "linear"]); + assert.equal(scenario.providers.intake.required, true); + assert.equal(scenario.providers.pullRequest.mode, "synthetic_pr_record"); + assert.equal(scenario.providers.ci.provider, "swebench_evaluator"); + assert.equal(scenario.providers.review.provider, "deterministic_review"); + assert.equal(scenario.retryPolicy.maxCiRetries, 2); + assert.equal(scenario.retryPolicy.maxReviewRetries, 1); + assert.ok(scenario.successCriteria.includes("upstream_evaluator_passed")); + assert.equal(scenario.tokenAccounting.sourceOfTruth, "provider_reported_usage"); + assert.equal(scenario.tokenAccounting.doNotEstimateMissingProviderFields, true); +}); + diff --git a/test/agentrail-swe-lifecycle-runner.test.js b/test/agentrail-swe-lifecycle-runner.test.js new file mode 100644 index 0000000..cd2a004 --- /dev/null +++ b/test/agentrail-swe-lifecycle-runner.test.js @@ -0,0 +1,714 @@ +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; + +import { + loadLifecycleDataset, + selectLifecycleTasks +} from "../benchmarks/agentrail-swe-lifecycle/lib/dataset.mjs"; +import { + buildHuggingFaceRowsUrl, + redactUpstreamRowForLifecycle, + resolveUpstreamRowsForTasks +} from "../benchmarks/agentrail-swe-lifecycle/lib/upstream.mjs"; +import { + buildLocalLifecycleArtifacts +} from "../benchmarks/agentrail-swe-lifecycle/lib/local-lifecycle.mjs"; +import { + aggregateLifecycleResults, + writeLifecycleResultFiles +} from "../benchmarks/agentrail-swe-lifecycle/lib/results.mjs"; + +const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const sampleDatasetPath = path.join(repoRoot, "benchmarks", "agentrail-swe-lifecycle", "fixtures", "sample-dataset.json"); + +test("selectLifecycleTasks takes a deterministic prefix by count", async () => { + const dataset = await loadLifecycleDataset(sampleDatasetPath); + const selected = selectLifecycleTasks({ ...dataset, tasks: [...dataset.tasks].reverse() }, { count: 1 }); + assert.deepEqual(selected.map((task) => task.id), ["agentrail-swe-pro-hard-aa-001"]); +}); + +test("selectLifecycleTasks can filter by repo, language, tag, and task id", async () => { + const dataset = await loadLifecycleDataset(sampleDatasetPath); + assert.deepEqual(selectLifecycleTasks(dataset, { language: "ts" }).map((task) => task.id), ["agentrail-swe-pro-hard-aa-002"]); + assert.deepEqual(selectLifecycleTasks(dataset, { tag: "bugfix" }).map((task) => task.id), ["agentrail-swe-pro-hard-aa-001"]); + assert.deepEqual(selectLifecycleTasks(dataset, { repo: "owner/repo" }).map((task) => task.id), ["agentrail-swe-pro-hard-aa-001"]); + assert.deepEqual(selectLifecycleTasks(dataset, { taskId: "agentrail-swe-pro-hard-aa-002" }).map((task) => task.id), [ + "agentrail-swe-pro-hard-aa-002" + ]); + assert.deepEqual(selectLifecycleTasks(dataset, { taskId: "instance_owner__repo-bbbb" }).map((task) => task.id), [ + "agentrail-swe-pro-hard-aa-002" + ]); + assert.deepEqual(selectLifecycleTasks(dataset, { taskId: "instance_owner__repo-selected-bbbb" }).map((task) => task.id), [ + "agentrail-swe-pro-hard-aa-002" + ]); +}); + +test("selectLifecycleTasks applies deterministic seeded shuffle before count", async () => { + const dataset = await loadLifecycleDataset(sampleDatasetPath); + assert.deepEqual(selectLifecycleTasks(dataset, { seed: "1", count: 2 }).map((task) => task.id), [ + "agentrail-swe-pro-hard-aa-002", + "agentrail-swe-pro-hard-aa-001" + ]); + assert.deepEqual(selectLifecycleTasks(dataset, { seed: "1", count: 2 }).map((task) => task.id), [ + "agentrail-swe-pro-hard-aa-002", + "agentrail-swe-pro-hard-aa-001" + ]); +}); + +test("loadLifecycleDataset validates version and tasks array", async () => { + const tempDir = await mkdtemp(path.join(os.tmpdir(), "agentrail-swe-dataset-")); + const invalidPath = path.join(tempDir, "invalid.json"); + try { + await writeFile(invalidPath, JSON.stringify({ version: 2, tasks: [] }), "utf8"); + await assert.rejects(loadLifecycleDataset(invalidPath), /Invalid AgentRail SWE lifecycle dataset/u); + + await writeFile(invalidPath, JSON.stringify({ version: 1 }), "utf8"); + await assert.rejects(loadLifecycleDataset(invalidPath), /Invalid AgentRail SWE lifecycle dataset/u); + } finally { + await rm(tempDir, { force: true, recursive: true }); + } +}); + +test("loadLifecycleDataset rejects malformed task entries", async () => { + const tempDir = await mkdtemp(path.join(os.tmpdir(), "agentrail-swe-task-")); + const invalidPath = path.join(tempDir, "invalid.json"); + const validTask = { + id: "task-1", + repo: "owner/repo", + language: null, + tags: ["bugfix"], + upstream: { + instanceId: "instance_owner__repo-1", + selectedInstanceId: null + } + }; + const invalidTasks = [ + null, + "task-1", + { ...validTask, id: "" }, + { ...validTask, repo: "" }, + { ...validTask, language: 1 }, + { ...validTask, tags: "bugfix" }, + { ...validTask, upstream: null }, + { ...validTask, upstream: { selectedInstanceId: "selected" } }, + { ...validTask, upstream: { instanceId: "", selectedInstanceId: null } }, + { ...validTask, upstream: { instanceId: "instance_owner__repo-1", selectedInstanceId: 1 } } + ]; + + try { + for (const task of invalidTasks) { + await writeFile(invalidPath, JSON.stringify({ version: 1, tasks: [task] }), "utf8"); + await assert.rejects(loadLifecycleDataset(invalidPath), /Invalid AgentRail SWE lifecycle dataset/u); + } + } finally { + await rm(tempDir, { force: true, recursive: true }); + } +}); + +test("selectLifecycleTasks rejects non-positive counts", async () => { + const dataset = await loadLifecycleDataset(sampleDatasetPath); + assert.throws(() => selectLifecycleTasks(dataset, { count: 0 }), /--count must be a positive integer/u); +}); + +test("selectLifecycleTasks rejects malformed counts", async () => { + const dataset = await loadLifecycleDataset(sampleDatasetPath); + assert.throws(() => selectLifecycleTasks(dataset, { count: "1abc" }), /--count must be a positive integer/u); + assert.throws(() => selectLifecycleTasks(dataset, { count: "1.9" }), /--count must be a positive integer/u); + assert.throws(() => selectLifecycleTasks(dataset, { count: " 2x" }), /--count must be a positive integer/u); + assert.throws( + () => selectLifecycleTasks(dataset, { count: String(Number.MAX_SAFE_INTEGER + 1) }), + /--count must be a positive integer/u + ); +}); + +test("buildHuggingFaceRowsUrl targets the SWE-bench Pro rows API", () => { + const url = buildHuggingFaceRowsUrl({ offset: 100, length: 50 }); + assert.equal(url.origin, "https://datasets-server.huggingface.co"); + assert.equal(url.pathname, "/rows"); + assert.equal(url.searchParams.get("dataset"), "ScaleAI/SWE-bench_Pro"); + assert.equal(url.searchParams.get("config"), "default"); + assert.equal(url.searchParams.get("split"), "test"); + assert.equal(url.searchParams.get("offset"), "100"); + assert.equal(url.searchParams.get("length"), "50"); +}); + +test("redactUpstreamRowForLifecycle keeps issue text but removes gold solution data", () => { + const row = { + repo: "owner/repo", + instance_id: "instance_owner__repo-aaaa", + base_commit: "1111111111111111111111111111111111111111", + problem_statement: "Fix the bug.", + requirements: "Keep the API stable.", + interface: "CLI", + patch: "gold patch", + test_patch: "hidden tests", + fail_to_pass: "[\"test_a\"]", + pass_to_pass: "[\"test_b\"]", + repo_language: "python" + }; + const redacted = redactUpstreamRowForLifecycle(row); + assert.deepEqual(redacted, { + repo: "owner/repo", + instanceId: "instance_owner__repo-aaaa", + baseCommit: "1111111111111111111111111111111111111111", + problemStatement: "Fix the bug.", + requirements: "Keep the API stable.", + interface: "CLI", + repoLanguage: "python" + }); + assert.equal(redacted.patch, undefined); + assert.equal(redacted.test_patch, undefined); + assert.equal(redacted.fail_to_pass, undefined); + assert.equal(redacted.pass_to_pass, undefined); +}); + +test("resolveUpstreamRowsForTasks fetches pages and returns rows in task order", async () => { + const fetchCalls = []; + const pageRows = [ + [ + { row: { instance_id: "instance_unselected__repo-0000", problem_statement: "Ignore me." } }, + { + row: { + instance_id: "instance_owner__repo-bbbb", + base_commit: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + problem_statement: "Second task.", + patch: "gold patch b", + test_patch: "hidden tests b", + fail_to_pass: "[\"test_b\"]", + pass_to_pass: "[\"test_existing_b\"]" + } + }, + ...Array.from({ length: 98 }, (_, index) => ({ + row: { instance_id: `instance_unselected__repo-${String(index + 1).padStart(4, "0")}` } + })) + ], + [ + { + row: { + instance_id: "instance_owner__repo-aaaa", + base_commit: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + problem_statement: "First task.", + patch: "gold patch a", + test_patch: "hidden tests a", + fail_to_pass: "[\"test_a\"]", + pass_to_pass: "[\"test_existing_a\"]" + } + } + ] + ]; + const fetchImpl = async (url) => { + fetchCalls.push(url); + const page = pageRows[fetchCalls.length - 1] ?? []; + return { + ok: true, + status: 200, + async text() { + return JSON.stringify({ rows: page }); + } + }; + }; + const tasks = [ + { upstream: { instanceId: "instance_owner__repo-aaaa" } }, + { upstream: { instanceId: "instance_owner__repo-bbbb" } } + ]; + + const rows = await resolveUpstreamRowsForTasks(tasks, { fetchImpl }); + + assert.deepEqual(rows.map((row) => row.problemStatement), ["First task.", "Second task."]); + assert.deepEqual(rows.map((row) => row.instanceId), ["instance_owner__repo-aaaa", "instance_owner__repo-bbbb"]); + for (const row of rows) { + assert.equal(Object.hasOwn(row, "patch"), false); + assert.equal(Object.hasOwn(row, "test_patch"), false); + assert.equal(Object.hasOwn(row, "fail_to_pass"), false); + assert.equal(Object.hasOwn(row, "pass_to_pass"), false); + } + assert.deepEqual(fetchCalls.map((url) => url.searchParams.get("offset")), ["0", "100"]); +}); + +test("resolveUpstreamRowsForTasks does not fetch beyond a partial final page", async () => { + const fetchCalls = []; + const fetchImpl = async (url) => { + fetchCalls.push(url); + return { + ok: true, + status: 200, + async text() { + return JSON.stringify({ + num_rows_total: 50, + rows: [ + { row: { instance_id: "instance_owner__repo-other", problem_statement: "Other task." } } + ] + }); + } + }; + }; + + await assert.rejects( + resolveUpstreamRowsForTasks([{ upstream: { instanceId: "instance_owner__repo-missing" } }], { fetchImpl }), + /Missing upstream rows: instance_owner__repo-missing/u + ); + assert.deepEqual(fetchCalls.map((url) => url.searchParams.get("offset")), ["0"]); +}); + +test("resolveUpstreamRowsForTasks throws when selected rows are exhausted", async () => { + const fetchImpl = async () => ({ + ok: true, + status: 200, + async text() { + return JSON.stringify({ rows: [] }); + } + }); + + await assert.rejects( + resolveUpstreamRowsForTasks([{ upstream: { instanceId: "instance_owner__repo-missing" } }], { fetchImpl }), + /Missing upstream rows: instance_owner__repo-missing/u + ); +}); + +test("buildLocalLifecycleArtifacts creates local provider records for AgentRail lifecycle", () => { + const task = { + id: "agentrail-swe-pro-hard-aa-001", + repo: "owner/repo", + baseCommit: "1111111111111111111111111111111111111111", + upstream: { instanceId: "instance_owner__repo-aaaa" }, + lifecycle: { issueProvider: "linear" }, + evaluation: { oracle: "swebench_fail_to_pass", maxCiRetries: 2, maxReviewRetries: 1 } + }; + const upstream = { + instanceId: "instance_owner__repo-aaaa", + problemStatement: "Fix the bug." + }; + + const artifacts = buildLocalLifecycleArtifacts({ task, upstream, runId: "run_001" }); + + assert.deepEqual(Object.keys(artifacts), ["issue", "agentrailTask", "pullRequest", "ci", "review", "final"]); + assert.deepEqual(artifacts.issue, { + provider: "linear", + mode: "local_mock", + externalId: "run_001:agentrail-swe-pro-hard-aa-001:issue", + title: "[instance_owner__repo-aaaa] owner/repo", + body: "Fix the bug.", + repo: "owner/repo", + baseCommit: "1111111111111111111111111111111111111111", + createdAt: "1970-01-01T00:00:00.000Z" + }); + assert.deepEqual(artifacts.agentrailTask, { + id: artifacts.agentrailTask.id, + identifier: "linear:owner/repo:issues/run_001-agentrail-swe-pro-hard-aa-001", + status: "todo", + sourceInstanceId: "instance_owner__repo-aaaa", + availableActions: ["start"], + createdAt: "1970-01-01T00:00:00.000Z" + }); + assert.match(artifacts.agentrailTask.id, /^tsk_[a-f0-9]{24}$/u); + assert.equal( + buildLocalLifecycleArtifacts({ task, upstream, runId: "run_001" }).agentrailTask.id, + artifacts.agentrailTask.id + ); + assert.deepEqual(artifacts.pullRequest, { + provider: "linear", + mode: "synthetic_pr_record", + status: "not_created", + number: null, + headSha: null, + branch: null + }); + assert.deepEqual(artifacts.ci, { + provider: "swebench_evaluator", + overallStatus: "not_run", + failureSummaries: [] + }); + assert.deepEqual(artifacts.review, { + provider: "deterministic_review", + outcome: "not_reviewed", + blockingFindings: [] + }); + assert.deepEqual(artifacts.final, { + status: "pending", + success: false, + reason: null + }); + + const defaultProviderArtifacts = buildLocalLifecycleArtifacts({ + task: { ...task, lifecycle: {} }, + upstream, + runId: "run_001" + }); + assert.equal(defaultProviderArtifacts.issue.provider, "github"); +}); + +test("buildLocalLifecycleArtifacts validates local lifecycle identifiers", () => { + const task = { + id: "agentrail-swe-pro-hard-aa-001", + repo: "owner/repo", + baseCommit: "1111111111111111111111111111111111111111", + lifecycle: { issueProvider: "github" } + }; + const upstream = { + instanceId: "instance_owner__repo-aaaa", + problemStatement: "Fix the bug." + }; + + assert.throws( + () => buildLocalLifecycleArtifacts({ task, upstream, runId: "bad run" }), + /runId must match/u + ); + assert.throws( + () => buildLocalLifecycleArtifacts({ task: { ...task, id: "agentrail_SWE" }, upstream, runId: "run_001" }), + /task\.id must match/u + ); + assert.throws( + () => buildLocalLifecycleArtifacts({ task: { ...task, repo: "owner" }, upstream, runId: "run_001" }), + /task\.repo must be an owner\/name repository/u + ); + assert.throws( + () => buildLocalLifecycleArtifacts({ + task: { ...task, lifecycle: { issueProvider: "jira" } }, + upstream, + runId: "run_001" + }), + /issueProvider must be github or linear/u + ); + assert.throws( + () => buildLocalLifecycleArtifacts({ task: { ...task, baseCommit: undefined }, upstream, runId: "run_001" }), + /task\.baseCommit must be a 40-character lowercase hex SHA/u + ); + assert.throws( + () => buildLocalLifecycleArtifacts({ task: { ...task, baseCommit: "not-a-sha" }, upstream, runId: "run_001" }), + /task\.baseCommit must be a 40-character lowercase hex SHA/u + ); + assert.throws( + () => buildLocalLifecycleArtifacts({ task: { ...task, lifecycle: "github" }, upstream, runId: "run_001" }), + /task\.lifecycle must be a non-null object when present/u + ); + assert.throws( + () => buildLocalLifecycleArtifacts({ task: { ...task, lifecycle: null }, upstream, runId: "run_001" }), + /task\.lifecycle must be a non-null object when present/u + ); + assert.throws( + () => buildLocalLifecycleArtifacts({ task: null, upstream, runId: "run_001" }), + /task must be a non-null object/u + ); + assert.throws( + () => buildLocalLifecycleArtifacts({ task, runId: "run_001" }), + /upstream must be a non-null object/u + ); + assert.throws( + () => buildLocalLifecycleArtifacts({ + task, + upstream: { ...upstream, instanceId: 123 }, + runId: "run_001" + }), + /upstream\.instanceId must be a non-empty string/u + ); + assert.throws( + () => buildLocalLifecycleArtifacts({ + task, + upstream: { ...upstream, problemStatement: 123 }, + runId: "run_001" + }), + /upstream\.problemStatement must be a non-empty string/u + ); + assert.throws( + () => buildLocalLifecycleArtifacts({ + task, + upstream: { ...upstream, problemStatement: "" }, + runId: "run_001" + }), + /upstream\.problemStatement must be a non-empty string/u + ); + assert.throws( + () => buildLocalLifecycleArtifacts({ + task: { ...task, upstream: { instanceId: 123 } }, + upstream, + runId: "run_001" + }), + /task\.upstream\.instanceId must be a non-empty string/u + ); + assert.throws( + () => buildLocalLifecycleArtifacts({ + task: { ...task, upstream: { instanceId: "" } }, + upstream, + runId: "run_001" + }), + /task\.upstream\.instanceId must be a non-empty string/u + ); + assert.throws( + () => buildLocalLifecycleArtifacts({ + task: { ...task, upstream: null }, + upstream, + runId: "run_001" + }), + /task\.upstream must be a non-null object when present/u + ); +}); + +test("buildLocalLifecycleArtifacts scopes synthetic task IDs by run", () => { + const task = { + id: "agentrail-swe-pro-hard-aa-001", + repo: "owner/repo", + baseCommit: "1111111111111111111111111111111111111111", + lifecycle: { issueProvider: "github" } + }; + const upstream = { + instanceId: "instance_owner__repo-aaaa", + problemStatement: "Fix the bug." + }; + + const firstRun = buildLocalLifecycleArtifacts({ task, upstream, runId: "run_001" }); + const secondRun = buildLocalLifecycleArtifacts({ task, upstream, runId: "run_002" }); + + assert.match(firstRun.agentrailTask.id, /^tsk_[a-f0-9]{24}$/u); + assert.match(secondRun.agentrailTask.id, /^tsk_[a-f0-9]{24}$/u); + assert.notEqual(firstRun.agentrailTask.id, secondRun.agentrailTask.id); +}); + +test("aggregateLifecycleResults reports success rate and token availability", () => { + const aggregate = aggregateLifecycleResults([ + { taskId: "a", final: { success: true }, tokenUsage: { totalTokens: 100 } }, + { taskId: "b", final: { success: false }, tokenUsage: null } + ]); + + assert.equal(aggregate.totalTasks, 2); + assert.equal(aggregate.successfulTasks, 1); + assert.equal(aggregate.successRate, 0.5); + assert.equal(aggregate.totalTokens, 100); + assert.equal(aggregate.tokenUsageUnavailableTasks, 1); + assert.equal(aggregate.tokensPerSuccessfulTask, 100); +}); + +test("aggregateLifecycleResults coerces numeric string token totals", () => { + const aggregate = aggregateLifecycleResults([ + { taskId: "a", final: { success: true }, tokenUsage: { totalTokens: "100" } }, + { taskId: "b", final: { success: true }, tokenUsage: { totalTokens: "50" } } + ]); + + assert.equal(aggregate.totalTokens, 150); + assert.equal(aggregate.tokensPerSuccessfulTask, 75); +}); + +test("aggregateLifecycleResults counts invalid token totals as unavailable and excludes them", () => { + const aggregate = aggregateLifecycleResults([ + { taskId: "a", final: { success: true }, tokenUsage: { totalTokens: "abc" } }, + { taskId: "b", final: { success: true }, tokenUsage: {} }, + { taskId: "c", final: { success: false }, tokenUsage: { totalTokens: Infinity } }, + { taskId: "d", final: { success: false }, tokenUsage: { totalTokens: -Infinity } }, + { taskId: "e", final: { success: true }, tokenUsage: { totalTokens: -1 } }, + { taskId: "f", final: { success: false }, tokenUsage: { totalTokens: null } }, + { taskId: "g", final: { success: false }, tokenUsage: { totalTokens: "" } }, + { taskId: "h", final: { success: false }, tokenUsage: { totalTokens: " " } }, + { taskId: "i", final: { success: false }, tokenUsage: { totalTokens: false } }, + { taskId: "j", final: { success: true }, tokenUsage: { totalTokens: "100" } }, + { taskId: "k", final: { success: true }, tokenUsage: { totalTokens: 25 } } + ]); + + assert.equal(aggregate.totalTokens, 125); + assert.equal(aggregate.tokenUsageUnavailableTasks, 9); + assert.equal(aggregate.tokensPerSuccessfulTask, 25); +}); + +test("aggregateLifecycleResults returns null tokens per successful task with no successes", () => { + const aggregate = aggregateLifecycleResults([ + { taskId: "a", final: { success: false }, tokenUsage: { totalTokens: 100 } } + ]); + + assert.equal(aggregate.totalTasks, 1); + assert.equal(aggregate.successfulTasks, 0); + assert.equal(aggregate.successRate, 0); + assert.equal(aggregate.totalTokens, 100); + assert.equal(aggregate.tokenUsageUnavailableTasks, 0); + assert.equal(aggregate.tokensPerSuccessfulTask, null); +}); + +test("writeLifecycleResultFiles writes JSON and Markdown summaries", async () => { + const tempDir = await mkdtemp(path.join(os.tmpdir(), "agentrail-swe-results-")); + try { + await writeLifecycleResultFiles({ + outputDir: tempDir, + run: { + runId: "run_001", + aggregate: { + totalTasks: 1, + successfulTasks: 0, + successRate: 0, + totalTokens: 0, + tokenUsageUnavailableTasks: 1, + tokensPerSuccessfulTask: null + }, + tasks: [] + } + }); + + const jsonText = await readFile(path.join(tempDir, "results.json"), "utf8"); + const json = JSON.parse(jsonText); + const markdown = await readFile(path.join(tempDir, "summary.md"), "utf8"); + const marker = await readFile(path.join(tempDir, "complete.marker"), "utf8"); + + assert.equal(jsonText.endsWith("\n"), true); + assert.equal(json.runId, "run_001"); + assert.match(markdown, /^# AgentRail SWE Lifecycle Benchmark/u); + assert.match(markdown, /- Tasks: 1/u); + assert.match(markdown, /- Successful tasks: 0/u); + assert.match(markdown, /- Success rate: 0\.00%/u); + assert.match(markdown, /- Token usage unavailable tasks: 1/u); + assert.equal(marker, "complete\n"); + } finally { + await rm(tempDir, { recursive: true, force: true }); + } +}); + +test("writeLifecycleResultFiles removes stale completion marker before failed writes", async () => { + const tempDir = await mkdtemp(path.join(os.tmpdir(), "agentrail-swe-results-stale-")); + try { + await writeFile(path.join(tempDir, "complete.marker"), "complete\n", "utf8"); + await mkdir(path.join(tempDir, "summary.md")); + + await assert.rejects( + writeLifecycleResultFiles({ + outputDir: tempDir, + run: { + runId: "run_001", + aggregate: { + totalTasks: 1, + successfulTasks: 0, + successRate: 0, + totalTokens: 0, + tokenUsageUnavailableTasks: 1, + tokensPerSuccessfulTask: null + }, + tasks: [] + } + }) + ); + await assert.rejects(readFile(path.join(tempDir, "complete.marker"), "utf8"), /ENOENT/u); + } finally { + await rm(tempDir, { recursive: true, force: true }); + } +}); + +test("run.mjs dry-run selects tasks and writes lifecycle artifacts", async () => { + const tempDir = await mkdtemp(path.join(os.tmpdir(), "agentrail-swe-run-")); + try { + const result = await runNode([ + path.join(repoRoot, "benchmarks", "agentrail-swe-lifecycle", "run.mjs"), + "--dataset", sampleDatasetPath, + "--count", "1", + "--dry-run", + "--output-dir", tempDir, + "--run-id", "run_001" + ]); + + assert.equal(result.code, 0, result.stderr); + assert.deepEqual(JSON.parse(result.stdout), { + status: "ok", + outputDir: tempDir, + tasks: 1 + }); + + const results = JSON.parse(await readFile(path.join(tempDir, "results.json"), "utf8")); + assert.equal(results.aggregate.totalTasks, 1); + assert.equal(results.tasks[0].taskId, "agentrail-swe-pro-hard-aa-001"); + assert.equal(results.tasks[0].lifecycle.issue.provider, "github"); + } finally { + await rm(tempDir, { recursive: true, force: true }); + } +}); + +test("run.mjs rejects invalid run IDs before writing outputs", async () => { + const tempDir = await mkdtemp(path.join(os.tmpdir(), "agentrail-swe-invalid-run-")); + try { + const result = await runNode([ + path.join(repoRoot, "benchmarks", "agentrail-swe-lifecycle", "run.mjs"), + "--dataset", sampleDatasetPath, + "--count", "1", + "--dry-run", + "--output-dir", tempDir, + "--run-id", "bad run" + ]); + + assert.notEqual(result.code, 0); + assert.match(result.stderr, /runId must match \/\^run_\[A-Za-z0-9_-\]\+\$\/\./u); + await assert.rejects(readFile(path.join(tempDir, "results.json"), "utf8"), /ENOENT/u); + } finally { + await rm(tempDir, { recursive: true, force: true }); + } +}); + +test("run.mjs rejects unmatched lifecycle task filters before writing outputs", async () => { + const tempDir = await mkdtemp(path.join(os.tmpdir(), "agentrail-swe-no-tasks-")); + try { + const result = await runNode([ + path.join(repoRoot, "benchmarks", "agentrail-swe-lifecycle", "run.mjs"), + "--dataset", sampleDatasetPath, + "--task", "missing-task", + "--dry-run", + "--output-dir", tempDir, + "--run-id", "run_001" + ]); + + assert.notEqual(result.code, 0); + assert.match(result.stderr, /No lifecycle tasks matched the provided filters\./u); + await assert.rejects(readFile(path.join(tempDir, "results.json"), "utf8"), /ENOENT/u); + } finally { + await rm(tempDir, { recursive: true, force: true }); + } +}); + +test("run.mjs dry-run writes default output files under ignored benchmark output root", async () => { + const runId = "run_cli_default_test"; + const outputDir = path.join(repoRoot, ".agentrail-bench", "agentrail-swe-lifecycle", runId); + await rm(outputDir, { recursive: true, force: true }); + try { + const result = await runNode([ + path.join(repoRoot, "benchmarks", "agentrail-swe-lifecycle", "run.mjs"), + "--dataset", sampleDatasetPath, + "--count", "1", + "--dry-run", + "--run-id", runId + ]); + + assert.equal(result.code, 0, result.stderr); + assert.deepEqual(JSON.parse(result.stdout), { + status: "ok", + outputDir, + tasks: 1 + }); + + const results = JSON.parse(await readFile(path.join(outputDir, "results.json"), "utf8")); + const summary = await readFile(path.join(outputDir, "summary.md"), "utf8"); + const marker = await readFile(path.join(outputDir, "complete.marker"), "utf8"); + + assert.equal(results.runId, runId); + assert.equal(results.aggregate.totalTasks, 1); + assert.match(summary, /AgentRail SWE Lifecycle Benchmark/u); + assert.equal(marker, "complete\n"); + } finally { + await rm(outputDir, { recursive: true, force: true }); + } +}); + +function runNode(args) { + return new Promise((resolve, reject) => { + const child = spawn(process.execPath, args, { cwd: repoRoot, stdio: ["ignore", "pipe", "pipe"] }); + let stdout = ""; + let stderr = ""; + + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk) => { + stdout += chunk; + }); + child.stderr.on("data", (chunk) => { + stderr += chunk; + }); + child.on("error", reject); + child.on("close", (code) => { + resolve({ code, stdout, stderr }); + }); + }); +} From 9fa681a8a48c5265885640b819b4c9ec56751611 Mon Sep 17 00:00:00 2001 From: Onyeka Nwamba Date: Mon, 25 May 2026 01:40:08 +0100 Subject: [PATCH 02/18] feat: add benchmark execution mode parsing --- .../agentrail-swe-lifecycle/lib/args.mjs | 104 ++++++++++++++++++ benchmarks/agentrail-swe-lifecycle/run.mjs | 46 +------- ...gentrail-swe-lifecycle-real-runner.test.js | 58 ++++++++++ 3 files changed, 166 insertions(+), 42 deletions(-) create mode 100644 benchmarks/agentrail-swe-lifecycle/lib/args.mjs create mode 100644 test/agentrail-swe-lifecycle-real-runner.test.js diff --git a/benchmarks/agentrail-swe-lifecycle/lib/args.mjs b/benchmarks/agentrail-swe-lifecycle/lib/args.mjs new file mode 100644 index 0000000..4f214cf --- /dev/null +++ b/benchmarks/agentrail-swe-lifecycle/lib/args.mjs @@ -0,0 +1,104 @@ +const VALID_MODES = new Set(["dry-run", "control", "agentrail", "paired"]); +const REAL_MODES = new Set(["control", "agentrail", "paired"]); +const VALID_RUNNERS = new Set(["codex", "cursor", "claude-code", "custom"]); +const VALID_EVALUATORS = new Set(["none", "command"]); + +export function parseBenchmarkArgs(argv) { + const flags = { + mode: "dry-run", + evaluator: "none", + cleanup: true + }; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === "--dry-run") { + flags.mode = "dry-run"; + } else if (arg === "--mode") { + flags.mode = readFlagValue({ argv, index: ++index, flag: arg }); + } else if (arg === "--runner") { + flags.runner = readFlagValue({ argv, index: ++index, flag: arg }); + } else if (arg === "--runner-command") { + flags.runnerCommand = readFlagValue({ argv, index: ++index, flag: arg }); + } else if (arg === "--model") { + flags.model = readFlagValue({ argv, index: ++index, flag: arg }); + } else if (arg === "--timeout-seconds") { + flags.timeoutSeconds = parsePositiveSafeInteger(readFlagValue({ argv, index: ++index, flag: arg }), arg); + } else if (arg === "--repo-cache") { + flags.repoCache = readFlagValue({ argv, index: ++index, flag: arg }); + } else if (arg === "--evaluator") { + flags.evaluator = readFlagValue({ argv, index: ++index, flag: arg }); + } else if (arg === "--eval-command-template") { + flags.evalCommandTemplate = readFlagValue({ argv, index: ++index, flag: arg }); + } else if (arg === "--require-token-usage") { + flags.requireTokenUsage = true; + } else if (arg === "--require-evaluator") { + flags.requireEvaluator = true; + } else if (arg === "--no-cleanup") { + flags.cleanup = false; + } else if (arg === "--dataset") { + flags.dataset = readFlagValue({ argv, index: ++index, flag: arg }); + } else if (arg === "--count") { + flags.count = readFlagValue({ argv, index: ++index, flag: arg }); + } else if (arg === "--repo") { + flags.repo = readFlagValue({ argv, index: ++index, flag: arg }); + } else if (arg === "--language") { + flags.language = readFlagValue({ argv, index: ++index, flag: arg }); + } else if (arg === "--tag") { + flags.tag = readFlagValue({ argv, index: ++index, flag: arg }); + } else if (arg === "--task") { + flags.taskId = readFlagValue({ argv, index: ++index, flag: arg }); + } else if (arg === "--seed") { + flags.seed = readFlagValue({ argv, index: ++index, flag: arg }); + } else if (arg === "--output-dir") { + flags.outputDir = readFlagValue({ argv, index: ++index, flag: arg }); + } else if (arg === "--run-id") { + flags.runId = readFlagValue({ argv, index: ++index, flag: arg }); + } else { + throw new Error(`Unknown argument: ${arg}`); + } + } + + validateBenchmarkArgs(flags); + return flags; +} + +function validateBenchmarkArgs(flags) { + if (!VALID_MODES.has(flags.mode)) { + throw new Error(`--mode must be one of: ${[...VALID_MODES].join(", ")}.`); + } + if (flags.runner !== undefined && !VALID_RUNNERS.has(flags.runner)) { + throw new Error(`--runner must be one of: ${[...VALID_RUNNERS].join(", ")}.`); + } + if (!VALID_EVALUATORS.has(flags.evaluator)) { + throw new Error(`--evaluator must be one of: ${[...VALID_EVALUATORS].join(", ")}.`); + } + if (REAL_MODES.has(flags.mode) && flags.runner === undefined) { + throw new Error("--runner is required for control, agentrail, and paired modes."); + } + if (flags.runner === "custom" && flags.runnerCommand === undefined) { + throw new Error("--runner-command is required when --runner custom is used."); + } + if (flags.evaluator === "command" && flags.evalCommandTemplate === undefined) { + throw new Error("--eval-command-template is required when --evaluator command is used."); + } +} + +function readFlagValue({ argv, index, flag }) { + const value = argv[index]; + if (value === undefined || value.startsWith("--")) { + throw new Error(`${flag} requires a value.`); + } + return value; +} + +function parsePositiveSafeInteger(value, flag) { + if (!/^[1-9]\d*$/u.test(value)) { + throw new Error(`${flag} must be a positive safe integer.`); + } + const parsed = Number(value); + if (!Number.isSafeInteger(parsed)) { + throw new Error(`${flag} must be a positive safe integer.`); + } + return parsed; +} diff --git a/benchmarks/agentrail-swe-lifecycle/run.mjs b/benchmarks/agentrail-swe-lifecycle/run.mjs index 4cd0db5..55013b7 100644 --- a/benchmarks/agentrail-swe-lifecycle/run.mjs +++ b/benchmarks/agentrail-swe-lifecycle/run.mjs @@ -2,6 +2,7 @@ import path from "node:path"; import { fileURLToPath } from "node:url"; +import { parseBenchmarkArgs } from "./lib/args.mjs"; import { loadLifecycleDataset, selectLifecycleTasks } from "./lib/dataset.mjs"; import { buildLocalLifecycleArtifacts } from "./lib/local-lifecycle.mjs"; import { aggregateLifecycleResults, writeLifecycleResultFiles } from "./lib/results.mjs"; @@ -10,10 +11,10 @@ const RUN_ID_PATTERN = /^run_[A-Za-z0-9_-]+$/u; const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", ".."); const defaultDatasetPath = path.join(repoRoot, "benchmarks", "agentrail-swe-lifecycle", "dataset.json"); -const flags = parseArgs(process.argv.slice(2)); +const flags = parseBenchmarkArgs(process.argv.slice(2)); -if (!flags.dryRun) { - throw new Error("Only --dry-run is implemented in this phase. Model execution is planned for the next phase."); +if (flags.mode !== "dry-run") { + throw new Error("Real benchmark execution is not wired yet. Continue with Task 7 after lane modules exist."); } const dataset = await loadLifecycleDataset(flags.dataset ?? defaultDatasetPath); @@ -55,45 +56,6 @@ function buildDryRunTaskResult({ task, runId }) { }; } -function parseArgs(argv) { - const flags = {}; - for (let index = 0; index < argv.length; index += 1) { - const arg = argv[index]; - if (arg === "--dry-run") { - flags.dryRun = true; - } else if (arg === "--dataset") { - flags.dataset = readFlagValue({ argv, index: ++index, flag: arg }); - } else if (arg === "--count") { - flags.count = readFlagValue({ argv, index: ++index, flag: arg }); - } else if (arg === "--repo") { - flags.repo = readFlagValue({ argv, index: ++index, flag: arg }); - } else if (arg === "--language") { - flags.language = readFlagValue({ argv, index: ++index, flag: arg }); - } else if (arg === "--tag") { - flags.tag = readFlagValue({ argv, index: ++index, flag: arg }); - } else if (arg === "--task") { - flags.taskId = readFlagValue({ argv, index: ++index, flag: arg }); - } else if (arg === "--seed") { - flags.seed = readFlagValue({ argv, index: ++index, flag: arg }); - } else if (arg === "--output-dir") { - flags.outputDir = readFlagValue({ argv, index: ++index, flag: arg }); - } else if (arg === "--run-id") { - flags.runId = readFlagValue({ argv, index: ++index, flag: arg }); - } else { - throw new Error(`Unknown argument: ${arg}`); - } - } - return flags; -} - -function readFlagValue({ argv, index, flag }) { - const value = argv[index]; - if (value === undefined || value.startsWith("--")) { - throw new Error(`${flag} requires a value.`); - } - return value; -} - function validateRunId(runId) { if (!RUN_ID_PATTERN.test(runId)) { throw new Error("runId must match /^run_[A-Za-z0-9_-]+$/."); diff --git a/test/agentrail-swe-lifecycle-real-runner.test.js b/test/agentrail-swe-lifecycle-real-runner.test.js new file mode 100644 index 0000000..9a9b4b5 --- /dev/null +++ b/test/agentrail-swe-lifecycle-real-runner.test.js @@ -0,0 +1,58 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { parseBenchmarkArgs } from "../benchmarks/agentrail-swe-lifecycle/lib/args.mjs"; + +test("parseBenchmarkArgs keeps --dry-run as a dry-run mode alias", () => { + const flags = parseBenchmarkArgs(["--dry-run", "--count", "1"]); + assert.equal(flags.mode, "dry-run"); + assert.equal(flags.count, "1"); +}); + +test("parseBenchmarkArgs accepts paired execution flags", () => { + const flags = parseBenchmarkArgs([ + "--mode", "paired", + "--runner", "custom", + "--runner-command", "node fake-runner.mjs", + "--model", "fake-model", + "--timeout-seconds", "30", + "--repo-cache", ".cache/repos", + "--evaluator", "command", + "--eval-command-template", "node fake-evaluator.mjs --patch {patchPath} --output {outputPath}", + "--require-token-usage", + "--require-evaluator", + "--no-cleanup", + ]); + assert.equal(flags.mode, "paired"); + assert.equal(flags.runner, "custom"); + assert.equal(flags.runnerCommand, "node fake-runner.mjs"); + assert.equal(flags.model, "fake-model"); + assert.equal(flags.timeoutSeconds, 30); + assert.equal(flags.repoCache, ".cache/repos"); + assert.equal(flags.evaluator, "command"); + assert.equal(flags.evalCommandTemplate, "node fake-evaluator.mjs --patch {patchPath} --output {outputPath}"); + assert.equal(flags.requireTokenUsage, true); + assert.equal(flags.requireEvaluator, true); + assert.equal(flags.cleanup, false); +}); + +test("parseBenchmarkArgs rejects real modes without a runner", () => { + assert.throws( + () => parseBenchmarkArgs(["--mode", "control"]), + /--runner is required for control, agentrail, and paired modes/u, + ); +}); + +test("parseBenchmarkArgs rejects custom runner without a command", () => { + assert.throws( + () => parseBenchmarkArgs(["--mode", "control", "--runner", "custom"]), + /--runner-command is required when --runner custom is used/u, + ); +}); + +test("parseBenchmarkArgs rejects command evaluator without a template", () => { + assert.throws( + () => parseBenchmarkArgs(["--mode", "control", "--runner", "codex", "--evaluator", "command"]), + /--eval-command-template is required when --evaluator command is used/u, + ); +}); From ab9e0b4401e17389f621834c1646ad1aba2faf7d Mon Sep 17 00:00:00 2001 From: Onyeka Nwamba Date: Mon, 25 May 2026 01:51:36 +0100 Subject: [PATCH 03/18] feat: parse benchmark runner token usage --- .../lib/token-usage.mjs | 233 ++++++++++++++++++ ...gentrail-swe-lifecycle-token-usage.test.js | 193 +++++++++++++++ 2 files changed, 426 insertions(+) create mode 100644 benchmarks/agentrail-swe-lifecycle/lib/token-usage.mjs create mode 100644 test/agentrail-swe-lifecycle-token-usage.test.js diff --git a/benchmarks/agentrail-swe-lifecycle/lib/token-usage.mjs b/benchmarks/agentrail-swe-lifecycle/lib/token-usage.mjs new file mode 100644 index 0000000..21f8161 --- /dev/null +++ b/benchmarks/agentrail-swe-lifecycle/lib/token-usage.mjs @@ -0,0 +1,233 @@ +import { readFile } from "node:fs/promises"; + +const TOKEN_VALUE_PATTERN = /^(0|[1-9]\d*)$/u; +const TOKEN_AGGREGATE_OVERFLOW_ERROR = "Token usage aggregate exceeds Number.MAX_SAFE_INTEGER."; +const TOKEN_TOTAL_VALUE_ERROR = "Token usage total_tokens must be a non-negative safe integer."; + +export async function parseTokenUsageFromFiles(paths) { + const aggregate = createEmptyAggregate(); + const sources = []; + + for (const filePath of paths) { + const text = await readTokenUsageFile(filePath); + if (text === null) continue; + + let contributed = false; + for (const record of parseJsonRecords(text)) { + for (const usage of findUsageObjects(record)) { + aggregateUsage(aggregate, usage); + contributed = true; + } + } + if (contributed) sources.push(filePath); + } + + if (!aggregate.available) { + return { + status: "unavailable", + inputTokens: null, + outputTokens: null, + reasoningTokens: null, + cachedInputTokens: null, + totalTokens: null, + sources: [] + }; + } + + return { + status: "available", + inputTokens: aggregate.hasInput ? aggregate.inputTokens : null, + outputTokens: aggregate.hasOutput ? aggregate.outputTokens : null, + reasoningTokens: aggregate.hasReasoning ? aggregate.reasoningTokens : null, + cachedInputTokens: aggregate.hasCachedInput ? aggregate.cachedInputTokens : null, + totalTokens: aggregate.hasTotal ? aggregate.totalTokens : null, + sources + }; +} + +async function readTokenUsageFile(filePath) { + try { + return await readFile(filePath, "utf8"); + } catch (error) { + if (error?.code === "ENOENT") return null; + throw error; + } +} + +function parseJsonRecords(text) { + const trimmedText = text.trimStart(); + if (trimmedText.startsWith("{") || trimmedText.startsWith("[")) { + try { + return [JSON.parse(text)]; + } catch { + // Fall through to JSONL scanning. + } + } + + const records = []; + for (const line of text.split(/\r?\n/u)) { + const trimmed = line.trimStart(); + if (!trimmed.startsWith("{") && !trimmed.startsWith("[")) continue; + try { + records.push(JSON.parse(trimmed)); + } catch { + // Runner logs can contain partial JSON-looking lines; ignore non-records. + } + } + + return records; +} + +function* findUsageObjects(value) { + if (Array.isArray(value)) { + for (const nested of value) { + yield* findUsageObjects(nested); + } + return; + } + + if (!isRecord(value)) return; + if (isUsageObject(value)) yield value; + + for (const [key, nested] of Object.entries(value)) { + if (isTokenDetailsKey(key)) continue; + yield* findUsageObjects(nested); + } +} + +function isTokenDetailsKey(key) { + return key === "input_tokens_details" + || key === "inputTokensDetails" + || key === "output_tokens_details" + || key === "outputTokensDetails"; +} + +function isUsageObject(value) { + return getTokenValue(value, "input_tokens", "inputTokens") !== null + || getTokenValue(value, "output_tokens", "outputTokens") !== null + || getTokenValue(value, "reasoning_tokens", "reasoningTokens") !== null + || getTokenValue(value, "cached_input_tokens", "cachedInputTokens") !== null + || hasTokenField(value, "total_tokens", "totalTokens") + || getCachedInputTokensFromDetails(value) !== null + || getReasoningTokensFromDetails(value) !== null; +} + +function aggregateUsage(aggregate, usage) { + const inputTokens = getTokenValue(usage, "input_tokens", "inputTokens"); + const outputTokens = getTokenValue(usage, "output_tokens", "outputTokens"); + const reasoningTokens = getTokenValue(usage, "reasoning_tokens", "reasoningTokens") + ?? getReasoningTokensFromDetails(usage); + const cachedInputTokens = getTokenValue(usage, "cached_input_tokens", "cachedInputTokens") + ?? getCachedInputTokensFromDetails(usage); + const totalTokens = getTotalTokenValue(usage); + + addComponent(aggregate, "inputTokens", "hasInput", inputTokens); + addComponent(aggregate, "outputTokens", "hasOutput", outputTokens); + addComponent(aggregate, "reasoningTokens", "hasReasoning", reasoningTokens); + addComponent(aggregate, "cachedInputTokens", "hasCachedInput", cachedInputTokens); + + if (totalTokens.present) { + addComponent(aggregate, "totalTokens", "hasTotal", totalTokens.value); + return; + } + + if (inputTokens !== null || outputTokens !== null || reasoningTokens !== null) { + addComponent( + aggregate, + "totalTokens", + "hasTotal", + sumTokenValues(inputTokens ?? 0, outputTokens ?? 0, reasoningTokens ?? 0) + ); + } +} + +function addComponent(aggregate, totalKey, flagKey, value) { + if (value === null) return; + aggregate[totalKey] = sumTokenValues(aggregate[totalKey], value); + aggregate[flagKey] = true; + aggregate.available = true; +} + +function sumTokenValues(...values) { + let sum = 0; + for (const value of values) { + sum += value; + if (!Number.isSafeInteger(sum) || sum < 0) { + throw new Error(TOKEN_AGGREGATE_OVERFLOW_ERROR); + } + } + return sum; +} + +function getTokenValue(record, snakeKey, camelKey) { + return normalizeTokenValue(record[snakeKey] ?? record[camelKey]); +} + +function getTotalTokenValue(record) { + if (!hasTokenField(record, "total_tokens", "totalTokens")) { + return { present: false, value: null }; + } + + let value = null; + for (const key of ["total_tokens", "totalTokens"]) { + if (!hasOwn(record, key)) continue; + const normalized = normalizeTokenValue(record[key]); + if (normalized === null) throw new Error(TOKEN_TOTAL_VALUE_ERROR); + value ??= normalized; + } + + return { present: true, value }; +} + +function hasTokenField(record, snakeKey, camelKey) { + return hasOwn(record, snakeKey) || hasOwn(record, camelKey); +} + +function getCachedInputTokensFromDetails(record) { + return getDetailTokenValue(record, "input_tokens_details", "inputTokensDetails", "cached_tokens", "cachedTokens"); +} + +function getReasoningTokensFromDetails(record) { + return getDetailTokenValue(record, "output_tokens_details", "outputTokensDetails", "reasoning_tokens", "reasoningTokens"); +} + +function getDetailTokenValue(record, snakeContainerKey, camelContainerKey, snakeTokenKey, camelTokenKey) { + const container = record[snakeContainerKey] ?? record[camelContainerKey]; + if (!isRecord(container)) return null; + return normalizeTokenValue(container[snakeTokenKey] ?? container[camelTokenKey]); +} + +function normalizeTokenValue(value) { + if (typeof value === "number") { + return Number.isSafeInteger(value) && value >= 0 ? value : null; + } + if (typeof value === "string" && TOKEN_VALUE_PATTERN.test(value)) { + const parsed = Number(value); + return Number.isSafeInteger(parsed) ? parsed : null; + } + return null; +} + +function createEmptyAggregate() { + return { + available: false, + inputTokens: 0, + outputTokens: 0, + reasoningTokens: 0, + cachedInputTokens: 0, + totalTokens: 0, + hasInput: false, + hasOutput: false, + hasReasoning: false, + hasCachedInput: false, + hasTotal: false + }; +} + +function isRecord(value) { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +function hasOwn(record, key) { + return Object.prototype.hasOwnProperty.call(record, key); +} diff --git a/test/agentrail-swe-lifecycle-token-usage.test.js b/test/agentrail-swe-lifecycle-token-usage.test.js new file mode 100644 index 0000000..f2dcdd5 --- /dev/null +++ b/test/agentrail-swe-lifecycle-token-usage.test.js @@ -0,0 +1,193 @@ +import assert from "node:assert/strict"; +import { mkdtemp, readFile, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; + +import { parseTokenUsageFromFiles } from "../benchmarks/agentrail-swe-lifecycle/lib/token-usage.mjs"; + +test("parseTokenUsageFromFiles aggregates OpenAI-style usage JSONL", async () => { + const dir = await mkdtemp(path.join(os.tmpdir(), "agentrail-token-usage-")); + const log = path.join(dir, "runner.jsonl"); + await writeFile(log, [ + JSON.stringify({ type: "response.completed", response: { usage: { input_tokens: 100, output_tokens: 40, reasoning_tokens: 10, input_tokens_details: { cached_tokens: 20 } } } }), + JSON.stringify({ type: "response.completed", response: { usage: { input_tokens: 30, output_tokens: 5 } } }), + "", + ].join("\n")); + const usage = await parseTokenUsageFromFiles([log]); + assert.deepEqual(usage, { + status: "available", + inputTokens: 130, + outputTokens: 45, + reasoningTokens: 10, + cachedInputTokens: 20, + totalTokens: 185, + sources: [log], + }); + assert.match(await readFile(log, "utf8"), /response.completed/u); +}); + +test("parseTokenUsageFromFiles accepts total token fields without double counting", async () => { + const dir = await mkdtemp(path.join(os.tmpdir(), "agentrail-token-total-")); + const log = path.join(dir, "stdout.log"); + await writeFile(log, `${JSON.stringify({ usage: { total_tokens: 77 } })}\n`); + const usage = await parseTokenUsageFromFiles([log]); + assert.equal(usage.status, "available"); + assert.equal(usage.totalTokens, 77); + assert.equal(usage.inputTokens, null); + assert.equal(usage.outputTokens, null); +}); + +test("parseTokenUsageFromFiles rejects invalid provider total instead of recomputing components", async () => { + const dir = await mkdtemp(path.join(os.tmpdir(), "agentrail-token-invalid-total-")); + const log = path.join(dir, "stdout.log"); + await writeFile(log, `${JSON.stringify({ + usage: { + input_tokens: 5, + output_tokens: 2, + total_tokens: String(Number.MAX_SAFE_INTEGER + 1), + }, + })}\n`); + await assert.rejects( + parseTokenUsageFromFiles([log]), + /Token usage total_tokens must be a non-negative safe integer\./u + ); +}); + +test("parseTokenUsageFromFiles does not double-count nested OpenAI detail token fields", async () => { + const dir = await mkdtemp(path.join(os.tmpdir(), "agentrail-token-details-")); + const log = path.join(dir, "stdout.log"); + await writeFile(log, `${JSON.stringify({ + usage: { + input_tokens: 100, + output_tokens: 25, + total_tokens: 140, + output_tokens_details: { reasoning_tokens: 15 }, + }, + })}\n`); + const usage = await parseTokenUsageFromFiles([log]); + assert.deepEqual(usage, { + status: "available", + inputTokens: 100, + outputTokens: 25, + reasoningTokens: 15, + cachedInputTokens: null, + totalTokens: 140, + sources: [log], + }); +}); + +test("parseTokenUsageFromFiles aggregates top-level JSON array usage entries", async () => { + const dir = await mkdtemp(path.join(os.tmpdir(), "agentrail-token-array-")); + const log = path.join(dir, "runner.json"); + await writeFile(log, JSON.stringify([ + { usage: { input_tokens: 12, output_tokens: 3 } }, + { response: { usage: { total_tokens: 50 } } }, + ], null, 2)); + const usage = await parseTokenUsageFromFiles([log]); + assert.deepEqual(usage, { + status: "available", + inputTokens: 12, + outputTokens: 3, + reasoningTokens: null, + cachedInputTokens: null, + totalTokens: 65, + sources: [log], + }); +}); + +test("parseTokenUsageFromFiles parses whole-document arrays before JSONL scanning", async () => { + const dir = await mkdtemp(path.join(os.tmpdir(), "agentrail-token-pretty-array-")); + const log = path.join(dir, "runner.json"); + await writeFile(log, [ + "[", + JSON.stringify({ usage: { input_tokens: 4, output_tokens: 1 } }) + ",", + JSON.stringify({ usage: { input_tokens: 6, output_tokens: 2 } }), + "]", + "", + ].join("\n")); + const usage = await parseTokenUsageFromFiles([log]); + assert.deepEqual(usage, { + status: "available", + inputTokens: 10, + outputTokens: 3, + reasoningTokens: null, + cachedInputTokens: null, + totalTokens: 13, + sources: [log], + }); +}); + +test("parseTokenUsageFromFiles aggregates JSONL array line usage entries", async () => { + const dir = await mkdtemp(path.join(os.tmpdir(), "agentrail-token-jsonl-array-")); + const log = path.join(dir, "runner.jsonl"); + await writeFile(log, [ + "runner started", + JSON.stringify([ + { usage: { input_tokens: 8, output_tokens: 2 } }, + { usage: { input_tokens: 3, output_tokens: 1 } }, + ]), + "", + ].join("\n")); + const usage = await parseTokenUsageFromFiles([log]); + assert.deepEqual(usage, { + status: "available", + inputTokens: 11, + outputTokens: 3, + reasoningTokens: null, + cachedInputTokens: null, + totalTokens: 14, + sources: [log], + }); +}); + +test("parseTokenUsageFromFiles aggregates multiple nested usage objects in one record", async () => { + const dir = await mkdtemp(path.join(os.tmpdir(), "agentrail-token-nested-")); + const log = path.join(dir, "runner.jsonl"); + await writeFile(log, `${JSON.stringify({ + events: [ + { response: { usage: { input_tokens: 5, output_tokens: 2 } } }, + { response: { usage: { inputTokens: 7, outputTokens: 1, reasoningTokens: 4 } } }, + ], + })}\n`); + const usage = await parseTokenUsageFromFiles([log]); + assert.deepEqual(usage, { + status: "available", + inputTokens: 12, + outputTokens: 3, + reasoningTokens: 4, + cachedInputTokens: null, + totalTokens: 19, + sources: [log], + }); +}); + +test("parseTokenUsageFromFiles throws when aggregate tokens exceed safe integer range", async () => { + const dir = await mkdtemp(path.join(os.tmpdir(), "agentrail-token-overflow-")); + const log = path.join(dir, "runner.jsonl"); + await writeFile(log, [ + JSON.stringify({ usage: { total_tokens: Number.MAX_SAFE_INTEGER } }), + JSON.stringify({ usage: { total_tokens: 1 } }), + "", + ].join("\n")); + await assert.rejects( + parseTokenUsageFromFiles([log]), + /Token usage aggregate exceeds Number\.MAX_SAFE_INTEGER\./u + ); +}); + +test("parseTokenUsageFromFiles reports unavailable when no usage appears", async () => { + const dir = await mkdtemp(path.join(os.tmpdir(), "agentrail-token-missing-")); + const log = path.join(dir, "stderr.log"); + await writeFile(log, "plain text with no usage\n"); + const usage = await parseTokenUsageFromFiles([log]); + assert.deepEqual(usage, { + status: "unavailable", + inputTokens: null, + outputTokens: null, + reasoningTokens: null, + cachedInputTokens: null, + totalTokens: null, + sources: [], + }); +}); From c5bcf7f9ae693a3fc0056d53feae9db096364bb6 Mon Sep 17 00:00:00 2001 From: Onyeka Nwamba Date: Mon, 25 May 2026 02:24:03 +0100 Subject: [PATCH 04/18] feat: prepare benchmark workspaces and patches --- .../agentrail-swe-lifecycle/lib/patch.mjs | 132 ++++ .../agentrail-swe-lifecycle/lib/workspace.mjs | 156 +++++ .../agentrail-swe-lifecycle-workspace.test.js | 599 ++++++++++++++++++ 3 files changed, 887 insertions(+) create mode 100644 benchmarks/agentrail-swe-lifecycle/lib/patch.mjs create mode 100644 benchmarks/agentrail-swe-lifecycle/lib/workspace.mjs create mode 100644 test/agentrail-swe-lifecycle-workspace.test.js diff --git a/benchmarks/agentrail-swe-lifecycle/lib/patch.mjs b/benchmarks/agentrail-swe-lifecycle/lib/patch.mjs new file mode 100644 index 0000000..5794c5a --- /dev/null +++ b/benchmarks/agentrail-swe-lifecycle/lib/patch.mjs @@ -0,0 +1,132 @@ +import { execFile } from "node:child_process"; +import { lstat, mkdir, readlink, realpath, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { promisify } from "node:util"; + +const execFileAsync = promisify(execFile); +const GIT_MAX_BUFFER = 1024 * 1024 * 20; +const BASE_REF = "refs/agentrail-bench/base"; +const OUTPUT_PATH_ERROR = "capturePatch outputPath must be outside the benchmark worktree."; + +export async function capturePatch({ worktreePath, outputPath }) { + await assertOutputPathOutsideWorktree({ worktreePath, outputPath }); + + if (!(await baseRefExists(worktreePath))) { + throw new Error("capturePatch requires refs/agentrail-bench/base in benchmark worktrees."); + } + + await includeUntrackedFiles(worktreePath); + + const diff = await git(["-C", worktreePath, "diff", "--binary", BASE_REF]); + const changedFilesOutput = await git(["-C", worktreePath, "diff", "--name-only", "-z", BASE_REF]); + const commitSha = (await git(["-C", worktreePath, "rev-parse", "HEAD"])).trim(); + const changedFiles = changedFilesOutput.split("\0").filter((line) => line.length > 0); + + await mkdir(path.dirname(outputPath), { recursive: true }); + await writeFile(outputPath, diff, "utf8"); + + return { + status: diff.trim().length > 0 ? "available" : "empty", + path: outputPath, + changedFiles, + commitSha + }; +} + +async function assertOutputPathOutsideWorktree({ worktreePath, outputPath }) { + const realWorktreePath = await realpath(worktreePath); + const realOutputLocation = await resolveOutputLocation(outputPath); + if (isInsideOrEqual(realWorktreePath, realOutputLocation)) { + throw new Error(OUTPUT_PATH_ERROR); + } +} + +async function resolveOutputLocation(outputPath) { + const outputAbsolutePath = path.resolve(outputPath); + + const symlinkTarget = await outputSymlinkTarget(outputAbsolutePath); + if (symlinkTarget) return resolveSymlinkTargetLocation({ symlinkPath: outputAbsolutePath, symlinkTarget }); + + try { + return await realpath(outputAbsolutePath); + } catch (error) { + if (error?.code !== "ENOENT") throw error; + } + + const outputParentPath = path.dirname(outputAbsolutePath); + const { existingPath, missingSegments } = await nearestExistingPath(outputParentPath); + const realExistingPath = await realpath(existingPath); + return path.resolve(realExistingPath, ...missingSegments, path.basename(outputAbsolutePath)); +} + +async function outputSymlinkTarget(outputAbsolutePath) { + try { + const entry = await lstat(outputAbsolutePath); + if (!entry.isSymbolicLink()) return null; + return await readlink(outputAbsolutePath); + } catch (error) { + if (error?.code === "ENOENT") return null; + throw error; + } +} + +async function resolveSymlinkTargetLocation({ symlinkPath, symlinkTarget }) { + const targetPath = path.resolve(path.dirname(symlinkPath), symlinkTarget); + + try { + return await realpath(targetPath); + } catch (error) { + if (error?.code !== "ENOENT") throw error; + } + + const targetParentPath = path.dirname(targetPath); + const { existingPath, missingSegments } = await nearestExistingPath(targetParentPath); + const realExistingPath = await realpath(existingPath); + return path.resolve(realExistingPath, ...missingSegments, path.basename(targetPath)); +} + +async function nearestExistingPath(targetPath) { + const missingSegments = []; + let currentPath = path.resolve(targetPath); + + while (true) { + try { + await realpath(currentPath); + return { existingPath: currentPath, missingSegments: missingSegments.reverse() }; + } catch (error) { + if (error?.code !== "ENOENT") throw error; + } + + const parentPath = path.dirname(currentPath); + if (parentPath === currentPath) throw new Error(OUTPUT_PATH_ERROR); + missingSegments.push(path.basename(currentPath)); + currentPath = parentPath; + } +} + +function isInsideOrEqual(parentPath, candidatePath) { + const relative = path.relative(parentPath, candidatePath); + return relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative)); +} + +async function includeUntrackedFiles(worktreePath) { + const output = await git(["-C", worktreePath, "ls-files", "--others", "--exclude-standard", "-z"]); + const files = output.split("\0").filter((file) => file.length > 0); + if (files.length === 0) return; + await git(["-C", worktreePath, "add", "--intent-to-add", "--", ...files]); +} + +async function baseRefExists(worktreePath) { + try { + await git(["-C", worktreePath, "rev-parse", "--verify", `${BASE_REF}^{commit}`]); + return true; + } catch (error) { + if (typeof error?.code === "number") return false; + throw error; + } +} + +async function git(args) { + const { stdout } = await execFileAsync("git", args, { encoding: "utf8", maxBuffer: GIT_MAX_BUFFER }); + return stdout; +} diff --git a/benchmarks/agentrail-swe-lifecycle/lib/workspace.mjs b/benchmarks/agentrail-swe-lifecycle/lib/workspace.mjs new file mode 100644 index 0000000..ed5a4a8 --- /dev/null +++ b/benchmarks/agentrail-swe-lifecycle/lib/workspace.mjs @@ -0,0 +1,156 @@ +import { createHash } from "node:crypto"; +import { execFile } from "node:child_process"; +import { lstat, mkdir, rm, stat } from "node:fs/promises"; +import path from "node:path"; +import { promisify } from "node:util"; + +const execFileAsync = promisify(execFile); +const GIT_MAX_BUFFER = 1024 * 1024 * 20; +const BASE_REF = "refs/agentrail-bench/base"; +const ORIGIN_MAIN_REF = "refs/heads/main"; +const LANE_PATTERN = /^[A-Za-z0-9._-]+$/u; +const REPO_PATTERN = /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u; +const BASE_COMMIT_PATTERN = /^[a-f0-9]{40}$/u; +const TASK_ID_SEGMENT_ERROR = "prepareLaneWorkspace requires task.id to resolve to a safe path segment."; +const SYMLINK_WORKSPACE_PATH_ERROR = "prepareLaneWorkspace refuses to operate through symlinked workspace paths."; +const REPO_PATTERN_ERROR = "prepareLaneWorkspace requires task.repo to be an owner/name repository."; +const BASE_COMMIT_PATTERN_ERROR = "prepareLaneWorkspace requires upstream.baseCommit to be a 40-character lowercase hex SHA."; + +export async function prepareLaneWorkspace({ task, upstream, lane, runDir, repoCache, sourceRepoPath }) { + if (!task?.id || !task?.repo) { + throw new Error("prepareLaneWorkspace requires task.id and task.repo."); + } + if (typeof task.repo !== "string" || !isSafeRepo(task.repo)) { + throw new Error(REPO_PATTERN_ERROR); + } + if (typeof upstream?.baseCommit !== "string" || !BASE_COMMIT_PATTERN.test(upstream.baseCommit)) { + throw new Error(BASE_COMMIT_PATTERN_ERROR); + } + if (typeof lane !== "string" || !LANE_PATTERN.test(lane) || lane === "." || lane === "..") { + throw new Error("prepareLaneWorkspace requires lane to match /^[A-Za-z0-9._-]+$/."); + } + + const taskId = String(task.id); + const safeTaskId = taskId.replaceAll(/[^A-Za-z0-9._-]/gu, "_"); + if (safeTaskId.length === 0 || safeTaskId === "." || safeTaskId === "..") { + throw new Error(TASK_ID_SEGMENT_ERROR); + } + const taskPathSegment = `${safeTaskId}__${shortHash(taskId)}`; + const tasksDir = path.join(runDir, "tasks"); + const legacyTaskDir = path.join(tasksDir, safeTaskId); + const taskDir = path.join(tasksDir, taskPathSegment); + const laneDir = path.join(taskDir, lane); + assertPathInsideTaskDir({ laneDir, taskDir }); + const worktreePath = path.join(laneDir, "worktree"); + const remotePath = path.join(laneDir, "remote.git"); + + await assertNoSymlinkWorkspacePaths([tasksDir, legacyTaskDir, taskDir, laneDir]); + await rm(laneDir, { recursive: true, force: true }); + await mkdir(laneDir, { recursive: true }); + + const cloneSource = sourceRepoPath ?? (await prepareCachedRepo({ repo: task.repo, repoCache })); + await git(["clone", "--no-local", "--no-hardlinks", cloneSource, worktreePath]); + await git(["-C", worktreePath, "checkout", "--detach", upstream.baseCommit]); + await git(["-C", worktreePath, "update-ref", BASE_REF, upstream.baseCommit]); + await git(["-C", worktreePath, "update-ref", ORIGIN_MAIN_REF, upstream.baseCommit]); + await git(["init", "--bare", remotePath]); + await git(["-C", worktreePath, "remote", "set-url", "origin", remotePath]); + await git(["-C", worktreePath, "push", "origin", `${upstream.baseCommit}:${ORIGIN_MAIN_REF}`]); + await git(["--git-dir", remotePath, "symbolic-ref", "HEAD", ORIGIN_MAIN_REF]); + + return { laneDir, worktreePath, remotePath }; +} + +function shortHash(value) { + return createHash("sha256").update(value).digest("hex").slice(0, 12); +} + +function isSafeRepo(repo) { + if (!REPO_PATTERN.test(repo)) return false; + return repo.split("/").every((segment) => segment !== "." && segment !== ".."); +} + +function assertPathInsideTaskDir({ laneDir, taskDir }) { + const resolvedLaneDir = path.resolve(laneDir); + const resolvedTaskDir = path.resolve(taskDir); + const relative = path.relative(resolvedTaskDir, resolvedLaneDir); + if (relative === "" || relative.startsWith("..") || path.isAbsolute(relative)) { + throw new Error("prepareLaneWorkspace resolved lane outside task directory."); + } +} + +async function assertNoSymlinkWorkspacePaths(paths) { + for (const workspacePath of paths) { + let entry; + try { + entry = await lstat(workspacePath); + } catch (error) { + if (error?.code === "ENOENT") continue; + throw error; + } + + if (entry.isSymbolicLink()) { + throw new Error(SYMLINK_WORKSPACE_PATH_ERROR); + } + } +} + +async function prepareCachedRepo({ repo, repoCache }) { + if (!repoCache) { + throw new Error("prepareLaneWorkspace requires repoCache when sourceRepoPath is absent."); + } + + const cachePath = path.join(repoCache, String(repo).replaceAll("/", "__")); + const remoteUrl = `https://github.com/${repo}.git`; + if (await pathExists(cachePath)) { + if (await isBareGitRepo(cachePath)) { + await git(["-C", cachePath, "remote", "update", "--prune"]); + } else { + const existingRemoteUrl = (await originUrl(cachePath)) ?? remoteUrl; + await rm(cachePath, { recursive: true, force: true }); + await git(["clone", "--mirror", existingRemoteUrl, cachePath]); + } + } else { + await mkdir(repoCache, { recursive: true }); + await git(["clone", "--mirror", remoteUrl, cachePath]); + } + return cachePath; +} + +async function isBareGitRepo(repoPath) { + try { + return (await gitOutput(["-C", repoPath, "rev-parse", "--is-bare-repository"])).trim() === "true"; + } catch (error) { + if (typeof error?.code === "number") return false; + throw error; + } +} + +async function originUrl(repoPath) { + try { + const output = await gitOutput(["-C", repoPath, "remote", "get-url", "origin"]); + return output.trim() || null; + } catch (error) { + if (typeof error?.code === "number") return null; + throw error; + } +} + +async function pathExists(targetPath) { + try { + await stat(targetPath); + return true; + } catch (error) { + if (error?.code === "ENOENT") return false; + throw error; + } +} + +async function git(args) { + await execFileAsync("git", args, { maxBuffer: GIT_MAX_BUFFER }); +} + +async function gitOutput(args) { + const { stdout } = await execFileAsync("git", args, { encoding: "utf8", maxBuffer: GIT_MAX_BUFFER }); + return stdout; +} diff --git a/test/agentrail-swe-lifecycle-workspace.test.js b/test/agentrail-swe-lifecycle-workspace.test.js new file mode 100644 index 0000000..6926075 --- /dev/null +++ b/test/agentrail-swe-lifecycle-workspace.test.js @@ -0,0 +1,599 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { lstat, mkdir, mkdtemp, readFile, symlink, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; + +import { capturePatch } from "../benchmarks/agentrail-swe-lifecycle/lib/patch.mjs"; +import { prepareLaneWorkspace } from "../benchmarks/agentrail-swe-lifecycle/lib/workspace.mjs"; + +test("prepareLaneWorkspace clones a source repo and checks out the base commit", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "agentrail-workspace-")); + const source = path.join(root, "source"); + execFileSync("git", ["init", source]); + execFileSync("git", ["-C", source, "config", "user.email", "test@example.com"]); + execFileSync("git", ["-C", source, "config", "user.name", "Test User"]); + await writeFile(path.join(source, "README.md"), "base\n"); + execFileSync("git", ["-C", source, "add", "README.md"]); + execFileSync("git", ["-C", source, "commit", "-m", "base"]); + const baseCommit = execFileSync("git", ["-C", source, "rev-parse", "HEAD"], { encoding: "utf8" }).trim(); + await writeFile(path.join(source, "README.md"), "after\n"); + execFileSync("git", ["-C", source, "commit", "-am", "after"]); + + const workspace = await prepareLaneWorkspace({ + task: { id: "task_1", repo: "fixture/repo" }, + upstream: { baseCommit }, + lane: "control", + runDir: root, + repoCache: path.join(root, "cache"), + sourceRepoPath: source + }); + + const checkedOut = execFileSync("git", ["-C", workspace.worktreePath, "rev-parse", "HEAD"], { encoding: "utf8" }).trim(); + assert.equal(checkedOut, baseCommit); + assert.equal(await readFile(path.join(workspace.worktreePath, "README.md"), "utf8"), "base\n"); +}); + +test("prepareLaneWorkspace seeds local origin main with the base commit", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "agentrail-workspace-")); + const { baseCommit, source } = await createFixtureRepo(root); + + const workspace = await prepareLaneWorkspace({ + task: { id: "task_1", repo: "fixture/repo" }, + upstream: { baseCommit }, + lane: "control", + runDir: root, + repoCache: path.join(root, "cache"), + sourceRepoPath: source + }); + + const remoteMain = execFileSync("git", ["-C", workspace.worktreePath, "ls-remote", "origin", "refs/heads/main"], { + encoding: "utf8" + }).trim(); + assert.equal(remoteMain, `${baseCommit}\trefs/heads/main`); +}); + +test("prepareLaneWorkspace points local main at the base commit", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "agentrail-workspace-")); + const { baseCommit, source } = await createFixtureRepo(root); + + const workspace = await prepareLaneWorkspace({ + task: { id: "task_1", repo: "fixture/repo" }, + upstream: { baseCommit }, + lane: "control", + runDir: root, + repoCache: path.join(root, "cache"), + sourceRepoPath: source + }); + + const localMain = execFileSync("git", ["-C", workspace.worktreePath, "rev-parse", "refs/heads/main"], { + encoding: "utf8" + }).trim(); + assert.equal(localMain, baseCommit); + + execFileSync("git", ["-C", workspace.worktreePath, "switch", "main"], { stdio: "ignore" }); + const switchedHead = execFileSync("git", ["-C", workspace.worktreePath, "rev-parse", "HEAD"], { + encoding: "utf8" + }).trim(); + assert.equal(switchedHead, baseCommit); +}); + +test("prepareLaneWorkspace points bare remote HEAD at main", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "agentrail-workspace-")); + const { baseCommit, source } = await createFixtureRepo(root); + + const workspace = await prepareLaneWorkspace({ + task: { id: "task_1", repo: "fixture/repo" }, + upstream: { baseCommit }, + lane: "control", + runDir: root, + repoCache: path.join(root, "cache"), + sourceRepoPath: source + }); + + const remoteHead = execFileSync("git", ["--git-dir", workspace.remotePath, "symbolic-ref", "HEAD"], { + encoding: "utf8" + }).trim(); + assert.equal(remoteHead, "refs/heads/main"); +}); + +test("prepareLaneWorkspace rejects mutable base commits before removing workspace paths", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "agentrail-workspace-")); + const { source } = await createFixtureRepo(root); + const laneDir = path.join(root, "tasks", taskPathSegment("task_1"), "control"); + const sentinelPath = path.join(laneDir, "sentinel.txt"); + await mkdir(laneDir, { recursive: true }); + await writeFile(sentinelPath, "keep\n"); + + await assert.rejects( + prepareLaneWorkspace({ + task: { id: "task_1", repo: "fixture/repo" }, + upstream: { baseCommit: "main" }, + lane: "control", + runDir: root, + repoCache: path.join(root, "cache"), + sourceRepoPath: source + }), + (error) => { + assert.equal(error.message, "prepareLaneWorkspace requires upstream.baseCommit to be a 40-character lowercase hex SHA."); + return true; + } + ); + assert.equal(await readFile(sentinelPath, "utf8"), "keep\n"); +}); + +test("prepareLaneWorkspace rejects unsafe lane names before removing paths", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "agentrail-workspace-")); + const { baseCommit, source } = await createFixtureRepo(root); + const outsideTaskDir = path.join(root, "tasks", "escape"); + const sentinelPath = path.join(outsideTaskDir, "sentinel.txt"); + await mkdir(outsideTaskDir, { recursive: true }); + await writeFile(sentinelPath, "keep\n"); + + await assert.rejects( + prepareLaneWorkspace({ + task: { id: "task_1", repo: "fixture/repo" }, + upstream: { baseCommit }, + lane: "../escape", + runDir: root, + repoCache: path.join(root, "cache"), + sourceRepoPath: source + }), + (error) => { + assert.equal(error.message, "prepareLaneWorkspace requires lane to match /^[A-Za-z0-9._-]+$/."); + return true; + } + ); + assert.equal(await readFile(sentinelPath, "utf8"), "keep\n"); +}); + +test("prepareLaneWorkspace rejects dot lane before removing task paths", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "agentrail-workspace-")); + const { baseCommit, source } = await createFixtureRepo(root); + const siblingLaneDir = path.join(root, "tasks", "task_1", "control"); + const sentinelPath = path.join(siblingLaneDir, "sentinel.txt"); + await mkdir(siblingLaneDir, { recursive: true }); + await writeFile(sentinelPath, "keep\n"); + + await assert.rejects( + prepareLaneWorkspace({ + task: { id: "task_1", repo: "fixture/repo" }, + upstream: { baseCommit }, + lane: ".", + runDir: root, + repoCache: path.join(root, "cache"), + sourceRepoPath: source + }), + (error) => { + assert.equal(error.message, "prepareLaneWorkspace requires lane to match /^[A-Za-z0-9._-]+$/."); + return true; + } + ); + assert.equal(await readFile(sentinelPath, "utf8"), "keep\n"); +}); + +test("prepareLaneWorkspace rejects parent task id before removing run paths", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "agentrail-workspace-")); + const { baseCommit, source } = await createFixtureRepo(root); + const sentinelPath = path.join(root, "control", "sentinel.txt"); + await mkdir(path.dirname(sentinelPath), { recursive: true }); + await writeFile(sentinelPath, "keep\n"); + + await assert.rejects( + prepareLaneWorkspace({ + task: { id: "..", repo: "fixture/repo" }, + upstream: { baseCommit }, + lane: "control", + runDir: root, + repoCache: path.join(root, "cache"), + sourceRepoPath: source + }), + (error) => { + assert.equal(error.message, "prepareLaneWorkspace requires task.id to resolve to a safe path segment."); + return true; + } + ); + assert.equal(await readFile(sentinelPath, "utf8"), "keep\n"); +}); + +test("prepareLaneWorkspace rejects dot task id before removing task paths", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "agentrail-workspace-")); + const { baseCommit, source } = await createFixtureRepo(root); + const sentinelPath = path.join(root, "tasks", "control", "sentinel.txt"); + await mkdir(path.dirname(sentinelPath), { recursive: true }); + await writeFile(sentinelPath, "keep\n"); + + await assert.rejects( + prepareLaneWorkspace({ + task: { id: ".", repo: "fixture/repo" }, + upstream: { baseCommit }, + lane: "control", + runDir: root, + repoCache: path.join(root, "cache"), + sourceRepoPath: source + }), + (error) => { + assert.equal(error.message, "prepareLaneWorkspace requires task.id to resolve to a safe path segment."); + return true; + } + ); + assert.equal(await readFile(sentinelPath, "utf8"), "keep\n"); +}); + +test("prepareLaneWorkspace rejects symlinked task workspace paths before removing lanes", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "agentrail-workspace-")); + const outside = await mkdtemp(path.join(os.tmpdir(), "agentrail-outside-")); + const { baseCommit, source } = await createFixtureRepo(root); + const outsideSentinelPath = path.join(outside, "control", "sentinel.txt"); + await mkdir(path.dirname(outsideSentinelPath), { recursive: true }); + await writeFile(outsideSentinelPath, "keep\n"); + await mkdir(path.join(root, "tasks"), { recursive: true }); + await symlink(outside, path.join(root, "tasks", "task_1")); + + await assert.rejects( + prepareLaneWorkspace({ + task: { id: "task_1", repo: "fixture/repo" }, + upstream: { baseCommit }, + lane: "control", + runDir: root, + repoCache: path.join(root, "cache"), + sourceRepoPath: source + }), + (error) => { + assert.equal(error.message, "prepareLaneWorkspace refuses to operate through symlinked workspace paths."); + return true; + } + ); + assert.equal(await readFile(outsideSentinelPath, "utf8"), "keep\n"); +}); + +test("prepareLaneWorkspace uses collision-resistant task path segments", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "agentrail-workspace-")); + const { baseCommit, source } = await createFixtureRepo(root); + + const slashTaskWorkspace = await prepareLaneWorkspace({ + task: { id: "a/b", repo: "fixture/repo" }, + upstream: { baseCommit }, + lane: "control", + runDir: root, + repoCache: path.join(root, "cache"), + sourceRepoPath: source + }); + const markerPath = path.join(slashTaskWorkspace.laneDir, "marker.txt"); + await writeFile(markerPath, "keep\n"); + + const underscoreTaskWorkspace = await prepareLaneWorkspace({ + task: { id: "a_b", repo: "fixture/repo" }, + upstream: { baseCommit }, + lane: "control", + runDir: root, + repoCache: path.join(root, "cache"), + sourceRepoPath: source + }); + + assert.notEqual(slashTaskWorkspace.laneDir, underscoreTaskWorkspace.laneDir); + assert.equal(await readFile(markerPath, "utf8"), "keep\n"); +}); + +test("prepareLaneWorkspace rejects repo names without owner and name before creating workspace", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "agentrail-workspace-")); + const { baseCommit, source } = await createFixtureRepo(root); + + await assert.rejects( + prepareLaneWorkspace({ + task: { id: "task_1", repo: "not-owner-name" }, + upstream: { baseCommit }, + lane: "control", + runDir: root, + repoCache: path.join(root, "cache"), + sourceRepoPath: source + }), + (error) => { + assert.equal(error.message, "prepareLaneWorkspace requires task.repo to be an owner/name repository."); + return true; + } + ); + + await assert.rejects(lstat(path.join(root, "tasks")), /ENOENT/u); +}); + +test("prepareLaneWorkspace rejects repo names with unsafe path components before creating workspace", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "agentrail-workspace-")); + const { baseCommit, source } = await createFixtureRepo(root); + + await assert.rejects( + prepareLaneWorkspace({ + task: { id: "task_1", repo: "../repo" }, + upstream: { baseCommit }, + lane: "control", + runDir: root, + repoCache: path.join(root, "cache"), + sourceRepoPath: source + }), + (error) => { + assert.equal(error.message, "prepareLaneWorkspace requires task.repo to be an owner/name repository."); + return true; + } + ); + + await assert.rejects(lstat(path.join(root, "tasks")), /ENOENT/u); +}); + +test("prepareLaneWorkspace migrates stale non-bare repo caches to mirrors before cloning", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "agentrail-workspace-")); + const source = path.join(root, "source"); + execFileSync("git", ["init", source]); + execFileSync("git", ["-C", source, "config", "user.email", "test@example.com"]); + execFileSync("git", ["-C", source, "config", "user.name", "Test User"]); + execFileSync("git", ["-C", source, "branch", "-M", "main"]); + await writeFile(path.join(source, "README.md"), "base\n"); + execFileSync("git", ["-C", source, "add", "README.md"]); + execFileSync("git", ["-C", source, "commit", "-m", "base"]); + const repoCache = path.join(root, "cache"); + const cachePath = path.join(repoCache, "fixture__repo"); + await mkdir(repoCache, { recursive: true }); + execFileSync("git", ["clone", source, cachePath], { stdio: "ignore" }); + + await writeFile(path.join(source, "README.md"), "after\n"); + execFileSync("git", ["-C", source, "commit", "-am", "after"]); + const updatedBaseCommit = execFileSync("git", ["-C", source, "rev-parse", "HEAD"], { encoding: "utf8" }).trim(); + + const workspace = await prepareLaneWorkspace({ + task: { id: "task_1", repo: "fixture/repo" }, + upstream: { baseCommit: updatedBaseCommit }, + lane: "control", + runDir: root, + repoCache + }); + + const checkedOut = execFileSync("git", ["-C", workspace.worktreePath, "rev-parse", "HEAD"], { encoding: "utf8" }).trim(); + const isBareCache = execFileSync("git", ["-C", cachePath, "rev-parse", "--is-bare-repository"], { + encoding: "utf8" + }).trim(); + assert.equal(checkedOut, updatedBaseCommit); + assert.equal(isBareCache, "true"); +}); + +test("prepareLaneWorkspace refreshes existing mirror caches before cloning newer base commits", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "agentrail-workspace-")); + const source = path.join(root, "source"); + execFileSync("git", ["init", source]); + execFileSync("git", ["-C", source, "config", "user.email", "test@example.com"]); + execFileSync("git", ["-C", source, "config", "user.name", "Test User"]); + execFileSync("git", ["-C", source, "branch", "-M", "main"]); + await writeFile(path.join(source, "README.md"), "base\n"); + execFileSync("git", ["-C", source, "add", "README.md"]); + execFileSync("git", ["-C", source, "commit", "-m", "base"]); + const repoCache = path.join(root, "cache"); + const cachePath = path.join(repoCache, "fixture__repo"); + await mkdir(repoCache, { recursive: true }); + execFileSync("git", ["clone", "--mirror", source, cachePath], { stdio: "ignore" }); + + await writeFile(path.join(source, "README.md"), "after\n"); + execFileSync("git", ["-C", source, "commit", "-am", "after"]); + const updatedBaseCommit = execFileSync("git", ["-C", source, "rev-parse", "HEAD"], { encoding: "utf8" }).trim(); + + const workspace = await prepareLaneWorkspace({ + task: { id: "task_1", repo: "fixture/repo" }, + upstream: { baseCommit: updatedBaseCommit }, + lane: "control", + runDir: root, + repoCache + }); + + const checkedOut = execFileSync("git", ["-C", workspace.worktreePath, "rev-parse", "HEAD"], { encoding: "utf8" }).trim(); + assert.equal(checkedOut, updatedBaseCommit); +}); + +test("capturePatch writes a diff and changed file list", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "agentrail-patch-")); + const outputRoot = await mkdtemp(path.join(os.tmpdir(), "agentrail-patch-output-")); + execFileSync("git", ["init", root]); + execFileSync("git", ["-C", root, "config", "user.email", "test@example.com"]); + execFileSync("git", ["-C", root, "config", "user.name", "Test User"]); + await writeFile(path.join(root, "README.md"), "base\n"); + execFileSync("git", ["-C", root, "add", "README.md"]); + execFileSync("git", ["-C", root, "commit", "-m", "base"]); + const baseCommit = execFileSync("git", ["-C", root, "rev-parse", "HEAD"], { encoding: "utf8" }).trim(); + execFileSync("git", ["-C", root, "update-ref", "refs/agentrail-bench/base", baseCommit]); + await writeFile(path.join(root, "README.md"), "changed\n"); + + const patch = await capturePatch({ + worktreePath: root, + outputPath: path.join(outputRoot, "patch.diff") + }); + + assert.equal(patch.status, "available"); + assert.deepEqual(patch.changedFiles, ["README.md"]); + assert.match(await readFile(patch.path, "utf8"), /changed/u); +}); + +test("capturePatch preserves leading and trailing whitespace in changed file names", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "agentrail-patch-")); + const outputRoot = await mkdtemp(path.join(os.tmpdir(), "agentrail-patch-output-")); + const fileName = " leading and trailing .md"; + execFileSync("git", ["init", root]); + execFileSync("git", ["-C", root, "config", "user.email", "test@example.com"]); + execFileSync("git", ["-C", root, "config", "user.name", "Test User"]); + await writeFile(path.join(root, "README.md"), "base\n"); + execFileSync("git", ["-C", root, "add", "README.md"]); + execFileSync("git", ["-C", root, "commit", "-m", "base"]); + const baseCommit = execFileSync("git", ["-C", root, "rev-parse", "HEAD"], { encoding: "utf8" }).trim(); + execFileSync("git", ["-C", root, "update-ref", "refs/agentrail-bench/base", baseCommit]); + await writeFile(path.join(root, fileName), "changed\n"); + + const patch = await capturePatch({ + worktreePath: root, + outputPath: path.join(outputRoot, "patch.diff") + }); + + assert.deepEqual(patch.changedFiles, [fileName]); +}); + +test("capturePatch rejects output paths inside the worktree", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "agentrail-patch-")); + execFileSync("git", ["init", root]); + execFileSync("git", ["-C", root, "config", "user.email", "test@example.com"]); + execFileSync("git", ["-C", root, "config", "user.name", "Test User"]); + await writeFile(path.join(root, "README.md"), "base\n"); + execFileSync("git", ["-C", root, "add", "README.md"]); + execFileSync("git", ["-C", root, "commit", "-m", "base"]); + const baseCommit = execFileSync("git", ["-C", root, "rev-parse", "HEAD"], { encoding: "utf8" }).trim(); + execFileSync("git", ["-C", root, "update-ref", "refs/agentrail-bench/base", baseCommit]); + await writeFile(path.join(root, "README.md"), "changed\n"); + + await assert.rejects( + capturePatch({ + worktreePath: root, + outputPath: path.join(root, "patch.diff") + }), + /capturePatch outputPath must be outside the benchmark worktree\./u + ); +}); + +test("capturePatch rejects output paths that resolve through symlinks into the worktree", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "agentrail-patch-")); + const { baseCommit, source } = await createFixtureRepo(root); + const workspace = await prepareLaneWorkspace({ + task: { id: "task_1", repo: "fixture/repo" }, + upstream: { baseCommit }, + lane: "control", + runDir: root, + repoCache: path.join(root, "cache"), + sourceRepoPath: source + }); + await writeFile(path.join(workspace.worktreePath, "README.md"), "changed\n"); + const worktreeLink = path.join(root, "worktree-link"); + await symlink(workspace.worktreePath, worktreeLink); + + await assert.rejects( + capturePatch({ + worktreePath: workspace.worktreePath, + outputPath: path.join(worktreeLink, "patch.diff") + }), + /capturePatch outputPath must be outside the benchmark worktree\./u + ); +}); + +test("capturePatch rejects dangling output symlinks that point into the worktree", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "agentrail-patch-")); + const outside = await mkdtemp(path.join(os.tmpdir(), "agentrail-patch-output-")); + const { baseCommit, source } = await createFixtureRepo(root); + const workspace = await prepareLaneWorkspace({ + task: { id: "task_1", repo: "fixture/repo" }, + upstream: { baseCommit }, + lane: "control", + runDir: root, + repoCache: path.join(root, "cache"), + sourceRepoPath: source + }); + await writeFile(path.join(workspace.worktreePath, "README.md"), "changed\n"); + const worktreePatchPath = path.join(workspace.worktreePath, "patch.diff"); + const outputPath = path.join(outside, "patch.diff"); + await symlink(worktreePatchPath, outputPath); + + await assert.rejects( + capturePatch({ + worktreePath: workspace.worktreePath, + outputPath + }), + /capturePatch outputPath must be outside the benchmark worktree\./u + ); + await assert.rejects(readFile(worktreePatchPath, "utf8"), /ENOENT/u); +}); + +test("capturePatch rejects benchmark worktrees without a base ref", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "agentrail-patch-")); + const outputRoot = await mkdtemp(path.join(os.tmpdir(), "agentrail-patch-output-")); + execFileSync("git", ["init", root]); + execFileSync("git", ["-C", root, "config", "user.email", "test@example.com"]); + execFileSync("git", ["-C", root, "config", "user.name", "Test User"]); + await writeFile(path.join(root, "README.md"), "base\n"); + execFileSync("git", ["-C", root, "add", "README.md"]); + execFileSync("git", ["-C", root, "commit", "-m", "base"]); + await writeFile(path.join(root, "README.md"), "changed\n"); + + await assert.rejects( + capturePatch({ + worktreePath: root, + outputPath: path.join(outputRoot, "patch.diff") + }), + /capturePatch requires refs\/agentrail-bench\/base in benchmark worktrees\./u + ); +}); + +test("capturePatch includes committed lane changes against the prepared base ref", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "agentrail-patch-")); + const { baseCommit, source } = await createFixtureRepo(root); + const workspace = await prepareLaneWorkspace({ + task: { id: "task_1", repo: "fixture/repo" }, + upstream: { baseCommit }, + lane: "control", + runDir: root, + repoCache: path.join(root, "cache"), + sourceRepoPath: source + }); + execFileSync("git", ["-C", workspace.worktreePath, "config", "user.email", "test@example.com"]); + execFileSync("git", ["-C", workspace.worktreePath, "config", "user.name", "Test User"]); + await writeFile(path.join(workspace.worktreePath, "README.md"), "committed\n"); + execFileSync("git", ["-C", workspace.worktreePath, "commit", "-am", "lane change"]); + + const patch = await capturePatch({ + worktreePath: workspace.worktreePath, + outputPath: path.join(root, "patch.diff") + }); + + assert.equal(patch.status, "available"); + assert.deepEqual(patch.changedFiles, ["README.md"]); + assert.match(await readFile(patch.path, "utf8"), /committed/u); +}); + +test("capturePatch includes staged changes and untracked files", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "agentrail-patch-")); + const { baseCommit, source } = await createFixtureRepo(root); + const workspace = await prepareLaneWorkspace({ + task: { id: "task_1", repo: "fixture/repo" }, + upstream: { baseCommit }, + lane: "control", + runDir: root, + repoCache: path.join(root, "cache"), + sourceRepoPath: source + }); + await writeFile(path.join(workspace.worktreePath, "README.md"), "staged\n"); + execFileSync("git", ["-C", workspace.worktreePath, "add", "README.md"]); + await writeFile(path.join(workspace.worktreePath, "NEW.md"), "untracked\n"); + + const patch = await capturePatch({ + worktreePath: workspace.worktreePath, + outputPath: path.join(root, "patch.diff") + }); + + assert.equal(patch.status, "available"); + assert.deepEqual(patch.changedFiles.toSorted(), ["NEW.md", "README.md"]); + const diff = await readFile(patch.path, "utf8"); + assert.match(diff, /staged/u); + assert.match(diff, /untracked/u); +}); + +async function createFixtureRepo(root) { + const source = path.join(root, "source"); + execFileSync("git", ["init", source]); + execFileSync("git", ["-C", source, "config", "user.email", "test@example.com"]); + execFileSync("git", ["-C", source, "config", "user.name", "Test User"]); + execFileSync("git", ["-C", source, "branch", "-M", "main"]); + await writeFile(path.join(source, "README.md"), "base\n"); + execFileSync("git", ["-C", source, "add", "README.md"]); + execFileSync("git", ["-C", source, "commit", "-m", "base"]); + const baseCommit = execFileSync("git", ["-C", source, "rev-parse", "HEAD"], { encoding: "utf8" }).trim(); + await writeFile(path.join(source, "README.md"), "after\n"); + execFileSync("git", ["-C", source, "commit", "-am", "after"]); + return { baseCommit, source }; +} + +function taskPathSegment(taskId) { + const safeTaskId = taskId.replaceAll(/[^A-Za-z0-9._-]/gu, "_"); + const hash = createHash("sha256").update(taskId).digest("hex").slice(0, 12); + return `${safeTaskId}__${hash}`; +} From 73b01b56994e862ad3fd92c7c5f67cc3c1a91c04 Mon Sep 17 00:00:00 2001 From: Onyeka Nwamba Date: Mon, 25 May 2026 04:00:37 +0100 Subject: [PATCH 05/18] feat: add benchmark prompts and runner commands --- .../fixtures/fake-runner.mjs | 36 ++++++ .../agentrail-swe-lifecycle/lib/prompt.mjs | 43 +++++++ .../lib/runner-command.mjs | 109 ++++++++++++++++++ ...gentrail-swe-lifecycle-real-runner.test.js | 53 +++++++++ 4 files changed, 241 insertions(+) create mode 100644 benchmarks/agentrail-swe-lifecycle/fixtures/fake-runner.mjs create mode 100644 benchmarks/agentrail-swe-lifecycle/lib/prompt.mjs create mode 100644 benchmarks/agentrail-swe-lifecycle/lib/runner-command.mjs diff --git a/benchmarks/agentrail-swe-lifecycle/fixtures/fake-runner.mjs b/benchmarks/agentrail-swe-lifecycle/fixtures/fake-runner.mjs new file mode 100644 index 0000000..a362853 --- /dev/null +++ b/benchmarks/agentrail-swe-lifecycle/fixtures/fake-runner.mjs @@ -0,0 +1,36 @@ +#!/usr/bin/env node +import { execFileSync } from "node:child_process"; +import { appendFile, readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; + +const repoPath = process.cwd(); +const readmePath = path.join(repoPath, "README.md"); +const previous = await readFile(readmePath, "utf8").catch(() => ""); +await writeFile(readmePath, `${previous.trimEnd()}\nfake runner completed\n`, "utf8"); + +execFileSync("git", ["-C", repoPath, "add", "README.md"]); +execFileSync("git", ["-C", repoPath, "config", "user.email", "bench@example.com"]); +execFileSync("git", ["-C", repoPath, "config", "user.name", "Benchmark Runner"]); +execFileSync("git", ["-C", repoPath, "commit", "-m", "fake runner change"]); +const commitSha = execFileSync("git", ["-C", repoPath, "rev-parse", "HEAD"], { encoding: "utf8" }).trim(); + +if (process.env.AGENTRAIL_HANDOFF_PATH) { + await writeFile(process.env.AGENTRAIL_HANDOFF_PATH, `${JSON.stringify({ + version: 1, + status: "completed", + summary: "Fake runner completed benchmark task.", + checks: [], + artifacts: [], + blocker: null, + commitSha, + }, null, 2)}\n`, "utf8"); +} + +if (process.env.AGENTRAIL_BENCH_EXTRA_LOG_PATH) { + await appendFile( + process.env.AGENTRAIL_BENCH_EXTRA_LOG_PATH, + `${JSON.stringify({ usage: { input_tokens: 100, output_tokens: 20, reasoning_tokens: 5 } })}\n`, + ); +} + +process.stdout.write(`${JSON.stringify({ usage: { input_tokens: 100, output_tokens: 20, reasoning_tokens: 5, total_tokens: 125 } })}\n`); diff --git a/benchmarks/agentrail-swe-lifecycle/lib/prompt.mjs b/benchmarks/agentrail-swe-lifecycle/lib/prompt.mjs new file mode 100644 index 0000000..9ca185f --- /dev/null +++ b/benchmarks/agentrail-swe-lifecycle/lib/prompt.mjs @@ -0,0 +1,43 @@ +export function buildControlPrompt({ task, upstream }) { + const lines = [ + "You are solving a software engineering benchmark task.", + "", + `Repository: ${task.repo}`, + `Instance: ${upstream.instanceId}`, + `Base commit: ${upstream.baseCommit}`, + "", + "Problem statement:", + upstream.problemStatement, + ]; + + if (upstream.requirements) lines.push("", "Requirements:", upstream.requirements); + if (upstream.interface) lines.push("", "Interface notes:", upstream.interface); + + lines.push( + "", + "Instructions:", + "- Modify the repository to solve the problem.", + "- Do not invent external services or credentials.", + "- Keep changes focused on the requested fix.", + "- When finished, leave the final patch in the git working tree.", + ); + + return lines.join("\n"); +} + +export function buildAgentRailTaskFields({ task, upstream }) { + return { + identifier: `swe:${upstream.instanceId}`, + title: `SWE benchmark task ${upstream.instanceId}`, + description: buildControlPrompt({ task, upstream }), + acceptanceCriteria: [ + "The repository is patched to solve the problem statement.", + "The final patch is left in the git working tree.", + "No gold patch or hidden test metadata is used.", + ], + context: { + project: task.repo, + goal: "Solve a SWE-style benchmark issue through AgentRail.", + }, + }; +} diff --git a/benchmarks/agentrail-swe-lifecycle/lib/runner-command.mjs b/benchmarks/agentrail-swe-lifecycle/lib/runner-command.mjs new file mode 100644 index 0000000..04071da --- /dev/null +++ b/benchmarks/agentrail-swe-lifecycle/lib/runner-command.mjs @@ -0,0 +1,109 @@ +import { spawn } from "node:child_process"; +import { access, mkdir, writeFile } from "node:fs/promises"; +import path from "node:path"; + +export async function runRunnerCommand({ runner, runnerCommand, model, repoPath, prompt, logDir, timeoutSeconds, env = {} }) { + await mkdir(logDir, { recursive: true }); + const promptPath = path.join(logDir, "prompt.md"); + const stdoutPath = path.join(logDir, "stdout.log"); + const stderrPath = path.join(logDir, "stderr.log"); + await writeFile(promptPath, prompt, "utf8"); + + const command = await resolveRunnerCommand({ runner, runnerCommand, model, promptPath }); + const startedAt = Date.now(); + const result = await spawnWithTimeout({ + command: command.executable, + args: command.args, + cwd: repoPath, + env: { ...process.env, ...env, AGENTRAIL_BENCH_PROMPT_PATH: promptPath }, + timeoutMs: timeoutSeconds * 1000, + }); + + await writeFile(stdoutPath, result.stdout, "utf8"); + await writeFile(stderrPath, result.stderr, "utf8"); + + return { + status: result.timedOut ? "timed_out" : result.exitCode === 0 ? "completed" : "failed", + exitCode: result.exitCode, + durationMs: Date.now() - startedAt, + promptPath, + stdoutPath, + stderrPath, + command: command.redacted, + }; +} + +export async function resolveRunnerCommand({ runner, runnerCommand, model, promptPath }) { + if (runner === "custom") { + const [executable, ...args] = splitCommand(runnerCommand); + return { + executable, + args: await resolveRepoRelativeArgs(args), + redacted: [executable, ...args].join(" "), + }; + } + if (runner === "codex") { + const args = ["--model", model, "--sandbox", "workspace-write", "exec", "--prompt-file", promptPath].filter(Boolean); + return { executable: "codex", args, redacted: ["codex", ...args].join(" ") }; + } + if (runner === "cursor") { + const args = ["agent", "--model", model, "--prompt-file", promptPath].filter(Boolean); + return { executable: "cursor", args, redacted: ["cursor", ...args].join(" ") }; + } + if (runner === "claude-code") { + const args = ["--model", model, "--print", `@${promptPath}`].filter(Boolean); + return { executable: "claude", args, redacted: ["claude", ...args].join(" ") }; + } + throw new Error(`Unsupported runner: ${runner}`); +} + +function splitCommand(command) { + const parts = String(command).match(/(?:[^\s"]+|"[^"]*")+/gu)?.map((part) => part.replace(/^"|"$/gu, "")) ?? []; + if (parts.length === 0) throw new Error("--runner-command must not be empty."); + return parts; +} + +async function resolveRepoRelativeArgs(args) { + const resolved = []; + for (const arg of args) { + resolved.push(await resolveExistingRelativePathArg(arg)); + } + return resolved; +} + +async function resolveExistingRelativePathArg(arg) { + if (arg.startsWith("-") || path.isAbsolute(arg)) return arg; + const absolute = path.resolve(process.cwd(), arg); + try { + await access(absolute); + return absolute; + } catch (error) { + if (error?.code === "ENOENT") return arg; + throw error; + } +} + +async function spawnWithTimeout({ command, args, cwd, env, timeoutMs }) { + return await new Promise((resolve, reject) => { + const child = spawn(command, args, { cwd, env, stdio: ["ignore", "pipe", "pipe"] }); + const stdout = []; + const stderr = []; + let timedOut = false; + const timer = setTimeout(() => { + timedOut = true; + child.kill("SIGTERM"); + }, timeoutMs); + child.stdout.on("data", (chunk) => stdout.push(Buffer.from(chunk))); + child.stderr.on("data", (chunk) => stderr.push(Buffer.from(chunk))); + child.on("error", reject); + child.on("close", (exitCode) => { + clearTimeout(timer); + resolve({ + exitCode, + timedOut, + stdout: Buffer.concat(stdout).toString("utf8"), + stderr: Buffer.concat(stderr).toString("utf8"), + }); + }); + }); +} diff --git a/test/agentrail-swe-lifecycle-real-runner.test.js b/test/agentrail-swe-lifecycle-real-runner.test.js index 9a9b4b5..65486c2 100644 --- a/test/agentrail-swe-lifecycle-real-runner.test.js +++ b/test/agentrail-swe-lifecycle-real-runner.test.js @@ -1,7 +1,13 @@ import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { mkdtemp, readFile, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; import test from "node:test"; import { parseBenchmarkArgs } from "../benchmarks/agentrail-swe-lifecycle/lib/args.mjs"; +import { buildControlPrompt } from "../benchmarks/agentrail-swe-lifecycle/lib/prompt.mjs"; +import { runRunnerCommand } from "../benchmarks/agentrail-swe-lifecycle/lib/runner-command.mjs"; test("parseBenchmarkArgs keeps --dry-run as a dry-run mode alias", () => { const flags = parseBenchmarkArgs(["--dry-run", "--count", "1"]); @@ -56,3 +62,50 @@ test("parseBenchmarkArgs rejects command evaluator without a template", () => { /--eval-command-template is required when --evaluator command is used/u, ); }); + +test("buildControlPrompt includes problem input and excludes gold fields", () => { + const prompt = buildControlPrompt({ + task: { id: "task_1", repo: "owner/repo" }, + upstream: { + instanceId: "owner__repo-1", + problemStatement: "Fix the bug.", + requirements: "Keep API stable.", + interface: "Public function stays the same.", + baseCommit: "abc", + patch: "must not appear", + test_patch: "must not appear", + fail_to_pass: ["must not appear"], + }, + }); + assert.match(prompt, /Fix the bug/u); + assert.match(prompt, /Keep API stable/u); + assert.doesNotMatch(prompt, /test_patch/u); + assert.doesNotMatch(prompt, /must not appear/u); +}); + +test("runRunnerCommand invokes a custom runner and stores stdout and stderr", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "agentrail-runner-command-")); + const repo = path.join(root, "repo"); + execFileSync("git", ["init", repo]); + execFileSync("git", ["-C", repo, "config", "user.email", "test@example.com"]); + execFileSync("git", ["-C", repo, "config", "user.name", "Test User"]); + await writeFile(path.join(repo, "README.md"), "base\n"); + execFileSync("git", ["-C", repo, "add", "README.md"]); + execFileSync("git", ["-C", repo, "commit", "-m", "base"]); + + const result = await runRunnerCommand({ + runner: "custom", + runnerCommand: "node benchmarks/agentrail-swe-lifecycle/fixtures/fake-runner.mjs", + model: "fake-model", + repoPath: repo, + prompt: "Change README", + logDir: root, + timeoutSeconds: 30, + env: {}, + }); + + assert.equal(result.exitCode, 0); + assert.equal(result.status, "completed"); + assert.match(await readFile(path.join(repo, "README.md"), "utf8"), /fake runner completed/u); + assert.match(await readFile(result.stdoutPath, "utf8"), /total_tokens/u); +}); From 02a373389445c24eda8d5d5ae183076de6f3879f Mon Sep 17 00:00:00 2001 From: Onyeka Nwamba Date: Mon, 25 May 2026 04:01:50 +0100 Subject: [PATCH 06/18] feat: add benchmark command evaluator --- .../fixtures/fake-evaluator.mjs | 19 +++++++ .../agentrail-swe-lifecycle/lib/evaluator.mjs | 51 +++++++++++++++++++ ...gentrail-swe-lifecycle-real-runner.test.js | 24 +++++++++ 3 files changed, 94 insertions(+) create mode 100644 benchmarks/agentrail-swe-lifecycle/fixtures/fake-evaluator.mjs create mode 100644 benchmarks/agentrail-swe-lifecycle/lib/evaluator.mjs diff --git a/benchmarks/agentrail-swe-lifecycle/fixtures/fake-evaluator.mjs b/benchmarks/agentrail-swe-lifecycle/fixtures/fake-evaluator.mjs new file mode 100644 index 0000000..eb43c0b --- /dev/null +++ b/benchmarks/agentrail-swe-lifecycle/fixtures/fake-evaluator.mjs @@ -0,0 +1,19 @@ +#!/usr/bin/env node +import { readFile, writeFile } from "node:fs/promises"; + +const args = parseArgs(process.argv.slice(2)); +const patch = await readFile(args.patch, "utf8"); +const passed = patch.includes("fake runner completed"); +await writeFile(args.output, `${JSON.stringify({ passed, reason: passed ? "expected change found" : "expected change missing" }, null, 2)}\n`, "utf8"); +process.exitCode = passed ? 0 : 1; + +function parseArgs(argv) { + const parsed = {}; + for (let index = 0; index < argv.length; index += 1) { + if (argv[index] === "--patch") parsed.patch = argv[++index]; + else if (argv[index] === "--output") parsed.output = argv[++index]; + else throw new Error(`Unknown evaluator argument: ${argv[index]}`); + } + if (!parsed.patch || !parsed.output) throw new Error("--patch and --output are required."); + return parsed; +} diff --git a/benchmarks/agentrail-swe-lifecycle/lib/evaluator.mjs b/benchmarks/agentrail-swe-lifecycle/lib/evaluator.mjs new file mode 100644 index 0000000..fd1f090 --- /dev/null +++ b/benchmarks/agentrail-swe-lifecycle/lib/evaluator.mjs @@ -0,0 +1,51 @@ +import { spawn } from "node:child_process"; + +export async function runEvaluator({ evaluator, evalCommandTemplate, patchPath, outputPath, cwd = process.cwd(), timeoutSeconds = 300 }) { + if (evaluator === "none" || !evaluator) { + return { status: "unavailable", exitCode: null, outputPath: null }; + } + if (evaluator !== "command") throw new Error(`Unsupported evaluator: ${evaluator}`); + + const command = materializeTemplate(evalCommandTemplate, { patchPath, outputPath }); + const [executable, ...args] = splitCommand(command); + const result = await spawnWithTimeout({ executable, args, cwd, timeoutMs: timeoutSeconds * 1000 }); + return { + status: result.exitCode === 0 ? "passed" : "failed", + exitCode: result.exitCode, + outputPath, + stdout: result.stdout, + stderr: result.stderr, + }; +} + +function materializeTemplate(template, values) { + return String(template) + .replaceAll("{patchPath}", values.patchPath) + .replaceAll("{outputPath}", values.outputPath); +} + +function splitCommand(command) { + const parts = String(command).match(/(?:[^\s"]+|"[^"]*")+/gu)?.map((part) => part.replace(/^"|"$/gu, "")) ?? []; + if (parts.length === 0) throw new Error("Evaluator command must not be empty."); + return parts; +} + +async function spawnWithTimeout({ executable, args, cwd, timeoutMs }) { + return await new Promise((resolve, reject) => { + const child = spawn(executable, args, { cwd, stdio: ["ignore", "pipe", "pipe"] }); + const stdout = []; + const stderr = []; + const timer = setTimeout(() => child.kill("SIGTERM"), timeoutMs); + child.stdout.on("data", (chunk) => stdout.push(Buffer.from(chunk))); + child.stderr.on("data", (chunk) => stderr.push(Buffer.from(chunk))); + child.on("error", reject); + child.on("close", (exitCode) => { + clearTimeout(timer); + resolve({ + exitCode, + stdout: Buffer.concat(stdout).toString("utf8"), + stderr: Buffer.concat(stderr).toString("utf8"), + }); + }); + }); +} diff --git a/test/agentrail-swe-lifecycle-real-runner.test.js b/test/agentrail-swe-lifecycle-real-runner.test.js index 65486c2..c14ad4b 100644 --- a/test/agentrail-swe-lifecycle-real-runner.test.js +++ b/test/agentrail-swe-lifecycle-real-runner.test.js @@ -6,6 +6,7 @@ import path from "node:path"; import test from "node:test"; import { parseBenchmarkArgs } from "../benchmarks/agentrail-swe-lifecycle/lib/args.mjs"; +import { runEvaluator } from "../benchmarks/agentrail-swe-lifecycle/lib/evaluator.mjs"; import { buildControlPrompt } from "../benchmarks/agentrail-swe-lifecycle/lib/prompt.mjs"; import { runRunnerCommand } from "../benchmarks/agentrail-swe-lifecycle/lib/runner-command.mjs"; @@ -109,3 +110,26 @@ test("runRunnerCommand invokes a custom runner and stores stdout and stderr", as assert.match(await readFile(path.join(repo, "README.md"), "utf8"), /fake runner completed/u); assert.match(await readFile(result.stdoutPath, "utf8"), /total_tokens/u); }); + +test("runEvaluator returns unavailable for evaluator none", async () => { + const result = await runEvaluator({ evaluator: "none" }); + assert.deepEqual(result, { status: "unavailable", exitCode: null, outputPath: null }); +}); + +test("runEvaluator runs a command template with patch and output paths", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "agentrail-evaluator-")); + const patchPath = path.join(root, "patch.diff"); + const outputPath = path.join(root, "evaluator.json"); + await writeFile(patchPath, "diff --git a/README.md b/README.md\n+fake runner completed\n"); + const result = await runEvaluator({ + evaluator: "command", + evalCommandTemplate: "node benchmarks/agentrail-swe-lifecycle/fixtures/fake-evaluator.mjs --patch {patchPath} --output {outputPath}", + patchPath, + outputPath, + cwd: path.resolve("."), + timeoutSeconds: 30, + }); + assert.equal(result.status, "passed"); + assert.equal(result.exitCode, 0); + assert.equal(JSON.parse(await readFile(outputPath, "utf8")).passed, true); +}); From 15676d011075713b4b0dfd999a787497bf13ff31 Mon Sep 17 00:00:00 2001 From: Onyeka Nwamba Date: Mon, 25 May 2026 04:02:59 +0100 Subject: [PATCH 07/18] feat: run direct benchmark control lane --- .../lib/control-lane.mjs | 58 +++++++++++++++++++ ...gentrail-swe-lifecycle-real-runner.test.js | 35 +++++++++++ 2 files changed, 93 insertions(+) create mode 100644 benchmarks/agentrail-swe-lifecycle/lib/control-lane.mjs diff --git a/benchmarks/agentrail-swe-lifecycle/lib/control-lane.mjs b/benchmarks/agentrail-swe-lifecycle/lib/control-lane.mjs new file mode 100644 index 0000000..13b9bf9 --- /dev/null +++ b/benchmarks/agentrail-swe-lifecycle/lib/control-lane.mjs @@ -0,0 +1,58 @@ +import path from "node:path"; + +import { runEvaluator } from "./evaluator.mjs"; +import { capturePatch } from "./patch.mjs"; +import { buildControlPrompt } from "./prompt.mjs"; +import { runRunnerCommand } from "./runner-command.mjs"; +import { parseTokenUsageFromFiles } from "./token-usage.mjs"; +import { prepareLaneWorkspace } from "./workspace.mjs"; + +export async function runControlLane({ task, upstream, runDir, repoCache, sourceRepoPath, flags }) { + const workspace = await prepareLaneWorkspace({ + task, + upstream, + lane: "control", + runDir, + repoCache, + sourceRepoPath, + }); + const prompt = buildControlPrompt({ task, upstream }); + const runner = await runRunnerCommand({ + runner: flags.runner, + runnerCommand: flags.runnerCommand, + model: flags.model ?? null, + repoPath: workspace.worktreePath, + prompt, + logDir: workspace.laneDir, + timeoutSeconds: flags.timeoutSeconds ?? 1800, + env: {}, + }); + const patch = await capturePatch({ + worktreePath: workspace.worktreePath, + outputPath: path.join(workspace.laneDir, "patch.diff"), + }); + const tokenUsage = await parseTokenUsageFromFiles([runner.stdoutPath, runner.stderrPath]); + const evaluator = await runEvaluator({ + evaluator: flags.evaluator, + evalCommandTemplate: flags.evalCommandTemplate, + patchPath: patch.path, + outputPath: path.join(workspace.laneDir, "evaluator.json"), + timeoutSeconds: flags.timeoutSeconds ?? 1800, + }); + const success = runner.status === "completed" + && patch.status === "available" + && (evaluator.status === "passed" || evaluator.status === "unavailable"); + + return { + status: runner.status, + success, + runner, + tokenUsage, + patch, + evaluator, + workspace: { + laneDir: workspace.laneDir, + worktreePath: workspace.worktreePath, + }, + }; +} diff --git a/test/agentrail-swe-lifecycle-real-runner.test.js b/test/agentrail-swe-lifecycle-real-runner.test.js index c14ad4b..45f6a68 100644 --- a/test/agentrail-swe-lifecycle-real-runner.test.js +++ b/test/agentrail-swe-lifecycle-real-runner.test.js @@ -6,6 +6,7 @@ import path from "node:path"; import test from "node:test"; import { parseBenchmarkArgs } from "../benchmarks/agentrail-swe-lifecycle/lib/args.mjs"; +import { runControlLane } from "../benchmarks/agentrail-swe-lifecycle/lib/control-lane.mjs"; import { runEvaluator } from "../benchmarks/agentrail-swe-lifecycle/lib/evaluator.mjs"; import { buildControlPrompt } from "../benchmarks/agentrail-swe-lifecycle/lib/prompt.mjs"; import { runRunnerCommand } from "../benchmarks/agentrail-swe-lifecycle/lib/runner-command.mjs"; @@ -133,3 +134,37 @@ test("runEvaluator runs a command template with patch and output paths", async ( assert.equal(result.exitCode, 0); assert.equal(JSON.parse(await readFile(outputPath, "utf8")).passed, true); }); + +test("runControlLane executes a fake runner, captures tokens, patch, and evaluator result", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "agentrail-control-lane-")); + const source = path.join(root, "source"); + execFileSync("git", ["init", source]); + execFileSync("git", ["-C", source, "config", "user.email", "test@example.com"]); + execFileSync("git", ["-C", source, "config", "user.name", "Test User"]); + await writeFile(path.join(source, "README.md"), "base\n"); + execFileSync("git", ["-C", source, "add", "README.md"]); + execFileSync("git", ["-C", source, "commit", "-m", "base"]); + const baseCommit = execFileSync("git", ["-C", source, "rev-parse", "HEAD"], { encoding: "utf8" }).trim(); + + const result = await runControlLane({ + task: { id: "task_1", repo: "fixture/repo" }, + upstream: { instanceId: "fixture__repo-1", baseCommit, problemStatement: "Change README" }, + runDir: root, + repoCache: path.join(root, "cache"), + sourceRepoPath: source, + flags: { + runner: "custom", + runnerCommand: "node benchmarks/agentrail-swe-lifecycle/fixtures/fake-runner.mjs", + model: "fake-model", + timeoutSeconds: 30, + evaluator: "command", + evalCommandTemplate: "node benchmarks/agentrail-swe-lifecycle/fixtures/fake-evaluator.mjs --patch {patchPath} --output {outputPath}", + }, + }); + + assert.equal(result.status, "completed"); + assert.equal(result.success, true); + assert.equal(result.tokenUsage.totalTokens, 125); + assert.equal(result.patch.status, "available"); + assert.equal(result.evaluator.status, "passed"); +}); From 225aa0391def6318565eb7580d011ef6afdfd354 Mon Sep 17 00:00:00 2001 From: Onyeka Nwamba Date: Mon, 25 May 2026 04:07:02 +0100 Subject: [PATCH 08/18] feat: add local AgentRail benchmark harness --- .../lib/agentrail-harness.mjs | 144 ++++++++++++++++++ ...gentrail-swe-lifecycle-real-runner.test.js | 26 ++++ 2 files changed, 170 insertions(+) create mode 100644 benchmarks/agentrail-swe-lifecycle/lib/agentrail-harness.mjs diff --git a/benchmarks/agentrail-swe-lifecycle/lib/agentrail-harness.mjs b/benchmarks/agentrail-swe-lifecycle/lib/agentrail-harness.mjs new file mode 100644 index 0000000..3a3daab --- /dev/null +++ b/benchmarks/agentrail-swe-lifecycle/lib/agentrail-harness.mjs @@ -0,0 +1,144 @@ +import { once } from "node:events"; +import { mkdir, writeFile } from "node:fs/promises"; +import path from "node:path"; + +import { AgentAuthStore } from "../../../src/agent-auth-store.ts"; +import { AgentRunStore } from "../../../src/agent-run-store.ts"; +import { AgentTaskQueue } from "../../../src/agent-task-queue.ts"; +import { createServer } from "../../../src/app.ts"; +import { TaskEventStore } from "../../../src/task-event-store.ts"; +import { buildAgentRailTaskFields } from "./prompt.mjs"; + +export async function withAgentRailBenchmarkHarness({ runDir, task, upstream, agentId = "agt_bench", flags = {} }, handler) { + const storeDir = path.join(runDir, "agentrail-store"); + await mkdir(storeDir, { recursive: true }); + + const now = () => new Date("2026-05-25T00:00:00.000Z"); + const eventStore = new TaskEventStore({ now, storagePath: path.join(storeDir, "events.json") }); + const submissions = []; + let baseUrl = "http://127.0.0.1:0"; + const taskQueue = new AgentTaskQueue({ + now, + eventStore, + storagePath: path.join(storeDir, "tasks.json"), + delegate: { + submitTask: async (taskId, payload) => { + const payloadObject = isRecord(payload) ? payload : {}; + const pullRequest = isRecord(payloadObject.pullRequest) ? payloadObject.pullRequest : {}; + const submissionId = `sub_bench_${submissions.length + 1}`; + const response = { + data: { + taskId, + submissionId, + prUrl: `${baseUrl}/benchmark/pr/${submissions.length + 1}`, + prNumber: submissions.length + 1, + head: typeof pullRequest.head === "string" ? pullRequest.head : null, + base: typeof pullRequest.base === "string" ? pullRequest.base : null, + headSha: typeof pullRequest.headSha === "string" ? pullRequest.headSha : null, + }, + availableActions: ["view_ci_status", "view_review_feedback"], + }; + submissions.push(response.data); + return response; + }, + }, + }); + const authStore = new AgentAuthStore({ now, storagePath: path.join(storeDir, "auth.json") }); + const { data } = authStore.createKey({ + agent: { + id: agentId, + displayName: "Benchmark Agent", + role: "benchmark_agent", + externalIdentities: [], + }, + scopes: ["tasks:read", "tasks:write", "events:read", "ci:read", "reviews:read", "ship:write"], + }, `benchmark-key:${agentId}`); + const agentRailTask = taskQueue.createTask(buildTaskRecord({ task, upstream, agentId })); + + const server = createServer({ + store: eventStore, + authStore, + agentRunStore: new AgentRunStore({ now, storagePath: path.join(storeDir, "agent-runs.json") }), + taskLifecycleStore: taskQueue, + ciStatusAdapter: null, + reviewFeedbackAdapter: null, + now, + }); + + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("Missing AgentRail benchmark server address."); + baseUrl = `http://${address.address}:${address.port}`; + + const envFilePath = path.join(runDir, "agentrail.env"); + await writeAgentEnv({ + envFilePath, + baseUrl, + apiKey: data.apiKey, + agentId, + task, + flags, + }); + + try { + return await handler({ + baseUrl, + apiKey: data.apiKey, + agentId, + agentRailTaskId: agentRailTask.id, + envFilePath, + submissions, + }); + } finally { + await new Promise((resolve, reject) => { + server.close((error) => { + if (error) reject(error); + else resolve(); + }); + }); + } +} + +function buildTaskRecord({ task, upstream, agentId }) { + const fields = buildAgentRailTaskFields({ task, upstream }); + const [owner, repo] = task.repo.split("/"); + return { + identifier: fields.identifier, + title: fields.title, + description: fields.description, + status: "todo", + priority: "medium", + assignee: { id: agentId, name: "Benchmark Agent" }, + assigneeAgentId: agentId, + acceptanceCriteria: fields.acceptanceCriteria, + context: fields.context, + availableActions: ["start"], + source: { + provider: "github", + owner, + repo, + baseBranch: "main", + }, + }; +} + +async function writeAgentEnv({ envFilePath, baseUrl, apiKey, agentId, task, flags }) { + await mkdir(path.dirname(envFilePath), { recursive: true }); + await writeFile(envFilePath, [ + `AGENTRAIL_BASE_URL=${baseUrl}`, + `AGENTRAIL_API_KEY=${apiKey}`, + `AGENTRAIL_AGENT_ID=${agentId}`, + `AGENTRAIL_AGENT_RUNNER=${flags.runner ?? "custom"}`, + `AGENTRAIL_AGENT_MODEL=${flags.model ?? "fake-model"}`, + "AGENTRAIL_MAX_CONCURRENT_TASKS=1", + `AGENTRAIL_REPO_ALLOWLIST=${task.repo}`, + `AGENTRAIL_RUNNER_TIMEOUT_SECONDS=${flags.timeoutSeconds ?? 30}`, + flags.runnerCommand ? `AGENTRAIL_BENCH_RUNNER_COMMAND=${flags.runnerCommand}` : null, + "", + ].filter((line) => line !== null).join("\n"), { encoding: "utf8", mode: 0o600 }); +} + +function isRecord(value) { + return value !== null && typeof value === "object" && !Array.isArray(value); +} diff --git a/test/agentrail-swe-lifecycle-real-runner.test.js b/test/agentrail-swe-lifecycle-real-runner.test.js index 45f6a68..5c9e06b 100644 --- a/test/agentrail-swe-lifecycle-real-runner.test.js +++ b/test/agentrail-swe-lifecycle-real-runner.test.js @@ -6,6 +6,7 @@ import path from "node:path"; import test from "node:test"; import { parseBenchmarkArgs } from "../benchmarks/agentrail-swe-lifecycle/lib/args.mjs"; +import { withAgentRailBenchmarkHarness } from "../benchmarks/agentrail-swe-lifecycle/lib/agentrail-harness.mjs"; import { runControlLane } from "../benchmarks/agentrail-swe-lifecycle/lib/control-lane.mjs"; import { runEvaluator } from "../benchmarks/agentrail-swe-lifecycle/lib/evaluator.mjs"; import { buildControlPrompt } from "../benchmarks/agentrail-swe-lifecycle/lib/prompt.mjs"; @@ -168,3 +169,28 @@ test("runControlLane executes a fake runner, captures tokens, patch, and evaluat assert.equal(result.patch.status, "available"); assert.equal(result.evaluator.status, "passed"); }); + +test("withAgentRailBenchmarkHarness exposes a task and records synthetic submission", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "agentrail-harness-")); + const result = await withAgentRailBenchmarkHarness({ + runDir: root, + task: { id: "task_1", repo: "fixture/repo" }, + upstream: { instanceId: "fixture__repo-1", problemStatement: "Change README", baseCommit: "0123456789abcdef0123456789abcdef01234567" }, + agentId: "agt_bench", + }, async (harness) => { + const response = await fetch(`${harness.baseUrl}/tasks/${harness.agentRailTaskId}`, { + headers: { authorization: `Bearer ${harness.apiKey}` }, + }); + assert.equal(response.status, 200); + const body = await response.json(); + assert.equal(body.data.id, harness.agentRailTaskId); + return { + baseUrl: harness.baseUrl, + taskId: harness.agentRailTaskId, + envFilePath: harness.envFilePath, + }; + }); + assert.match(result.baseUrl, /^http:\/\/127\.0\.0\.1:/u); + assert.match(result.taskId, /^tsk_/u); + assert.match(await readFile(result.envFilePath, "utf8"), /AGENTRAIL_AGENT_ID=agt_bench/u); +}); From d6d151f90240d16328aba5b6fbba6c4a082e7edd Mon Sep 17 00:00:00 2001 From: Onyeka Nwamba Date: Mon, 25 May 2026 04:09:56 +0100 Subject: [PATCH 09/18] feat: run AgentRail benchmark lane --- .../lib/agentrail-lane.mjs | 188 ++++++++++++++++++ ...gentrail-swe-lifecycle-real-runner.test.js | 37 ++++ 2 files changed, 225 insertions(+) create mode 100644 benchmarks/agentrail-swe-lifecycle/lib/agentrail-lane.mjs diff --git a/benchmarks/agentrail-swe-lifecycle/lib/agentrail-lane.mjs b/benchmarks/agentrail-swe-lifecycle/lib/agentrail-lane.mjs new file mode 100644 index 0000000..dff280b --- /dev/null +++ b/benchmarks/agentrail-swe-lifecycle/lib/agentrail-lane.mjs @@ -0,0 +1,188 @@ +import { execFile } from "node:child_process"; +import path from "node:path"; +import { promisify } from "node:util"; + +import { defaultPrepareWorktree } from "../../../src/cli/agent-runner.ts"; +import { runCli } from "../../../src/cli/index.ts"; +import { createSetupConfig } from "../../../src/cli/setup-config.ts"; +import { writeSetupFiles } from "../../../src/cli/setup-files.ts"; +import { capturePatch } from "./patch.mjs"; +import { runEvaluator } from "./evaluator.mjs"; +import { runRunnerCommand } from "./runner-command.mjs"; +import { parseTokenUsageFromFiles } from "./token-usage.mjs"; +import { prepareLaneWorkspace } from "./workspace.mjs"; +import { withAgentRailBenchmarkHarness } from "./agentrail-harness.mjs"; + +const execFileAsync = promisify(execFile); +const BASE_REF = "refs/agentrail-bench/base"; + +export async function runAgentRailLane({ task, upstream, runDir, repoCache, sourceRepoPath, flags }) { + const workspace = await prepareLaneWorkspace({ + task, + upstream, + lane: "agentrail", + runDir, + repoCache, + sourceRepoPath, + }); + const agentRailHome = path.join(workspace.laneDir, "agentrail-home"); + const stdout = createMemoryWriter(); + const stderr = createMemoryWriter(); + let runnerResult = null; + let managedWorktreePath = null; + + const result = await withAgentRailBenchmarkHarness({ + runDir: workspace.laneDir, + task, + upstream, + flags, + }, async (harness) => { + await writeBenchmarkSetup({ + homePath: agentRailHome, + repoPath: workspace.worktreePath, + task, + baseUrl: harness.baseUrl, + flags, + }); + + const previousHome = process.env.AGENTRAIL_HOME; + const previousTelemetryDisabled = process.env.AGENTRAIL_TELEMETRY_DISABLED; + process.env.AGENTRAIL_HOME = agentRailHome; + process.env.AGENTRAIL_TELEMETRY_DISABLED = "1"; + try { + const exitCode = await runCli([ + "agent", + "run", + "--once", + "--env-file", + harness.envFilePath, + "--worktree-root", + path.join(workspace.laneDir, "managed-worktrees"), + "--json", + ], { + cwd: workspace.worktreePath, + stdout, + stderr, + agentRunner: { + prepareWorktree: async (params) => { + await defaultPrepareWorktree(params); + await git(["-C", params.worktreePath, "update-ref", BASE_REF, upstream.baseCommit]); + }, + launchRunner: async ({ worktreePath, prompt, logPath, handoffPath, env }) => { + managedWorktreePath = worktreePath; + runnerResult = await runRunnerCommand({ + runner: flags.runner, + runnerCommand: flags.runnerCommand, + model: flags.model ?? null, + repoPath: worktreePath, + prompt, + logDir: path.dirname(logPath), + timeoutSeconds: flags.timeoutSeconds ?? 1800, + env: { + ...env, + ...(handoffPath ? { + AGENTRAIL_HANDOFF_PATH: handoffPath, + AGENTRAIL_RESULT_PATH: handoffPath, + } : {}), + }, + }); + return { + status: runnerResult.status === "completed" ? "succeeded" : "failed", + exitCode: runnerResult.exitCode, + summary: runnerResult.status === "completed" + ? "Benchmark runner completed through AgentRail." + : "Benchmark runner failed through AgentRail.", + }; + }, + publishBranch: async () => {}, + }, + }); + return { exitCode, submissions: harness.submissions }; + } finally { + restoreEnv("AGENTRAIL_HOME", previousHome); + restoreEnv("AGENTRAIL_TELEMETRY_DISABLED", previousTelemetryDisabled); + } + }); + + const patch = managedWorktreePath + ? await capturePatch({ + worktreePath: managedWorktreePath, + outputPath: path.join(workspace.laneDir, "patch.diff"), + }) + : { status: "unavailable", path: null, changedFiles: [] }; + const tokenUsage = runnerResult + ? await parseTokenUsageFromFiles([runnerResult.stdoutPath, runnerResult.stderrPath]) + : { status: "unavailable", totalTokens: null, inputTokens: null, outputTokens: null, reasoningTokens: null, sourceFiles: [] }; + const evaluator = await runEvaluator({ + evaluator: flags.evaluator, + evalCommandTemplate: flags.evalCommandTemplate, + patchPath: patch.path, + outputPath: path.join(workspace.laneDir, "evaluator.json"), + timeoutSeconds: flags.timeoutSeconds ?? 1800, + }); + const status = result.exitCode === 0 && runnerResult?.status === "completed" ? "completed" : "failed"; + const success = status === "completed" + && patch.status === "available" + && (evaluator.status === "passed" || evaluator.status === "unavailable"); + + return { + status, + success, + exitCode: result.exitCode, + runner: runnerResult, + tokenUsage, + patch, + evaluator, + stdout: stdout.toString(), + stderr: stderr.toString(), + submissions: result.submissions, + workspace: { + laneDir: workspace.laneDir, + worktreePath: managedWorktreePath, + sourceWorktreePath: workspace.worktreePath, + }, + }; +} + +async function writeBenchmarkSetup({ homePath, repoPath, task, baseUrl, flags }) { + const config = createSetupConfig({ + cwd: repoPath, + detectedRepo: { + repoPath, + remoteSlug: task.repo, + defaultBranch: "main", + gitIgnoreHasAgentrail: true, + }, + interactionMode: "non_interactive", + acceptedDefaults: true, + baseUrl, + providerMode: "disabled", + repoAllowlist: [task.repo], + runnerPolicy: flags.runner === "custom" + ? { preset: "advisory", enforcementMode: "advisory" } + : undefined, + }); + await writeSetupFiles({ homePath, config }); +} + +function createMemoryWriter() { + const chunks = []; + return { + write(chunk) { + chunks.push(Buffer.isBuffer(chunk) ? chunk.toString("utf8") : String(chunk)); + return true; + }, + toString() { + return chunks.join(""); + }, + }; +} + +function restoreEnv(key, value) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; +} + +async function git(args) { + await execFileAsync("git", args, { maxBuffer: 1024 * 1024 * 20 }); +} diff --git a/test/agentrail-swe-lifecycle-real-runner.test.js b/test/agentrail-swe-lifecycle-real-runner.test.js index 5c9e06b..9abf350 100644 --- a/test/agentrail-swe-lifecycle-real-runner.test.js +++ b/test/agentrail-swe-lifecycle-real-runner.test.js @@ -7,6 +7,7 @@ import test from "node:test"; import { parseBenchmarkArgs } from "../benchmarks/agentrail-swe-lifecycle/lib/args.mjs"; import { withAgentRailBenchmarkHarness } from "../benchmarks/agentrail-swe-lifecycle/lib/agentrail-harness.mjs"; +import { runAgentRailLane } from "../benchmarks/agentrail-swe-lifecycle/lib/agentrail-lane.mjs"; import { runControlLane } from "../benchmarks/agentrail-swe-lifecycle/lib/control-lane.mjs"; import { runEvaluator } from "../benchmarks/agentrail-swe-lifecycle/lib/evaluator.mjs"; import { buildControlPrompt } from "../benchmarks/agentrail-swe-lifecycle/lib/prompt.mjs"; @@ -194,3 +195,39 @@ test("withAgentRailBenchmarkHarness exposes a task and records synthetic submiss assert.match(result.taskId, /^tsk_/u); assert.match(await readFile(result.envFilePath, "utf8"), /AGENTRAIL_AGENT_ID=agt_bench/u); }); + +test("runAgentRailLane executes a fake runner through AgentRail lifecycle", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "agentrail-agentrail-lane-")); + const source = path.join(root, "source"); + execFileSync("git", ["init", source]); + execFileSync("git", ["-C", source, "config", "user.email", "test@example.com"]); + execFileSync("git", ["-C", source, "config", "user.name", "Test User"]); + await writeFile(path.join(source, "README.md"), "base\n"); + execFileSync("git", ["-C", source, "add", "README.md"]); + execFileSync("git", ["-C", source, "commit", "-m", "base"]); + const baseCommit = execFileSync("git", ["-C", source, "rev-parse", "HEAD"], { encoding: "utf8" }).trim(); + + const result = await runAgentRailLane({ + task: { id: "task_1", repo: "fixture/repo" }, + upstream: { instanceId: "fixture__repo-1", baseCommit, problemStatement: "Change README" }, + runDir: root, + repoCache: path.join(root, "cache"), + sourceRepoPath: source, + flags: { + runner: "custom", + runnerCommand: "node benchmarks/agentrail-swe-lifecycle/fixtures/fake-runner.mjs", + model: "fake-model", + timeoutSeconds: 30, + evaluator: "command", + evalCommandTemplate: "node benchmarks/agentrail-swe-lifecycle/fixtures/fake-evaluator.mjs --patch {patchPath} --output {outputPath}", + }, + }); + + assert.equal(result.status, "completed", result.stderr); + assert.equal(result.success, true, result.stderr); + assert.equal(result.tokenUsage.totalTokens, 125); + assert.equal(result.patch.status, "available"); + assert.equal(result.evaluator.status, "passed"); + assert.equal(result.submissions.length, 1); + assert.match(result.stdout, /"status":"succeeded"/u); +}); From 63e8d35c9919d932448366d0a7399c52f31ca702 Mon Sep 17 00:00:00 2001 From: Onyeka Nwamba Date: Mon, 25 May 2026 04:12:52 +0100 Subject: [PATCH 10/18] feat: orchestrate paired benchmark runs --- .../agentrail-swe-lifecycle/lib/args.mjs | 2 + .../agentrail-swe-lifecycle/lib/results.mjs | 31 +++- benchmarks/agentrail-swe-lifecycle/run.mjs | 148 +++++++++++++++++- ...gentrail-swe-lifecycle-real-runner.test.js | 59 +++++++ 4 files changed, 232 insertions(+), 8 deletions(-) diff --git a/benchmarks/agentrail-swe-lifecycle/lib/args.mjs b/benchmarks/agentrail-swe-lifecycle/lib/args.mjs index 4f214cf..55386a0 100644 --- a/benchmarks/agentrail-swe-lifecycle/lib/args.mjs +++ b/benchmarks/agentrail-swe-lifecycle/lib/args.mjs @@ -26,6 +26,8 @@ export function parseBenchmarkArgs(argv) { flags.timeoutSeconds = parsePositiveSafeInteger(readFlagValue({ argv, index: ++index, flag: arg }), arg); } else if (arg === "--repo-cache") { flags.repoCache = readFlagValue({ argv, index: ++index, flag: arg }); + } else if (arg === "--source-repo-path") { + flags.sourceRepoPath = readFlagValue({ argv, index: ++index, flag: arg }); } else if (arg === "--evaluator") { flags.evaluator = readFlagValue({ argv, index: ++index, flag: arg }); } else if (arg === "--eval-command-template") { diff --git a/benchmarks/agentrail-swe-lifecycle/lib/results.mjs b/benchmarks/agentrail-swe-lifecycle/lib/results.mjs index 637db17..f073698 100644 --- a/benchmarks/agentrail-swe-lifecycle/lib/results.mjs +++ b/benchmarks/agentrail-swe-lifecycle/lib/results.mjs @@ -7,6 +7,17 @@ export function aggregateLifecycleResults(results) { const normalizedTokenTotals = results.map((result) => normalizeTokenTotal(result.tokenUsage)); const totalTokens = normalizedTokenTotals.reduce((sum, total) => sum + (total ?? 0), 0); const tokenUsageUnavailableTasks = normalizedTokenTotals.filter((total) => total === null).length; + const comparisons = results + .map((result) => result.comparison) + .filter((comparison) => comparison && typeof comparison === "object"); + const controlTokenTotals = comparisons.map((comparison) => normalizeTokenTotal(comparison.control?.tokenUsage)); + const agentRailTokenTotals = comparisons.map((comparison) => normalizeTokenTotal(comparison.agentrail?.tokenUsage)); + const comparableTokenTasks = comparisons.filter((_, index) => ( + controlTokenTotals[index] !== null && agentRailTokenTotals[index] !== null + )).length; + const controlTotalTokens = controlTokenTotals.reduce((sum, total) => sum + (total ?? 0), 0); + const agentRailTotalTokens = agentRailTokenTotals.reduce((sum, total) => sum + (total ?? 0), 0); + const tokenDelta = comparableTokenTasks > 0 ? agentRailTotalTokens - controlTotalTokens : null; return { totalTasks, @@ -14,7 +25,18 @@ export function aggregateLifecycleResults(results) { successRate: totalTasks > 0 ? successfulTasks / totalTasks : 0, totalTokens, tokenUsageUnavailableTasks, - tokensPerSuccessfulTask: successfulTasks > 0 ? totalTokens / successfulTasks : null + tokensPerSuccessfulTask: successfulTasks > 0 ? totalTokens / successfulTasks : null, + paired: comparisons.length > 0 + ? { + comparableTokenTasks, + controlTotalTokens, + agentRailTotalTokens, + tokenDelta, + tokenReductionRate: tokenDelta !== null && controlTotalTokens > 0 + ? (controlTotalTokens - agentRailTotalTokens) / controlTotalTokens + : null + } + : null }; } @@ -51,6 +73,13 @@ function renderSummary(run) { `- Total tokens: ${aggregate.totalTokens ?? "unavailable"}`, `- Token usage unavailable tasks: ${aggregate.tokenUsageUnavailableTasks}`, `- Tokens per successful task: ${aggregate.tokensPerSuccessfulTask ?? "unavailable"}`, + ...(aggregate.paired ? [ + `- Paired comparable token tasks: ${aggregate.paired.comparableTokenTasks}`, + `- Control total tokens: ${aggregate.paired.controlTotalTokens}`, + `- AgentRail total tokens: ${aggregate.paired.agentRailTotalTokens}`, + `- AgentRail token delta: ${aggregate.paired.tokenDelta ?? "unavailable"}`, + `- AgentRail token reduction: ${aggregate.paired.tokenReductionRate === null ? "unavailable" : `${(aggregate.paired.tokenReductionRate * 100).toFixed(2)}%`}` + ] : []), "" ].join("\n"); } diff --git a/benchmarks/agentrail-swe-lifecycle/run.mjs b/benchmarks/agentrail-swe-lifecycle/run.mjs index 55013b7..0de45fa 100644 --- a/benchmarks/agentrail-swe-lifecycle/run.mjs +++ b/benchmarks/agentrail-swe-lifecycle/run.mjs @@ -3,20 +3,18 @@ import path from "node:path"; import { fileURLToPath } from "node:url"; import { parseBenchmarkArgs } from "./lib/args.mjs"; +import { runAgentRailLane } from "./lib/agentrail-lane.mjs"; +import { runControlLane } from "./lib/control-lane.mjs"; import { loadLifecycleDataset, selectLifecycleTasks } from "./lib/dataset.mjs"; import { buildLocalLifecycleArtifacts } from "./lib/local-lifecycle.mjs"; import { aggregateLifecycleResults, writeLifecycleResultFiles } from "./lib/results.mjs"; +import { resolveUpstreamRowsForTasks } from "./lib/upstream.mjs"; const RUN_ID_PATTERN = /^run_[A-Za-z0-9_-]+$/u; const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", ".."); const defaultDatasetPath = path.join(repoRoot, "benchmarks", "agentrail-swe-lifecycle", "dataset.json"); const flags = parseBenchmarkArgs(process.argv.slice(2)); - -if (flags.mode !== "dry-run") { - throw new Error("Real benchmark execution is not wired yet. Continue with Task 7 after lane modules exist."); -} - const dataset = await loadLifecycleDataset(flags.dataset ?? defaultDatasetPath); const tasks = selectLifecycleTasks(dataset, flags); if (tasks.length === 0) { @@ -25,11 +23,13 @@ if (tasks.length === 0) { const runId = flags.runId ?? `run_${new Date().toISOString().replaceAll(/[:.]/gu, "-")}`; validateRunId(runId); const outputDir = flags.outputDir ?? path.join(repoRoot, ".agentrail-bench", "agentrail-swe-lifecycle", runId); -const results = tasks.map((task) => buildDryRunTaskResult({ task, runId })); +const results = flags.mode === "dry-run" + ? tasks.map((task) => buildDryRunTaskResult({ task, runId })) + : await runRealBenchmarkTasks({ tasks, runId, outputDir, flags }); const run = { version: 1, runId, - mode: "dry_run", + mode: flags.mode === "dry-run" ? "dry_run" : flags.mode, dataset: dataset.id, scenario: dataset.defaultScenario, aggregate: aggregateLifecycleResults(results), @@ -40,6 +40,140 @@ await writeLifecycleResultFiles({ outputDir, run }); console.log(JSON.stringify({ status: "ok", outputDir, tasks: tasks.length }, null, 2)); +async function runRealBenchmarkTasks({ tasks, runId, outputDir, flags }) { + const upstreams = await resolveUpstreamsForExecution(tasks); + const results = []; + for (let index = 0; index < tasks.length; index += 1) { + const task = tasks[index]; + const upstream = upstreams[index]; + process.stderr.write(`[agentrail-swe] ${flags.mode} ${index + 1}/${tasks.length} ${task.id}\n`); + if (flags.mode === "control") { + const control = await runControlLane(buildLaneOptions({ task, upstream, outputDir, flags })); + validateRequiredOutputs({ label: "control", lane: control, flags }); + results.push(buildSingleLaneTaskResult({ task, upstream, runId, laneName: "control", lane: control })); + } else if (flags.mode === "agentrail") { + const agentrail = await runAgentRailLane(buildLaneOptions({ task, upstream, outputDir, flags })); + validateRequiredOutputs({ label: "agentrail", lane: agentrail, flags }); + results.push(buildSingleLaneTaskResult({ task, upstream, runId, laneName: "agentrail", lane: agentrail })); + } else if (flags.mode === "paired") { + const control = await runControlLane(buildLaneOptions({ task, upstream, outputDir, flags })); + validateRequiredOutputs({ label: "control", lane: control, flags }); + const agentrail = await runAgentRailLane(buildLaneOptions({ task, upstream, outputDir, flags })); + validateRequiredOutputs({ label: "agentrail", lane: agentrail, flags }); + results.push(buildPairedTaskResult({ task, upstream, runId, control, agentrail })); + } else { + throw new Error(`Unsupported execution mode: ${flags.mode}`); + } + } + return results; +} + +async function resolveUpstreamsForExecution(tasks) { + const manifestUpstreams = tasks.map(upstreamFromTaskManifest); + if (manifestUpstreams.every(Boolean)) return manifestUpstreams; + return await resolveUpstreamRowsForTasks(tasks); +} + +function upstreamFromTaskManifest(task) { + const upstream = task.upstream ?? {}; + const baseCommit = upstream.baseCommit ?? task.baseCommit; + const problemStatement = upstream.problemStatement ?? task.problemStatement; + if (typeof baseCommit !== "string" || typeof problemStatement !== "string") return null; + return { + repo: task.repo, + instanceId: upstream.instanceId, + baseCommit, + problemStatement, + requirements: upstream.requirements ?? null, + interface: upstream.interface ?? null, + repoLanguage: task.language ?? null + }; +} + +function buildLaneOptions({ task, upstream, outputDir, flags }) { + return { + task, + upstream, + runDir: outputDir, + repoCache: flags.repoCache ?? path.join(repoRoot, ".agentrail-bench", "repo-cache"), + sourceRepoPath: flags.sourceRepoPath, + flags + }; +} + +function buildSingleLaneTaskResult({ task, upstream, runId, laneName, lane }) { + return { + taskId: task.id, + upstreamInstanceId: upstream.instanceId, + repo: task.repo, + lifecycle: buildLocalLifecycleArtifacts({ task, upstream, runId }), + lanes: { [laneName]: summarizeLane(lane) }, + final: { success: lane.success, status: lane.status }, + tokenUsage: lane.tokenUsage + }; +} + +function buildPairedTaskResult({ task, upstream, runId, control, agentrail }) { + return { + taskId: task.id, + upstreamInstanceId: upstream.instanceId, + repo: task.repo, + lifecycle: buildLocalLifecycleArtifacts({ task, upstream, runId }), + lanes: { + control: summarizeLane(control), + agentrail: summarizeLane(agentrail) + }, + comparison: { + control: { tokenUsage: control.tokenUsage }, + agentrail: { tokenUsage: agentrail.tokenUsage }, + tokenDelta: tokenDelta({ control, agentrail }), + tokenReductionRate: tokenReductionRate({ control, agentrail }) + }, + final: { + success: control.success && agentrail.success, + status: control.success && agentrail.success ? "completed" : "failed" + }, + tokenUsage: agentrail.tokenUsage + }; +} + +function summarizeLane(lane) { + return { + status: lane.status, + success: lane.success, + tokenUsage: lane.tokenUsage, + patch: lane.patch, + evaluator: lane.evaluator, + workspace: lane.workspace + }; +} + +function tokenDelta({ control, agentrail }) { + const controlTokens = tokenTotal(control.tokenUsage); + const agentRailTokens = tokenTotal(agentrail.tokenUsage); + return controlTokens === null || agentRailTokens === null ? null : agentRailTokens - controlTokens; +} + +function tokenReductionRate({ control, agentrail }) { + const controlTokens = tokenTotal(control.tokenUsage); + const agentRailTokens = tokenTotal(agentrail.tokenUsage); + if (controlTokens === null || agentRailTokens === null || controlTokens === 0) return null; + return (controlTokens - agentRailTokens) / controlTokens; +} + +function tokenTotal(tokenUsage) { + return typeof tokenUsage?.totalTokens === "number" ? tokenUsage.totalTokens : null; +} + +function validateRequiredOutputs({ label, lane, flags }) { + if (flags.requireTokenUsage && lane.tokenUsage?.status !== "available") { + throw new Error(`${label} lane did not produce parseable token usage.`); + } + if (flags.requireEvaluator && lane.evaluator?.status === "unavailable") { + throw new Error(`${label} lane did not produce an evaluator result.`); + } +} + function buildDryRunTaskResult({ task, runId }) { const upstream = { instanceId: task.upstream.instanceId, diff --git a/test/agentrail-swe-lifecycle-real-runner.test.js b/test/agentrail-swe-lifecycle-real-runner.test.js index 9abf350..3db6adf 100644 --- a/test/agentrail-swe-lifecycle-real-runner.test.js +++ b/test/agentrail-swe-lifecycle-real-runner.test.js @@ -27,6 +27,7 @@ test("parseBenchmarkArgs accepts paired execution flags", () => { "--model", "fake-model", "--timeout-seconds", "30", "--repo-cache", ".cache/repos", + "--source-repo-path", "fixtures/source-repo", "--evaluator", "command", "--eval-command-template", "node fake-evaluator.mjs --patch {patchPath} --output {outputPath}", "--require-token-usage", @@ -39,6 +40,7 @@ test("parseBenchmarkArgs accepts paired execution flags", () => { assert.equal(flags.model, "fake-model"); assert.equal(flags.timeoutSeconds, 30); assert.equal(flags.repoCache, ".cache/repos"); + assert.equal(flags.sourceRepoPath, "fixtures/source-repo"); assert.equal(flags.evaluator, "command"); assert.equal(flags.evalCommandTemplate, "node fake-evaluator.mjs --patch {patchPath} --output {outputPath}"); assert.equal(flags.requireTokenUsage, true); @@ -231,3 +233,60 @@ test("runAgentRailLane executes a fake runner through AgentRail lifecycle", asyn assert.equal(result.submissions.length, 1); assert.match(result.stdout, /"status":"succeeded"/u); }); + +test("run.mjs paired mode writes control versus AgentRail token comparison", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "agentrail-paired-run-")); + const source = path.join(root, "source"); + const datasetPath = path.join(root, "dataset.json"); + const outputDir = path.join(root, "output"); + execFileSync("git", ["init", source]); + execFileSync("git", ["-C", source, "config", "user.email", "test@example.com"]); + execFileSync("git", ["-C", source, "config", "user.name", "Test User"]); + await writeFile(path.join(source, "README.md"), "base\n"); + execFileSync("git", ["-C", source, "add", "README.md"]); + execFileSync("git", ["-C", source, "commit", "-m", "base"]); + const baseCommit = execFileSync("git", ["-C", source, "rev-parse", "HEAD"], { encoding: "utf8" }).trim(); + await writeFile(datasetPath, `${JSON.stringify({ + version: 1, + id: "paired-fixture", + defaultScenario: "local-mock-lifecycle", + tasks: [{ + id: "task-1", + repo: "fixture/repo", + baseCommit, + language: "js", + tags: ["fixture"], + upstream: { + dataset: "fixture", + instanceId: "fixture__repo-1", + problemStatement: "Change README", + baseCommit, + }, + }], + }, null, 2)}\n`, "utf8"); + + execFileSync("node", [ + "benchmarks/agentrail-swe-lifecycle/run.mjs", + "--mode", "paired", + "--dataset", datasetPath, + "--source-repo-path", source, + "--output-dir", outputDir, + "--run-id", "run_fixture", + "--runner", "custom", + "--runner-command", "node benchmarks/agentrail-swe-lifecycle/fixtures/fake-runner.mjs", + "--model", "fake-model", + "--timeout-seconds", "30", + "--evaluator", "command", + "--eval-command-template", "node benchmarks/agentrail-swe-lifecycle/fixtures/fake-evaluator.mjs --patch {patchPath} --output {outputPath}", + "--require-token-usage", + "--require-evaluator", + ], { cwd: path.resolve("."), encoding: "utf8" }); + + const results = JSON.parse(await readFile(path.join(outputDir, "results.json"), "utf8")); + assert.equal(results.mode, "paired"); + assert.equal(results.tasks.length, 1); + assert.equal(results.tasks[0].final.success, true); + assert.equal(results.tasks[0].comparison.control.tokenUsage.totalTokens, 125); + assert.equal(results.tasks[0].comparison.agentrail.tokenUsage.totalTokens, 125); + assert.equal(results.aggregate.paired.tokenDelta, 0); +}); From 4e17ea820a0cf75e809ed1b262c629f33ba56766 Mon Sep 17 00:00:00 2001 From: Onyeka Nwamba Date: Mon, 25 May 2026 04:13:56 +0100 Subject: [PATCH 11/18] docs: document paired token benchmark --- benchmarks/agentrail-swe-lifecycle/README.md | 32 ++++++++++++++++++++ package.json | 4 ++- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/benchmarks/agentrail-swe-lifecycle/README.md b/benchmarks/agentrail-swe-lifecycle/README.md index 093d186..c90c0ea 100644 --- a/benchmarks/agentrail-swe-lifecycle/README.md +++ b/benchmarks/agentrail-swe-lifecycle/README.md @@ -45,3 +45,35 @@ Validate the checked-in manifest shape: npm run benchmark:agentrail-swe:validate ``` +## Run + +Dry-run the selected task manifest without launching a runner: + +```bash +npm run benchmark:agentrail-swe:dry-run -- --count 5 +``` + +Run the paired token benchmark. This runs the same task once through the direct +control lane and once through AgentRail, then writes `results.json` and +`summary.md` under `.agentrail-bench/agentrail-swe-lifecycle//`. + +```bash +npm run benchmark:agentrail-swe:paired -- \ + --runner codex \ + --model gpt-5 \ + --count 1 \ + --repo-cache .agentrail-bench/repo-cache \ + --require-token-usage +``` + +For local smoke tests, use `--runner custom --runner-command ` with a +fixture repository supplied through `--source-repo-path`. Real SWE-bench Pro +runs omit `--source-repo-path`; the benchmark clones the task repository into +`--repo-cache` and checks out the upstream base commit. + +The paired result records: + +- control lane status, patch, evaluator result, and token usage +- AgentRail lane status, patch, evaluator result, and token usage +- `tokenDelta`, where positive means AgentRail used more tokens +- `tokenReductionRate`, where positive means AgentRail used fewer tokens diff --git a/package.json b/package.json index 84e31d5..5fb614b 100644 --- a/package.json +++ b/package.json @@ -28,8 +28,10 @@ "smoke:sdk": "node scripts/smoke-test-sdk-packages.mjs --all", "smoke:sandbox:live": "npm --prefix sdk/typescript ci && npm --prefix sdk/typescript run build && node scripts/agentrail-sandbox-smoke.mjs --mode live", "benchmark:agentrail-swe:refresh": "node scripts/refresh-agentrail-swe-lifecycle-dataset.mjs", + "benchmark:agentrail-swe:dry-run": "node benchmarks/agentrail-swe-lifecycle/run.mjs --dry-run", + "benchmark:agentrail-swe:paired": "node benchmarks/agentrail-swe-lifecycle/run.mjs --mode paired", "benchmark:agentrail-swe:run": "node benchmarks/agentrail-swe-lifecycle/run.mjs --dry-run", - "benchmark:agentrail-swe:validate": "node --test test/agentrail-swe-lifecycle-dataset.test.js test/agentrail-swe-lifecycle-runner.test.js", + "benchmark:agentrail-swe:validate": "node --test test/agentrail-swe-lifecycle-dataset.test.js test/agentrail-swe-lifecycle-runner.test.js test/agentrail-swe-lifecycle-real-runner.test.js", "lint:openapi": "spectral lint docs/api/task-lifecycle.openapi.yaml docs/api/intake-routing-admin.openapi.yaml -r .spectral.yaml", "test": "npm --prefix sdk/typescript ci && npm --prefix sdk/typescript run build && npm run test:service", "test:service": "node --test 'test/*.test.js' 'test/*.test.ts'", From 43238cfad98458bd8c0738a32b49d5e691b2665b Mon Sep 17 00:00:00 2001 From: Onyeka Nwamba Date: Mon, 25 May 2026 04:18:41 +0100 Subject: [PATCH 12/18] test: make benchmark token fixture prompt-sensitive --- .../fixtures/fake-runner.mjs | 13 +++++++++++-- test/agentrail-swe-lifecycle-real-runner.test.js | 14 +++++++++----- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/benchmarks/agentrail-swe-lifecycle/fixtures/fake-runner.mjs b/benchmarks/agentrail-swe-lifecycle/fixtures/fake-runner.mjs index a362853..bd89399 100644 --- a/benchmarks/agentrail-swe-lifecycle/fixtures/fake-runner.mjs +++ b/benchmarks/agentrail-swe-lifecycle/fixtures/fake-runner.mjs @@ -4,6 +4,15 @@ import { appendFile, readFile, writeFile } from "node:fs/promises"; import path from "node:path"; const repoPath = process.cwd(); +const promptPath = process.env.AGENTRAIL_BENCH_PROMPT_PATH; +const prompt = promptPath ? await readFile(promptPath, "utf8").catch(() => "") : ""; +const usage = { + input_tokens: Math.max(1, Math.ceil(prompt.length / 4)), + output_tokens: 20, + reasoning_tokens: 5, +}; +usage.total_tokens = usage.input_tokens + usage.output_tokens + usage.reasoning_tokens; + const readmePath = path.join(repoPath, "README.md"); const previous = await readFile(readmePath, "utf8").catch(() => ""); await writeFile(readmePath, `${previous.trimEnd()}\nfake runner completed\n`, "utf8"); @@ -29,8 +38,8 @@ if (process.env.AGENTRAIL_HANDOFF_PATH) { if (process.env.AGENTRAIL_BENCH_EXTRA_LOG_PATH) { await appendFile( process.env.AGENTRAIL_BENCH_EXTRA_LOG_PATH, - `${JSON.stringify({ usage: { input_tokens: 100, output_tokens: 20, reasoning_tokens: 5 } })}\n`, + `${JSON.stringify({ usage })}\n`, ); } -process.stdout.write(`${JSON.stringify({ usage: { input_tokens: 100, output_tokens: 20, reasoning_tokens: 5, total_tokens: 125 } })}\n`); +process.stdout.write(`${JSON.stringify({ usage })}\n`); diff --git a/test/agentrail-swe-lifecycle-real-runner.test.js b/test/agentrail-swe-lifecycle-real-runner.test.js index 3db6adf..f5ef625 100644 --- a/test/agentrail-swe-lifecycle-real-runner.test.js +++ b/test/agentrail-swe-lifecycle-real-runner.test.js @@ -168,7 +168,8 @@ test("runControlLane executes a fake runner, captures tokens, patch, and evaluat assert.equal(result.status, "completed"); assert.equal(result.success, true); - assert.equal(result.tokenUsage.totalTokens, 125); + assert.equal(result.tokenUsage.status, "available"); + assert.ok(result.tokenUsage.totalTokens > 25); assert.equal(result.patch.status, "available"); assert.equal(result.evaluator.status, "passed"); }); @@ -227,7 +228,8 @@ test("runAgentRailLane executes a fake runner through AgentRail lifecycle", asyn assert.equal(result.status, "completed", result.stderr); assert.equal(result.success, true, result.stderr); - assert.equal(result.tokenUsage.totalTokens, 125); + assert.equal(result.tokenUsage.status, "available"); + assert.ok(result.tokenUsage.totalTokens > 25); assert.equal(result.patch.status, "available"); assert.equal(result.evaluator.status, "passed"); assert.equal(result.submissions.length, 1); @@ -286,7 +288,9 @@ test("run.mjs paired mode writes control versus AgentRail token comparison", asy assert.equal(results.mode, "paired"); assert.equal(results.tasks.length, 1); assert.equal(results.tasks[0].final.success, true); - assert.equal(results.tasks[0].comparison.control.tokenUsage.totalTokens, 125); - assert.equal(results.tasks[0].comparison.agentrail.tokenUsage.totalTokens, 125); - assert.equal(results.aggregate.paired.tokenDelta, 0); + assert.notEqual( + results.tasks[0].comparison.control.tokenUsage.totalTokens, + results.tasks[0].comparison.agentrail.tokenUsage.totalTokens, + ); + assert.notEqual(results.aggregate.paired.tokenDelta, 0); }); From 7df507332aaaed55f4ad619ee145ac35855033b4 Mon Sep 17 00:00:00 2001 From: Onyeka Nwamba Date: Mon, 25 May 2026 04:28:24 +0100 Subject: [PATCH 13/18] fix: run codex benchmark through stdin json mode --- benchmarks/agentrail-swe-lifecycle/README.md | 5 +++- .../lib/runner-command.mjs | 19 ++++++++---- .../lib/token-usage.mjs | 2 ++ package.json | 2 +- ...gentrail-swe-lifecycle-real-runner.test.js | 29 ++++++++++++++++++- ...gentrail-swe-lifecycle-token-usage.test.js | 29 +++++++++++++++++++ 6 files changed, 78 insertions(+), 8 deletions(-) diff --git a/benchmarks/agentrail-swe-lifecycle/README.md b/benchmarks/agentrail-swe-lifecycle/README.md index c90c0ea..140c527 100644 --- a/benchmarks/agentrail-swe-lifecycle/README.md +++ b/benchmarks/agentrail-swe-lifecycle/README.md @@ -60,12 +60,15 @@ control lane and once through AgentRail, then writes `results.json` and ```bash npm run benchmark:agentrail-swe:paired -- \ --runner codex \ - --model gpt-5 \ --count 1 \ --repo-cache .agentrail-bench/repo-cache \ --require-token-usage ``` +Add `--model ` only if your local Codex CLI accepts that +model. The benchmark omits `--model` by default so Codex can use your configured +default. + For local smoke tests, use `--runner custom --runner-command ` with a fixture repository supplied through `--source-repo-path`. Real SWE-bench Pro runs omit `--source-repo-path`; the benchmark clones the task repository into diff --git a/benchmarks/agentrail-swe-lifecycle/lib/runner-command.mjs b/benchmarks/agentrail-swe-lifecycle/lib/runner-command.mjs index 04071da..ff32ef2 100644 --- a/benchmarks/agentrail-swe-lifecycle/lib/runner-command.mjs +++ b/benchmarks/agentrail-swe-lifecycle/lib/runner-command.mjs @@ -1,5 +1,5 @@ import { spawn } from "node:child_process"; -import { access, mkdir, writeFile } from "node:fs/promises"; +import { access, mkdir, readFile, writeFile } from "node:fs/promises"; import path from "node:path"; export async function runRunnerCommand({ runner, runnerCommand, model, repoPath, prompt, logDir, timeoutSeconds, env = {} }) { @@ -14,6 +14,7 @@ export async function runRunnerCommand({ runner, runnerCommand, model, repoPath, const result = await spawnWithTimeout({ command: command.executable, args: command.args, + stdin: command.stdin, cwd: repoPath, env: { ...process.env, ...env, AGENTRAIL_BENCH_PROMPT_PATH: promptPath }, timeoutMs: timeoutSeconds * 1000, @@ -43,8 +44,9 @@ export async function resolveRunnerCommand({ runner, runnerCommand, model, promp }; } if (runner === "codex") { - const args = ["--model", model, "--sandbox", "workspace-write", "exec", "--prompt-file", promptPath].filter(Boolean); - return { executable: "codex", args, redacted: ["codex", ...args].join(" ") }; + const prompt = await readPromptFile(promptPath); + const args = ["-a", "never", ...(model ? ["--model", model] : []), "--sandbox", "workspace-write", "exec", "--json", "-"]; + return { executable: "codex", args, stdin: prompt, redacted: ["codex", ...args].join(" ") }; } if (runner === "cursor") { const args = ["agent", "--model", model, "--prompt-file", promptPath].filter(Boolean); @@ -83,9 +85,13 @@ async function resolveExistingRelativePathArg(arg) { } } -async function spawnWithTimeout({ command, args, cwd, env, timeoutMs }) { +async function readPromptFile(promptPath) { + return await readFile(promptPath, "utf8"); +} + +async function spawnWithTimeout({ command, args, stdin, cwd, env, timeoutMs }) { return await new Promise((resolve, reject) => { - const child = spawn(command, args, { cwd, env, stdio: ["ignore", "pipe", "pipe"] }); + const child = spawn(command, args, { cwd, env, stdio: [stdin === undefined ? "ignore" : "pipe", "pipe", "pipe"] }); const stdout = []; const stderr = []; let timedOut = false; @@ -96,6 +102,9 @@ async function spawnWithTimeout({ command, args, cwd, env, timeoutMs }) { child.stdout.on("data", (chunk) => stdout.push(Buffer.from(chunk))); child.stderr.on("data", (chunk) => stderr.push(Buffer.from(chunk))); child.on("error", reject); + if (stdin !== undefined) { + child.stdin.end(stdin); + } child.on("close", (exitCode) => { clearTimeout(timer); resolve({ diff --git a/benchmarks/agentrail-swe-lifecycle/lib/token-usage.mjs b/benchmarks/agentrail-swe-lifecycle/lib/token-usage.mjs index 21f8161..8816b51 100644 --- a/benchmarks/agentrail-swe-lifecycle/lib/token-usage.mjs +++ b/benchmarks/agentrail-swe-lifecycle/lib/token-usage.mjs @@ -106,6 +106,7 @@ function isUsageObject(value) { return getTokenValue(value, "input_tokens", "inputTokens") !== null || getTokenValue(value, "output_tokens", "outputTokens") !== null || getTokenValue(value, "reasoning_tokens", "reasoningTokens") !== null + || getTokenValue(value, "reasoning_output_tokens", "reasoningOutputTokens") !== null || getTokenValue(value, "cached_input_tokens", "cachedInputTokens") !== null || hasTokenField(value, "total_tokens", "totalTokens") || getCachedInputTokensFromDetails(value) !== null @@ -116,6 +117,7 @@ function aggregateUsage(aggregate, usage) { const inputTokens = getTokenValue(usage, "input_tokens", "inputTokens"); const outputTokens = getTokenValue(usage, "output_tokens", "outputTokens"); const reasoningTokens = getTokenValue(usage, "reasoning_tokens", "reasoningTokens") + ?? getTokenValue(usage, "reasoning_output_tokens", "reasoningOutputTokens") ?? getReasoningTokensFromDetails(usage); const cachedInputTokens = getTokenValue(usage, "cached_input_tokens", "cachedInputTokens") ?? getCachedInputTokensFromDetails(usage); diff --git a/package.json b/package.json index 5fb614b..f97094e 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,7 @@ "benchmark:agentrail-swe:dry-run": "node benchmarks/agentrail-swe-lifecycle/run.mjs --dry-run", "benchmark:agentrail-swe:paired": "node benchmarks/agentrail-swe-lifecycle/run.mjs --mode paired", "benchmark:agentrail-swe:run": "node benchmarks/agentrail-swe-lifecycle/run.mjs --dry-run", - "benchmark:agentrail-swe:validate": "node --test test/agentrail-swe-lifecycle-dataset.test.js test/agentrail-swe-lifecycle-runner.test.js test/agentrail-swe-lifecycle-real-runner.test.js", + "benchmark:agentrail-swe:validate": "node --test test/agentrail-swe-lifecycle-dataset.test.js test/agentrail-swe-lifecycle-runner.test.js test/agentrail-swe-lifecycle-token-usage.test.js test/agentrail-swe-lifecycle-real-runner.test.js", "lint:openapi": "spectral lint docs/api/task-lifecycle.openapi.yaml docs/api/intake-routing-admin.openapi.yaml -r .spectral.yaml", "test": "npm --prefix sdk/typescript ci && npm --prefix sdk/typescript run build && npm run test:service", "test:service": "node --test 'test/*.test.js' 'test/*.test.ts'", diff --git a/test/agentrail-swe-lifecycle-real-runner.test.js b/test/agentrail-swe-lifecycle-real-runner.test.js index f5ef625..69db54a 100644 --- a/test/agentrail-swe-lifecycle-real-runner.test.js +++ b/test/agentrail-swe-lifecycle-real-runner.test.js @@ -11,7 +11,7 @@ import { runAgentRailLane } from "../benchmarks/agentrail-swe-lifecycle/lib/agen import { runControlLane } from "../benchmarks/agentrail-swe-lifecycle/lib/control-lane.mjs"; import { runEvaluator } from "../benchmarks/agentrail-swe-lifecycle/lib/evaluator.mjs"; import { buildControlPrompt } from "../benchmarks/agentrail-swe-lifecycle/lib/prompt.mjs"; -import { runRunnerCommand } from "../benchmarks/agentrail-swe-lifecycle/lib/runner-command.mjs"; +import { resolveRunnerCommand, runRunnerCommand } from "../benchmarks/agentrail-swe-lifecycle/lib/runner-command.mjs"; test("parseBenchmarkArgs keeps --dry-run as a dry-run mode alias", () => { const flags = parseBenchmarkArgs(["--dry-run", "--count", "1"]); @@ -116,6 +116,33 @@ test("runRunnerCommand invokes a custom runner and stores stdout and stderr", as assert.match(await readFile(result.stdoutPath, "utf8"), /total_tokens/u); }); +test("resolveRunnerCommand invokes Codex through stdin JSON mode", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "agentrail-codex-command-")); + const promptPath = path.join(root, "prompt.md"); + await writeFile(promptPath, "Change README\n", "utf8"); + + const command = await resolveRunnerCommand({ + runner: "codex", + model: "codex-default", + promptPath, + }); + + assert.equal(command.executable, "codex"); + assert.deepEqual(command.args, [ + "-a", + "never", + "--model", + "codex-default", + "--sandbox", + "workspace-write", + "exec", + "--json", + "-", + ]); + assert.equal(command.stdin, "Change README\n"); + assert.equal(command.args.includes("--prompt-file"), false); +}); + test("runEvaluator returns unavailable for evaluator none", async () => { const result = await runEvaluator({ evaluator: "none" }); assert.deepEqual(result, { status: "unavailable", exitCode: null, outputPath: null }); diff --git a/test/agentrail-swe-lifecycle-token-usage.test.js b/test/agentrail-swe-lifecycle-token-usage.test.js index f2dcdd5..3a299d1 100644 --- a/test/agentrail-swe-lifecycle-token-usage.test.js +++ b/test/agentrail-swe-lifecycle-token-usage.test.js @@ -38,6 +38,35 @@ test("parseTokenUsageFromFiles accepts total token fields without double countin assert.equal(usage.outputTokens, null); }); +test("parseTokenUsageFromFiles accepts Codex JSON turn usage", async () => { + const dir = await mkdtemp(path.join(os.tmpdir(), "agentrail-token-codex-json-")); + const log = path.join(dir, "stdout.log"); + await writeFile(log, [ + JSON.stringify({ type: "thread.started", thread_id: "thread_1" }), + JSON.stringify({ + type: "turn.completed", + usage: { + input_tokens: 19931, + cached_input_tokens: 3456, + output_tokens: 17, + reasoning_output_tokens: 10, + }, + }), + "", + ].join("\n")); + const usage = await parseTokenUsageFromFiles([log]); + assert.deepEqual(usage, { + status: "available", + inputTokens: 19931, + outputTokens: 17, + reasoningTokens: 10, + cachedInputTokens: 3456, + totalTokens: 19958, + sources: [log], + }); +}); + + test("parseTokenUsageFromFiles rejects invalid provider total instead of recomputing components", async () => { const dir = await mkdtemp(path.join(os.tmpdir(), "agentrail-token-invalid-total-")); const log = path.join(dir, "stdout.log"); From c47acb62a15608dea61277693eb08eaa70c2e3ce Mon Sep 17 00:00:00 2001 From: Onyeka Nwamba Date: Mon, 25 May 2026 05:01:00 +0100 Subject: [PATCH 14/18] fix: allow SWE benchmark repos through AgentRail lane --- .../agentrail-swe-lifecycle/lib/agentrail-lane.mjs | 8 +++++--- test/agentrail-swe-lifecycle-real-runner.test.js | 9 ++++++++- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/benchmarks/agentrail-swe-lifecycle/lib/agentrail-lane.mjs b/benchmarks/agentrail-swe-lifecycle/lib/agentrail-lane.mjs index dff280b..f027f4e 100644 --- a/benchmarks/agentrail-swe-lifecycle/lib/agentrail-lane.mjs +++ b/benchmarks/agentrail-swe-lifecycle/lib/agentrail-lane.mjs @@ -158,13 +158,15 @@ async function writeBenchmarkSetup({ homePath, repoPath, task, baseUrl, flags }) baseUrl, providerMode: "disabled", repoAllowlist: [task.repo], - runnerPolicy: flags.runner === "custom" - ? { preset: "advisory", enforcementMode: "advisory" } - : undefined, + runnerPolicy: benchmarkRunnerPolicy(), }); await writeSetupFiles({ homePath, config }); } +export function benchmarkRunnerPolicy() { + return { preset: "advisory", enforcementMode: "advisory" }; +} + function createMemoryWriter() { const chunks = []; return { diff --git a/test/agentrail-swe-lifecycle-real-runner.test.js b/test/agentrail-swe-lifecycle-real-runner.test.js index 69db54a..113f3b4 100644 --- a/test/agentrail-swe-lifecycle-real-runner.test.js +++ b/test/agentrail-swe-lifecycle-real-runner.test.js @@ -7,7 +7,7 @@ import test from "node:test"; import { parseBenchmarkArgs } from "../benchmarks/agentrail-swe-lifecycle/lib/args.mjs"; import { withAgentRailBenchmarkHarness } from "../benchmarks/agentrail-swe-lifecycle/lib/agentrail-harness.mjs"; -import { runAgentRailLane } from "../benchmarks/agentrail-swe-lifecycle/lib/agentrail-lane.mjs"; +import { benchmarkRunnerPolicy, runAgentRailLane } from "../benchmarks/agentrail-swe-lifecycle/lib/agentrail-lane.mjs"; import { runControlLane } from "../benchmarks/agentrail-swe-lifecycle/lib/control-lane.mjs"; import { runEvaluator } from "../benchmarks/agentrail-swe-lifecycle/lib/evaluator.mjs"; import { buildControlPrompt } from "../benchmarks/agentrail-swe-lifecycle/lib/prompt.mjs"; @@ -263,6 +263,13 @@ test("runAgentRailLane executes a fake runner through AgentRail lifecycle", asyn assert.match(result.stdout, /"status":"succeeded"/u); }); +test("benchmarkRunnerPolicy keeps SWE fixture secret files from blocking launch", () => { + assert.deepEqual(benchmarkRunnerPolicy(), { + preset: "advisory", + enforcementMode: "advisory", + }); +}); + test("run.mjs paired mode writes control versus AgentRail token comparison", async () => { const root = await mkdtemp(path.join(os.tmpdir(), "agentrail-paired-run-")); const source = path.join(root, "source"); From 7cf6772d4e8b92f3cff24eb9238a95cffe3259e5 Mon Sep 17 00:00:00 2001 From: Onyeka Nwamba Date: Mon, 25 May 2026 09:10:08 +0100 Subject: [PATCH 15/18] feat: continue full benchmark after task failures --- .../agentrail-swe-lifecycle/lib/args.mjs | 2 + benchmarks/agentrail-swe-lifecycle/run.mjs | 52 +++++++++++++------ ...gentrail-swe-lifecycle-real-runner.test.js | 2 + 3 files changed, 40 insertions(+), 16 deletions(-) diff --git a/benchmarks/agentrail-swe-lifecycle/lib/args.mjs b/benchmarks/agentrail-swe-lifecycle/lib/args.mjs index 55386a0..95b89ac 100644 --- a/benchmarks/agentrail-swe-lifecycle/lib/args.mjs +++ b/benchmarks/agentrail-swe-lifecycle/lib/args.mjs @@ -36,6 +36,8 @@ export function parseBenchmarkArgs(argv) { flags.requireTokenUsage = true; } else if (arg === "--require-evaluator") { flags.requireEvaluator = true; + } else if (arg === "--continue-on-error") { + flags.continueOnError = true; } else if (arg === "--no-cleanup") { flags.cleanup = false; } else if (arg === "--dataset") { diff --git a/benchmarks/agentrail-swe-lifecycle/run.mjs b/benchmarks/agentrail-swe-lifecycle/run.mjs index 0de45fa..5fb680a 100644 --- a/benchmarks/agentrail-swe-lifecycle/run.mjs +++ b/benchmarks/agentrail-swe-lifecycle/run.mjs @@ -47,22 +47,29 @@ async function runRealBenchmarkTasks({ tasks, runId, outputDir, flags }) { const task = tasks[index]; const upstream = upstreams[index]; process.stderr.write(`[agentrail-swe] ${flags.mode} ${index + 1}/${tasks.length} ${task.id}\n`); - if (flags.mode === "control") { - const control = await runControlLane(buildLaneOptions({ task, upstream, outputDir, flags })); - validateRequiredOutputs({ label: "control", lane: control, flags }); - results.push(buildSingleLaneTaskResult({ task, upstream, runId, laneName: "control", lane: control })); - } else if (flags.mode === "agentrail") { - const agentrail = await runAgentRailLane(buildLaneOptions({ task, upstream, outputDir, flags })); - validateRequiredOutputs({ label: "agentrail", lane: agentrail, flags }); - results.push(buildSingleLaneTaskResult({ task, upstream, runId, laneName: "agentrail", lane: agentrail })); - } else if (flags.mode === "paired") { - const control = await runControlLane(buildLaneOptions({ task, upstream, outputDir, flags })); - validateRequiredOutputs({ label: "control", lane: control, flags }); - const agentrail = await runAgentRailLane(buildLaneOptions({ task, upstream, outputDir, flags })); - validateRequiredOutputs({ label: "agentrail", lane: agentrail, flags }); - results.push(buildPairedTaskResult({ task, upstream, runId, control, agentrail })); - } else { - throw new Error(`Unsupported execution mode: ${flags.mode}`); + try { + if (flags.mode === "control") { + const control = await runControlLane(buildLaneOptions({ task, upstream, outputDir, flags })); + validateRequiredOutputs({ label: "control", lane: control, flags }); + results.push(buildSingleLaneTaskResult({ task, upstream, runId, laneName: "control", lane: control })); + } else if (flags.mode === "agentrail") { + const agentrail = await runAgentRailLane(buildLaneOptions({ task, upstream, outputDir, flags })); + validateRequiredOutputs({ label: "agentrail", lane: agentrail, flags }); + results.push(buildSingleLaneTaskResult({ task, upstream, runId, laneName: "agentrail", lane: agentrail })); + } else if (flags.mode === "paired") { + const control = await runControlLane(buildLaneOptions({ task, upstream, outputDir, flags })); + validateRequiredOutputs({ label: "control", lane: control, flags }); + const agentrail = await runAgentRailLane(buildLaneOptions({ task, upstream, outputDir, flags })); + validateRequiredOutputs({ label: "agentrail", lane: agentrail, flags }); + results.push(buildPairedTaskResult({ task, upstream, runId, control, agentrail })); + } else { + throw new Error(`Unsupported execution mode: ${flags.mode}`); + } + } catch (error) { + if (!flags.continueOnError) throw error; + const message = error instanceof Error ? error.message : String(error); + process.stderr.write(`[agentrail-swe] failed ${task.id}: ${message}\n`); + results.push(buildErrorTaskResult({ task, upstream, runId, error })); } } return results; @@ -137,6 +144,19 @@ function buildPairedTaskResult({ task, upstream, runId, control, agentrail }) { }; } +function buildErrorTaskResult({ task, upstream, runId, error }) { + const message = error instanceof Error ? error.message : String(error); + return { + taskId: task.id, + upstreamInstanceId: upstream.instanceId, + repo: task.repo, + lifecycle: buildLocalLifecycleArtifacts({ task, upstream, runId }), + lanes: {}, + final: { success: false, status: "error", error: message }, + tokenUsage: null + }; +} + function summarizeLane(lane) { return { status: lane.status, diff --git a/test/agentrail-swe-lifecycle-real-runner.test.js b/test/agentrail-swe-lifecycle-real-runner.test.js index 113f3b4..5d8fe27 100644 --- a/test/agentrail-swe-lifecycle-real-runner.test.js +++ b/test/agentrail-swe-lifecycle-real-runner.test.js @@ -32,6 +32,7 @@ test("parseBenchmarkArgs accepts paired execution flags", () => { "--eval-command-template", "node fake-evaluator.mjs --patch {patchPath} --output {outputPath}", "--require-token-usage", "--require-evaluator", + "--continue-on-error", "--no-cleanup", ]); assert.equal(flags.mode, "paired"); @@ -45,6 +46,7 @@ test("parseBenchmarkArgs accepts paired execution flags", () => { assert.equal(flags.evalCommandTemplate, "node fake-evaluator.mjs --patch {patchPath} --output {outputPath}"); assert.equal(flags.requireTokenUsage, true); assert.equal(flags.requireEvaluator, true); + assert.equal(flags.continueOnError, true); assert.equal(flags.cleanup, false); }); From 3c56776e777d404905f2febd2c69bb56c3ab9b85 Mon Sep 17 00:00:00 2001 From: Onyeka Nwamba Date: Tue, 26 May 2026 02:26:00 +0100 Subject: [PATCH 16/18] feat: add phase-attribution and full-lifecycle benchmark modes Adds two new SWE-lifecycle benchmark modes that measure where AgentRail reduces agent-side token spend. - Phase-attribution mode runs the same task twice: a control lane where the agent owns every phase (intake -> triage -> plan -> implement -> CI -> review -> ship), and an AgentRail lane where only implement and revise phases are charged to the agent. Per-phase token accounting produces apples-to-apples reduction numbers. - Full-lifecycle paired mode runs both lanes end-to-end against real ansible/ansible SWE-bench-Pro tasks for headline aggregate numbers. Engine pieces: phase-runner, phase-prompts, phase-accounting, phase-definitions, lifecycle-provider-harness, lifecycle-control-prompt, plus fake-lifecycle-runner / fake-phase-runner fixtures and a mock lifecycle CLI for tests. Real-runner test suite expanded; new tests cover both modes. Also wires npm scripts: npm run benchmark:agentrail-swe:phase-attribution npm run benchmark:agentrail-swe:paired:full-lifecycle --- benchmarks/agentrail-swe-lifecycle/README.md | 71 +++- .../fixtures/fake-lifecycle-runner.mjs | 67 +++ .../fixtures/fake-phase-runner.mjs | 66 +++ .../fixtures/mock-lifecycle-cli.mjs | 168 ++++++++ .../lib/agentrail-harness.mjs | 56 ++- .../lib/agentrail-lane.mjs | 199 ++++++++- .../agentrail-swe-lifecycle/lib/args.mjs | 40 +- .../lib/control-lane.mjs | 91 ++++ .../lib/lifecycle-control-prompt.mjs | 30 ++ .../lib/lifecycle-provider-harness.mjs | 183 ++++++++ .../agentrail-swe-lifecycle/lib/patch.mjs | 24 +- .../lib/phase-accounting.mjs | 53 +++ .../lib/phase-definitions.mjs | 141 +++++++ .../lib/phase-prompts.mjs | 77 ++++ .../lib/phase-runner.mjs | 165 ++++++++ .../agentrail-swe-lifecycle/lib/results.mjs | 76 +++- .../lib/runner-command.mjs | 43 +- .../lib/token-usage.mjs | 10 +- benchmarks/agentrail-swe-lifecycle/run.mjs | 33 +- package.json | 4 +- src/cli/agent-runner.ts | 19 +- src/cli/setup-files.ts | 8 +- ...trail-swe-lifecycle-full-lifecycle.test.js | 118 ++++++ ...il-swe-lifecycle-phase-attribution.test.js | 399 ++++++++++++++++++ ...gentrail-swe-lifecycle-real-runner.test.js | 360 +++++++++++++++- ...gentrail-swe-lifecycle-token-usage.test.js | 8 + 26 files changed, 2465 insertions(+), 44 deletions(-) create mode 100644 benchmarks/agentrail-swe-lifecycle/fixtures/fake-lifecycle-runner.mjs create mode 100644 benchmarks/agentrail-swe-lifecycle/fixtures/fake-phase-runner.mjs create mode 100644 benchmarks/agentrail-swe-lifecycle/fixtures/mock-lifecycle-cli.mjs create mode 100644 benchmarks/agentrail-swe-lifecycle/lib/lifecycle-control-prompt.mjs create mode 100644 benchmarks/agentrail-swe-lifecycle/lib/lifecycle-provider-harness.mjs create mode 100644 benchmarks/agentrail-swe-lifecycle/lib/phase-accounting.mjs create mode 100644 benchmarks/agentrail-swe-lifecycle/lib/phase-definitions.mjs create mode 100644 benchmarks/agentrail-swe-lifecycle/lib/phase-prompts.mjs create mode 100644 benchmarks/agentrail-swe-lifecycle/lib/phase-runner.mjs create mode 100644 test/agentrail-swe-lifecycle-full-lifecycle.test.js create mode 100644 test/agentrail-swe-lifecycle-phase-attribution.test.js diff --git a/benchmarks/agentrail-swe-lifecycle/README.md b/benchmarks/agentrail-swe-lifecycle/README.md index 140c527..800d151 100644 --- a/benchmarks/agentrail-swe-lifecycle/README.md +++ b/benchmarks/agentrail-swe-lifecycle/README.md @@ -53,9 +53,10 @@ Dry-run the selected task manifest without launching a runner: npm run benchmark:agentrail-swe:dry-run -- --count 5 ``` -Run the paired token benchmark. This runs the same task once through the direct -control lane and once through AgentRail, then writes `results.json` and -`summary.md` under `.agentrail-bench/agentrail-swe-lifecycle//`. +Run the paired patch-only token benchmark. This runs the same coding task once +through the direct control lane and once through AgentRail, then writes +`results.json` and `summary.md` under +`.agentrail-bench/agentrail-swe-lifecycle//`. ```bash npm run benchmark:agentrail-swe:paired -- \ @@ -65,6 +66,68 @@ npm run benchmark:agentrail-swe:paired -- \ --require-token-usage ``` +Run the paired full-lifecycle token benchmark when you want the control lane to +pay LLM tokens for lifecycle work that AgentRail handles outside the runner: + +```bash +npm run benchmark:agentrail-swe:paired:full-lifecycle -- \ + --runner codex \ + --count 1 \ + --repo-cache .agentrail-bench/repo-cache \ + --require-token-usage +``` + +The full-lifecycle surface requires the control runner to ingest the issue from +the local provider, create the branch and pull request, check CI, check review +feedback, and complete the task. The AgentRail lane records the equivalent +lifecycle through the harness, so the runner prompt only carries the coding +work and AgentRail handoff contract. + +## Phase Attribution Benchmark + +Use phase attribution when measuring token savings from lifecycle orchestration. +This mode avoids comparing two independent autonomous agent trajectories. It +runs a deterministic lifecycle script with separate measured model calls for +each phase, then computes two accounting views: + +- Control: provider lifecycle phases plus coding phases. +- AgentRail: coding phases plus AgentRail handoff. + +```bash +npm run benchmark:agentrail-swe:phase-attribution -- \ + --runner codex \ + --count 1 \ + --repo-cache .agentrail-bench/repo-cache \ + --require-phase-token-usage +``` + +The default scenario is `ci-and-review-feedback`, which measures: + +- issue intake +- initial coding +- PR creation +- CI polling +- CI failure triage +- CI fix +- review polling +- review feedback triage +- review fix +- PR update +- final CI check +- final review check +- AgentRail handoff +- provider completion + +The result file includes per-phase token usage, command logs, Control +accounting, AgentRail accounting, and token deltas. + +Use `--phase-repeat ` to run the full phase sequence multiple times and sum +all measured phase calls into the same accounting views. + +Cursor Agent can be used for non-token smoke runs, but its CLI JSON output does +not currently expose token usage, so `--require-token-usage` and +`--require-phase-token-usage` are rejected for `--runner cursor`. + Add `--model ` only if your local Codex CLI accepts that model. The benchmark omits `--model` by default so Codex can use your configured default. @@ -78,5 +141,7 @@ The paired result records: - control lane status, patch, evaluator result, and token usage - AgentRail lane status, patch, evaluator result, and token usage +- lifecycle event summaries for each lane when `--benchmark-surface full-lifecycle` is used - `tokenDelta`, where positive means AgentRail used more tokens - `tokenReductionRate`, where positive means AgentRail used fewer tokens +- cached, non-cached input, output, and reasoning token deltas when the runner exposes them diff --git a/benchmarks/agentrail-swe-lifecycle/fixtures/fake-lifecycle-runner.mjs b/benchmarks/agentrail-swe-lifecycle/fixtures/fake-lifecycle-runner.mjs new file mode 100644 index 0000000..a2d0754 --- /dev/null +++ b/benchmarks/agentrail-swe-lifecycle/fixtures/fake-lifecycle-runner.mjs @@ -0,0 +1,67 @@ +#!/usr/bin/env node +import { execFileSync } from "node:child_process"; +import { appendFile, readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; + +const repoPath = process.cwd(); +const promptPath = process.env.AGENTRAIL_BENCH_PROMPT_PATH; +const prompt = promptPath ? await readFile(promptPath, "utf8").catch(() => "") : ""; +const hasLifecycleProvider = Boolean(process.env.AGENTRAIL_BENCH_LIFECYCLE_URL); +const lifecycleCli = process.env.AGENTRAIL_BENCH_LIFECYCLE_CLI + ?? path.resolve("benchmarks/agentrail-swe-lifecycle/fixtures/mock-lifecycle-cli.mjs"); + +const issue = hasLifecycleProvider ? runLifecycle(["issue", "get"]) : {}; +if (hasLifecycleProvider) { + runLifecycle(["branch", "create", "--name", "benchmark-fix"]); +} + +const readmePath = path.join(repoPath, "README.md"); +const previous = await readFile(readmePath, "utf8").catch(() => ""); +await writeFile(readmePath, `${previous.trimEnd()}\nfake lifecycle runner completed ${issue.instanceId ?? ""}\n`, "utf8"); + +execFileSync("git", ["-C", repoPath, "add", "README.md"]); +execFileSync("git", ["-C", repoPath, "config", "user.email", "bench@example.com"]); +execFileSync("git", ["-C", repoPath, "config", "user.name", "Benchmark Runner"]); +execFileSync("git", ["-C", repoPath, "commit", "-m", "fake lifecycle runner change"]); +const commitSha = execFileSync("git", ["-C", repoPath, "rev-parse", "HEAD"], { encoding: "utf8" }).trim(); + +if (hasLifecycleProvider) { + const pr = runLifecycle(["pr", "create", "--head", "benchmark-fix", "--base", "main", "--title", "Benchmark fix"]); + runLifecycle(["ci", "get", "--pr", String(pr.number)]); + runLifecycle(["review", "get", "--pr", String(pr.number)]); + runLifecycle(["complete", "--pr", String(pr.number), "--summary", "Fake lifecycle runner completed benchmark task."]); +} + +if (process.env.AGENTRAIL_HANDOFF_PATH) { + await writeFile(process.env.AGENTRAIL_HANDOFF_PATH, `${JSON.stringify({ + version: 1, + status: "completed", + summary: "Fake lifecycle runner completed benchmark task.", + checks: [], + artifacts: [], + blocker: null, + commitSha, + }, null, 2)}\n`, "utf8"); +} + +const usage = { + input_tokens: Math.max(1, Math.ceil(prompt.length / 4)), + output_tokens: 35, + reasoning_tokens: 7, +}; +usage.total_tokens = usage.input_tokens + usage.output_tokens + usage.reasoning_tokens; + +if (process.env.AGENTRAIL_BENCH_EXTRA_LOG_PATH) { + await appendFile(process.env.AGENTRAIL_BENCH_EXTRA_LOG_PATH, `${JSON.stringify({ usage })}\n`); +} + +process.stdout.write(`${JSON.stringify({ usage })}\n`); + +function runLifecycle(args) { + const stdout = execFileSync("node", [lifecycleCli, ...args], { + cwd: repoPath, + env: process.env, + encoding: "utf8", + }); + return JSON.parse(stdout); +} diff --git a/benchmarks/agentrail-swe-lifecycle/fixtures/fake-phase-runner.mjs b/benchmarks/agentrail-swe-lifecycle/fixtures/fake-phase-runner.mjs new file mode 100644 index 0000000..1c450e7 --- /dev/null +++ b/benchmarks/agentrail-swe-lifecycle/fixtures/fake-phase-runner.mjs @@ -0,0 +1,66 @@ +#!/usr/bin/env node +import { execFileSync } from "node:child_process"; +import { appendFile, mkdir, readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; + +const phaseId = process.env.AGENTRAIL_BENCH_PHASE_ID ?? "unknown"; +const promptPath = process.env.AGENTRAIL_BENCH_PROMPT_PATH; +const prompt = promptPath ? await readFile(promptPath, "utf8").catch(() => "") : ""; + +runLifecycleStep(phaseId); + +if (phaseId.includes("code") || phaseId.startsWith("fix_")) { + const readmePath = path.join(process.cwd(), "README.md"); + const previous = await readFile(readmePath, "utf8").catch(() => ""); + await writeFile(readmePath, `${previous.trimEnd()}\nphase runner coding change ${phaseId}\n`, "utf8"); +} + +if (phaseId === "agentrail_handoff") { + const resultPath = process.env.AGENTRAIL_RESULT_PATH; + if (resultPath) { + await mkdir(path.dirname(resultPath), { recursive: true }); + await writeFile(resultPath, `${JSON.stringify({ + version: 1, + status: "completed", + summary: "phase attribution benchmark complete", + checks: [], + artifacts: [], + })}\n`, "utf8"); + } +} + +const base = phaseId.startsWith("provider_") || phaseId.endsWith("_triage") ? 100 : 25; +const usage = { + input_tokens: Math.max(base, Math.ceil(prompt.length / 4)), + output_tokens: phaseId === "agentrail_handoff" ? 5 : 10, + reasoning_tokens: phaseId.includes("fix") ? 4 : 2, +}; +usage.total_tokens = usage.input_tokens + usage.output_tokens + usage.reasoning_tokens; + +if (process.env.AGENTRAIL_BENCH_EXTRA_LOG_PATH) { + await appendFile(process.env.AGENTRAIL_BENCH_EXTRA_LOG_PATH, `${JSON.stringify({ phaseId, usage })}\n`, "utf8"); +} + +process.stdout.write(`${JSON.stringify({ phaseId, usage })}\n`); + +function runLifecycleStep(id) { + const lifecycleCli = process.env.AGENTRAIL_BENCH_LIFECYCLE_CLI; + if (!lifecycleCli) return; + const argsByPhase = { + provider_issue_intake: ["issue", "get"], + provider_create_pr: ["pr", "create", "--head", "benchmark-fix", "--base", "main", "--title", "Benchmark fix"], + provider_wait_for_ci: ["ci", "get", "--pr", "1"], + provider_wait_for_review: ["review", "get", "--pr", "1"], + provider_update_pr: ["pr", "update", "--pr", "1", "--summary", "phase attribution update"], + provider_final_ci_check: ["ci", "get", "--pr", "1"], + provider_final_review_check: ["review", "get", "--pr", "1"], + provider_complete: ["complete", "--pr", "1", "--summary", "phase attribution benchmark complete"], + }; + const args = argsByPhase[id]; + if (!args) return; + execFileSync("node", [lifecycleCli, ...args], { + cwd: process.cwd(), + env: process.env, + stdio: "pipe", + }); +} diff --git a/benchmarks/agentrail-swe-lifecycle/fixtures/mock-lifecycle-cli.mjs b/benchmarks/agentrail-swe-lifecycle/fixtures/mock-lifecycle-cli.mjs new file mode 100644 index 0000000..87dae1c --- /dev/null +++ b/benchmarks/agentrail-swe-lifecycle/fixtures/mock-lifecycle-cli.mjs @@ -0,0 +1,168 @@ +#!/usr/bin/env node +import { readFile, writeFile } from "node:fs/promises"; + +const baseUrl = process.env.AGENTRAIL_BENCH_LIFECYCLE_URL; +const statePath = process.env.AGENTRAIL_BENCH_LIFECYCLE_STATE_PATH; +if (!baseUrl && !statePath) { + process.stderr.write("AGENTRAIL_BENCH_LIFECYCLE_URL or AGENTRAIL_BENCH_LIFECYCLE_STATE_PATH is required.\n"); + process.exit(2); +} + +const [resource, action, ...args] = process.argv.slice(2); +const flags = parseFlags(args); + +try { + if (resource === "issue" && action === "get") { + await printJson(await lifecycleOperation("issue.get", {}, () => request("GET", "/issue"))); + } else if (resource === "branch" && action === "create") { + await printJson(await lifecycleOperation("branch.create", { name: flags.name ?? "benchmark-fix" }, () => ( + request("POST", "/branches", { name: flags.name ?? "benchmark-fix" }) + ))); + } else if (resource === "pr" && action === "create") { + const body = { + head: flags.head ?? "benchmark-fix", + base: flags.base ?? "main", + title: flags.title ?? "Benchmark fix", + }; + await printJson(await lifecycleOperation("pr.create", body, () => request("POST", "/pulls", body))); + } else if (resource === "pr" && action === "update") { + const body = { + pr: flags.pr ?? "1", + summary: flags.summary ?? "", + }; + await printJson(await lifecycleOperation("pr.update", body, () => request("POST", "/pulls/update", body))); + } else if (resource === "ci" && action === "get") { + await printJson(await lifecycleOperation("ci.get", { pr: flags.pr ?? "" }, () => ( + request("GET", `/ci?pr=${encodeURIComponent(flags.pr ?? "")}`) + ))); + } else if (resource === "review" && action === "get") { + await printJson(await lifecycleOperation("review.get", { pr: flags.pr ?? "" }, () => ( + request("GET", `/review?pr=${encodeURIComponent(flags.pr ?? "")}`) + ))); + } else if (resource === "complete") { + const body = { + pr: flags.pr ?? "", + summary: flags.summary ?? "", + }; + await printJson(await lifecycleOperation("complete", body, () => request("POST", "/complete", body))); + } else { + process.stderr.write("Usage: issue get | branch create | pr create | pr update | ci get | review get | complete\n"); + process.exit(2); + } +} catch (error) { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + process.exit(1); +} + +async function lifecycleOperation(type, body, httpRequest) { + if (statePath) { + return await fileLifecycleOperation(type, body); + } + return await httpRequest(); +} + +async function fileLifecycleOperation(type, body) { + const state = JSON.parse(await readFile(statePath, "utf8")); + state.events = Array.isArray(state.events) ? state.events : []; + state.pullRequests = Array.isArray(state.pullRequests) ? state.pullRequests : []; + + if (type === "issue.get") { + state.events.push({ type }); + await writeState(state); + return { + provider: state.task?.lifecycle?.issueProvider ?? "github", + repo: state.task?.repo, + instanceId: state.upstream?.instanceId, + baseCommit: state.upstream?.baseCommit, + problemStatement: state.upstream?.problemStatement, + requirements: state.upstream?.requirements ?? null, + interface: state.upstream?.interface ?? null, + }; + } + + if (type === "branch.create") { + const name = stringValue(body.name) || "benchmark-fix"; + state.events.push({ type, name }); + await writeState(state); + return { status: "created", name }; + } + + if (type === "pr.create") { + const number = state.pullRequests.length + 1; + const pullRequest = { + number, + url: `file://${statePath}#pull-${number}`, + head: stringValue(body.head) || "benchmark-fix", + base: stringValue(body.base) || "main", + title: stringValue(body.title) || `Benchmark fix ${number}`, + }; + state.pullRequests.push(pullRequest); + state.events.push({ type, pullRequest }); + await writeState(state); + return pullRequest; + } + + if (type === "pr.update") { + const pr = stringValue(body.pr) || "1"; + state.events.push({ type, pr, summary: stringValue(body.summary) }); + await writeState(state); + return { status: "updated", pr }; + } + + if (type === "ci.get") { + const pr = stringValue(body.pr); + state.events.push({ type, pr }); + await writeState(state); + return { provider: "mock-ci", pr, status: state.ciStatus ?? "passed" }; + } + + if (type === "review.get") { + const pr = stringValue(body.pr); + state.events.push({ type, pr }); + await writeState(state); + return { provider: "mock-review", pr, state: state.reviewState ?? "approved", comments: [] }; + } + + if (type === "complete") { + state.completed = true; + state.events.push({ type, pr: stringValue(body.pr), summary: stringValue(body.summary) }); + await writeState(state); + return { status: "completed" }; + } + + throw new Error(`Unsupported lifecycle operation: ${type}`); +} + +async function writeState(state) { + await writeFile(statePath, `${JSON.stringify(state, null, 2)}\n`, "utf8"); +} + +async function request(method, pathname, body) { + const response = await fetch(new URL(pathname, baseUrl), { + method, + headers: body ? { "content-type": "application/json" } : {}, + body: body ? JSON.stringify(body) : undefined, + }); + const text = await response.text(); + if (!response.ok) throw new Error(text); + return JSON.parse(text); +} + +function parseFlags(args) { + const result = {}; + for (let index = 0; index < args.length; index += 1) { + const key = args[index]; + if (!key.startsWith("--")) continue; + result[key.slice(2)] = args[index + 1] ?? ""; + index += 1; + } + return result; +} + +async function printJson(value) { + process.stdout.write(`${JSON.stringify(value)}\n`); +} + +function stringValue(value) { + return typeof value === "string" ? value : ""; +} diff --git a/benchmarks/agentrail-swe-lifecycle/lib/agentrail-harness.mjs b/benchmarks/agentrail-swe-lifecycle/lib/agentrail-harness.mjs index 3a3daab..8833638 100644 --- a/benchmarks/agentrail-swe-lifecycle/lib/agentrail-harness.mjs +++ b/benchmarks/agentrail-swe-lifecycle/lib/agentrail-harness.mjs @@ -8,6 +8,7 @@ import { AgentTaskQueue } from "../../../src/agent-task-queue.ts"; import { createServer } from "../../../src/app.ts"; import { TaskEventStore } from "../../../src/task-event-store.ts"; import { buildAgentRailTaskFields } from "./prompt.mjs"; +import { summarizeLifecycleEvents, AGENTRAIL_LIFECYCLE_EVENTS } from "./lifecycle-provider-harness.mjs"; export async function withAgentRailBenchmarkHarness({ runDir, task, upstream, agentId = "agt_bench", flags = {} }, handler) { const storeDir = path.join(runDir, "agentrail-store"); @@ -15,6 +16,11 @@ export async function withAgentRailBenchmarkHarness({ runDir, task, upstream, ag const now = () => new Date("2026-05-25T00:00:00.000Z"); const eventStore = new TaskEventStore({ now, storagePath: path.join(storeDir, "events.json") }); + const lifecycleEvents = []; + const lifecycleEventsPath = path.join(runDir, "lifecycle-events.json"); + const recordLifecycleEvent = (type, data = {}) => { + lifecycleEvents.push({ type, ...data }); + }; const submissions = []; let baseUrl = "http://127.0.0.1:0"; const taskQueue = new AgentTaskQueue({ @@ -39,6 +45,12 @@ export async function withAgentRailBenchmarkHarness({ runDir, task, upstream, ag availableActions: ["view_ci_status", "view_review_feedback"], }; submissions.push(response.data); + if (flags.benchmarkSurface === "full-lifecycle") { + recordLifecycleEvent("pr.created", response.data); + recordLifecycleEvent("ci.checked", { provider: "mock-ci", status: "passed", prNumber: response.data.prNumber }); + recordLifecycleEvent("review.checked", { provider: "mock-review", state: "approved", prNumber: response.data.prNumber }); + recordLifecycleEvent("task.completed", { taskId }); + } return response; }, }, @@ -54,14 +66,17 @@ export async function withAgentRailBenchmarkHarness({ runDir, task, upstream, ag scopes: ["tasks:read", "tasks:write", "events:read", "ci:read", "reviews:read", "ship:write"], }, `benchmark-key:${agentId}`); const agentRailTask = taskQueue.createTask(buildTaskRecord({ task, upstream, agentId })); + if (flags.benchmarkSurface === "full-lifecycle") { + recordLifecycleEvent("issue.ingested", { taskId: agentRailTask.id, repo: task.repo, instanceId: upstream.instanceId }); + } const server = createServer({ store: eventStore, authStore, agentRunStore: new AgentRunStore({ now, storagePath: path.join(storeDir, "agent-runs.json") }), taskLifecycleStore: taskQueue, - ciStatusAdapter: null, - reviewFeedbackAdapter: null, + ciStatusAdapter: flags.benchmarkSurface === "full-lifecycle" ? mockCiStatusAdapter() : null, + reviewFeedbackAdapter: flags.benchmarkSurface === "full-lifecycle" ? mockReviewFeedbackAdapter() : null, now, }); @@ -89,8 +104,17 @@ export async function withAgentRailBenchmarkHarness({ runDir, task, upstream, ag agentRailTaskId: agentRailTask.id, envFilePath, submissions, + lifecycleEvents, + lifecycleEventsPath, + recordLifecycleEvent, + lifecycleSummary: () => summarizeLifecycleEvents({ + eventsPath: lifecycleEventsPath, + events: lifecycleEvents, + requiredEvents: AGENTRAIL_LIFECYCLE_EVENTS, + }), }); } finally { + await writeFile(lifecycleEventsPath, `${JSON.stringify(lifecycleEvents, null, 2)}\n`, "utf8"); await new Promise((resolve, reject) => { server.close((error) => { if (error) reject(error); @@ -100,6 +124,34 @@ export async function withAgentRailBenchmarkHarness({ runDir, task, upstream, ag } } +function mockCiStatusAdapter() { + return { + async getTaskCiStatus() { + return { + data: { + provider: "mock-ci", + status: "passed", + overallStatus: "passed", + }, + }; + }, + }; +} + +function mockReviewFeedbackAdapter() { + return { + async getTaskReviewFeedback() { + return { + data: { + provider: "mock-review", + state: "approved", + comments: [], + }, + }; + }, + }; +} + function buildTaskRecord({ task, upstream, agentId }) { const fields = buildAgentRailTaskFields({ task, upstream }); const [owner, repo] = task.repo.split("/"); diff --git a/benchmarks/agentrail-swe-lifecycle/lib/agentrail-lane.mjs b/benchmarks/agentrail-swe-lifecycle/lib/agentrail-lane.mjs index f027f4e..3025e53 100644 --- a/benchmarks/agentrail-swe-lifecycle/lib/agentrail-lane.mjs +++ b/benchmarks/agentrail-swe-lifecycle/lib/agentrail-lane.mjs @@ -1,4 +1,5 @@ import { execFile } from "node:child_process"; +import { appendFile, mkdir, readFile, writeFile } from "node:fs/promises"; import path from "node:path"; import { promisify } from "node:util"; @@ -50,6 +51,9 @@ export async function runAgentRailLane({ task, upstream, runDir, repoCache, sour process.env.AGENTRAIL_HOME = agentRailHome; process.env.AGENTRAIL_TELEMETRY_DISABLED = "1"; try { + if (flags.benchmarkSurface === "full-lifecycle") { + harness.recordLifecycleEvent("task.started", { taskId: harness.agentRailTaskId }); + } const exitCode = await runCli([ "agent", "run", @@ -70,12 +74,20 @@ export async function runAgentRailLane({ task, upstream, runDir, repoCache, sour }, launchRunner: async ({ worktreePath, prompt, logPath, handoffPath, env }) => { managedWorktreePath = worktreePath; + const preparedPrompt = await prepareBenchmarkPromptVariant({ + worktreePath, + prompt, + handoffPath, + task, + upstream, + flags, + }); runnerResult = await runRunnerCommand({ runner: flags.runner, runnerCommand: flags.runnerCommand, model: flags.model ?? null, repoPath: worktreePath, - prompt, + prompt: preparedPrompt, logDir: path.dirname(logPath), timeoutSeconds: flags.timeoutSeconds ?? 1800, env: { @@ -97,7 +109,7 @@ export async function runAgentRailLane({ task, upstream, runDir, repoCache, sour publishBranch: async () => {}, }, }); - return { exitCode, submissions: harness.submissions }; + return { exitCode, submissions: harness.submissions, lifecycle: harness.lifecycleSummary() }; } finally { restoreEnv("AGENTRAIL_HOME", previousHome); restoreEnv("AGENTRAIL_TELEMETRY_DISABLED", previousTelemetryDisabled); @@ -123,6 +135,7 @@ export async function runAgentRailLane({ task, upstream, runDir, repoCache, sour const status = result.exitCode === 0 && runnerResult?.status === "completed" ? "completed" : "failed"; const success = status === "completed" && patch.status === "available" + && (flags.benchmarkSurface !== "full-lifecycle" || result.lifecycle?.complete === true) && (evaluator.status === "passed" || evaluator.status === "unavailable"); return { @@ -133,6 +146,7 @@ export async function runAgentRailLane({ task, upstream, runDir, repoCache, sour tokenUsage, patch, evaluator, + lifecycle: result.lifecycle ?? null, stdout: stdout.toString(), stderr: stderr.toString(), submissions: result.submissions, @@ -167,6 +181,187 @@ export function benchmarkRunnerPolicy() { return { preset: "advisory", enforcementMode: "advisory" }; } +async function prepareBenchmarkPromptVariant({ worktreePath, prompt, handoffPath, task, upstream, flags }) { + const variant = flags.agentrailPromptVariant ?? "current"; + if (variant === "current") return prompt; + if (variant !== "contract" && variant !== "skill") { + throw new Error(`Unsupported AgentRail prompt variant: ${variant}`); + } + + const paths = await writeAgentRailRunContractFiles({ worktreePath, handoffPath, task, upstream }); + if (variant === "skill") { + await writeAgentRailSkillFiles({ worktreePath, contractPath: paths.agentrailPath }); + } + return buildContractPrompt({ + task, + paths, + includeSkill: variant === "skill", + }); +} + +async function writeAgentRailRunContractFiles({ worktreePath, handoffPath, task, upstream }) { + const runDir = path.join(worktreePath, ".agentrail-run"); + await mkdir(runDir, { recursive: true }); + const taskPath = path.join(runDir, "task.md"); + const schemaPath = path.join(runDir, "result.schema.json"); + const agentrailPath = path.join(runDir, "agentrail.md"); + await writeFile(taskPath, renderTaskFile({ task, upstream }), "utf8"); + await writeFile(schemaPath, `${JSON.stringify(resultSchema(), null, 2)}\n`, "utf8"); + await writeFile(agentrailPath, renderAgentRailContract({ resultPath: relativePath(worktreePath, handoffPath) }), "utf8"); + return { + taskPath: relativePath(worktreePath, taskPath), + schemaPath: relativePath(worktreePath, schemaPath), + agentrailPath: relativePath(worktreePath, agentrailPath), + resultPath: relativePath(worktreePath, handoffPath), + }; +} + +async function writeAgentRailSkillFiles({ worktreePath, contractPath }) { + const skillDir = path.join(worktreePath, ".codex", "skills", "agentrail-run"); + const referencesDir = path.join(skillDir, "references"); + await mkdir(referencesDir, { recursive: true }); + await addGitExcludeEntry(worktreePath, ".codex/skills/agentrail-run/"); + await writeFile(path.join(skillDir, "SKILL.md"), [ + "---", + "name: agentrail-run", + "description: Use inside AgentRail-managed benchmark runs only when lifecycle, reporting, or result-file details are needed.", + "---", + "", + "# AgentRail Run", + "", + "You are in an AgentRail-managed run. AgentRail owns task assignment, pull requests, CI/review polling, shipping, and rollback.", + "", + "Read `.agentrail-run/task.md` for the assigned task. Write the result JSON to `$AGENTRAIL_RESULT_PATH` using `.agentrail-run/result.schema.json`.", + "", + "Do repository work only. Do not push, create pull requests, ship, roll back, commit manually, or call broad AgentRail APIs.", + "", + "For details, read `references/contract.md`.", + "", + ].join("\n"), "utf8"); + await writeFile(path.join(referencesDir, "contract.md"), [ + "# AgentRail Contract Reference", + "", + `The canonical run contract for this benchmark task is \`${contractPath}\`.`, + "", + "Completion means the repository patch is left in the worktree and the result file says `completed`.", + "Blocked means no safe repository patch can be produced without user action; the result file must include a blocker object.", + "", + ].join("\n"), "utf8"); +} + +function buildContractPrompt({ task, paths, includeSkill }) { + return [ + `You are working on AgentRail task ${task.upstream?.instanceId ? `swe:${task.upstream.instanceId}` : task.id}.`, + "", + `Read ${paths.taskPath}.`, + `Write ${paths.resultPath} matching ${paths.schemaPath}.`, + includeSkill + ? "If you need AgentRail lifecycle/reporting details, use the agentrail-run skill or read .codex/skills/agentrail-run/SKILL.md." + : `If you need AgentRail lifecycle/reporting details, read ${paths.agentrailPath}.`, + "Modify only repository files needed to solve the task.", + "Do not push, create pull requests, ship, roll back, run git add, or run git commit.", + "Leave the final patch in the working tree for AgentRail to collect.", + "Do not expose secrets in output.", + "", + ].join("\n"); +} + +function renderTaskFile({ task, upstream }) { + return [ + "# AgentRail Benchmark Task", + "", + `Repository: ${task.repo}`, + `Instance: ${upstream.instanceId}`, + `Base commit: ${upstream.baseCommit}`, + "", + "## Problem Statement", + "", + upstream.problemStatement, + upstream.requirements ? ["", "## Requirements", "", upstream.requirements].join("\n") : "", + upstream.interface ? ["", "## Interface Notes", "", upstream.interface].join("\n") : "", + "", + "## Acceptance Criteria", + "", + "- The repository is patched to solve the problem statement.", + "- The final patch is left in the git working tree.", + "- No gold patch or hidden test metadata is used.", + "", + ].filter(Boolean).join("\n"); +} + +function renderAgentRailContract({ resultPath }) { + return [ + "# AgentRail Run Contract", + "", + "AgentRail owns lifecycle state, pull request creation, CI reads, review reads, shipping, and rollback.", + "The runner only edits repository files, validates the smallest relevant surface, and writes a structured result.", + "", + "## Rules", + "", + "- Work only inside this git worktree.", + "- Do not hand-roll AgentRail API calls.", + "- Do not run `git add`, `git commit`, `git push`, create pull requests, ship, or roll back.", + "- Leave completed changes in place for AgentRail to collect.", + `- Write the result JSON to \`${resultPath}\` or \`$AGENTRAIL_RESULT_PATH\`.`, + "", + "## Result Examples", + "", + "Completed:", + "```json", + "{\"version\":1,\"status\":\"completed\",\"summary\":\"what changed\",\"checks\":[],\"artifacts\":[]}", + "```", + "", + "Blocked:", + "```json", + "{\"version\":1,\"status\":\"blocked\",\"summary\":\"what is blocked\",\"blocker\":{\"reason\":\"short reason\",\"actionRequired\":\"what the user must do\",\"resumeInstructions\":\"how to continue after the user acts\"}}", + "```", + "", + ].join("\n"); +} + +function resultSchema() { + return { + $schema: "https://json-schema.org/draft/2020-12/schema", + type: "object", + required: ["version", "status", "summary"], + properties: { + version: { const: 1 }, + status: { enum: ["completed", "blocked"] }, + summary: { type: "string" }, + checks: { type: "array", items: { type: "string" }, default: [] }, + artifacts: { type: "array", items: { type: "string" }, default: [] }, + blocker: { + type: "object", + required: ["reason", "actionRequired", "resumeInstructions"], + properties: { + reason: { type: "string" }, + actionRequired: { type: "string" }, + resumeInstructions: { type: "string" }, + }, + additionalProperties: true, + }, + }, + additionalProperties: true, + }; +} + +async function addGitExcludeEntry(worktreePath, entry) { + const excludePath = (await execFileAsync("git", ["-C", worktreePath, "rev-parse", "--git-path", "info/exclude"])).stdout.trim(); + const resolvedExcludePath = path.isAbsolute(excludePath) ? excludePath : path.join(worktreePath, excludePath); + await mkdir(path.dirname(resolvedExcludePath), { recursive: true }); + const existing = await readFile(resolvedExcludePath, "utf8").catch((error) => { + if (error?.code === "ENOENT") return ""; + throw error; + }); + if (existing.split(/\r?\n/u).map((line) => line.trim()).includes(entry)) return; + const prefix = existing.length === 0 || existing.endsWith("\n") ? "" : "\n"; + await appendFile(resolvedExcludePath, `${prefix}${entry}\n`, "utf8"); +} + +function relativePath(from, to) { + return path.relative(from, to).replaceAll(path.sep, "/") || "."; +} + function createMemoryWriter() { const chunks = []; return { diff --git a/benchmarks/agentrail-swe-lifecycle/lib/args.mjs b/benchmarks/agentrail-swe-lifecycle/lib/args.mjs index 95b89ac..3d6bc01 100644 --- a/benchmarks/agentrail-swe-lifecycle/lib/args.mjs +++ b/benchmarks/agentrail-swe-lifecycle/lib/args.mjs @@ -1,13 +1,19 @@ -const VALID_MODES = new Set(["dry-run", "control", "agentrail", "paired"]); -const REAL_MODES = new Set(["control", "agentrail", "paired"]); +const VALID_MODES = new Set(["dry-run", "control", "agentrail", "paired", "phase-attribution"]); +const REAL_MODES = new Set(["control", "agentrail", "paired", "phase-attribution"]); const VALID_RUNNERS = new Set(["codex", "cursor", "claude-code", "custom"]); const VALID_EVALUATORS = new Set(["none", "command"]); +const VALID_AGENTRAIL_PROMPT_VARIANTS = new Set(["current", "contract", "skill"]); +const VALID_BENCHMARK_SURFACES = new Set(["patch-only", "full-lifecycle"]); +const VALID_PHASE_SCENARIOS = new Set(["ci-and-review-feedback"]); export function parseBenchmarkArgs(argv) { const flags = { mode: "dry-run", evaluator: "none", - cleanup: true + cleanup: true, + benchmarkSurface: "patch-only", + phaseScenario: "ci-and-review-feedback", + phaseRepeat: 1 }; for (let index = 0; index < argv.length; index += 1) { @@ -38,6 +44,16 @@ export function parseBenchmarkArgs(argv) { flags.requireEvaluator = true; } else if (arg === "--continue-on-error") { flags.continueOnError = true; + } else if (arg === "--agentrail-prompt-variant") { + flags.agentrailPromptVariant = readFlagValue({ argv, index: ++index, flag: arg }); + } else if (arg === "--benchmark-surface") { + flags.benchmarkSurface = readFlagValue({ argv, index: ++index, flag: arg }); + } else if (arg === "--phase-scenario") { + flags.phaseScenario = readFlagValue({ argv, index: ++index, flag: arg }); + } else if (arg === "--phase-repeat") { + flags.phaseRepeat = parsePositiveSafeInteger(readFlagValue({ argv, index: ++index, flag: arg }), arg); + } else if (arg === "--require-phase-token-usage") { + flags.requirePhaseTokenUsage = true; } else if (arg === "--no-cleanup") { flags.cleanup = false; } else if (arg === "--dataset") { @@ -86,6 +102,24 @@ function validateBenchmarkArgs(flags) { if (flags.evaluator === "command" && flags.evalCommandTemplate === undefined) { throw new Error("--eval-command-template is required when --evaluator command is used."); } + if (flags.runner === "cursor" && (flags.requireTokenUsage || flags.requirePhaseTokenUsage)) { + throw new Error( + "Cursor Agent CLI does not expose token usage in its JSON output. " + + "Use --runner codex for token-required benchmarks, or omit token usage requirements for Cursor smoke runs.", + ); + } + if ( + flags.agentrailPromptVariant !== undefined + && !VALID_AGENTRAIL_PROMPT_VARIANTS.has(flags.agentrailPromptVariant) + ) { + throw new Error(`--agentrail-prompt-variant must be one of: ${[...VALID_AGENTRAIL_PROMPT_VARIANTS].join(", ")}.`); + } + if (!VALID_BENCHMARK_SURFACES.has(flags.benchmarkSurface)) { + throw new Error(`--benchmark-surface must be one of: ${[...VALID_BENCHMARK_SURFACES].join(", ")}.`); + } + if (!VALID_PHASE_SCENARIOS.has(flags.phaseScenario)) { + throw new Error(`--phase-scenario must be one of: ${[...VALID_PHASE_SCENARIOS].join(", ")}.`); + } } function readFlagValue({ argv, index, flag }) { diff --git a/benchmarks/agentrail-swe-lifecycle/lib/control-lane.mjs b/benchmarks/agentrail-swe-lifecycle/lib/control-lane.mjs index 13b9bf9..d221e7b 100644 --- a/benchmarks/agentrail-swe-lifecycle/lib/control-lane.mjs +++ b/benchmarks/agentrail-swe-lifecycle/lib/control-lane.mjs @@ -1,4 +1,5 @@ import path from "node:path"; +import { fileURLToPath } from "node:url"; import { runEvaluator } from "./evaluator.mjs"; import { capturePatch } from "./patch.mjs"; @@ -6,8 +7,21 @@ import { buildControlPrompt } from "./prompt.mjs"; import { runRunnerCommand } from "./runner-command.mjs"; import { parseTokenUsageFromFiles } from "./token-usage.mjs"; import { prepareLaneWorkspace } from "./workspace.mjs"; +import { buildFullLifecycleControlPrompt } from "./lifecycle-control-prompt.mjs"; +import { + CONTROL_LIFECYCLE_EVENTS, + summarizeLifecycleEvents, + withLifecycleProviderHarness, +} from "./lifecycle-provider-harness.mjs"; export async function runControlLane({ task, upstream, runDir, repoCache, sourceRepoPath, flags }) { + if (flags.benchmarkSurface === "full-lifecycle") { + return await runFullLifecycleControlLane({ task, upstream, runDir, repoCache, sourceRepoPath, flags }); + } + return await runPatchOnlyControlLane({ task, upstream, runDir, repoCache, sourceRepoPath, flags }); +} + +async function runPatchOnlyControlLane({ task, upstream, runDir, repoCache, sourceRepoPath, flags }) { const workspace = await prepareLaneWorkspace({ task, upstream, @@ -56,3 +70,80 @@ export async function runControlLane({ task, upstream, runDir, repoCache, source }, }; } + +async function runFullLifecycleControlLane({ task, upstream, runDir, repoCache, sourceRepoPath, flags }) { + const workspace = await prepareLaneWorkspace({ + task, + upstream, + lane: "control", + runDir, + repoCache, + sourceRepoPath, + }); + + return await withLifecycleProviderHarness({ + runDir: workspace.laneDir, + stateDir: workspace.worktreePath, + task, + upstream, + }, async (lifecycle) => { + const lifecycleCli = path.join(repoRoot(), "benchmarks", "agentrail-swe-lifecycle", "fixtures", "mock-lifecycle-cli.mjs"); + const prompt = buildFullLifecycleControlPrompt({ + repo: task.repo, + lifecycleCli: `node ${lifecycleCli}`, + }); + const runner = await runRunnerCommand({ + runner: flags.runner, + runnerCommand: flags.runnerCommand, + model: flags.model ?? null, + repoPath: workspace.worktreePath, + prompt, + logDir: workspace.laneDir, + timeoutSeconds: flags.timeoutSeconds ?? 1800, + env: { + AGENTRAIL_BENCH_LIFECYCLE_URL: lifecycle.baseUrl, + AGENTRAIL_BENCH_LIFECYCLE_STATE_PATH: lifecycle.statePath, + AGENTRAIL_BENCH_LIFECYCLE_CLI: lifecycleCli, + }, + }); + const patch = await capturePatch({ + worktreePath: workspace.worktreePath, + outputPath: path.join(workspace.laneDir, "patch.diff"), + }); + const tokenUsage = await parseTokenUsageFromFiles([runner.stdoutPath, runner.stderrPath]); + const evaluator = await runEvaluator({ + evaluator: flags.evaluator, + evalCommandTemplate: flags.evalCommandTemplate, + patchPath: patch.path, + outputPath: path.join(workspace.laneDir, "evaluator.json"), + timeoutSeconds: flags.timeoutSeconds ?? 1800, + }); + const lifecycleSummary = summarizeLifecycleEvents({ + eventsPath: lifecycle.eventsPath, + events: await lifecycle.readEvents(), + requiredEvents: CONTROL_LIFECYCLE_EVENTS, + }); + const success = runner.status === "completed" + && patch.status === "available" + && lifecycleSummary.complete + && (evaluator.status === "passed" || evaluator.status === "unavailable"); + + return { + status: runner.status, + success, + runner, + tokenUsage, + patch, + evaluator, + lifecycle: lifecycleSummary, + workspace: { + laneDir: workspace.laneDir, + worktreePath: workspace.worktreePath, + }, + }; + }); +} + +function repoRoot() { + return path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", "..", ".."); +} diff --git a/benchmarks/agentrail-swe-lifecycle/lib/lifecycle-control-prompt.mjs b/benchmarks/agentrail-swe-lifecycle/lib/lifecycle-control-prompt.mjs new file mode 100644 index 0000000..259df02 --- /dev/null +++ b/benchmarks/agentrail-swe-lifecycle/lib/lifecycle-control-prompt.mjs @@ -0,0 +1,30 @@ +export function buildFullLifecycleControlPrompt({ repo, lifecycleCli }) { + return [ + "You are solving a software engineering task through the normal provider lifecycle.", + "", + `Repository: ${repo}`, + "", + "You must use the lifecycle CLI to fetch the issue and complete the provider workflow.", + "", + "Lifecycle CLI:", + `- Fetch issue: ${lifecycleCli} issue get`, + `- Create branch record: ${lifecycleCli} branch create --name benchmark-fix`, + `- Create pull request: ${lifecycleCli} pr create --head benchmark-fix --base main --title "Benchmark fix"`, + `- Check CI: ${lifecycleCli} ci get --pr `, + `- Check code review: ${lifecycleCli} review get --pr `, + `- Complete lifecycle: ${lifecycleCli} complete --pr --summary "short summary"`, + "", + "Required flow:", + "1. Fetch the issue through the lifecycle CLI before inspecting the repository.", + "2. Modify the repository to solve the fetched issue.", + "3. Leave the final patch in the git working tree.", + "4. Create a pull request through the lifecycle CLI.", + "5. Check CI through the lifecycle CLI.", + "6. Check code review through the lifecycle CLI.", + "7. Complete the lifecycle through the lifecycle CLI.", + "", + "Do not use gold patches or hidden test metadata.", + "Do not invent credentials or external services.", + "", + ].join("\n"); +} diff --git a/benchmarks/agentrail-swe-lifecycle/lib/lifecycle-provider-harness.mjs b/benchmarks/agentrail-swe-lifecycle/lib/lifecycle-provider-harness.mjs new file mode 100644 index 0000000..48c1e97 --- /dev/null +++ b/benchmarks/agentrail-swe-lifecycle/lib/lifecycle-provider-harness.mjs @@ -0,0 +1,183 @@ +import { once } from "node:events"; +import { createServer } from "node:http"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; + +export const CONTROL_LIFECYCLE_EVENTS = ["issue.get", "branch.create", "pr.create", "ci.get", "review.get", "complete"]; +export const PHASE_ATTRIBUTION_LIFECYCLE_EVENTS = [ + "issue.get", + "pr.create", + "ci.get", + "review.get", + "pr.update", + "complete", +]; +export const AGENTRAIL_LIFECYCLE_EVENTS = ["issue.ingested", "task.started", "pr.created", "ci.checked", "review.checked", "task.completed"]; + +export async function withLifecycleProviderHarness({ runDir, stateDir, task, upstream }, handler) { + const events = []; + const resolvedRunDir = path.resolve(runDir); + const resolvedStateDir = path.resolve(stateDir ?? runDir); + const eventsPath = path.join(resolvedRunDir, "lifecycle-events.json"); + const statePath = path.join(resolvedStateDir, "lifecycle-state.json"); + await mkdir(resolvedRunDir, { recursive: true }); + await mkdir(resolvedStateDir, { recursive: true }); + + let baseUrl = "http://127.0.0.1:0"; + const state = { + task, + upstream, + pullRequests: [], + ciStatus: "passed", + reviewState: "approved", + completed: false, + }; + const recordEvent = (type, data = {}) => { + events.push({ type, ...data }); + }; + await writeLifecycleState({ statePath, state, events }); + + const server = createServer(async (request, response) => { + try { + const url = new URL(request.url ?? "/", "http://127.0.0.1"); + const body = request.method === "POST" ? await readJsonBody(request) : {}; + + if (request.method === "GET" && url.pathname === "/issue") { + recordEvent("issue.get"); + return sendJson(response, { + provider: task.lifecycle?.issueProvider ?? "github", + repo: task.repo, + instanceId: upstream.instanceId, + baseCommit: upstream.baseCommit, + problemStatement: upstream.problemStatement, + requirements: upstream.requirements ?? null, + interface: upstream.interface ?? null, + }); + } + + if (request.method === "POST" && url.pathname === "/branches") { + const name = stringValue(body.name) || "benchmark-fix"; + recordEvent("branch.create", { name }); + return sendJson(response, { status: "created", name }); + } + + if (request.method === "POST" && url.pathname === "/pulls") { + const number = state.pullRequests.length + 1; + const pullRequest = { + number, + url: `${baseUrl}/pull/${number}`, + head: stringValue(body.head) || "benchmark-fix", + base: stringValue(body.base) || "main", + title: stringValue(body.title) || `Benchmark fix ${number}`, + }; + state.pullRequests.push(pullRequest); + recordEvent("pr.create", { pullRequest }); + return sendJson(response, pullRequest); + } + + if (request.method === "POST" && url.pathname === "/pulls/update") { + const pr = stringValue(body.pr) || "1"; + recordEvent("pr.update", { pr, summary: stringValue(body.summary) }); + return sendJson(response, { status: "updated", pr }); + } + + if (request.method === "GET" && url.pathname === "/ci") { + const pr = url.searchParams.get("pr") ?? ""; + recordEvent("ci.get", { pr }); + return sendJson(response, { provider: "mock-ci", pr, status: state.ciStatus }); + } + + if (request.method === "GET" && url.pathname === "/review") { + const pr = url.searchParams.get("pr") ?? ""; + recordEvent("review.get", { pr }); + return sendJson(response, { provider: "mock-review", pr, state: state.reviewState, comments: [] }); + } + + if (request.method === "POST" && url.pathname === "/complete") { + state.completed = true; + recordEvent("complete", { pr: stringValue(body.pr), summary: stringValue(body.summary) }); + return sendJson(response, { status: "completed" }); + } + + response.writeHead(404, { "content-type": "application/json" }); + response.end(`${JSON.stringify({ error: "not_found" })}\n`); + } catch (error) { + response.writeHead(500, { "content-type": "application/json" }); + response.end(`${JSON.stringify({ error: error instanceof Error ? error.message : String(error) })}\n`); + } + }); + + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("Missing lifecycle provider address."); + baseUrl = `http://${address.address}:${address.port}`; + + try { + return await handler({ + baseUrl, + statePath, + eventsPath, + events, + recordEvent, + readEvents: async () => await readLifecycleStateEvents({ statePath, fallbackEvents: events }), + state, + }); + } finally { + const recordedEvents = await readLifecycleStateEvents({ statePath, fallbackEvents: events }); + await writeFile(eventsPath, `${JSON.stringify(recordedEvents, null, 2)}\n`, "utf8"); + await new Promise((resolve, reject) => { + server.close((error) => error ? reject(error) : resolve()); + }); + } +} + +async function writeLifecycleState({ statePath, state, events }) { + await writeFile(statePath, `${JSON.stringify({ + task: state.task, + upstream: state.upstream, + pullRequests: state.pullRequests, + ciStatus: state.ciStatus, + reviewState: state.reviewState, + completed: state.completed, + events, + }, null, 2)}\n`, "utf8"); +} + +async function readLifecycleStateEvents({ statePath, fallbackEvents }) { + try { + const state = JSON.parse(await readFile(statePath, "utf8")); + if (Array.isArray(state.events) && state.events.length > 0) return state.events; + return fallbackEvents; + } catch { + return fallbackEvents; + } +} + +export function summarizeLifecycleEvents({ eventsPath, events, requiredEvents }) { + const eventTypes = Array.isArray(events) ? events.map((event) => event.type).filter(Boolean) : []; + const missingEvents = requiredEvents.filter((eventType) => !eventTypes.includes(eventType)); + return { + status: "available", + eventsPath, + events: eventTypes, + missingEvents, + complete: missingEvents.length === 0, + }; +} + +async function readJsonBody(request) { + const chunks = []; + for await (const chunk of request) chunks.push(Buffer.from(chunk)); + const text = Buffer.concat(chunks).toString("utf8").trim(); + return text ? JSON.parse(text) : {}; +} + +function sendJson(response, data) { + response.writeHead(200, { "content-type": "application/json" }); + response.end(`${JSON.stringify(data)}\n`); +} + +function stringValue(value) { + return typeof value === "string" ? value : ""; +} diff --git a/benchmarks/agentrail-swe-lifecycle/lib/patch.mjs b/benchmarks/agentrail-swe-lifecycle/lib/patch.mjs index 5794c5a..12111ed 100644 --- a/benchmarks/agentrail-swe-lifecycle/lib/patch.mjs +++ b/benchmarks/agentrail-swe-lifecycle/lib/patch.mjs @@ -7,6 +7,10 @@ const execFileAsync = promisify(execFile); const GIT_MAX_BUFFER = 1024 * 1024 * 20; const BASE_REF = "refs/agentrail-bench/base"; const OUTPUT_PATH_ERROR = "capturePatch outputPath must be outside the benchmark worktree."; +const BENCHMARK_PATCH_EXCLUDES = new Set([ + "lifecycle-state.json", + ".agentrail-run/result.json", +]); export async function capturePatch({ worktreePath, outputPath }) { await assertOutputPathOutsideWorktree({ worktreePath, outputPath }); @@ -17,10 +21,12 @@ export async function capturePatch({ worktreePath, outputPath }) { await includeUntrackedFiles(worktreePath); - const diff = await git(["-C", worktreePath, "diff", "--binary", BASE_REF]); - const changedFilesOutput = await git(["-C", worktreePath, "diff", "--name-only", "-z", BASE_REF]); + const diff = await git(["-C", worktreePath, "diff", "--binary", BASE_REF, ...patchPathspecArgs()]); + const changedFilesOutput = await git(["-C", worktreePath, "diff", "--name-only", "-z", BASE_REF, ...patchPathspecArgs()]); const commitSha = (await git(["-C", worktreePath, "rev-parse", "HEAD"])).trim(); - const changedFiles = changedFilesOutput.split("\0").filter((line) => line.length > 0); + const changedFiles = changedFilesOutput + .split("\0") + .filter((line) => line.length > 0 && !BENCHMARK_PATCH_EXCLUDES.has(line)); await mkdir(path.dirname(outputPath), { recursive: true }); await writeFile(outputPath, diff, "utf8"); @@ -111,11 +117,21 @@ function isInsideOrEqual(parentPath, candidatePath) { async function includeUntrackedFiles(worktreePath) { const output = await git(["-C", worktreePath, "ls-files", "--others", "--exclude-standard", "-z"]); - const files = output.split("\0").filter((file) => file.length > 0); + const files = output + .split("\0") + .filter((file) => file.length > 0 && !BENCHMARK_PATCH_EXCLUDES.has(file)); if (files.length === 0) return; await git(["-C", worktreePath, "add", "--intent-to-add", "--", ...files]); } +function patchPathspecArgs() { + return [ + "--", + ".", + ...[...BENCHMARK_PATCH_EXCLUDES].map((file) => `:(exclude)${file}`), + ]; +} + async function baseRefExists(worktreePath) { try { await git(["-C", worktreePath, "rev-parse", "--verify", `${BASE_REF}^{commit}`]); diff --git a/benchmarks/agentrail-swe-lifecycle/lib/phase-accounting.mjs b/benchmarks/agentrail-swe-lifecycle/lib/phase-accounting.mjs new file mode 100644 index 0000000..8054df0 --- /dev/null +++ b/benchmarks/agentrail-swe-lifecycle/lib/phase-accounting.mjs @@ -0,0 +1,53 @@ +const TOKEN_FIELDS = [ + "inputTokens", + "cachedInputTokens", + "nonCachedInputTokens", + "outputTokens", + "reasoningTokens", + "totalTokens", +]; + +export function computePhaseAccounting({ phases }) { + const controlPhases = phases.filter((phase) => phase.controlIncluded); + const agentRailPhases = phases.filter((phase) => phase.agentRailIncluded); + const control = sumTokenUsage(controlPhases.map((phase) => phase.tokenUsage)); + const agentrail = sumTokenUsage(agentRailPhases.map((phase) => phase.tokenUsage)); + const tokenDelta = control.status === "available" && agentrail.status === "available" + ? agentrail.totalTokens - control.totalTokens + : null; + + return { + control, + agentrail, + tokenDelta, + tokenReductionRate: tokenDelta !== null && control.totalTokens > 0 + ? (control.totalTokens - agentrail.totalTokens) / control.totalTokens + : null, + controlPhaseIds: controlPhases.map((phase) => phase.id), + agentRailPhaseIds: agentRailPhases.map((phase) => phase.id), + }; +} + +export function sumTokenUsage(tokenUsages) { + const totals = { + status: "available", + inputTokens: 0, + cachedInputTokens: 0, + nonCachedInputTokens: 0, + outputTokens: 0, + reasoningTokens: 0, + totalTokens: 0, + }; + + for (const usage of tokenUsages) { + if (usage?.status !== "available") { + totals.status = "unavailable"; + continue; + } + for (const field of TOKEN_FIELDS) { + totals[field] += typeof usage[field] === "number" ? usage[field] : 0; + } + } + + return totals; +} diff --git a/benchmarks/agentrail-swe-lifecycle/lib/phase-definitions.mjs b/benchmarks/agentrail-swe-lifecycle/lib/phase-definitions.mjs new file mode 100644 index 0000000..d79e5a1 --- /dev/null +++ b/benchmarks/agentrail-swe-lifecycle/lib/phase-definitions.mjs @@ -0,0 +1,141 @@ +const CI_AND_REVIEW_FEEDBACK_PHASES = [ + { + id: "provider_issue_intake", + title: "Read provider issue", + measured: true, + actor: "llm_provider_lifecycle", + controlIncluded: true, + agentRailIncluded: false, + }, + { + id: "initial_code_change", + title: "Implement initial code change", + measured: true, + actor: "llm_coding", + controlIncluded: true, + agentRailIncluded: true, + }, + { + id: "provider_create_pr", + title: "Create provider pull request", + measured: true, + actor: "llm_provider_lifecycle", + controlIncluded: true, + agentRailIncluded: false, + }, + { + id: "provider_wait_for_ci", + title: "Wait for CI and read status", + measured: true, + actor: "llm_provider_lifecycle", + controlIncluded: true, + agentRailIncluded: false, + }, + { + id: "ci_failure_triage", + title: "Read CI failure and plan fix", + measured: true, + actor: "llm_provider_lifecycle", + controlIncluded: true, + agentRailIncluded: false, + }, + { + id: "fix_ci_failure", + title: "Fix CI failure", + measured: true, + actor: "llm_coding", + controlIncluded: true, + agentRailIncluded: true, + }, + { + id: "provider_wait_for_review", + title: "Wait for review and read comments", + measured: true, + actor: "llm_provider_lifecycle", + controlIncluded: true, + agentRailIncluded: false, + }, + { + id: "review_feedback_triage", + title: "Read review feedback and plan fix", + measured: true, + actor: "llm_provider_lifecycle", + controlIncluded: true, + agentRailIncluded: false, + }, + { + id: "fix_review_feedback", + title: "Fix review feedback", + measured: true, + actor: "llm_coding", + controlIncluded: true, + agentRailIncluded: true, + }, + { + id: "provider_update_pr", + title: "Update pull request after fixes", + measured: true, + actor: "llm_provider_lifecycle", + controlIncluded: true, + agentRailIncluded: false, + }, + { + id: "provider_final_ci_check", + title: "Check final CI", + measured: true, + actor: "llm_provider_lifecycle", + controlIncluded: true, + agentRailIncluded: false, + }, + { + id: "provider_final_review_check", + title: "Check final review approval", + measured: true, + actor: "llm_provider_lifecycle", + controlIncluded: true, + agentRailIncluded: false, + }, + { + id: "agentrail_handoff", + title: "Write AgentRail result handoff", + measured: true, + actor: "llm_agentrail_handoff", + controlIncluded: false, + agentRailIncluded: true, + }, + { + id: "provider_complete", + title: "Complete provider lifecycle", + measured: true, + actor: "llm_provider_lifecycle", + controlIncluded: true, + agentRailIncluded: false, + }, +]; + +export const PHASE_SCENARIOS = new Map([ + ["ci-and-review-feedback", { + id: "ci-and-review-feedback", + phases: CI_AND_REVIEW_FEEDBACK_PHASES, + }], +]); + +export const CONTROL_INCLUDED_PHASES = new Set( + CI_AND_REVIEW_FEEDBACK_PHASES.filter((phase) => phase.controlIncluded).map((phase) => phase.id), +); + +export const AGENTRAIL_INCLUDED_PHASES = new Set( + CI_AND_REVIEW_FEEDBACK_PHASES.filter((phase) => phase.agentRailIncluded).map((phase) => phase.id), +); + +export function getPhaseScenario(id) { + const scenario = PHASE_SCENARIOS.get(id); + if (!scenario) throw new Error(`Unknown phase scenario: ${id}`); + return scenario; +} + +export function validatePhaseResult({ phase, result, requireTokenUsage }) { + if (requireTokenUsage && phase.measured && result.tokenUsage?.status !== "available") { + throw new Error(`${phase.id} did not produce parseable token usage.`); + } +} diff --git a/benchmarks/agentrail-swe-lifecycle/lib/phase-prompts.mjs b/benchmarks/agentrail-swe-lifecycle/lib/phase-prompts.mjs new file mode 100644 index 0000000..2224809 --- /dev/null +++ b/benchmarks/agentrail-swe-lifecycle/lib/phase-prompts.mjs @@ -0,0 +1,77 @@ +export function buildPhasePrompt({ phase, task, upstream, artifacts = {} }) { + if (phase.actor === "llm_coding") { + return buildCodingPrompt({ phase, task, upstream, artifacts }); + } + if (phase.actor === "llm_provider_lifecycle") { + return buildProviderLifecyclePrompt({ phase, task, artifacts }); + } + if (phase.actor === "llm_agentrail_handoff") { + return buildAgentRailHandoffPrompt({ artifacts }); + } + throw new Error(`Unsupported phase actor: ${phase.actor}`); +} + +function buildCodingPrompt({ phase, task, upstream, artifacts }) { + return [ + `You are working on benchmark task ${task.id}.`, + `Repository: ${task.repo}`, + `Phase: ${phase.id}`, + "", + "Modify the repository only for this coding phase.", + "Do not operate provider lifecycle tooling or lifecycle state.", + "Leave changes in the working tree.", + "", + `Instance: ${upstream.instanceId}`, + `Base commit: ${upstream.baseCommit}`, + "", + `Problem statement:\n${upstream.problemStatement}`, + upstream.requirements ? `\nRequirements:\n${upstream.requirements}` : "", + upstream.interface ? `\nInterface notes:\n${upstream.interface}` : "", + artifacts.ciFailure ? `\nCI failure to address:\n${artifacts.ciFailure}` : "", + artifacts.reviewFeedback ? `\nReview feedback to address:\n${artifacts.reviewFeedback}` : "", + "", + "When this phase is complete, stop. Do not continue into later phases.", + ].filter(Boolean).join("\n"); +} + +function buildProviderLifecyclePrompt({ phase, task, artifacts }) { + const lifecycleCli = artifacts.lifecycleCli ?? "node benchmarks/agentrail-swe-lifecycle/fixtures/mock-lifecycle-cli.mjs"; + const instructions = { + provider_issue_intake: `Fetch the issue with: ${lifecycleCli} issue get`, + provider_create_pr: `Use ${lifecycleCli} to create the pull request.`, + provider_wait_for_ci: `Check CI with: ${lifecycleCli} ci get --pr 1`, + ci_failure_triage: `Read the provided CI failure and summarize the fix needed. CI failure:\n${artifacts.ciFailure ?? "Synthetic CI failure."}`, + provider_wait_for_review: `Check review with: ${lifecycleCli} review get --pr 1`, + review_feedback_triage: `Read the provided review feedback and summarize the fix needed. Review feedback:\n${artifacts.reviewFeedback ?? "Synthetic review feedback."}`, + provider_update_pr: `Update the pull request status with: ${lifecycleCli} pr update --pr 1`, + provider_final_ci_check: `Check final CI with: ${lifecycleCli} ci get --pr 1`, + provider_final_review_check: `Check final review with: ${lifecycleCli} review get --pr 1`, + provider_complete: `Complete the lifecycle with: ${lifecycleCli} complete --pr 1`, + }; + + const instruction = instructions[phase.id]; + if (!instruction) throw new Error(`Unsupported provider lifecycle phase: ${phase.id}`); + + return [ + `You are performing only a provider lifecycle phase for ${task.repo}.`, + `Phase: ${phase.id}`, + "", + instruction, + "", + "Do not edit source files in this phase.", + "Do not solve coding tasks in this phase.", + "When this phase is complete, stop.", + ].join("\n"); +} + +function buildAgentRailHandoffPrompt({ artifacts }) { + if (typeof artifacts.resultPath !== "string" || artifacts.resultPath.length === 0) { + throw new Error("AgentRail handoff phase requires artifacts.resultPath."); + } + + return [ + "Write the completed AgentRail result JSON.", + `Result path: ${artifacts.resultPath}`, + `JSON: {"version":1,"status":"completed","summary":"phase attribution benchmark complete","checks":[],"artifacts":[]}`, + ].join("\n"); +} diff --git a/benchmarks/agentrail-swe-lifecycle/lib/phase-runner.mjs b/benchmarks/agentrail-swe-lifecycle/lib/phase-runner.mjs new file mode 100644 index 0000000..b46b721 --- /dev/null +++ b/benchmarks/agentrail-swe-lifecycle/lib/phase-runner.mjs @@ -0,0 +1,165 @@ +import { mkdir, readFile } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { capturePatch } from "./patch.mjs"; +import { + PHASE_ATTRIBUTION_LIFECYCLE_EVENTS, + summarizeLifecycleEvents, + withLifecycleProviderHarness, +} from "./lifecycle-provider-harness.mjs"; +import { computePhaseAccounting } from "./phase-accounting.mjs"; +import { getPhaseScenario, validatePhaseResult } from "./phase-definitions.mjs"; +import { buildPhasePrompt } from "./phase-prompts.mjs"; +import { runRunnerCommand } from "./runner-command.mjs"; +import { parseTokenUsageFromFiles } from "./token-usage.mjs"; +import { prepareLaneWorkspace } from "./workspace.mjs"; + +export async function runPhaseAttributionTask({ + task, + upstream, + runId, + runDir, + repoCache, + sourceRepoPath, + flags, +}) { + const workspace = await prepareLaneWorkspace({ + task, + upstream, + lane: "phase-attribution", + runDir, + repoCache, + sourceRepoPath, + }); + + return await withLifecycleProviderHarness({ + runDir: workspace.laneDir, + stateDir: workspace.worktreePath, + task, + upstream, + }, async (lifecycle) => await runPhaseAttributionInWorkspace({ + task, + upstream, + runId, + workspace, + flags, + lifecycle, + })); +} + +async function runPhaseAttributionInWorkspace({ + task, + upstream, + runId, + workspace, + flags, + lifecycle, +}) { + const scenario = getPhaseScenario(flags.phaseScenario ?? "ci-and-review-feedback"); + const resultPath = path.join(workspace.worktreePath, ".agentrail-run", "result.json"); + const lifecycleCli = path.join(repoRoot(), "benchmarks", "agentrail-swe-lifecycle", "fixtures", "mock-lifecycle-cli.mjs"); + const phases = []; + const artifacts = { + lifecycleCli: `node ${lifecycleCli}`, + ciFailure: "Synthetic CI failure: targeted test failed after initial patch.", + reviewFeedback: "Synthetic review feedback: simplify the fix and add focused regression coverage.", + resultPath, + }; + + const repeatCount = flags.phaseRepeat ?? 1; + for (let repeatIndex = 0; repeatIndex < repeatCount; repeatIndex += 1) { + for (const phase of scenario.phases) { + const phaseDir = repeatCount === 1 + ? path.join(workspace.laneDir, "phases", phase.id) + : path.join(workspace.laneDir, "phases", `repeat-${repeatIndex + 1}`, phase.id); + await mkdir(phaseDir, { recursive: true }); + const prompt = buildPhasePrompt({ phase, task, upstream, artifacts }); + const runner = await runRunnerCommand({ + runner: flags.runner, + runnerCommand: flags.runnerCommand, + model: flags.model ?? null, + repoPath: workspace.worktreePath, + prompt, + logDir: phaseDir, + timeoutSeconds: flags.timeoutSeconds ?? 1800, + env: { + AGENTRAIL_BENCH_PHASE_ID: phase.id, + AGENTRAIL_BENCH_PHASE_REPEAT: String(repeatIndex + 1), + AGENTRAIL_RESULT_PATH: resultPath, + AGENTRAIL_BENCH_LIFECYCLE_URL: lifecycle.baseUrl, + AGENTRAIL_BENCH_LIFECYCLE_STATE_PATH: lifecycle.statePath, + AGENTRAIL_BENCH_LIFECYCLE_CLI: lifecycleCli, + }, + }); + await validateCompletedRunner({ phase, runner }); + const tokenUsage = await parseTokenUsageFromFiles([runner.stdoutPath, runner.stderrPath]); + const phaseResult = { + ...phase, + repeat: repeatIndex + 1, + runner, + tokenUsage, + status: runner.status, + success: runner.status === "completed", + }; + validatePhaseResult({ + phase, + result: phaseResult, + requireTokenUsage: flags.requirePhaseTokenUsage, + }); + phases.push(phaseResult); + } + } + + const patch = await capturePatch({ + worktreePath: workspace.worktreePath, + outputPath: path.join(workspace.laneDir, "patch.diff"), + }); + const accounting = computePhaseAccounting({ phases }); + const lifecycleSummary = summarizeLifecycleEvents({ + eventsPath: lifecycle.eventsPath, + events: await lifecycle.readEvents(), + requiredEvents: PHASE_ATTRIBUTION_LIFECYCLE_EVENTS, + }); + const success = phases.every((phase) => phase.success) + && patch.status === "available" + && lifecycleSummary.complete; + + return { + taskId: task.id, + upstreamInstanceId: upstream.instanceId, + repo: task.repo, + runId, + mode: "phase-attribution", + scenario: scenario.id, + phases, + accounting, + patch, + lifecycle: lifecycleSummary, + final: { + success, + status: success ? "completed" : "failed", + }, + tokenUsage: accounting.agentrail, + }; +} + +async function validateCompletedRunner({ phase, runner }) { + if (runner.status === "completed") return; + const stderr = await readTextFile(runner.stderrPath); + const suffix = stderr.trim().length > 0 ? ` Stderr: ${stderr.trim().slice(-1000)}` : ""; + throw new Error(`${phase.id} runner ${runner.status}.${suffix}`); +} + +async function readTextFile(filePath) { + try { + return await readFile(filePath, "utf8"); + } catch (error) { + if (error?.code === "ENOENT") return ""; + throw error; + } +} + +function repoRoot() { + return path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", "..", ".."); +} diff --git a/benchmarks/agentrail-swe-lifecycle/lib/results.mjs b/benchmarks/agentrail-swe-lifecycle/lib/results.mjs index f073698..4c22cfb 100644 --- a/benchmarks/agentrail-swe-lifecycle/lib/results.mjs +++ b/benchmarks/agentrail-swe-lifecycle/lib/results.mjs @@ -18,6 +18,18 @@ export function aggregateLifecycleResults(results) { const controlTotalTokens = controlTokenTotals.reduce((sum, total) => sum + (total ?? 0), 0); const agentRailTotalTokens = agentRailTokenTotals.reduce((sum, total) => sum + (total ?? 0), 0); const tokenDelta = comparableTokenTasks > 0 ? agentRailTotalTokens - controlTotalTokens : null; + const tokenFieldTotals = (lane, field) => comparisons.reduce((sum, comparison, index) => { + if (controlTokenTotals[index] === null || agentRailTokenTotals[index] === null) return sum; + return sum + normalizeTokenComponent(comparison[lane]?.tokenUsage, field); + }, 0); + const controlCachedInputTokens = tokenFieldTotals("control", "cachedInputTokens"); + const agentRailCachedInputTokens = tokenFieldTotals("agentrail", "cachedInputTokens"); + const controlNonCachedInputTokens = tokenFieldTotals("control", "nonCachedInputTokens"); + const agentRailNonCachedInputTokens = tokenFieldTotals("agentrail", "nonCachedInputTokens"); + const controlOutputTokens = tokenFieldTotals("control", "outputTokens"); + const agentRailOutputTokens = tokenFieldTotals("agentrail", "outputTokens"); + const controlReasoningTokens = tokenFieldTotals("control", "reasoningTokens"); + const agentRailReasoningTokens = tokenFieldTotals("agentrail", "reasoningTokens"); return { totalTasks, @@ -26,6 +38,7 @@ export function aggregateLifecycleResults(results) { totalTokens, tokenUsageUnavailableTasks, tokensPerSuccessfulTask: successfulTasks > 0 ? totalTokens / successfulTasks : null, + phaseAttribution: aggregatePhaseAttributionResults(results), paired: comparisons.length > 0 ? { comparableTokenTasks, @@ -34,12 +47,53 @@ export function aggregateLifecycleResults(results) { tokenDelta, tokenReductionRate: tokenDelta !== null && controlTotalTokens > 0 ? (controlTotalTokens - agentRailTotalTokens) / controlTotalTokens - : null + : null, + controlCachedInputTokens, + agentRailCachedInputTokens, + cachedInputDelta: agentRailCachedInputTokens - controlCachedInputTokens, + controlNonCachedInputTokens, + agentRailNonCachedInputTokens, + nonCachedInputDelta: agentRailNonCachedInputTokens - controlNonCachedInputTokens, + controlOutputTokens, + agentRailOutputTokens, + outputDelta: agentRailOutputTokens - controlOutputTokens, + controlReasoningTokens, + agentRailReasoningTokens, + reasoningDelta: agentRailReasoningTokens - controlReasoningTokens, } : null }; } +function aggregatePhaseAttributionResults(results) { + const phaseResults = results.filter((result) => result.accounting); + if (phaseResults.length === 0) return null; + + let comparableTasks = 0; + let controlTotalTokens = 0; + let agentRailTotalTokens = 0; + + for (const result of phaseResults) { + const controlTokens = normalizeTokenTotal(result.accounting?.control); + const agentRailTokens = normalizeTokenTotal(result.accounting?.agentrail); + if (controlTokens === null || agentRailTokens === null) continue; + comparableTasks += 1; + controlTotalTokens += controlTokens; + agentRailTotalTokens += agentRailTokens; + } + + const tokenDelta = comparableTasks > 0 ? agentRailTotalTokens - controlTotalTokens : null; + return { + comparableTasks, + controlTotalTokens, + agentRailTotalTokens, + tokenDelta, + tokenReductionRate: tokenDelta !== null && controlTotalTokens > 0 + ? (controlTotalTokens - agentRailTotalTokens) / controlTotalTokens + : null, + }; +} + export async function writeLifecycleResultFiles({ outputDir, run }) { await mkdir(outputDir, { recursive: true }); const completeMarkerPath = path.join(outputDir, "complete.marker"); @@ -51,6 +105,7 @@ export async function writeLifecycleResultFiles({ outputDir, run }) { function normalizeTokenTotal(tokenUsage) { if (!tokenUsage || tokenUsage.totalTokens === undefined) return null; + if (tokenUsage.status === "unavailable") return null; if (typeof tokenUsage.totalTokens === "number") { return Number.isSafeInteger(tokenUsage.totalTokens) && tokenUsage.totalTokens >= 0 ? tokenUsage.totalTokens : null; } @@ -61,12 +116,18 @@ function normalizeTokenTotal(tokenUsage) { return null; } +function normalizeTokenComponent(tokenUsage, field) { + const value = tokenUsage?.[field]; + return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : 0; +} + function renderSummary(run) { const aggregate = run.aggregate; return [ "# AgentRail SWE Lifecycle Benchmark", "", `- Run: ${run.runId}`, + `- Benchmark surface: ${run.benchmarkSurface ?? "patch-only"}`, `- Tasks: ${aggregate.totalTasks}`, `- Successful tasks: ${aggregate.successfulTasks}`, `- Success rate: ${(aggregate.successRate * 100).toFixed(2)}%`, @@ -78,7 +139,18 @@ function renderSummary(run) { `- Control total tokens: ${aggregate.paired.controlTotalTokens}`, `- AgentRail total tokens: ${aggregate.paired.agentRailTotalTokens}`, `- AgentRail token delta: ${aggregate.paired.tokenDelta ?? "unavailable"}`, - `- AgentRail token reduction: ${aggregate.paired.tokenReductionRate === null ? "unavailable" : `${(aggregate.paired.tokenReductionRate * 100).toFixed(2)}%`}` + `- AgentRail token reduction: ${aggregate.paired.tokenReductionRate === null ? "unavailable" : `${(aggregate.paired.tokenReductionRate * 100).toFixed(2)}%`}`, + `- Cached input token delta: ${aggregate.paired.cachedInputDelta}`, + `- Non-cached input token delta: ${aggregate.paired.nonCachedInputDelta}`, + `- Output token delta: ${aggregate.paired.outputDelta}`, + `- Reasoning token delta: ${aggregate.paired.reasoningDelta}` + ] : []), + ...(aggregate.phaseAttribution ? [ + `- Phase attribution comparable tasks: ${aggregate.phaseAttribution.comparableTasks}`, + `- Phase attribution control total tokens: ${aggregate.phaseAttribution.controlTotalTokens}`, + `- Phase attribution AgentRail total tokens: ${aggregate.phaseAttribution.agentRailTotalTokens}`, + `- Phase attribution AgentRail token delta: ${aggregate.phaseAttribution.tokenDelta ?? "unavailable"}`, + `- Phase attribution AgentRail token reduction: ${aggregate.phaseAttribution.tokenReductionRate === null ? "unavailable" : `${(aggregate.phaseAttribution.tokenReductionRate * 100).toFixed(2)}%`}`, ] : []), "" ].join("\n"); diff --git a/benchmarks/agentrail-swe-lifecycle/lib/runner-command.mjs b/benchmarks/agentrail-swe-lifecycle/lib/runner-command.mjs index ff32ef2..a1f4eab 100644 --- a/benchmarks/agentrail-swe-lifecycle/lib/runner-command.mjs +++ b/benchmarks/agentrail-swe-lifecycle/lib/runner-command.mjs @@ -45,16 +45,49 @@ export async function resolveRunnerCommand({ runner, runnerCommand, model, promp } if (runner === "codex") { const prompt = await readPromptFile(promptPath); - const args = ["-a", "never", ...(model ? ["--model", model] : []), "--sandbox", "workspace-write", "exec", "--json", "-"]; + const args = [ + "-a", + "never", + "-c", + "features.hooks=false", + "exec", + "--sandbox", + "workspace-write", + "--ephemeral", + "--ignore-user-config", + "--ignore-rules", + ...(model ? ["--model", model] : []), + "--json", + "-", + ]; return { executable: "codex", args, stdin: prompt, redacted: ["codex", ...args].join(" ") }; } if (runner === "cursor") { - const args = ["agent", "--model", model, "--prompt-file", promptPath].filter(Boolean); - return { executable: "cursor", args, redacted: ["cursor", ...args].join(" ") }; + const prompt = await readPromptFile(promptPath); + const args = [ + "agent", + "--print", + "--output-format", + "stream-json", + "--trust", + "--force", + ...(model ? ["--model", model] : []), + ]; + return { executable: "cursor", args, stdin: prompt, redacted: ["cursor", ...args].join(" ") }; } if (runner === "claude-code") { - const args = ["--model", model, "--print", `@${promptPath}`].filter(Boolean); - return { executable: "claude", args, redacted: ["claude", ...args].join(" ") }; + const prompt = await readPromptFile(promptPath); + const args = [ + "--print", + "--output-format", + "stream-json", + "--verbose", + "--permission-mode", + "bypassPermissions", + "--dangerously-skip-permissions", + ...(model ? ["--model", model] : []), + ]; + return { executable: "claude", args, stdin: prompt, redacted: ["claude", ...args].join(" ") }; } throw new Error(`Unsupported runner: ${runner}`); } diff --git a/benchmarks/agentrail-swe-lifecycle/lib/token-usage.mjs b/benchmarks/agentrail-swe-lifecycle/lib/token-usage.mjs index 8816b51..153bef2 100644 --- a/benchmarks/agentrail-swe-lifecycle/lib/token-usage.mjs +++ b/benchmarks/agentrail-swe-lifecycle/lib/token-usage.mjs @@ -29,17 +29,23 @@ export async function parseTokenUsageFromFiles(paths) { outputTokens: null, reasoningTokens: null, cachedInputTokens: null, + nonCachedInputTokens: null, totalTokens: null, sources: [] }; } + const inputTokens = aggregate.hasInput ? aggregate.inputTokens : null; + const cachedInputTokens = aggregate.hasCachedInput ? aggregate.cachedInputTokens : null; return { status: "available", - inputTokens: aggregate.hasInput ? aggregate.inputTokens : null, + inputTokens, outputTokens: aggregate.hasOutput ? aggregate.outputTokens : null, reasoningTokens: aggregate.hasReasoning ? aggregate.reasoningTokens : null, - cachedInputTokens: aggregate.hasCachedInput ? aggregate.cachedInputTokens : null, + cachedInputTokens, + nonCachedInputTokens: inputTokens === null || cachedInputTokens === null + ? null + : Math.max(0, inputTokens - cachedInputTokens), totalTokens: aggregate.hasTotal ? aggregate.totalTokens : null, sources }; diff --git a/benchmarks/agentrail-swe-lifecycle/run.mjs b/benchmarks/agentrail-swe-lifecycle/run.mjs index 5fb680a..9272339 100644 --- a/benchmarks/agentrail-swe-lifecycle/run.mjs +++ b/benchmarks/agentrail-swe-lifecycle/run.mjs @@ -7,6 +7,7 @@ import { runAgentRailLane } from "./lib/agentrail-lane.mjs"; import { runControlLane } from "./lib/control-lane.mjs"; import { loadLifecycleDataset, selectLifecycleTasks } from "./lib/dataset.mjs"; import { buildLocalLifecycleArtifacts } from "./lib/local-lifecycle.mjs"; +import { runPhaseAttributionTask } from "./lib/phase-runner.mjs"; import { aggregateLifecycleResults, writeLifecycleResultFiles } from "./lib/results.mjs"; import { resolveUpstreamRowsForTasks } from "./lib/upstream.mjs"; @@ -22,14 +23,19 @@ if (tasks.length === 0) { } const runId = flags.runId ?? `run_${new Date().toISOString().replaceAll(/[:.]/gu, "-")}`; validateRunId(runId); -const outputDir = flags.outputDir ?? path.join(repoRoot, ".agentrail-bench", "agentrail-swe-lifecycle", runId); +const outputDir = flags.outputDir + ? path.resolve(flags.outputDir) + : path.join(repoRoot, ".agentrail-bench", "agentrail-swe-lifecycle", runId); const results = flags.mode === "dry-run" ? tasks.map((task) => buildDryRunTaskResult({ task, runId })) - : await runRealBenchmarkTasks({ tasks, runId, outputDir, flags }); + : flags.mode === "phase-attribution" + ? await runPhaseAttributionBenchmarkTasks({ tasks, runId, outputDir, flags }) + : await runRealBenchmarkTasks({ tasks, runId, outputDir, flags }); const run = { version: 1, runId, mode: flags.mode === "dry-run" ? "dry_run" : flags.mode, + benchmarkSurface: flags.benchmarkSurface, dataset: dataset.id, scenario: dataset.defaultScenario, aggregate: aggregateLifecycleResults(results), @@ -40,6 +46,25 @@ await writeLifecycleResultFiles({ outputDir, run }); console.log(JSON.stringify({ status: "ok", outputDir, tasks: tasks.length }, null, 2)); +async function runPhaseAttributionBenchmarkTasks({ tasks, runId, outputDir, flags }) { + const upstreams = await resolveUpstreamsForExecution(tasks); + const results = []; + for (let index = 0; index < tasks.length; index += 1) { + const task = tasks[index]; + const upstream = upstreams[index]; + process.stderr.write(`[agentrail-swe] phase-attribution ${index + 1}/${tasks.length} ${task.id}\n`); + try { + results.push(await runPhaseAttributionTask(buildLaneOptions({ task, upstream, outputDir, flags, runId }))); + } catch (error) { + if (!flags.continueOnError) throw error; + const message = error instanceof Error ? error.message : String(error); + process.stderr.write(`[agentrail-swe] failed ${task.id}: ${message}\n`); + results.push(buildErrorTaskResult({ task, upstream, runId, error })); + } + } + return results; +} + async function runRealBenchmarkTasks({ tasks, runId, outputDir, flags }) { const upstreams = await resolveUpstreamsForExecution(tasks); const results = []; @@ -97,10 +122,11 @@ function upstreamFromTaskManifest(task) { }; } -function buildLaneOptions({ task, upstream, outputDir, flags }) { +function buildLaneOptions({ task, upstream, outputDir, flags, runId }) { return { task, upstream, + runId, runDir: outputDir, repoCache: flags.repoCache ?? path.join(repoRoot, ".agentrail-bench", "repo-cache"), sourceRepoPath: flags.sourceRepoPath, @@ -164,6 +190,7 @@ function summarizeLane(lane) { tokenUsage: lane.tokenUsage, patch: lane.patch, evaluator: lane.evaluator, + lifecycle: lane.lifecycle ?? null, workspace: lane.workspace }; } diff --git a/package.json b/package.json index f97094e..572fd19 100644 --- a/package.json +++ b/package.json @@ -30,8 +30,10 @@ "benchmark:agentrail-swe:refresh": "node scripts/refresh-agentrail-swe-lifecycle-dataset.mjs", "benchmark:agentrail-swe:dry-run": "node benchmarks/agentrail-swe-lifecycle/run.mjs --dry-run", "benchmark:agentrail-swe:paired": "node benchmarks/agentrail-swe-lifecycle/run.mjs --mode paired", + "benchmark:agentrail-swe:paired:full-lifecycle": "node benchmarks/agentrail-swe-lifecycle/run.mjs --mode paired --benchmark-surface full-lifecycle", + "benchmark:agentrail-swe:phase-attribution": "node benchmarks/agentrail-swe-lifecycle/run.mjs --mode phase-attribution", "benchmark:agentrail-swe:run": "node benchmarks/agentrail-swe-lifecycle/run.mjs --dry-run", - "benchmark:agentrail-swe:validate": "node --test test/agentrail-swe-lifecycle-dataset.test.js test/agentrail-swe-lifecycle-runner.test.js test/agentrail-swe-lifecycle-token-usage.test.js test/agentrail-swe-lifecycle-real-runner.test.js", + "benchmark:agentrail-swe:validate": "node --test test/agentrail-swe-lifecycle-dataset.test.js test/agentrail-swe-lifecycle-runner.test.js test/agentrail-swe-lifecycle-token-usage.test.js test/agentrail-swe-lifecycle-full-lifecycle.test.js test/agentrail-swe-lifecycle-real-runner.test.js test/agentrail-swe-lifecycle-phase-attribution.test.js", "lint:openapi": "spectral lint docs/api/task-lifecycle.openapi.yaml docs/api/intake-routing-admin.openapi.yaml -r .spectral.yaml", "test": "npm --prefix sdk/typescript ci && npm --prefix sdk/typescript run build && npm run test:service", "test:service": "node --test 'test/*.test.js' 'test/*.test.ts'", diff --git a/src/cli/agent-runner.ts b/src/cli/agent-runner.ts index f011eca..ea6592c 100644 --- a/src/cli/agent-runner.ts +++ b/src/cli/agent-runner.ts @@ -2598,25 +2598,24 @@ function renderManagedAgentRecipe(): string { "", "This file is generated for one AgentRail-managed run.", "AgentRail owns task assignment, lifecycle state, provider status, pull request creation, shipping, and rollback.", - "Your job is limited to repository work inside the provided git worktree and reporting the result.", + "Your job is limited to repository work inside the provided git worktree and writing the result file.", "", "## Required Flow", "", "1. Work only inside the provided git worktree for this task.", - "2. Use only run-scoped AgentRail commands from the child process: `agentrail run current`, `agentrail run actions`, and `agentrail agent report`.", + "2. Use only read-only run-scoped AgentRail commands from the child process: `agentrail run current` and `agentrail run actions`.", "3. Do not hand-roll AgentRail API calls, query broad task lifecycle endpoints, push branches, create pull requests, ship, or roll back directly.", - "4. Report meaningful progress with `agentrail agent report --status progress --summary \"short update\"`.", + "4. Optional: report progress only if the task will run for a long time and the report adds useful information.", "5. Run the smallest relevant validation before reporting completion.", "6. Leave completed worktree changes in place for AgentRail to commit; do not run `git add`, `git commit`, `git push`, or create a pull request directly.", "7. Write the structured result JSON file at `$AGENTRAIL_RESULT_PATH` before exiting.", - "8. Report completion with `agentrail agent report --status completed --result-file \"$AGENTRAIL_RESULT_PATH\"`.", - "9. If blocked, write a blocked result file and report it with `agentrail agent report --status blocked --result-file \"$AGENTRAIL_RESULT_PATH\"`.", + "8. After writing the result file, exit normally; AgentRail will consume the result file and complete the lifecycle.", + "9. If blocked, write a blocked result file and exit normally; AgentRail will consume the blocker from the result file.", "", "## Run-Scoped AgentRail Commands", "", "- `agentrail run current` re-reads only this run's assigned task and current action.", "- `agentrail run actions` shows only the allowed next AgentRail actions for this run.", - "- `agentrail agent report` reports progress, blockers, or completion for this run.", "", "## Result File Contract", "", @@ -2676,13 +2675,13 @@ function buildPrompt({ `Use the recipe file as your standing instructions.`, `Work only inside the provided git worktree.`, `Do not edit AGENTS.md, CLAUDE.md, AgentRail recipe files, or other instruction/config files unless the task explicitly requires it.`, - `AgentRail commands available to you: agentrail run current, agentrail run actions, and agentrail agent report.`, + `AgentRail commands available to you: agentrail run current and agentrail run actions.`, `Use agentrail run current or agentrail run actions only if you need to re-read this run's assignment or allowed next actions.`, `Do not hand-roll AgentRail API calls, run git add, run git commit, push branches, or create pull requests directly; use the run-scoped AgentRail commands instead.`, - `Report meaningful progress with: agentrail agent report --status progress --summary "short update".`, - `If blocked by missing credentials, provider access, sandbox limits, or validation failures, write a blocked result file and report it with: agentrail agent report --status blocked --result-file "$AGENTRAIL_RESULT_PATH".`, + `Optional: only report progress if the task will run for a long time and the report adds useful information.`, + `If blocked by missing credentials, provider access, sandbox limits, or validation failures, write a blocked result file and exit normally; AgentRail will consume the blocker from the result file.`, `When the work is ready, leave the validated worktree changes in place and write the completed result JSON file at the result path; AgentRail will commit and publish the changes.`, - `After writing the result file, report completion with: agentrail agent report --status completed --result-file "$AGENTRAIL_RESULT_PATH".`, + `After writing the result file, exit normally; AgentRail will consume the result file and complete the lifecycle.`, `Completed result shape: {"version":1,"status":"completed","summary":"...","checks":[],"artifacts":[]}.`, `Blocked result shape: {"version":1,"status":"blocked","summary":"...","blocker":{"reason":"...","actionRequired":"...","resumeInstructions":"..."}}.`, `Do not expose secrets in output.`, diff --git a/src/cli/setup-files.ts b/src/cli/setup-files.ts index 4674b03..05d467d 100644 --- a/src/cli/setup-files.ts +++ b/src/cli/setup-files.ts @@ -210,13 +210,13 @@ function renderDefaultRecipe(): string { "", "1. Read the assigned task from the AgentRail-provided run prompt before editing anything.", "2. Work only inside the provided git worktree for that task.", - "3. Do not hand-roll AgentRail API calls, push branches, create pull requests, ship, or roll back directly; use `agentrail agent report` for status updates.", - "4. Report meaningful progress with `agentrail agent report --status progress --summary \"short update\"`.", + "3. Do not hand-roll AgentRail API calls, push branches, create pull requests, ship, or roll back directly.", + "4. Optional: report progress only if the task will run for a long time and the report adds useful information.", "5. Run the smallest relevant validation before reporting completion.", "6. Leave completed worktree changes in place for AgentRail to commit; do not run `git add`, `git commit`, `git push`, or create a pull request directly.", "7. Write the structured result JSON file at `$AGENTRAIL_RESULT_PATH` before exiting. AgentRail provides this environment variable at runtime; users do not set it manually.", - "8. Report completion with `agentrail agent report --status completed --result-file \"$AGENTRAIL_RESULT_PATH\"`.", - "9. If blocked, write a blocked result file and report it with `agentrail agent report --status blocked --result-file \"$AGENTRAIL_RESULT_PATH\"`.", + "8. After writing the result file, exit normally; AgentRail will consume the result file and complete the lifecycle.", + "9. If blocked, write a blocked result file and exit normally; AgentRail will consume the blocker from the result file.", "", "## Result File Contract", "", diff --git a/test/agentrail-swe-lifecycle-full-lifecycle.test.js b/test/agentrail-swe-lifecycle-full-lifecycle.test.js new file mode 100644 index 0000000..07e9b6f --- /dev/null +++ b/test/agentrail-swe-lifecycle-full-lifecycle.test.js @@ -0,0 +1,118 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { mkdtemp, readFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { promisify } from "node:util"; + +import { buildFullLifecycleControlPrompt } from "../benchmarks/agentrail-swe-lifecycle/lib/lifecycle-control-prompt.mjs"; +import { + CONTROL_LIFECYCLE_EVENTS, + summarizeLifecycleEvents, + withLifecycleProviderHarness +} from "../benchmarks/agentrail-swe-lifecycle/lib/lifecycle-provider-harness.mjs"; + +const execFileAsync = promisify(execFile); + +test("lifecycle provider exposes issue, PR, CI, review, and completion operations", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "agentrail-lifecycle-provider-")); + const task = { + id: "task-1", + repo: "fixture/repo", + upstream: { instanceId: "fixture__repo-1" }, + }; + const upstream = { + instanceId: "fixture__repo-1", + baseCommit: "0123456789abcdef0123456789abcdef01234567", + problemStatement: "Change README.", + requirements: "Keep it small.", + interface: null, + }; + + await withLifecycleProviderHarness({ runDir: root, task, upstream }, async (harness) => { + const env = { ...process.env, AGENTRAIL_BENCH_LIFECYCLE_URL: harness.baseUrl }; + const cli = "benchmarks/agentrail-swe-lifecycle/fixtures/mock-lifecycle-cli.mjs"; + + const issue = JSON.parse((await execFileAsync("node", [cli, "issue", "get"], { env })).stdout); + assert.equal(issue.repo, "fixture/repo"); + assert.equal(issue.problemStatement, "Change README."); + + const branch = JSON.parse((await execFileAsync("node", [cli, "branch", "create", "--name", "bench"], { env })).stdout); + assert.equal(branch.name, "bench"); + + const pr = JSON.parse((await execFileAsync("node", [cli, "pr", "create", "--head", "bench", "--base", "main", "--title", "Fix README"], { env })).stdout); + assert.equal(pr.number, 1); + assert.match(pr.url, /\/pull\/1$/u); + + const ci = JSON.parse((await execFileAsync("node", [cli, "ci", "get", "--pr", "1"], { env })).stdout); + assert.equal(ci.status, "passed"); + + const review = JSON.parse((await execFileAsync("node", [cli, "review", "get", "--pr", "1"], { env })).stdout); + assert.equal(review.state, "approved"); + + const done = JSON.parse((await execFileAsync("node", [cli, "complete", "--pr", "1", "--summary", "done"], { env })).stdout); + assert.equal(done.status, "completed"); + }); + + const events = JSON.parse(await readFile(path.join(root, "lifecycle-events.json"), "utf8")); + assert.deepEqual(events.map((event) => event.type), CONTROL_LIFECYCLE_EVENTS); +}); + +test("lifecycle CLI records operations through file-backed state", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "agentrail-lifecycle-file-provider-")); + const task = { + id: "task-1", + repo: "fixture/repo", + upstream: { instanceId: "fixture__repo-1" }, + }; + const upstream = { + instanceId: "fixture__repo-1", + baseCommit: "0123456789abcdef0123456789abcdef01234567", + problemStatement: "Change README.", + requirements: "Keep it small.", + interface: null, + }; + + await withLifecycleProviderHarness({ runDir: root, task, upstream }, async (harness) => { + const env = { ...process.env, AGENTRAIL_BENCH_LIFECYCLE_STATE_PATH: harness.statePath }; + const cli = "benchmarks/agentrail-swe-lifecycle/fixtures/mock-lifecycle-cli.mjs"; + + const issue = JSON.parse((await execFileAsync("node", [cli, "issue", "get"], { env })).stdout); + assert.equal(issue.repo, "fixture/repo"); + + await execFileAsync("node", [cli, "branch", "create", "--name", "bench"], { env }); + const pr = JSON.parse((await execFileAsync("node", [cli, "pr", "create", "--head", "bench"], { env })).stdout); + await execFileAsync("node", [cli, "ci", "get", "--pr", String(pr.number)], { env }); + await execFileAsync("node", [cli, "review", "get", "--pr", String(pr.number)], { env }); + await execFileAsync("node", [cli, "complete", "--pr", String(pr.number), "--summary", "done"], { env }); + + const events = await harness.readEvents(); + assert.deepEqual(events.map((event) => event.type), CONTROL_LIFECYCLE_EVENTS); + }); +}); + +test("full lifecycle control prompt requires provider intake, PR, CI, review, and completion", () => { + const prompt = buildFullLifecycleControlPrompt({ + repo: "fixture/repo", + lifecycleCli: "node benchmarks/agentrail-swe-lifecycle/fixtures/mock-lifecycle-cli.mjs", + }); + + assert.match(prompt, /issue get/u); + assert.match(prompt, /pr create/u); + assert.match(prompt, /ci get/u); + assert.match(prompt, /review get/u); + assert.match(prompt, /complete/u); + assert.doesNotMatch(prompt, /Problem statement:/u); +}); + +test("summarizeLifecycleEvents reports missing lifecycle operations", () => { + const summary = summarizeLifecycleEvents({ + eventsPath: "/tmp/events.json", + events: [{ type: "issue.get" }, { type: "pr.create" }], + requiredEvents: CONTROL_LIFECYCLE_EVENTS, + }); + + assert.equal(summary.complete, false); + assert.deepEqual(summary.missingEvents, ["branch.create", "ci.get", "review.get", "complete"]); +}); diff --git a/test/agentrail-swe-lifecycle-phase-attribution.test.js b/test/agentrail-swe-lifecycle-phase-attribution.test.js new file mode 100644 index 0000000..571fe4b --- /dev/null +++ b/test/agentrail-swe-lifecycle-phase-attribution.test.js @@ -0,0 +1,399 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { mkdtemp, readFile, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; + +import { + AGENTRAIL_INCLUDED_PHASES, + CONTROL_INCLUDED_PHASES, + getPhaseScenario, + validatePhaseResult, +} from "../benchmarks/agentrail-swe-lifecycle/lib/phase-definitions.mjs"; +import { + computePhaseAccounting, + sumTokenUsage, +} from "../benchmarks/agentrail-swe-lifecycle/lib/phase-accounting.mjs"; +import { buildPhasePrompt } from "../benchmarks/agentrail-swe-lifecycle/lib/phase-prompts.mjs"; +import { runPhaseAttributionTask } from "../benchmarks/agentrail-swe-lifecycle/lib/phase-runner.mjs"; +import { capturePatch } from "../benchmarks/agentrail-swe-lifecycle/lib/patch.mjs"; +import { aggregateLifecycleResults } from "../benchmarks/agentrail-swe-lifecycle/lib/results.mjs"; + +test("ci-and-review-feedback scenario defines full lifecycle phases", () => { + const scenario = getPhaseScenario("ci-and-review-feedback"); + assert.deepEqual(scenario.phases.map((phase) => phase.id), [ + "provider_issue_intake", + "initial_code_change", + "provider_create_pr", + "provider_wait_for_ci", + "ci_failure_triage", + "fix_ci_failure", + "provider_wait_for_review", + "review_feedback_triage", + "fix_review_feedback", + "provider_update_pr", + "provider_final_ci_check", + "provider_final_review_check", + "agentrail_handoff", + "provider_complete", + ]); +}); + +test("phase definitions expose control and AgentRail accounting sets", () => { + assert.deepEqual([...CONTROL_INCLUDED_PHASES], [ + "provider_issue_intake", + "initial_code_change", + "provider_create_pr", + "provider_wait_for_ci", + "ci_failure_triage", + "fix_ci_failure", + "provider_wait_for_review", + "review_feedback_triage", + "fix_review_feedback", + "provider_update_pr", + "provider_final_ci_check", + "provider_final_review_check", + "provider_complete", + ]); + + assert.deepEqual([...AGENTRAIL_INCLUDED_PHASES], [ + "initial_code_change", + "fix_ci_failure", + "fix_review_feedback", + "agentrail_handoff", + ]); +}); + +test("validatePhaseResult rejects missing token usage for measured phases", () => { + assert.throws( + () => validatePhaseResult({ + phase: { id: "initial_code_change", measured: true }, + result: { tokenUsage: { status: "unavailable" } }, + requireTokenUsage: true, + }), + /initial_code_change did not produce parseable token usage/u, + ); +}); + +test("sumTokenUsage adds token components", () => { + assert.deepEqual(sumTokenUsage([ + { + status: "available", + inputTokens: 10, + cachedInputTokens: 4, + nonCachedInputTokens: 6, + outputTokens: 2, + reasoningTokens: 1, + totalTokens: 13, + }, + { + status: "available", + inputTokens: 20, + cachedInputTokens: 5, + nonCachedInputTokens: 15, + outputTokens: 3, + reasoningTokens: 2, + totalTokens: 25, + }, + ]), { + status: "available", + inputTokens: 30, + cachedInputTokens: 9, + nonCachedInputTokens: 21, + outputTokens: 5, + reasoningTokens: 3, + totalTokens: 38, + }); +}); + +test("sumTokenUsage reports unavailable if any included token usage is unavailable", () => { + assert.deepEqual(sumTokenUsage([ + tokenUsage(10), + { status: "unavailable", totalTokens: null }, + tokenUsage(5), + ]), { + status: "unavailable", + inputTokens: 15, + cachedInputTokens: 0, + nonCachedInputTokens: 15, + outputTokens: 0, + reasoningTokens: 0, + totalTokens: 15, + }); +}); + +test("computePhaseAccounting compares control and AgentRail phase inclusion", () => { + const accounting = computePhaseAccounting({ + phases: [ + { + id: "provider_issue_intake", + controlIncluded: true, + agentRailIncluded: false, + tokenUsage: tokenUsage(10), + }, + { + id: "initial_code_change", + controlIncluded: true, + agentRailIncluded: true, + tokenUsage: tokenUsage(50), + }, + { + id: "agentrail_handoff", + controlIncluded: false, + agentRailIncluded: true, + tokenUsage: tokenUsage(5), + }, + { + id: "provider_complete", + controlIncluded: true, + agentRailIncluded: false, + tokenUsage: tokenUsage(15), + }, + ], + }); + + assert.equal(accounting.control.totalTokens, 75); + assert.equal(accounting.agentrail.totalTokens, 55); + assert.equal(accounting.tokenDelta, -20); + assert.equal(accounting.tokenReductionRate, 20 / 75); +}); + +test("coding phase prompt contains task but no provider lifecycle commands", () => { + const prompt = buildPhasePrompt({ + phase: { id: "initial_code_change", actor: "llm_coding" }, + task: { id: "task-1", repo: "fixture/repo" }, + upstream: { + instanceId: "fixture__repo-1", + baseCommit: "abc", + problemStatement: "Fix README.", + requirements: "Keep it small.", + interface: null, + }, + artifacts: {}, + }); + + assert.match(prompt, /Fix README/u); + assert.doesNotMatch(prompt, /mock-lifecycle-cli/u); + assert.doesNotMatch(prompt, /create pull request/u); +}); + +test("provider phase prompt contains lifecycle instruction but no coding instruction", () => { + const prompt = buildPhasePrompt({ + phase: { id: "provider_create_pr", actor: "llm_provider_lifecycle" }, + task: { id: "task-1", repo: "fixture/repo" }, + upstream: { + instanceId: "fixture__repo-1", + baseCommit: "abc", + problemStatement: "Fix README.", + requirements: null, + interface: null, + }, + artifacts: { lifecycleCli: "node mock-lifecycle-cli.mjs" }, + }); + + assert.match(prompt, /create the pull request/u); + assert.match(prompt, /node mock-lifecycle-cli.mjs/u); + assert.doesNotMatch(prompt, /modify the repository/u); +}); + +test("buildPhasePrompt rejects unknown actors and provider phases", () => { + assert.throws( + () => buildPhasePrompt({ + phase: { id: "initial_code_change", actor: "llm_typo" }, + task: { id: "task-1", repo: "fixture/repo" }, + upstream: { instanceId: "fixture__repo-1", baseCommit: "abc", problemStatement: "Fix README." }, + }), + /Unsupported phase actor/u, + ); + + assert.throws( + () => buildPhasePrompt({ + phase: { id: "provider_typo", actor: "llm_provider_lifecycle" }, + task: { id: "task-1", repo: "fixture/repo" }, + upstream: { instanceId: "fixture__repo-1", baseCommit: "abc", problemStatement: "Fix README." }, + }), + /Unsupported provider lifecycle phase/u, + ); +}); + +test("AgentRail handoff prompt requires result path", () => { + assert.throws( + () => buildPhasePrompt({ + phase: { id: "agentrail_handoff", actor: "llm_agentrail_handoff" }, + task: { id: "task-1", repo: "fixture/repo" }, + upstream: { instanceId: "fixture__repo-1", baseCommit: "abc", problemStatement: "Fix README." }, + artifacts: {}, + }), + /requires artifacts.resultPath/u, + ); +}); + +test("runPhaseAttributionTask records phase tokens and accounting", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "agentrail-phase-attribution-")); + const source = path.join(root, "source"); + execFileSync("git", ["init", source]); + execFileSync("git", ["-C", source, "config", "user.email", "test@example.com"]); + execFileSync("git", ["-C", source, "config", "user.name", "Test User"]); + await writeFile(path.join(source, "README.md"), "base\n", "utf8"); + execFileSync("git", ["-C", source, "add", "README.md"]); + execFileSync("git", ["-C", source, "commit", "-m", "base"]); + const baseCommit = execFileSync("git", ["-C", source, "rev-parse", "HEAD"], { encoding: "utf8" }).trim(); + + const result = await runPhaseAttributionTask({ + task: { id: "task-1", repo: "fixture/repo" }, + upstream: { instanceId: "fixture__repo-1", baseCommit, problemStatement: "Change README." }, + runId: "run_phase_fixture", + runDir: root, + repoCache: path.join(root, "cache"), + sourceRepoPath: source, + flags: { + runner: "custom", + runnerCommand: "node benchmarks/agentrail-swe-lifecycle/fixtures/fake-phase-runner.mjs", + model: "fake-model", + timeoutSeconds: 30, + phaseScenario: "ci-and-review-feedback", + requirePhaseTokenUsage: true, + }, + }); + + assert.equal(result.final.success, true); + assert.equal(result.phases.length, 14); + assert.equal(result.lifecycle.complete, true); + assert.equal(result.lifecycle.events.includes("pr.update"), true); + assert.equal(result.accounting.control.totalTokens > result.accounting.agentrail.totalTokens, true); + assert.equal( + result.phases.some((phase) => phase.id === "provider_issue_intake" && phase.controlIncluded && !phase.agentRailIncluded), + true, + ); + assert.match(await readFile(result.patch.path, "utf8"), /phase runner coding change/u); +}); + +test("run.mjs phase-attribution mode writes accounting results", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "agentrail-phase-run-")); + const source = path.join(root, "source"); + const datasetPath = path.join(root, "dataset.json"); + const outputDir = path.join(root, "output"); + execFileSync("git", ["init", source]); + execFileSync("git", ["-C", source, "config", "user.email", "test@example.com"]); + execFileSync("git", ["-C", source, "config", "user.name", "Test User"]); + await writeFile(path.join(source, "README.md"), "base\n", "utf8"); + execFileSync("git", ["-C", source, "add", "README.md"]); + execFileSync("git", ["-C", source, "commit", "-m", "base"]); + const baseCommit = execFileSync("git", ["-C", source, "rev-parse", "HEAD"], { encoding: "utf8" }).trim(); + + await writeFile(datasetPath, `${JSON.stringify({ + version: 1, + id: "phase-fixture", + defaultScenario: "local-mock-lifecycle", + tasks: [{ + id: "task-1", + repo: "fixture/repo", + baseCommit, + language: "js", + tags: ["fixture"], + upstream: { + dataset: "fixture", + instanceId: "fixture__repo-1", + problemStatement: "Change README", + baseCommit, + }, + }], + }, null, 2)}\n`, "utf8"); + + execFileSync("node", [ + "benchmarks/agentrail-swe-lifecycle/run.mjs", + "--mode", "phase-attribution", + "--dataset", datasetPath, + "--source-repo-path", source, + "--output-dir", path.relative(path.resolve("."), outputDir), + "--run-id", "run_phase_fixture", + "--runner", "custom", + "--runner-command", "node benchmarks/agentrail-swe-lifecycle/fixtures/fake-phase-runner.mjs", + "--model", "fake-model", + "--timeout-seconds", "30", + "--phase-repeat", "2", + "--require-phase-token-usage", + ], { cwd: path.resolve("."), encoding: "utf8" }); + + const results = JSON.parse(await readFile(path.join(outputDir, "results.json"), "utf8")); + assert.equal(results.mode, "phase-attribution"); + assert.equal(results.tasks.length, 1); + assert.equal(results.tasks[0].phases.length, 28); + assert.equal(results.tasks[0].phases[14].repeat, 2); + assert.equal(results.tasks[0].lifecycle.complete, true); + assert.equal(results.tasks[0].accounting.control.totalTokens > results.tasks[0].accounting.agentrail.totalTokens, true); + assert.equal(results.aggregate.phaseAttribution.controlTotalTokens > results.aggregate.phaseAttribution.agentRailTotalTokens, true); +}); + +test("aggregateLifecycleResults summarizes phase attribution runs", () => { + const aggregate = aggregateLifecycleResults([{ + final: { success: true }, + tokenUsage: tokenUsage(55), + accounting: { + control: tokenUsage(75), + agentrail: tokenUsage(55), + tokenDelta: -20, + tokenReductionRate: 20 / 75, + }, + }]); + + assert.equal(aggregate.phaseAttribution.controlTotalTokens, 75); + assert.equal(aggregate.phaseAttribution.agentRailTotalTokens, 55); + assert.equal(aggregate.phaseAttribution.tokenDelta, -20); + assert.equal(aggregate.phaseAttribution.tokenReductionRate, 20 / 75); +}); + +test("aggregateLifecycleResults excludes unavailable phase attribution accounting from comparison", () => { + const aggregate = aggregateLifecycleResults([{ + final: { success: true }, + tokenUsage: { ...tokenUsage(55), status: "unavailable" }, + accounting: { + control: { ...tokenUsage(75), status: "unavailable" }, + agentrail: tokenUsage(55), + tokenDelta: null, + tokenReductionRate: null, + }, + }]); + + assert.equal(aggregate.tokenUsageUnavailableTasks, 1); + assert.equal(aggregate.phaseAttribution.comparableTasks, 0); + assert.equal(aggregate.phaseAttribution.tokenDelta, null); + assert.equal(aggregate.phaseAttribution.tokenReductionRate, null); +}); + +test("capturePatch excludes benchmark lifecycle state artifacts", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "agentrail-patch-exclude-")); + const outputDir = path.join(os.tmpdir(), "agentrail-patch-output"); + execFileSync("git", ["init", root]); + execFileSync("git", ["-C", root, "config", "user.email", "test@example.com"]); + execFileSync("git", ["-C", root, "config", "user.name", "Test User"]); + await writeFile(path.join(root, "README.md"), "base\n", "utf8"); + execFileSync("git", ["-C", root, "add", "README.md"]); + execFileSync("git", ["-C", root, "commit", "-m", "base"]); + const baseCommit = execFileSync("git", ["-C", root, "rev-parse", "HEAD"], { encoding: "utf8" }).trim(); + execFileSync("git", ["-C", root, "update-ref", "refs/agentrail-bench/base", baseCommit]); + await writeFile(path.join(root, "README.md"), "changed\n", "utf8"); + await writeFile(path.join(root, "lifecycle-state.json"), "{}\n", "utf8"); + + const patch = await capturePatch({ + worktreePath: root, + outputPath: path.join(outputDir, "patch.diff"), + }); + + assert.equal(patch.changedFiles.includes("README.md"), true); + assert.equal(patch.changedFiles.includes("lifecycle-state.json"), false); + assert.doesNotMatch(await readFile(patch.path, "utf8"), /lifecycle-state/u); +}); + +function tokenUsage(totalTokens) { + return { + status: "available", + inputTokens: totalTokens, + cachedInputTokens: 0, + nonCachedInputTokens: totalTokens, + outputTokens: 0, + reasoningTokens: 0, + totalTokens, + }; +} diff --git a/test/agentrail-swe-lifecycle-real-runner.test.js b/test/agentrail-swe-lifecycle-real-runner.test.js index 5d8fe27..5c1ea4d 100644 --- a/test/agentrail-swe-lifecycle-real-runner.test.js +++ b/test/agentrail-swe-lifecycle-real-runner.test.js @@ -33,6 +33,8 @@ test("parseBenchmarkArgs accepts paired execution flags", () => { "--require-token-usage", "--require-evaluator", "--continue-on-error", + "--agentrail-prompt-variant", "contract", + "--benchmark-surface", "full-lifecycle", "--no-cleanup", ]); assert.equal(flags.mode, "paired"); @@ -47,9 +49,46 @@ test("parseBenchmarkArgs accepts paired execution flags", () => { assert.equal(flags.requireTokenUsage, true); assert.equal(flags.requireEvaluator, true); assert.equal(flags.continueOnError, true); + assert.equal(flags.agentrailPromptVariant, "contract"); + assert.equal(flags.benchmarkSurface, "full-lifecycle"); assert.equal(flags.cleanup, false); }); +test("parseBenchmarkArgs defaults to patch-only benchmark surface", () => { + const flags = parseBenchmarkArgs(["--mode", "paired", "--runner", "custom", "--runner-command", "node fake-runner.mjs"]); + assert.equal(flags.benchmarkSurface, "patch-only"); +}); + +test("parseBenchmarkArgs accepts phase-attribution mode", () => { + const flags = parseBenchmarkArgs([ + "--mode", "phase-attribution", + "--runner", "custom", + "--runner-command", "node fake-runner.mjs", + "--phase-scenario", "ci-and-review-feedback", + "--phase-repeat", "3", + "--require-phase-token-usage", + ]); + + assert.equal(flags.mode, "phase-attribution"); + assert.equal(flags.runner, "custom"); + assert.equal(flags.runnerCommand, "node fake-runner.mjs"); + assert.equal(flags.phaseScenario, "ci-and-review-feedback"); + assert.equal(flags.phaseRepeat, 3); + assert.equal(flags.requirePhaseTokenUsage, true); +}); + +test("parseBenchmarkArgs rejects malformed phase repeat", () => { + assert.throws( + () => parseBenchmarkArgs([ + "--mode", "phase-attribution", + "--runner", "custom", + "--runner-command", "node fake-runner.mjs", + "--phase-repeat", "0", + ]), + /--phase-repeat must be a positive safe integer/u, + ); +}); + test("parseBenchmarkArgs rejects real modes without a runner", () => { assert.throws( () => parseBenchmarkArgs(["--mode", "control"]), @@ -71,6 +110,25 @@ test("parseBenchmarkArgs rejects command evaluator without a template", () => { ); }); +test("parseBenchmarkArgs rejects unknown AgentRail prompt variants", () => { + assert.throws( + () => parseBenchmarkArgs(["--mode", "agentrail", "--runner", "codex", "--agentrail-prompt-variant", "verbose"]), + /--agentrail-prompt-variant must be one of/u, + ); +}); + +test("parseBenchmarkArgs rejects unknown benchmark surfaces", () => { + assert.throws( + () => parseBenchmarkArgs([ + "--mode", "paired", + "--runner", "custom", + "--runner-command", "node fake-runner.mjs", + "--benchmark-surface", "fake", + ]), + /--benchmark-surface must be one of/u, + ); +}); + test("buildControlPrompt includes problem input and excludes gold fields", () => { const prompt = buildControlPrompt({ task: { id: "task_1", repo: "owner/repo" }, @@ -133,11 +191,16 @@ test("resolveRunnerCommand invokes Codex through stdin JSON mode", async () => { assert.deepEqual(command.args, [ "-a", "never", - "--model", - "codex-default", + "-c", + "features.hooks=false", + "exec", "--sandbox", "workspace-write", - "exec", + "--ephemeral", + "--ignore-user-config", + "--ignore-rules", + "--model", + "codex-default", "--json", "-", ]); @@ -145,6 +208,70 @@ test("resolveRunnerCommand invokes Codex through stdin JSON mode", async () => { assert.equal(command.args.includes("--prompt-file"), false); }); +test("resolveRunnerCommand invokes Cursor Agent through print stream JSON mode", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "agentrail-cursor-command-")); + const promptPath = path.join(root, "prompt.md"); + await writeFile(promptPath, "Change README\n", "utf8"); + + const command = await resolveRunnerCommand({ + runner: "cursor", + model: "composer-2-5", + promptPath, + }); + + assert.equal(command.executable, "cursor"); + assert.deepEqual(command.args, [ + "agent", + "--print", + "--output-format", + "stream-json", + "--trust", + "--force", + "--model", + "composer-2-5", + ]); + assert.equal(command.stdin, "Change README\n"); + assert.equal(command.args.includes("--prompt-file"), false); +}); + +test("resolveRunnerCommand invokes Claude Code through non-interactive stream JSON mode", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "agentrail-claude-command-")); + const promptPath = path.join(root, "prompt.md"); + await writeFile(promptPath, "Change README\n", "utf8"); + + const command = await resolveRunnerCommand({ + runner: "claude-code", + model: "claude-opus-4-7", + promptPath, + }); + + assert.equal(command.executable, "claude"); + assert.deepEqual(command.args, [ + "--print", + "--output-format", + "stream-json", + "--verbose", + "--permission-mode", + "bypassPermissions", + "--dangerously-skip-permissions", + "--model", + "claude-opus-4-7", + ]); + assert.equal(command.stdin, "Change README\n"); + assert.equal(command.args.includes("@prompt.md"), false); +}); + +test("parseBenchmarkArgs rejects Cursor token-required benchmarks", () => { + assert.throws( + () => parseBenchmarkArgs([ + "--mode", "phase-attribution", + "--runner", "cursor", + "--require-phase-token-usage", + ]), + /Cursor Agent CLI does not expose token usage/u, + ); +}); + test("runEvaluator returns unavailable for evaluator none", async () => { const result = await runEvaluator({ evaluator: "none" }); assert.deepEqual(result, { status: "unavailable", exitCode: null, outputPath: null }); @@ -203,6 +330,39 @@ test("runControlLane executes a fake runner, captures tokens, patch, and evaluat assert.equal(result.evaluator.status, "passed"); }); +test("runControlLane full-lifecycle requires provider issue, PR, CI, review, and completion", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "agentrail-control-lifecycle-lane-")); + const source = path.join(root, "source"); + execFileSync("git", ["init", source]); + execFileSync("git", ["-C", source, "config", "user.email", "test@example.com"]); + execFileSync("git", ["-C", source, "config", "user.name", "Test User"]); + await writeFile(path.join(source, "README.md"), "base\n"); + execFileSync("git", ["-C", source, "add", "README.md"]); + execFileSync("git", ["-C", source, "commit", "-m", "base"]); + const baseCommit = execFileSync("git", ["-C", source, "rev-parse", "HEAD"], { encoding: "utf8" }).trim(); + + const result = await runControlLane({ + task: { id: "task_1", repo: "fixture/repo" }, + upstream: { instanceId: "fixture__repo-1", baseCommit, problemStatement: "Change README" }, + runDir: root, + repoCache: path.join(root, "cache"), + sourceRepoPath: source, + flags: { + benchmarkSurface: "full-lifecycle", + runner: "custom", + runnerCommand: "node benchmarks/agentrail-swe-lifecycle/fixtures/fake-lifecycle-runner.mjs", + model: "fake-model", + timeoutSeconds: 30, + evaluator: "none", + }, + }); + + assert.equal(result.status, "completed"); + assert.equal(result.success, true); + assert.deepEqual(result.lifecycle.events, ["issue.get", "branch.create", "pr.create", "ci.get", "review.get", "complete"]); + assert.equal(result.lifecycle.complete, true); +}); + test("withAgentRailBenchmarkHarness exposes a task and records synthetic submission", async () => { const root = await mkdtemp(path.join(os.tmpdir(), "agentrail-harness-")); const result = await withAgentRailBenchmarkHarness({ @@ -265,6 +425,132 @@ test("runAgentRailLane executes a fake runner through AgentRail lifecycle", asyn assert.match(result.stdout, /"status":"succeeded"/u); }); +test("runAgentRailLane full-lifecycle keeps provider lifecycle outside the runner LLM", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "agentrail-agentrail-full-lifecycle-lane-")); + const source = path.join(root, "source"); + execFileSync("git", ["init", source]); + execFileSync("git", ["-C", source, "config", "user.email", "test@example.com"]); + execFileSync("git", ["-C", source, "config", "user.name", "Test User"]); + await writeFile(path.join(source, "README.md"), "base\n"); + execFileSync("git", ["-C", source, "add", "README.md"]); + execFileSync("git", ["-C", source, "commit", "-m", "base"]); + const baseCommit = execFileSync("git", ["-C", source, "rev-parse", "HEAD"], { encoding: "utf8" }).trim(); + + const result = await runAgentRailLane({ + task: { id: "task_1", repo: "fixture/repo" }, + upstream: { instanceId: "fixture__repo-1", baseCommit, problemStatement: "Change README" }, + runDir: root, + repoCache: path.join(root, "cache"), + sourceRepoPath: source, + flags: { + benchmarkSurface: "full-lifecycle", + runner: "custom", + runnerCommand: "node benchmarks/agentrail-swe-lifecycle/fixtures/fake-runner.mjs", + model: "fake-model", + timeoutSeconds: 30, + evaluator: "none", + }, + }); + + const prompt = await readFile(result.runner.promptPath, "utf8"); + assert.doesNotMatch(prompt, /agentrail agent report --status progress/u); + assert.doesNotMatch(prompt, /After writing the result file, report completion/u); + assert.match(prompt, /write the completed result JSON file/u); + assert.equal(result.success, true, result.stderr); + assert.deepEqual(result.lifecycle.events, ["issue.ingested", "task.started", "pr.created", "ci.checked", "review.checked", "task.completed"]); +}); + +test("runAgentRailLane contract variant writes run contract files and shrinks the prompt", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "agentrail-contract-lane-")); + const source = path.join(root, "source"); + execFileSync("git", ["init", source]); + execFileSync("git", ["-C", source, "config", "user.email", "test@example.com"]); + execFileSync("git", ["-C", source, "config", "user.name", "Test User"]); + await writeFile(path.join(source, "README.md"), "base\n"); + execFileSync("git", ["-C", source, "add", "README.md"]); + execFileSync("git", ["-C", source, "commit", "-m", "base"]); + const baseCommit = execFileSync("git", ["-C", source, "rev-parse", "HEAD"], { encoding: "utf8" }).trim(); + const task = { + id: "task_1", + repo: "fixture/repo", + upstream: { instanceId: "fixture__repo-1" }, + }; + const upstream = { instanceId: "fixture__repo-1", baseCommit, problemStatement: "Change README" }; + + const current = await runAgentRailLane({ + task, + upstream, + runDir: path.join(root, "current"), + repoCache: path.join(root, "cache"), + sourceRepoPath: source, + flags: { + runner: "custom", + runnerCommand: "node benchmarks/agentrail-swe-lifecycle/fixtures/fake-runner.mjs", + model: "fake-model", + timeoutSeconds: 30, + evaluator: "none", + }, + }); + const contract = await runAgentRailLane({ + task, + upstream, + runDir: path.join(root, "contract"), + repoCache: path.join(root, "cache"), + sourceRepoPath: source, + flags: { + runner: "custom", + runnerCommand: "node benchmarks/agentrail-swe-lifecycle/fixtures/fake-runner.mjs", + model: "fake-model", + timeoutSeconds: 30, + evaluator: "none", + agentrailPromptVariant: "contract", + }, + }); + + const prompt = await readFile(contract.runner.promptPath, "utf8"); + assert.match(prompt, /Read \.agentrail-run\/task\.md/u); + assert.ok(contract.tokenUsage.totalTokens < current.tokenUsage.totalTokens); + assert.match(await readFile(path.join(contract.workspace.worktreePath, ".agentrail-run", "task.md"), "utf8"), /Change README/u); + assert.match(await readFile(path.join(contract.workspace.worktreePath, ".agentrail-run", "result.schema.json"), "utf8"), /"completed"/u); +}); + +test("runAgentRailLane skill variant writes a thin SKILL.md adapter", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "agentrail-skill-lane-")); + const source = path.join(root, "source"); + execFileSync("git", ["init", source]); + execFileSync("git", ["-C", source, "config", "user.email", "test@example.com"]); + execFileSync("git", ["-C", source, "config", "user.name", "Test User"]); + await writeFile(path.join(source, "README.md"), "base\n"); + execFileSync("git", ["-C", source, "add", "README.md"]); + execFileSync("git", ["-C", source, "commit", "-m", "base"]); + const baseCommit = execFileSync("git", ["-C", source, "rev-parse", "HEAD"], { encoding: "utf8" }).trim(); + + const result = await runAgentRailLane({ + task: { + id: "task_1", + repo: "fixture/repo", + upstream: { instanceId: "fixture__repo-1" }, + }, + upstream: { instanceId: "fixture__repo-1", baseCommit, problemStatement: "Change README" }, + runDir: root, + repoCache: path.join(root, "cache"), + sourceRepoPath: source, + flags: { + runner: "custom", + runnerCommand: "node benchmarks/agentrail-swe-lifecycle/fixtures/fake-runner.mjs", + model: "fake-model", + timeoutSeconds: 30, + evaluator: "none", + agentrailPromptVariant: "skill", + }, + }); + + const skill = await readFile(path.join(result.workspace.worktreePath, ".codex", "skills", "agentrail-run", "SKILL.md"), "utf8"); + assert.match(skill, /name: agentrail-run/u); + assert.match(skill, /references\/contract\.md/u); + assert.match(await readFile(result.runner.promptPath, "utf8"), /agentrail-run skill/u); +}); + test("benchmarkRunnerPolicy keeps SWE fixture secret files from blocking launch", () => { assert.deepEqual(benchmarkRunnerPolicy(), { preset: "advisory", @@ -322,6 +608,7 @@ test("run.mjs paired mode writes control versus AgentRail token comparison", asy const results = JSON.parse(await readFile(path.join(outputDir, "results.json"), "utf8")); assert.equal(results.mode, "paired"); + assert.equal(results.benchmarkSurface, "patch-only"); assert.equal(results.tasks.length, 1); assert.equal(results.tasks[0].final.success, true); assert.notEqual( @@ -329,4 +616,71 @@ test("run.mjs paired mode writes control versus AgentRail token comparison", asy results.tasks[0].comparison.agentrail.tokenUsage.totalTokens, ); assert.notEqual(results.aggregate.paired.tokenDelta, 0); + assert.equal(typeof results.aggregate.paired.cachedInputDelta, "number"); +}); + +test("run.mjs paired full-lifecycle mode writes lifecycle summaries for both lanes", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "agentrail-paired-full-lifecycle-run-")); + const source = path.join(root, "source"); + const datasetPath = path.join(root, "dataset.json"); + const outputDir = path.join(root, "output"); + execFileSync("git", ["init", source]); + execFileSync("git", ["-C", source, "config", "user.email", "test@example.com"]); + execFileSync("git", ["-C", source, "config", "user.name", "Test User"]); + await writeFile(path.join(source, "README.md"), "base\n"); + execFileSync("git", ["-C", source, "add", "README.md"]); + execFileSync("git", ["-C", source, "commit", "-m", "base"]); + const baseCommit = execFileSync("git", ["-C", source, "rev-parse", "HEAD"], { encoding: "utf8" }).trim(); + await writeFile(datasetPath, `${JSON.stringify({ + version: 1, + id: "paired-full-lifecycle-fixture", + defaultScenario: "local-mock-lifecycle", + tasks: [{ + id: "task-1", + repo: "fixture/repo", + baseCommit, + language: "js", + tags: ["fixture"], + upstream: { + dataset: "fixture", + instanceId: "fixture__repo-1", + problemStatement: "Change README", + baseCommit, + }, + }], + }, null, 2)}\n`, "utf8"); + + execFileSync("node", [ + "benchmarks/agentrail-swe-lifecycle/run.mjs", + "--mode", "paired", + "--benchmark-surface", "full-lifecycle", + "--dataset", datasetPath, + "--source-repo-path", source, + "--output-dir", outputDir, + "--run-id", "run_full_lifecycle_fixture", + "--runner", "custom", + "--runner-command", "node benchmarks/agentrail-swe-lifecycle/fixtures/fake-lifecycle-runner.mjs", + "--model", "fake-model", + "--timeout-seconds", "30", + "--require-token-usage", + ], { cwd: path.resolve("."), encoding: "utf8" }); + + const results = JSON.parse(await readFile(path.join(outputDir, "results.json"), "utf8")); + assert.equal(results.mode, "paired"); + assert.equal(results.benchmarkSurface, "full-lifecycle"); + assert.equal(results.tasks.length, 1); + assert.equal(results.tasks[0].final.success, true); + assert.deepEqual( + results.tasks[0].lanes.control.lifecycle.events, + ["issue.get", "branch.create", "pr.create", "ci.get", "review.get", "complete"], + ); + assert.deepEqual( + results.tasks[0].lanes.agentrail.lifecycle.events, + ["issue.ingested", "task.started", "pr.created", "ci.checked", "review.checked", "task.completed"], + ); + assert.equal(results.tasks[0].lanes.control.lifecycle.complete, true); + assert.equal(results.tasks[0].lanes.agentrail.lifecycle.complete, true); + assert.equal(typeof results.aggregate.paired.nonCachedInputDelta, "number"); + assert.equal(typeof results.aggregate.paired.outputDelta, "number"); + assert.equal(typeof results.aggregate.paired.reasoningDelta, "number"); }); diff --git a/test/agentrail-swe-lifecycle-token-usage.test.js b/test/agentrail-swe-lifecycle-token-usage.test.js index 3a299d1..52ce98e 100644 --- a/test/agentrail-swe-lifecycle-token-usage.test.js +++ b/test/agentrail-swe-lifecycle-token-usage.test.js @@ -21,6 +21,7 @@ test("parseTokenUsageFromFiles aggregates OpenAI-style usage JSONL", async () => outputTokens: 45, reasoningTokens: 10, cachedInputTokens: 20, + nonCachedInputTokens: 110, totalTokens: 185, sources: [log], }); @@ -61,6 +62,7 @@ test("parseTokenUsageFromFiles accepts Codex JSON turn usage", async () => { outputTokens: 17, reasoningTokens: 10, cachedInputTokens: 3456, + nonCachedInputTokens: 16475, totalTokens: 19958, sources: [log], }); @@ -101,6 +103,7 @@ test("parseTokenUsageFromFiles does not double-count nested OpenAI detail token outputTokens: 25, reasoningTokens: 15, cachedInputTokens: null, + nonCachedInputTokens: null, totalTokens: 140, sources: [log], }); @@ -120,6 +123,7 @@ test("parseTokenUsageFromFiles aggregates top-level JSON array usage entries", a outputTokens: 3, reasoningTokens: null, cachedInputTokens: null, + nonCachedInputTokens: null, totalTokens: 65, sources: [log], }); @@ -142,6 +146,7 @@ test("parseTokenUsageFromFiles parses whole-document arrays before JSONL scannin outputTokens: 3, reasoningTokens: null, cachedInputTokens: null, + nonCachedInputTokens: null, totalTokens: 13, sources: [log], }); @@ -165,6 +170,7 @@ test("parseTokenUsageFromFiles aggregates JSONL array line usage entries", async outputTokens: 3, reasoningTokens: null, cachedInputTokens: null, + nonCachedInputTokens: null, totalTokens: 14, sources: [log], }); @@ -186,6 +192,7 @@ test("parseTokenUsageFromFiles aggregates multiple nested usage objects in one r outputTokens: 3, reasoningTokens: 4, cachedInputTokens: null, + nonCachedInputTokens: null, totalTokens: 19, sources: [log], }); @@ -216,6 +223,7 @@ test("parseTokenUsageFromFiles reports unavailable when no usage appears", async outputTokens: null, reasoningTokens: null, cachedInputTokens: null, + nonCachedInputTokens: null, totalTokens: null, sources: [], }); From f0466ba6b2cefb0a0f8f9ae450ba399f00d793a7 Mon Sep 17 00:00:00 2001 From: Onyeka Nwamba Date: Tue, 26 May 2026 02:26:27 +0100 Subject: [PATCH 17/18] feat: publish benchmark results on landing page and /benchmarks Adds a "04 / Benchmarks" hero section to the landing page with the two-number headline ("Up to 60% fewer tokens. 21% lower cost.") and a dedicated /benchmarks report page covering the 5-task run-set across Cursor Composer 2.5, Claude Opus 4.7, and GPT-5.5. Landing section: - Editorial layout matching existing numbered sections - Display serif italic headline; only the 60% and 21% percentages are accent green, rest in ink - Links to /benchmarks for the full report /benchmarks page: - tl;dr strip with per-runner deltas - 01 / tasks: 5 real ansible/ansible SWE-bench-Pro tasks - 02 / lanes: control vs AgentRail explanation - 03 / results: full 14-row metrics table with emphasised cost rows - 03b / paired bars: hand-rolled CSS bars, normalised per metric - 04 / per-runner notes including Cursor's plan-billing caveat - 05 / metric definitions - 06 / reproduce with git clone command Cursor cost is intentionally not claimed since Composer 2.5 is plan-billed rather than per-token. GPT-5.5 cost reduction (~21%) is computed from the captured token mix at published GPT-5-class rates; Claude Opus cost is taken directly from the benchmark's Anthropic billing output. Also adds an npm downloads shields.io badge to the project README. --- README.md | 1 + landing-next/app/benchmarks/page.tsx | 1400 ++++++++++++++++++++++++ landing-next/app/page.tsx | 2 + landing-next/components/Benchmarks.tsx | 115 ++ 4 files changed, 1518 insertions(+) create mode 100644 landing-next/app/benchmarks/page.tsx create mode 100644 landing-next/components/Benchmarks.tsx diff --git a/README.md b/README.md index 4e3eebf..241cf31 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # AgentRail [![CI](https://github.com/oxnw/agentrail/actions/workflows/ci.yml/badge.svg)](https://github.com/oxnw/agentrail/actions/workflows/ci.yml) +[![npm downloads](https://img.shields.io/npm/dm/%40agentrail-core/cli.svg)](https://www.npmjs.com/package/@agentrail-core/cli) [![License: Source-Available](https://img.shields.io/badge/License-Source--Available-blue.svg)](./LICENSE) [![Node.js](https://img.shields.io/badge/node-%E2%89%A524-brightgreen)](https://nodejs.org/) diff --git a/landing-next/app/benchmarks/page.tsx b/landing-next/app/benchmarks/page.tsx new file mode 100644 index 0000000..1fbc9a7 --- /dev/null +++ b/landing-next/app/benchmarks/page.tsx @@ -0,0 +1,1400 @@ +import type { Metadata } from "next"; +import Link from "next/link"; +import Nav from "@/components/Nav"; +import Footer from "@/components/Footer"; + +export const metadata: Metadata = { + title: "Benchmarks — AgentRail", + description: + "AgentRail SWE-lifecycle benchmark: 60% fewer tokens on Cursor Composer 2.5, 21% lower cost on Claude Opus 4.7 and GPT-5.5, across 5 real software-engineering tasks.", + alternates: { canonical: "/benchmarks" }, +}; + +type RunnerRow = { + metric: string; + control: string; + agentrail: string; + deltaAbs: string; + deltaPct: string; + emphasised?: boolean; +}; + +type Runner = { + id: string; + name: string; + provider: string; + via: string; + headline: string; + headlineMetric: string; + rows: RunnerRow[]; + runId: string; + note: string; +}; + +const runners: Runner[] = [ + { + id: "cursor", + name: "Cursor Composer 2.5", + provider: "Cursor", + via: "native agent", + headline: "−60.0%", + headlineMetric: "non-cache tokens", + rows: [ + { + metric: "non-cache tokens", + control: "8,800,719", + agentrail: "3,520,245", + deltaAbs: "−5,280,474", + deltaPct: "−60.0%", + emphasised: true, + }, + { + metric: "cache-inclusive tokens", + control: "100,969,521", + agentrail: "77,663,576", + deltaAbs: "−23,305,945", + deltaPct: "−23.1%", + }, + { + metric: "input tokens", + control: "8,524,884", + agentrail: "3,312,805", + deltaAbs: "−5,212,079", + deltaPct: "−61.1%", + }, + { + metric: "output tokens", + control: "275,835", + agentrail: "207,440", + deltaAbs: "−68,395", + deltaPct: "−24.8%", + }, + ], + runId: "run_2026-05-25T18-15-35-314Z", + note: "Non-cache tokens are the headline figure for Cursor — they correspond to billable activity outside the prompt-cache hit path. Composer 2.5 is plan-billed; per-token dollar cost is not directly comparable to API providers.", + }, + { + id: "opus", + name: "Claude Opus 4.7", + provider: "Anthropic", + via: "via Claude Code", + headline: "−21.3%", + headlineMetric: "dollar cost", + rows: [ + { + metric: "dollar cost (USD)", + control: "$48.93", + agentrail: "$38.53", + deltaAbs: "−$10.40", + deltaPct: "−21.3%", + emphasised: true, + }, + { + metric: "cache-inclusive tokens", + control: "49,367,189", + agentrail: "45,202,488", + deltaAbs: "−4,164,701", + deltaPct: "−8.4%", + }, + { + metric: "cache creation", + control: "2,729,630", + agentrail: "1,388,016", + deltaAbs: "−1,341,614", + deltaPct: "−49.2%", + }, + { + metric: "cache read", + control: "46,374,169", + agentrail: "43,575,956", + deltaAbs: "−2,798,213", + deltaPct: "−6.0%", + }, + { + metric: "non-cache tokens", + control: "263,390", + agentrail: "238,516", + deltaAbs: "−24,874", + deltaPct: "−9.4%", + }, + ], + runId: "run_phase_opus47_5_20260525_213304", + note: "Cost reduction outpaces raw token reduction because AgentRail shifts work from cache-creation (priced higher) toward cache-read. Direct dollar figures come from Anthropic's billed rates at run time.", + }, + { + id: "gpt55", + name: "GPT-5.5", + provider: "OpenAI", + via: "via Codex", + headline: "−21.0%", + headlineMetric: "dollar cost", + rows: [ + { + metric: "dollar cost (USD)", + control: "~$8.02", + agentrail: "~$6.34", + deltaAbs: "−$1.68", + deltaPct: "−21.0%", + emphasised: true, + }, + { + metric: "total tokens", + control: "27,392,589", + agentrail: "24,228,912", + deltaAbs: "−3,163,677", + deltaPct: "−11.5%", + }, + { + metric: "input tokens", + control: "27,131,547", + agentrail: "24,003,102", + deltaAbs: "−3,128,445", + deltaPct: "−11.5%", + }, + { + metric: "cached input", + control: "24,862,976", + agentrail: "22,605,952", + deltaAbs: "−2,257,024", + deltaPct: "−9.1%", + }, + { + metric: "output tokens", + control: "207,314", + agentrail: "176,400", + deltaAbs: "−30,914", + deltaPct: "−14.9%", + }, + ], + runId: "run_phase_gpt55_5_20260525_212153", + note: "Output-token reductions matter most here — output is the priciest unit on GPT-5.5 and the one most sensitive to scaffolding overhead. Dollar cost computed at OpenAI's published GPT-5-class rates.", + }, +]; + +const tasks = [ + { + id: "T-01", + repo: "ansible/ansible", + summary: "Password lookup + encrypt: fix subjects_alt_name handling", + files: 5, + loc: "+143 / −89", + }, + { + id: "T-02", + repo: "ansible/ansible", + summary: "YAML dumper: propagate filter trust through templating", + files: 6, + loc: "+129 / −19", + }, + { + id: "T-03", + repo: "ansible/ansible", + summary: "Linux facts: correct nproc/cpuinfo on container hosts", + files: 3, + loc: "+73 / −1", + }, + { + id: "T-04", + repo: "ansible/ansible", + summary: "mount_facts module: introduce structured mount inventory", + files: 7, + loc: "+1622 / −0", + }, + { + id: "T-05", + repo: "ansible/ansible", + summary: "Module respawn + SELinux compat shim (libselinux removal)", + files: 13, + loc: "+649 / −304", + }, +]; + +const metricDefs = [ + { + term: "non-cache tokens", + desc: "Tokens billed at the model's full input rate — i.e. not served from the provider's prompt cache. The cleanest signal of net new work the model had to do. Where AgentRail's lifecycle routing has the largest effect.", + }, + { + term: "cache-inclusive tokens", + desc: "Every token the agent sent or received, including cache reads. Larger absolute numbers and smaller percentage deltas — context-window-bound, not cost-bound.", + }, + { + term: "cache creation", + desc: "First-time write of a prompt prefix into the provider's cache. Priced higher than cache-read for Anthropic models; AgentRail's stable role briefs cut repeat creations.", + }, + { + term: "cache read", + desc: "Reuse of a previously cached prefix. Discounted; counts toward cache-inclusive but not non-cache. Higher is generally better — it means the cache is doing its job.", + }, + { + term: "output tokens", + desc: "Tokens the model generated. Priced highest on most providers. Disproportionate driver of cost on GPT-5.5 and a tight proxy for agent verbosity.", + }, + { + term: "dollar cost", + desc: "Provider-reported USD spend, summed across all five tasks in the run-set. For Claude Opus, computed from Anthropic's billed rates at the time of the run. For GPT-5.5, computed from OpenAI's published GPT-5-class rates applied to the captured token mix.", + }, +]; + +const fontMono = "var(--font-mono)"; +const fontSerif = "var(--font-serif)"; + +// ── Layout primitives ────────────────────────────────────────────────────── + +function SectionLabel({ num, title }: { num: string; title: string }) { + return ( +
    + + + {num} / {title} + + +
    + ); +} + +// ── Page sections ────────────────────────────────────────────────────────── + +function PageHeader() { + return ( +
    +
    +
    + agentrail / benchmarks / run-set 5 + + 2026-05-26 + +
    +

    + Run-set 5. +

    +

    + Three coding agents — Cursor Composer 2.5, Claude Opus 4.7 (via Claude + Code), and GPT-5.5 (via Codex) — on five real software-engineering + tasks, full issue-to-ship lifecycle, two lanes per task. All totals + below are summed across the run-set. +

    + + {/* TL;DR strip */} +
    + + tl;dr + + {[ + { + runner: "cursor 2.5", + stat: "−60.0%", + metric: "non-cache tokens", + detail: "8.80M → 3.52M", + }, + { + runner: "opus 4.7", + stat: "−21.3%", + metric: "cost", + detail: "$48.93 → $38.53", + }, + { + runner: "gpt-5.5", + stat: "−21.0%", + metric: "cost", + detail: "$8.02 → $6.34", + }, + ].map((entry) => ( +
    +
    + {entry.runner} +
    +
    + {entry.stat}{" "} + + {entry.metric} + +
    +
    + {entry.detail} +
    +
    + ))} +
    +
    +
    + ); +} + +function TasksSection() { + return ( +
    +
    + +

    + Five real Ansible tasks. +

    +

    + Drawn from the SWE-bench-Pro hard set against the{" "} + + ansible/ansible + {" "} + repository. Each task runs the full 14-phase lifecycle: issue intake → + plan → implement → CI → review → ship. +

    + +
    +
    +
    id
    +
    task
    +
    files
    +
    loc
    +
    + {tasks.map((t, i) => ( +
    +
    + {t.id} +
    +
    {t.summary}
    +
    + {t.files} +
    +
    + {t.loc} +
    +
    + ))} +
    +
    +
    + ); +} + +function LanesSection() { + return ( +
    +
    + +

    + Control vs AgentRail. +

    +
    + {[ + { + label: "control", + body: "The agent owns every phase: intake, triage, plan, implement, CI watch, fix CI, review, revise, ship. Tokens are charged for every prompt the agent issues across the loop.", + }, + { + label: "agentrail", + body: "The agent is invoked only for the phases AgentRail's lifecycle leaves to it (predominantly implement and revise). Other phases are handled by AgentRail and incur no agent-side spend.", + }, + ].map((lane, i, arr) => ( +
    +
    + {lane.label} +
    +

    + {lane.body} +

    +
    + ))} +
    +

    + Both lanes share the same model, same repository state, and same + acceptance criteria. They differ only in which phases the agent runs + — mirroring the deployment decision in production, not a controlled + lab study. +

    +
    +
    + ); +} + +function ResultsTable() { + return ( +
    +
    + +

    + Every metric, every cell. +

    +

    + 3 runners × 2 lanes +

    + +
    + {/* Header */} +
    +
    runner
    +
    metric
    +
    control
    +
    agentrail
    +
    Δ
    +
    Δ%
    +
    + + {runners.map((runner, ri) => ( +
    + {runner.rows.map((row, i) => ( +
    +
    + {i === 0 ? runner.name : "·"} +
    +
    + {row.metric} +
    +
    + {row.control} +
    +
    + {row.agentrail} +
    +
    + {row.deltaAbs} +
    +
    + {row.deltaPct} +
    +
    + ))} +
    + ))} +
    +
    +
    + ); +} + +// ── Paired-bar chart (CSS-only) ──────────────────────────────────────────── + +function PairedBar({ + controlLabel, + controlValue, + agentrailLabel, + agentrailValue, + controlWidthPct, + agentrailWidthPct, + deltaLabel, +}: { + controlLabel: string; + controlValue: string; + agentrailLabel: string; + agentrailValue: string; + controlWidthPct: number; + agentrailWidthPct: number; + deltaLabel: string; +}) { + return ( +
    +
    + + {controlLabel} + +
    +
    +
    + + {controlValue} + +
    +
    + + {agentrailLabel} + +
    +
    + + {deltaLabel} + +
    + + {agentrailValue} + +
    +
    + ); +} + +function chartData(runner: Runner) { + // For each row, compute normalised widths against the row's control value. + return runner.rows.map((row) => { + const parse = (s: string) => + Number(s.replace(/[$,~Mk]/g, "")) || 0; + const c = parse(row.control); + const a = parse(row.agentrail); + const max = Math.max(c, a) || 1; + return { + ...row, + controlWidthPct: (c / max) * 100, + agentrailWidthPct: (a / max) * 100, + }; + }); +} + +function PerRunnerSection({ runner, idx }: { runner: Runner; idx: number }) { + const data = chartData(runner); + return ( +
    +
    + + 0{idx + 1} + +

    + {runner.name} +

    + + {runner.provider} · {runner.via} + + + {runner.headline}{" "} + + {runner.headlineMetric} + + +
    + +
    + {data.map((row) => ( +
    +
    + {row.metric} +
    + +
    + ))} +
    + +
    + {runner.runId} +
    +
    + ); +} + +function ChartsSection() { + return ( +
    +
    + +

    + Control vs AgentRail, per metric. +

    +

    + ink = control · green = agentrail · normalised per row +

    + {runners.map((runner, i) => ( + + ))} +
    +
    + ); +} + +function NotesSection() { + return ( +
    +
    + +

    + Why the deltas differ. +

    +
    + {runners.map((runner, i, arr) => ( +
    +
    + {runner.provider} +
    +
    + {runner.name} +
    +

    + {runner.note} +

    +
    + {runner.runId} +
    +
    + ))} +
    +
    +
    + ); +} + +function DefinitionsSection() { + return ( +
    +
    + +

    + What each number means. +

    +
    + {metricDefs.map((def) => ( +
    +
    + {def.term} +
    +
    + {def.desc} +
    +
    + ))} +
    +
    +
    + ); +} + +function ReproduceSection() { + return ( +
    +
    + +

    + Run it yourself. +

    +

    + Every prompt, every task harness, and every run id above is in the + benchmark repo. Numbers should reproduce within ±2% on a fresh + sample. +

    + 
    +          $ git clone
+          https://github.com/oxnw/agentrail
+          {"\n"}
+          $ cd
+          agentrail/benchmarks/agentrail-swe-lifecycle
+          {"\n"}
+          $ npm run
+          benchmark:phase-attribution -- --runner claude-code --count 5
+
    +
    + byok · BUSL-1.1 +
    +
    +
    + ); +} + +// ── Page ────────────────────────────────────────────────────────────────── + +export default function BenchmarksPage() { + return ( + <> +