diff --git a/.gitignore b/.gitignore index da05d18..006475f 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ AGENTS.md !.env.example .agentrail/ .agentrail.legacy/ +.agentrail-bench/ .idea/ .vscode/ browser-artifacts/ @@ -49,7 +50,9 @@ verify-browser.mjs verify-waitlist.mjs verify-waitlist-api.mjs verify-landing-claims.mjs -benchmarks/ +benchmarks/* +!benchmarks/agentrail-swe-lifecycle/ +!benchmarks/agentrail-swe-lifecycle/** scripts/phase1-dogfood.mjs scripts/phase2-dogfood.mjs scripts/phase3-dogfood.mjs diff --git a/README.md b/README.md index 4e3eebf..241cf31 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # AgentRail [![CI](https://github.com/oxnw/agentrail/actions/workflows/ci.yml/badge.svg)](https://github.com/oxnw/agentrail/actions/workflows/ci.yml) +[![npm downloads](https://img.shields.io/npm/dm/%40agentrail-core/cli.svg)](https://www.npmjs.com/package/@agentrail-core/cli) [![License: Source-Available](https://img.shields.io/badge/License-Source--Available-blue.svg)](./LICENSE) [![Node.js](https://img.shields.io/badge/node-%E2%89%A524-brightgreen)](https://nodejs.org/) diff --git a/benchmarks/agentrail-swe-lifecycle/README.md b/benchmarks/agentrail-swe-lifecycle/README.md new file mode 100644 index 0000000..800d151 --- /dev/null +++ b/benchmarks/agentrail-swe-lifecycle/README.md @@ -0,0 +1,147 @@ +# AgentRail SWE Lifecycle Dataset + +This directory defines AgentRail's wrapper dataset for token-efficiency +benchmarking on SWE-bench-style coding tasks. + +The checked-in `dataset.json` does not vendor SWE-bench problem statements, +gold patches, test patches, or evaluator payloads. It stores only a stable +manifest of upstream task references plus AgentRail lifecycle metadata. + +## Sources + +- Task selection: Artificial Analysis Coding Agent Index methodology, + `SWE-Bench-Pro-Hard-AA`. +- Upstream task rows: `ScaleAI/SWE-bench_Pro`, `default/test` split on + Hugging Face. + +## Lifecycle Wrapper + +Each task is intended to run through an AgentRail lifecycle: + +1. create a synthetic GitHub or Linear issue from the upstream problem statement +2. ingest that issue into AgentRail +3. assign it to a managed AgentRail runner +4. produce a patch or pull-request artifact +5. run the SWE-bench evaluator as CI +6. convert deterministic review findings into AgentRail review feedback +7. retry through AgentRail when CI or review is blocking +8. mark final pass/fail from the evaluator and lifecycle state + +The first scenario is `local-mock-lifecycle`, which simulates GitHub/Linear, +PRs, CI, and code review locally. Live GitHub/Linear scenarios can be added +after the local lifecycle is repeatable. + +## Refresh + +Regenerate the manifest from upstream sources: + +```bash +npm run benchmark:agentrail-swe:refresh +``` + +Validate the checked-in manifest shape: + +```bash +npm run benchmark:agentrail-swe:validate +``` + +## Run + +Dry-run the selected task manifest without launching a runner: + +```bash +npm run benchmark:agentrail-swe:dry-run -- --count 5 +``` + +Run the paired patch-only token benchmark. This runs the same coding task once +through the direct control lane and once through AgentRail, then writes +`results.json` and `summary.md` under +`.agentrail-bench/agentrail-swe-lifecycle//`. + +```bash +npm run benchmark:agentrail-swe:paired -- \ + --runner codex \ + --count 1 \ + --repo-cache .agentrail-bench/repo-cache \ + --require-token-usage +``` + +Run the paired full-lifecycle token benchmark when you want the control lane to +pay LLM tokens for lifecycle work that AgentRail handles outside the runner: + +```bash +npm run benchmark:agentrail-swe:paired:full-lifecycle -- \ + --runner codex \ + --count 1 \ + --repo-cache .agentrail-bench/repo-cache \ + --require-token-usage +``` + +The full-lifecycle surface requires the control runner to ingest the issue from +the local provider, create the branch and pull request, check CI, check review +feedback, and complete the task. The AgentRail lane records the equivalent +lifecycle through the harness, so the runner prompt only carries the coding +work and AgentRail handoff contract. + +## Phase Attribution Benchmark + +Use phase attribution when measuring token savings from lifecycle orchestration. +This mode avoids comparing two independent autonomous agent trajectories. It +runs a deterministic lifecycle script with separate measured model calls for +each phase, then computes two accounting views: + +- Control: provider lifecycle phases plus coding phases. +- AgentRail: coding phases plus AgentRail handoff. + +```bash +npm run benchmark:agentrail-swe:phase-attribution -- \ + --runner codex \ + --count 1 \ + --repo-cache .agentrail-bench/repo-cache \ + --require-phase-token-usage +``` + +The default scenario is `ci-and-review-feedback`, which measures: + +- issue intake +- initial coding +- PR creation +- CI polling +- CI failure triage +- CI fix +- review polling +- review feedback triage +- review fix +- PR update +- final CI check +- final review check +- AgentRail handoff +- provider completion + +The result file includes per-phase token usage, command logs, Control +accounting, AgentRail accounting, and token deltas. + +Use `--phase-repeat ` to run the full phase sequence multiple times and sum +all measured phase calls into the same accounting views. + +Cursor Agent can be used for non-token smoke runs, but its CLI JSON output does +not currently expose token usage, so `--require-token-usage` and +`--require-phase-token-usage` are rejected for `--runner cursor`. + +Add `--model ` only if your local Codex CLI accepts that +model. The benchmark omits `--model` by default so Codex can use your configured +default. + +For local smoke tests, use `--runner custom --runner-command ` with a +fixture repository supplied through `--source-repo-path`. Real SWE-bench Pro +runs omit `--source-repo-path`; the benchmark clones the task repository into +`--repo-cache` and checks out the upstream base commit. + +The paired result records: + +- control lane status, patch, evaluator result, and token usage +- AgentRail lane status, patch, evaluator result, and token usage +- lifecycle event summaries for each lane when `--benchmark-surface full-lifecycle` is used +- `tokenDelta`, where positive means AgentRail used more tokens +- `tokenReductionRate`, where positive means AgentRail used fewer tokens +- cached, non-cached input, output, and reasoning token deltas when the runner exposes them diff --git a/benchmarks/agentrail-swe-lifecycle/dataset.json b/benchmarks/agentrail-swe-lifecycle/dataset.json new file mode 100644 index 0000000..bcb9cfc --- /dev/null +++ b/benchmarks/agentrail-swe-lifecycle/dataset.json @@ -0,0 +1,8304 @@ +{ + "version": 1, + "id": "agentrail-swe-lifecycle", + "name": "AgentRail SWE Lifecycle Token Efficiency Dataset", + "description": "AgentRail lifecycle wrapper around Artificial Analysis SWE-Bench-Pro-Hard-AA task selection and ScaleAI/SWE-bench_Pro upstream rows.", + "defaultScenario": "local-mock-lifecycle", + "upstream": { + "taskSelection": { + "name": "Artificial Analysis Coding Agent Index", + "benchmark": "SWE-Bench-Pro-Hard-AA", + "sourceUrl": "https://artificialanalysis.ai/methodology/coding-agents-benchmarking", + "evaluatedTaskCount": 150 + }, + "taskData": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + } + }, + "lifecycle": { + "issueProviders": [ + "github", + "linear" + ], + "defaultIssueProvider": "github", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record", + "finalOracle": "swebench_fail_to_pass" + }, + "metrics": { + "primary": "tokens_per_successful_task", + "guardrails": [ + "success_rate", + "cost_per_successful_task", + "wall_clock_time_per_successful_task", + "ci_retry_count", + "review_retry_count" + ] + }, + "tasks": [ + { + "id": "agentrail-swe-pro-hard-aa-001", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_ansible__ansible-1bd7dcf339dd8b6c50bc16670be2448a206f4fdb-vba6da65a0f3baefda7a058ebbd0a8dcafb8512f5", + "selectedInstanceId": "instance_ansible__ansible-1bd7dcf339dd8b6c50bc16670be2448a206f4fdb-vba6da65a0f3baefda7a058ebbd0a8dcafb8512f5", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "ansible/ansible", + "baseCommit": "20ef733ee02ba688757998404c1926381356b031", + "language": "python", + "tags": [ + "python", + "security_feat", + "performance_feat", + "security_knowledge", + "back_end_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "ansible.ansible-ansible__ansible-1bd7dcf339dd8b6c50bc16670be2448a206f4fdb-vba6da65a0f3baefda7a058ebbd0a8dcafb8512f5", + "beforeRepoSetCommand": "git reset --hard 20ef733ee02ba688757998404c1926381356b031\ngit clean -fd \ngit checkout 20ef733ee02ba688757998404c1926381356b031 \ngit checkout 1bd7dcf339dd8b6c50bc16670be2448a206f4fdb -- test/units/plugins/lookup/test_password.py test/units/utils/test_encrypt.py", + "selectedTestFiles": [ + "test/units/utils/test_encrypt.py", + "test/units/plugins/lookup/test_password.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-002", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_ansible__ansible-1c06c46cc14324df35ac4f39a45fb3ccd602195d-v0f01c69f1e2528b935359cfe578530722bca2c59", + "selectedInstanceId": "instance_ansible__ansible-1c06c46cc14324df35ac4f39a45fb3ccd602195d-v0f01c69f1e2528b935359cfe578530722bca2c59", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "ansible/ansible", + "baseCommit": "6198c7377f545207218fe8eb2e3cfa9673ff8f5e", + "language": "python", + "tags": [ + "python", + "major_bug", + "back_end_knowledge", + "security_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "ansible.ansible-ansible__ansible-1c06c46cc14324df35ac4f39a45fb3ccd602195d-v0f01c69f1e2528b935359cfe578530722bca2c59", + "beforeRepoSetCommand": "git reset --hard 6198c7377f545207218fe8eb2e3cfa9673ff8f5e\ngit clean -fd \ngit checkout 6198c7377f545207218fe8eb2e3cfa9673ff8f5e \ngit checkout 1c06c46cc14324df35ac4f39a45fb3ccd602195d -- test/integration/targets/templating/tasks/main.yml test/units/_internal/_yaml/__init__.py test/units/_internal/_yaml/test_dumper.py test/units/plugins/filter/test_core.py", + "selectedTestFiles": [ + "test/units/plugins/filter/test_core.py", + "test/units/_internal/_yaml/__init__.py", + "test/units/_internal/_yaml/test_dumper.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-003", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_ansible__ansible-34db57a47f875d11c4068567b9ec7ace174ec4cf-v1055803c3a812189a1133297f7f5468579283f86", + "selectedInstanceId": "instance_ansible__ansible-34db57a47f875d11c4068567b9ec7ace174ec4cf-v1055803c3a812189a1133297f7f5468579283f86", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "ansible/ansible", + "baseCommit": "d63a71e3f83fc23defb97393367859634881b8da", + "language": "python", + "tags": [ + "python", + "performance_bug", + "core_feat", + "performance_feat", + "customization_feat", + "infrastructure_knowledge", + "back_end_knowledge", + "devops_knowledge", + "performance_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "ansible.ansible-ansible__ansible-34db57a47f875d11c4068567b9ec7ace174ec4cf-v1055803c3a812189a1133297f7f5468579283f86", + "beforeRepoSetCommand": "git reset --hard d63a71e3f83fc23defb97393367859634881b8da\ngit clean -fd \ngit checkout d63a71e3f83fc23defb97393367859634881b8da \ngit checkout 34db57a47f875d11c4068567b9ec7ace174ec4cf -- test/units/module_utils/facts/hardware/linux_data.py test/units/module_utils/facts/hardware/test_linux_get_cpu_info.py", + "selectedTestFiles": [ + "test/units/module_utils/facts/hardware/test_linux_get_cpu_info.py", + "test/units/module_utils/facts/hardware/linux_data.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-004", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_ansible__ansible-40ade1f84b8bb10a63576b0ac320c13f57c87d34-v6382ea168a93d80a64aab1fbd8c4f02dc5ada5bf", + "selectedInstanceId": "instance_ansible__ansible-40ade1f84b8bb10a63576b0ac320c13f57c87d34-v6382ea168a93d80a64aab1fbd8c4f02dc5ada5bf", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "ansible/ansible", + "baseCommit": "9ab63986ad528a6ad5bf4c59fe104d5106d6ef9b", + "language": "python", + "tags": [ + "python", + "major_bug", + "data_bug", + "edge_case_bug", + "compatibility_bug", + "back_end_knowledge", + "infrastructure_knowledge", + "devops_knowledge", + "api_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "ansible.ansible-ansible__ansible-40ade1f84b8bb10a63576b0ac320c13f57c87d34-v6382ea168a93d80a64aab1fbd8c4f02dc5ada5bf", + "beforeRepoSetCommand": "git reset --hard 9ab63986ad528a6ad5bf4c59fe104d5106d6ef9b\ngit clean -fd \ngit checkout 9ab63986ad528a6ad5bf4c59fe104d5106d6ef9b \ngit checkout 40ade1f84b8bb10a63576b0ac320c13f57c87d34 -- test/integration/targets/mount_facts/aliases test/integration/targets/mount_facts/meta/main.yml test/integration/targets/mount_facts/tasks/main.yml test/units/modules/mount_facts_data.py test/units/modules/test_mount_facts.py", + "selectedTestFiles": [ + "test/units/modules/test_mount_facts.py::test_list_mounts[gen_fstab_entries-\\nUUID=6b8b920d-f334-426e-a440-1207d0d8725b", + "test/units/modules/test_mount_facts.py::test_handle_deduplication_warning[True-False]", + "test/units/modules/test_mount_facts.py::test_parse_mount_bin_stdout_unknown", + "test/units/modules/test_mount_facts.py::test_handle_deduplication", + "test/units/modules/test_mount_facts.py::test_gen_mounts_by_source_timeout[ignore-False-False]", + "test/units/modules/test_mount_facts.py::test_get_mount_size_timeout[error-False-True]", + "test/units/modules/test_mount_facts.py::test_parse_mount_bin_stdout[node", + "test/units/modules/test_mount_facts.py::test_etc_filesystems_linux", + "test/units/modules/test_mount_facts.py::test_gen_mounts_by_sources[sources8-file_data8-/dev/gpt/rootfs", + "test/units/modules/test_mount_facts.py::test_gen_mounts_by_source_timeout[warn-True-False]", + "test/units/modules/test_mount_facts.py::test_handle_deduplication_warning[None-True]", + "test/units/modules/test_mount_facts.py::test_gen_mounts_by_sources[sources3-file_data3-proc", + "test/units/modules/test_mount_facts.py::test_parse_mount_bin_stdout[proc", + "test/units/modules/test_mount_facts.py::test_list_mounts[gen_fstab_entries-#", + "test/units/modules/test_mount_facts.py::test_invocation", + "test/units/modules/test_mount_facts.py::test_invalid_mount_binary[invalid_binary2-argument", + "test/units/modules/test_mount_facts.py::test_gen_mounts_by_sources[sources7-file_data7-/dev/gpt/rootfs", + "test/units/modules/test_mount_facts.py::test_gen_mounts_by_sources[sources9-file_data9-/dev/gpt/rootfs", + "test/units/modules/test_mount_facts.py::test_invalid_timeout", + "test/units/modules/test_mount_facts.py::test_gen_mounts_by_sources[sources1-file_data1-proc", + "test/units/modules/test_mount_facts.py::test_handle_deduplication_warning[False-False]", + "test/units/modules/test_mount_facts.py::test_list_mounts[gen_fstab_entries-proc", + "test/units/modules/test_mount_facts.py::test_invalid_mount_binary[false-Failed", + "test/units/modules/test_mount_facts.py::test_get_mount_size_timeout[warn-True-False]", + "test/units/modules/test_mount_facts.py::test_get_mount_size_timeout[None-False-True]", + "test/units/modules/test_mount_facts.py::test_invalid_mount_binary[ansible_test_missing_binary-Failed", + "test/units/modules/test_mount_facts.py::test_gen_mounts_by_sources[sources2-file_data2-proc", + "test/units/modules/test_mount_facts.py::test_get_mount_size_timeout[ignore-False-False]", + "test/units/modules/test_mount_facts.py::test_list_mounts[gen_vfstab_entries-#device", + "test/units/modules/test_mount_facts.py::test_get_mount_facts", + "test/units/modules/test_mount_facts.py::test_list_mounts[gen_aix_filesystems_entries-*\\n*", + "test/units/modules/test_mount_facts.py::test_gen_mounts_by_source_timeout[None-False-True]", + "test/units/modules/test_mount_facts.py::test_invalid_source", + "test/units/modules/test_mount_facts.py::test_gen_mounts_by_sources[sources10-file_data10-node", + "test/units/modules/test_mount_facts.py::test_list_mounts[gen_mnttab_entries-rpool/ROOT/solaris", + "test/units/modules/test_mount_facts.py::test_gen_mounts_by_sources[sources12-file_data12--results12]", + "test/units/modules/test_mount_facts.py::test_get_mounts_facts_filtering[devices-proc-proc", + "test/units/modules/test_mount_facts.py::test_gen_mounts_by_sources[sources5-file_data5-proc", + "test/units/modules/test_mount_facts.py::test_parse_mount_bin_stdout[/dev/gpt/rootfs", + "test/units/modules/test_mount_facts.py::test_get_mounts_size", + "test/units/modules/test_mount_facts.py::test_get_mounts_facts_filtering[fstypes-proc-proc", + "test/units/modules/test_mount_facts.py::test_gen_mounts_by_sources[sources6-file_data6-/dev/gpt/rootfs", + "test/units/modules/test_mount_facts.py::test_gen_mounts_by_sources[sources4-file_data4-proc", + "test/units/modules/test_mount_facts.py::test_gen_mounts_by_sources[sources0-file_data0-proc", + "test/units/modules/test_mount_facts.py::test_gen_mounts_by_sources[sources11-file_data11-/dev/sd0a", + "test/units/modules/test_mount_facts.py::test_duplicate_source", + "test/units/modules/test_mount_facts.py::test_parse_mount_bin_stdout[/dev/sd0a", + "test/units/modules/test_mount_facts.py::test_get_partition_uuid" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-005", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_ansible__ansible-4c5ce5a1a9e79a845aff4978cfeb72a0d4ecf7d6-v1055803c3a812189a1133297f7f5468579283f86", + "selectedInstanceId": "instance_ansible__ansible-4c5ce5a1a9e79a845aff4978cfeb72a0d4ecf7d6-v1055803c3a812189a1133297f7f5468579283f86", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "ansible/ansible", + "baseCommit": "8a175f59c939ca29ad56f3fa9edbc37a8656879a", + "language": "python", + "tags": [ + "python", + "code_quality_enh", + "dev_ops_enh", + "back_end_knowledge", + "infrastructure_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "ansible.ansible-ansible__ansible-4c5ce5a1a9e79a845aff4978cfeb72a0d4ecf7d6-v1055803c3a812189a1133297f7f5468579283f86", + "beforeRepoSetCommand": "git reset --hard 8a175f59c939ca29ad56f3fa9edbc37a8656879a\ngit clean -fd \ngit checkout 8a175f59c939ca29ad56f3fa9edbc37a8656879a \ngit checkout 4c5ce5a1a9e79a845aff4978cfeb72a0d4ecf7d6 -- .azure-pipelines/templates/test.yml test/integration/targets/apt/tasks/apt.yml test/integration/targets/dnf/tasks/dnf.yml test/integration/targets/dnf/tasks/gpg.yml test/integration/targets/module_utils_respawn/aliases test/integration/targets/module_utils_respawn/library/respawnme.py test/integration/targets/module_utils_respawn/tasks/main.yml test/integration/targets/module_utils_selinux/aliases test/integration/targets/module_utils_selinux/tasks/main.yml test/integration/targets/module_utils_selinux/tasks/selinux.yml test/integration/targets/setup_rpm_repo/library/create_repo.py test/integration/targets/setup_rpm_repo/tasks/main.yml test/integration/targets/setup_rpm_repo/vars/RedHat-8.yml test/integration/targets/yum/tasks/yum.yml test/support/integration/plugins/modules/sefcontext.py test/units/executor/module_common/test_recursive_finder.py test/units/module_utils/basic/test_imports.py test/units/module_utils/basic/test_selinux.py test/utils/shippable/incidental/remote.sh test/utils/shippable/remote.sh", + "selectedTestFiles": [ + "test/integration/targets/module_utils_respawn/library/respawnme.py", + "test/units/module_utils/basic/test_selinux.py", + "test/integration/targets/setup_rpm_repo/library/create_repo.py", + "test/support/integration/plugins/modules/sefcontext.py", + "test/units/module_utils/basic/test_imports.py", + "test/units/executor/module_common/test_recursive_finder.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-006", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_ansible__ansible-5640093f1ca63fd6af231cc8a7fb7d40e1907b8c-vba6da65a0f3baefda7a058ebbd0a8dcafb8512f5", + "selectedInstanceId": "instance_ansible__ansible-5640093f1ca63fd6af231cc8a7fb7d40e1907b8c-vba6da65a0f3baefda7a058ebbd0a8dcafb8512f5", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "ansible/ansible", + "baseCommit": "a5a13246ce02fd4c4ce997627b4f42152e59d288", + "language": "python", + "tags": [ + "python", + "regression_bug", + "compatibility_bug", + "back_end_knowledge", + "infrastructure_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "ansible.ansible-ansible__ansible-5640093f1ca63fd6af231cc8a7fb7d40e1907b8c-vba6da65a0f3baefda7a058ebbd0a8dcafb8512f5", + "beforeRepoSetCommand": "git reset --hard a5a13246ce02fd4c4ce997627b4f42152e59d288\ngit clean -fd \ngit checkout a5a13246ce02fd4c4ce997627b4f42152e59d288 \ngit checkout 5640093f1ca63fd6af231cc8a7fb7d40e1907b8c -- test/integration/targets/gathering_facts/runme.sh test/integration/targets/gathering_facts/test_module_defaults.yml test/integration/targets/package/tasks/main.yml test/integration/targets/service/tasks/tests.yml test/units/plugins/action/test_gather_facts.py", + "selectedTestFiles": [ + "test/units/plugins/action/test_gather_facts.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-007", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_ansible__ansible-5e88cd9972f10b66dd97e1ee684c910c6a2dd25e-v906c969b551b346ef54a2c0b41e04f632b7b73c2", + "selectedInstanceId": "instance_ansible__ansible-5e88cd9972f10b66dd97e1ee684c910c6a2dd25e-v906c969b551b346ef54a2c0b41e04f632b7b73c2", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "ansible/ansible", + "baseCommit": "5fa2d29c9a06baa8cc5d271c41e23b1f99b3814a", + "language": "python", + "tags": [ + "python", + "core_feat", + "api_feat", + "integration_feat", + "security_feat", + "back_end_knowledge", + "infrastructure_knowledge", + "api_knowledge", + "security_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "ansible.ansible-ansible__ansible-5e88cd9972f10b66dd97e1ee684c910c6a2dd25e-v906c969b551b346ef54a2c0b41e04f632b7b73c2", + "beforeRepoSetCommand": "git reset --hard 5fa2d29c9a06baa8cc5d271c41e23b1f99b3814a\ngit clean -fd \ngit checkout 5fa2d29c9a06baa8cc5d271c41e23b1f99b3814a \ngit checkout 5e88cd9972f10b66dd97e1ee684c910c6a2dd25e -- test/units/modules/network/netvisor/test_pn_user.py", + "selectedTestFiles": [ + "test/units/modules/network/netvisor/test_pn_user.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-008", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_ansible__ansible-709484969c8a4ffd74b839a673431a8c5caa6457-vba6da65a0f3baefda7a058ebbd0a8dcafb8512f5", + "selectedInstanceId": "instance_ansible__ansible-709484969c8a4ffd74b839a673431a8c5caa6457-vba6da65a0f3baefda7a058ebbd0a8dcafb8512f5", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "ansible/ansible", + "baseCommit": "35809806d3ab5d66fbb9696dc6a0009383e50673", + "language": "python", + "tags": [ + "python", + "compatibility_bug", + "code_quality_enh", + "major_bug", + "back_end_knowledge", + "desktop_knowledge", + "infrastructure_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "ansible.ansible-ansible__ansible-709484969c8a4ffd74b839a673431a8c5caa6457-vba6da65a0f3baefda7a058ebbd0a8dcafb8512f5", + "beforeRepoSetCommand": "git reset --hard 35809806d3ab5d66fbb9696dc6a0009383e50673\ngit clean -fd \ngit checkout 35809806d3ab5d66fbb9696dc6a0009383e50673 \ngit checkout 709484969c8a4ffd74b839a673431a8c5caa6457 -- test/units/module_utils/facts/test_sysctl.py", + "selectedTestFiles": [ + "test/units/module_utils/facts/test_sysctl.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-009", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_ansible__ansible-7e1a347695c7987ae56ef1b6919156d9254010ad-v390e508d27db7a51eece36bb6d9698b63a5b638a", + "selectedInstanceId": "instance_ansible__ansible-7e1a347695c7987ae56ef1b6919156d9254010ad-v390e508d27db7a51eece36bb6d9698b63a5b638a", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "ansible/ansible", + "baseCommit": "20ec92728004bc94729ffc08cbc483b7496c0c1f", + "language": "python", + "tags": [ + "python", + "core_feat", + "integration_feat", + "infrastructure_knowledge", + "networking_knowledge", + "devops_knowledge", + "back_end_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "ansible.ansible-ansible__ansible-7e1a347695c7987ae56ef1b6919156d9254010ad-v390e508d27db7a51eece36bb6d9698b63a5b638a", + "beforeRepoSetCommand": "git reset --hard 20ec92728004bc94729ffc08cbc483b7496c0c1f\ngit clean -fd \ngit checkout 20ec92728004bc94729ffc08cbc483b7496c0c1f \ngit checkout 7e1a347695c7987ae56ef1b6919156d9254010ad -- test/units/modules/network/icx/fixtures/lag_running_config.txt test/units/modules/network/icx/test_icx_linkagg.py", + "selectedTestFiles": [ + "test/units/modules/network/icx/test_icx_linkagg.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-010", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_ansible__ansible-935528e22e5283ee3f63a8772830d3d01f55ed8c-vba6da65a0f3baefda7a058ebbd0a8dcafb8512f5", + "selectedInstanceId": "instance_ansible__ansible-935528e22e5283ee3f63a8772830d3d01f55ed8c-vba6da65a0f3baefda7a058ebbd0a8dcafb8512f5", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "ansible/ansible", + "baseCommit": "43300e22798e4c9bd8ec2e321d28c5e8d2018aeb", + "language": "python", + "tags": [ + "python", + "edge_case_bug", + "integration_bug", + "back_end_knowledge", + "devops_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "ansible.ansible-ansible__ansible-935528e22e5283ee3f63a8772830d3d01f55ed8c-vba6da65a0f3baefda7a058ebbd0a8dcafb8512f5", + "beforeRepoSetCommand": "git reset --hard 43300e22798e4c9bd8ec2e321d28c5e8d2018aeb\ngit clean -fd \ngit checkout 43300e22798e4c9bd8ec2e321d28c5e8d2018aeb \ngit checkout 935528e22e5283ee3f63a8772830d3d01f55ed8c -- test/integration/targets/connection_windows_ssh/runme.sh test/units/plugins/connection/test_ssh.py", + "selectedTestFiles": [ + "test/units/plugins/connection/test_ssh.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-011", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_ansible__ansible-942424e10b2095a173dbd78e7128f52f7995849b-v30a923fb5c164d6cd18280c02422f75e611e8fb2", + "selectedInstanceId": "instance_ansible__ansible-942424e10b2095a173dbd78e7128f52f7995849b-v30a923fb5c164d6cd18280c02422f75e611e8fb2", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "ansible/ansible", + "baseCommit": "dd44449b6ec454a46426a020608fe3763287101f", + "language": "python", + "tags": [ + "python", + "major_bug", + "integration_bug", + "edge_case_bug", + "back_end_knowledge", + "infrastructure_knowledge", + "devops_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "ansible.ansible-ansible__ansible-942424e10b2095a173dbd78e7128f52f7995849b-v30a923fb5c164d6cd18280c02422f75e611e8fb2", + "beforeRepoSetCommand": "git reset --hard dd44449b6ec454a46426a020608fe3763287101f\ngit clean -fd \ngit checkout dd44449b6ec454a46426a020608fe3763287101f \ngit checkout 942424e10b2095a173dbd78e7128f52f7995849b -- test/units/plugins/connection/test_winrm.py", + "selectedTestFiles": [ + "test/units/plugins/connection/test_winrm.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-012", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_ansible__ansible-949c503f2ef4b2c5d668af0492a5c0db1ab86140-v0f01c69f1e2528b935359cfe578530722bca2c59", + "selectedInstanceId": "instance_ansible__ansible-949c503f2ef4b2c5d668af0492a5c0db1ab86140-v0f01c69f1e2528b935359cfe578530722bca2c59", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "ansible/ansible", + "baseCommit": "375d3889de9f437bc120ade623c170198629375d", + "language": "python", + "tags": [ + "python", + "customization_feat", + "back_end_knowledge", + "infrastructure_knowledge", + "networking_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "ansible.ansible-ansible__ansible-949c503f2ef4b2c5d668af0492a5c0db1ab86140-v0f01c69f1e2528b935359cfe578530722bca2c59", + "beforeRepoSetCommand": "git reset --hard 375d3889de9f437bc120ade623c170198629375d\ngit clean -fd \ngit checkout 375d3889de9f437bc120ade623c170198629375d \ngit checkout 949c503f2ef4b2c5d668af0492a5c0db1ab86140 -- test/integration/targets/ansible-config/files/galaxy_server.ini test/integration/targets/ansible-config/tasks/main.yml test/units/galaxy/test_collection.py test/units/galaxy/test_token.py", + "selectedTestFiles": [ + "test/units/galaxy/test_collection.py", + "test/units/galaxy/test_token.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-013", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_ansible__ansible-9a21e247786ebd294dafafca1105fcd770ff46c6-v67cdaa49f89b34e42b69d5b7830b3c3ad3d8803f", + "selectedInstanceId": "instance_ansible__ansible-9a21e247786ebd294dafafca1105fcd770ff46c6-v67cdaa49f89b34e42b69d5b7830b3c3ad3d8803f", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "ansible/ansible", + "baseCommit": "4c8c40fd3d4a58defdc80e7d22aa8d26b731353e", + "language": "python", + "tags": [ + "python", + "compatibility_bug", + "edge_case_bug", + "minor_bug", + "integration_bug", + "back_end_knowledge", + "devops_knowledge", + "infrastructure_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "ansible.ansible-ansible__ansible-9a21e247786ebd294dafafca1105fcd770ff46c6-v67cdaa49f89b34e42b69d5b7830b3c3ad3d8803f", + "beforeRepoSetCommand": "git reset --hard 4c8c40fd3d4a58defdc80e7d22aa8d26b731353e\ngit clean -fd \ngit checkout 4c8c40fd3d4a58defdc80e7d22aa8d26b731353e \ngit checkout 9a21e247786ebd294dafafca1105fcd770ff46c6 -- test/units/module_utils/basic/test_platform_distribution.py test/units/module_utils/common/test_sys_info.py test/units/modules/test_service.py", + "selectedTestFiles": [ + "test/units/module_utils/common/test_sys_info.py::test_get_distribution_not_linux[SunOS-Solaris]", + "test/units/module_utils/common/test_sys_info.py::test_get_distribution_not_linux[Darwin-Darwin]", + "test/units/module_utils/common/test_sys_info.py::test_get_distribution_version_not_linux[SunOS-11.4]", + "test/units/module_utils/common/test_sys_info.py::test_get_distribution_version_not_linux[Darwin-19.6.0]", + "test/units/module_utils/common/test_sys_info.py::test_get_distribution_not_linux[FreeBSD-Freebsd]", + "test/units/module_utils/common/test_sys_info.py::test_get_distribution_version_not_linux[FreeBSD-12.1]" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-014", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_ansible__ansible-a1569ea4ca6af5480cf0b7b3135f5e12add28a44-v0f01c69f1e2528b935359cfe578530722bca2c59", + "selectedInstanceId": "instance_ansible__ansible-a1569ea4ca6af5480cf0b7b3135f5e12add28a44-v0f01c69f1e2528b935359cfe578530722bca2c59", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "ansible/ansible", + "baseCommit": "f10d11bcdc54c9b7edc0111eb38c59a88e396d0a", + "language": "python", + "tags": [ + "python", + "major_bug", + "integration_bug", + "performance_bug", + "back_end_knowledge", + "networking_knowledge", + "api_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "ansible.ansible-ansible__ansible-a1569ea4ca6af5480cf0b7b3135f5e12add28a44-v0f01c69f1e2528b935359cfe578530722bca2c59", + "beforeRepoSetCommand": "git reset --hard f10d11bcdc54c9b7edc0111eb38c59a88e396d0a\ngit clean -fd \ngit checkout f10d11bcdc54c9b7edc0111eb38c59a88e396d0a \ngit checkout a1569ea4ca6af5480cf0b7b3135f5e12add28a44 -- test/integration/targets/iptables/tasks/chain_management.yml test/units/modules/test_iptables.py", + "selectedTestFiles": [ + "test/units/modules/test_iptables.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-015", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_ansible__ansible-b6290e1d156af608bd79118d209a64a051c55001-v390e508d27db7a51eece36bb6d9698b63a5b638a", + "selectedInstanceId": "instance_ansible__ansible-b6290e1d156af608bd79118d209a64a051c55001-v390e508d27db7a51eece36bb6d9698b63a5b638a", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "ansible/ansible", + "baseCommit": "b6e71c5ffb8ba382b6f49fc9b25e6d8bc78a9a88", + "language": "python", + "tags": [ + "python", + "core_feat", + "integration_feat", + "back_end_knowledge", + "networking_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "ansible.ansible-ansible__ansible-b6290e1d156af608bd79118d209a64a051c55001-v390e508d27db7a51eece36bb6d9698b63a5b638a", + "beforeRepoSetCommand": "git reset --hard b6e71c5ffb8ba382b6f49fc9b25e6d8bc78a9a88\ngit clean -fd \ngit checkout b6e71c5ffb8ba382b6f49fc9b25e6d8bc78a9a88 \ngit checkout b6290e1d156af608bd79118d209a64a051c55001 -- test/units/modules/network/icx/fixtures/icx_logging_config.cfg test/units/modules/network/icx/test_icx_logging.py", + "selectedTestFiles": [ + "test/units/modules/network/icx/test_icx_logging.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-016", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_ansible__ansible-b8025ac160146319d2b875be3366b60c852dd35d-v0f01c69f1e2528b935359cfe578530722bca2c59", + "selectedInstanceId": "instance_ansible__ansible-b8025ac160146319d2b875be3366b60c852dd35d-v0f01c69f1e2528b935359cfe578530722bca2c59", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "ansible/ansible", + "baseCommit": "fa093d8adf03c88908caa38fe70e0db2711e801c", + "language": "python", + "tags": [ + "python", + "major_bug", + "integration_bug", + "compatibility_bug", + "back_end_knowledge", + "devops_knowledge", + "networking_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "ansible.ansible-ansible__ansible-b8025ac160146319d2b875be3366b60c852dd35d-v0f01c69f1e2528b935359cfe578530722bca2c59", + "beforeRepoSetCommand": "git reset --hard fa093d8adf03c88908caa38fe70e0db2711e801c\ngit clean -fd \ngit checkout fa093d8adf03c88908caa38fe70e0db2711e801c \ngit checkout b8025ac160146319d2b875be3366b60c852dd35d -- test/integration/targets/get_url/tasks/ciphers.yml test/integration/targets/get_url/tasks/main.yml test/integration/targets/lookup_url/tasks/main.yml test/integration/targets/uri/tasks/ciphers.yml test/integration/targets/uri/tasks/main.yml test/units/module_utils/urls/test_Request.py test/units/module_utils/urls/test_fetch_url.py", + "selectedTestFiles": [ + "test/units/module_utils/urls/test_fetch_url.py", + "test/units/module_utils/urls/test_Request.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-017", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_ansible__ansible-bf98f031f3f5af31a2d78dc2f0a58fe92ebae0bb-v1055803c3a812189a1133297f7f5468579283f86", + "selectedInstanceId": "instance_ansible__ansible-bf98f031f3f5af31a2d78dc2f0a58fe92ebae0bb-v1055803c3a812189a1133297f7f5468579283f86", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "ansible/ansible", + "baseCommit": "17332532973343248297e3d3c746738d1f3b2f56", + "language": "python", + "tags": [ + "python", + "major_bug", + "regression_bug", + "data_bug", + "back_end_knowledge", + "devops_knowledge", + "security_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "ansible.ansible-ansible__ansible-bf98f031f3f5af31a2d78dc2f0a58fe92ebae0bb-v1055803c3a812189a1133297f7f5468579283f86", + "beforeRepoSetCommand": "git reset --hard 17332532973343248297e3d3c746738d1f3b2f56\ngit clean -fd \ngit checkout 17332532973343248297e3d3c746738d1f3b2f56 \ngit checkout bf98f031f3f5af31a2d78dc2f0a58fe92ebae0bb -- test/units/module_utils/basic/test_no_log.py test/units/module_utils/basic/test_sanitize_keys.py", + "selectedTestFiles": [ + "test/units/module_utils/basic/test_sanitize_keys.py", + "test/units/module_utils/basic/test_no_log.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-018", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_ansible__ansible-cb94c0cc550df9e98f1247bc71d8c2b861c75049-v1055803c3a812189a1133297f7f5468579283f86", + "selectedInstanceId": "instance_ansible__ansible-cb94c0cc550df9e98f1247bc71d8c2b861c75049-v1055803c3a812189a1133297f7f5468579283f86", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "ansible/ansible", + "baseCommit": "96c19724394a32b9d3c596966be2f46e478681f8", + "language": "python", + "tags": [ + "python", + "performance_feat", + "customization_feat", + "back_end_knowledge", + "desktop_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "ansible.ansible-ansible__ansible-cb94c0cc550df9e98f1247bc71d8c2b861c75049-v1055803c3a812189a1133297f7f5468579283f86", + "beforeRepoSetCommand": "git reset --hard 96c19724394a32b9d3c596966be2f46e478681f8\ngit clean -fd \ngit checkout 96c19724394a32b9d3c596966be2f46e478681f8 \ngit checkout cb94c0cc550df9e98f1247bc71d8c2b861c75049 -- test/integration/targets/adhoc/aliases test/integration/targets/adhoc/runme.sh test/units/cli/test_adhoc.py", + "selectedTestFiles": [ + "test/units/cli/test_adhoc.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-019", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_ansible__ansible-cd473dfb2fdbc97acf3293c134b21cbbcfa89ec3-vba6da65a0f3baefda7a058ebbd0a8dcafb8512f5", + "selectedInstanceId": "instance_ansible__ansible-cd473dfb2fdbc97acf3293c134b21cbbcfa89ec3-vba6da65a0f3baefda7a058ebbd0a8dcafb8512f5", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "ansible/ansible", + "baseCommit": "e8ae7211dabbd07093f50d6dfb383da3bb14f13d", + "language": "python", + "tags": [ + "python", + "major_bug", + "edge_case_bug", + "data_bug", + "back_end_knowledge", + "devops_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "ansible.ansible-ansible__ansible-cd473dfb2fdbc97acf3293c134b21cbbcfa89ec3-vba6da65a0f3baefda7a058ebbd0a8dcafb8512f5", + "beforeRepoSetCommand": "git reset --hard e8ae7211dabbd07093f50d6dfb383da3bb14f13d\ngit clean -fd \ngit checkout e8ae7211dabbd07093f50d6dfb383da3bb14f13d \ngit checkout cd473dfb2fdbc97acf3293c134b21cbbcfa89ec3 -- test/integration/targets/playbook/runme.sh test/units/playbook/test_play.py", + "selectedTestFiles": [ + "test/units/playbook/test_play.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-020", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_ansible__ansible-cd9c4eb5a6b2bfaf4a6709f001ce3d0c92c1eed2-v0f01c69f1e2528b935359cfe578530722bca2c59", + "selectedInstanceId": "instance_ansible__ansible-cd9c4eb5a6b2bfaf4a6709f001ce3d0c92c1eed2-v0f01c69f1e2528b935359cfe578530722bca2c59", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "ansible/ansible", + "baseCommit": "585ef6c55e87c10c1ce7d59ebe9c33dd6dbe5afb", + "language": "python", + "tags": [ + "python", + "major_bug", + "compatibility_bug", + "back_end_knowledge", + "infrastructure_knowledge", + "devops_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "ansible.ansible-ansible__ansible-cd9c4eb5a6b2bfaf4a6709f001ce3d0c92c1eed2-v0f01c69f1e2528b935359cfe578530722bca2c59", + "beforeRepoSetCommand": "git reset --hard 585ef6c55e87c10c1ce7d59ebe9c33dd6dbe5afb\ngit clean -fd \ngit checkout 585ef6c55e87c10c1ce7d59ebe9c33dd6dbe5afb \ngit checkout cd9c4eb5a6b2bfaf4a6709f001ce3d0c92c1eed2 -- test/units/module_utils/facts/hardware/linux/fixtures/sysinfo test/units/module_utils/facts/hardware/linux/test_get_sysinfo_facts.py", + "selectedTestFiles": [ + "test/units/module_utils/facts/hardware/linux/test_get_sysinfo_facts.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-021", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_ansible__ansible-d30fc6c0b359f631130b0e979d9a78a7b3747d48-v1055803c3a812189a1133297f7f5468579283f86", + "selectedInstanceId": "instance_ansible__ansible-d30fc6c0b359f631130b0e979d9a78a7b3747d48-v1055803c3a812189a1133297f7f5468579283f86", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "ansible/ansible", + "baseCommit": "a58fcde3a0d7a93c363ae7af4e6ee03001b96d82", + "language": "python", + "tags": [ + "python", + "edge_case_bug", + "code_quality_enh", + "dev_ops_enh", + "back_end_knowledge", + "devops_knowledge", + "infrastructure_knowledge", + "security_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "ansible.ansible-ansible__ansible-d30fc6c0b359f631130b0e979d9a78a7b3747d48-v1055803c3a812189a1133297f7f5468579283f86", + "beforeRepoSetCommand": "git reset --hard a58fcde3a0d7a93c363ae7af4e6ee03001b96d82\ngit clean -fd \ngit checkout a58fcde3a0d7a93c363ae7af4e6ee03001b96d82 \ngit checkout d30fc6c0b359f631130b0e979d9a78a7b3747d48 -- test/integration/targets/ansible-galaxy-collection/library/setup_collections.py test/integration/targets/ansible-galaxy-collection/tasks/install.yml test/integration/targets/ansible-galaxy-collection/tasks/main.yml test/units/galaxy/test_collection.py", + "selectedTestFiles": [ + "test/units/galaxy/test_collection.py", + "test/integration/targets/ansible-galaxy-collection/library/setup_collections.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-022", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_ansible__ansible-d58e69c82d7edd0583dd8e78d76b075c33c3151e-v173091e2e36d38c978002990795f66cfc0af30ad", + "selectedInstanceId": "instance_ansible__ansible-d58e69c82d7edd0583dd8e78d76b075c33c3151e-v173091e2e36d38c978002990795f66cfc0af30ad", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "ansible/ansible", + "baseCommit": "a6e671db25381ed111bbad0ab3e7d97366395d05", + "language": "python", + "tags": [ + "python", + "regression_bug", + "major_bug", + "back_end_knowledge", + "api_knowledge", + "infrastructure_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "ansible.ansible-ansible__ansible-d58e69c82d7edd0583dd8e78d76b075c33c3151e-v173091e2e36d38c978002990795f66cfc0af30ad", + "beforeRepoSetCommand": "git reset --hard a6e671db25381ed111bbad0ab3e7d97366395d05\ngit clean -fd \ngit checkout a6e671db25381ed111bbad0ab3e7d97366395d05 \ngit checkout d58e69c82d7edd0583dd8e78d76b075c33c3151e -- test/integration/targets/get_url/tasks/main.yml test/integration/targets/uri/tasks/main.yml test/sanity/ignore.txt test/units/module_utils/urls/test_Request.py test/units/module_utils/urls/test_fetch_url.py test/units/module_utils/urls/test_gzip.py", + "selectedTestFiles": [ + "test/units/module_utils/urls/test_fetch_url.py", + "test/units/module_utils/urls/test_gzip.py", + "test/units/module_utils/urls/test_Request.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-023", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_ansible__ansible-d6d2251929c84c3aa883bad7db0f19cc9ff0339e-v30a923fb5c164d6cd18280c02422f75e611e8fb2", + "selectedInstanceId": "instance_ansible__ansible-d6d2251929c84c3aa883bad7db0f19cc9ff0339e-v30a923fb5c164d6cd18280c02422f75e611e8fb2", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "ansible/ansible", + "baseCommit": "02e00aba3fd7b646a4f6d6af72159c2b366536bf", + "language": "python", + "tags": [ + "python", + "performance_bug", + "edge_case_bug", + "back_end_knowledge", + "devops_knowledge", + "infrastructure_knowledge", + "performance_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "ansible.ansible-ansible__ansible-d6d2251929c84c3aa883bad7db0f19cc9ff0339e-v30a923fb5c164d6cd18280c02422f75e611e8fb2", + "beforeRepoSetCommand": "git reset --hard 02e00aba3fd7b646a4f6d6af72159c2b366536bf\ngit clean -fd \ngit checkout 02e00aba3fd7b646a4f6d6af72159c2b366536bf \ngit checkout d6d2251929c84c3aa883bad7db0f19cc9ff0339e -- test/integration/targets/ansible-playbook-callbacks/callbacks_list.expected test/integration/targets/handlers/handler_notify_earlier_handler.yml test/integration/targets/handlers/runme.sh test/integration/targets/old_style_vars_plugins/runme.sh test/units/executor/test_play_iterator.py test/units/plugins/strategy/test_linear.py", + "selectedTestFiles": [ + "test/units/executor/test_play_iterator.py", + "test/units/plugins/strategy/test_linear.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-024", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_ansible__ansible-de5858f48dc9e1ce9117034e0d7e76806f420ca8-v1055803c3a812189a1133297f7f5468579283f86", + "selectedInstanceId": "instance_ansible__ansible-de5858f48dc9e1ce9117034e0d7e76806f420ca8-v1055803c3a812189a1133297f7f5468579283f86", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "ansible/ansible", + "baseCommit": "a1730af91f94552e8aaff4bedfb8dcae00bd284d", + "language": "python", + "tags": [ + "python", + "core_feat", + "performance_feat", + "api_feat", + "back_end_knowledge", + "api_knowledge", + "performance_knowledge", + "devops_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "ansible.ansible-ansible__ansible-de5858f48dc9e1ce9117034e0d7e76806f420ca8-v1055803c3a812189a1133297f7f5468579283f86", + "beforeRepoSetCommand": "git reset --hard a1730af91f94552e8aaff4bedfb8dcae00bd284d\ngit clean -fd \ngit checkout a1730af91f94552e8aaff4bedfb8dcae00bd284d \ngit checkout de5858f48dc9e1ce9117034e0d7e76806f420ca8 -- test/integration/targets/ansible-galaxy-collection/library/setup_collections.py test/integration/targets/ansible-galaxy-collection/tasks/install.yml test/integration/targets/ansible-galaxy-collection/tasks/main.yml test/integration/targets/ansible-galaxy-collection/vars/main.yml test/units/galaxy/test_api.py", + "selectedTestFiles": [ + "test/integration/targets/ansible-galaxy-collection/library/setup_collections.py", + "test/units/galaxy/test_api.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-025", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_ansible__ansible-deb54e4c5b32a346f1f0b0a14f1c713d2cc2e961-vba6da65a0f3baefda7a058ebbd0a8dcafb8512f5", + "selectedInstanceId": "instance_ansible__ansible-deb54e4c5b32a346f1f0b0a14f1c713d2cc2e961-vba6da65a0f3baefda7a058ebbd0a8dcafb8512f5", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "ansible/ansible", + "baseCommit": "ac1ca40fb3ed0a0fce387ae70cb8937208a13e03", + "language": "python", + "tags": [ + "python", + "customization_feat", + "back_end_knowledge", + "devops_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "ansible.ansible-ansible__ansible-deb54e4c5b32a346f1f0b0a14f1c713d2cc2e961-vba6da65a0f3baefda7a058ebbd0a8dcafb8512f5", + "beforeRepoSetCommand": "git reset --hard ac1ca40fb3ed0a0fce387ae70cb8937208a13e03\ngit clean -fd \ngit checkout ac1ca40fb3ed0a0fce387ae70cb8937208a13e03 \ngit checkout deb54e4c5b32a346f1f0b0a14f1c713d2cc2e961 -- test/units/galaxy/test_collection.py", + "selectedTestFiles": [ + "test/units/galaxy/test_collection.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-026", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_ansible__ansible-e64c6c1ca50d7d26a8e7747d8eb87642e767cd74-v0f01c69f1e2528b935359cfe578530722bca2c59", + "selectedInstanceId": "instance_ansible__ansible-e64c6c1ca50d7d26a8e7747d8eb87642e767cd74-v0f01c69f1e2528b935359cfe578530722bca2c59", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "ansible/ansible", + "baseCommit": "a0aad17912da687a3b0b5a573ab6ed0394b569ad", + "language": "python", + "tags": [ + "python", + "major_bug", + "data_bug", + "compatibility_bug", + "edge_case_bug", + "back_end_knowledge", + "devops_knowledge", + "performance_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "ansible.ansible-ansible__ansible-e64c6c1ca50d7d26a8e7747d8eb87642e767cd74-v0f01c69f1e2528b935359cfe578530722bca2c59", + "beforeRepoSetCommand": "git reset --hard a0aad17912da687a3b0b5a573ab6ed0394b569ad\ngit clean -fd \ngit checkout a0aad17912da687a3b0b5a573ab6ed0394b569ad \ngit checkout e64c6c1ca50d7d26a8e7747d8eb87642e767cd74 -- test/units/modules/test_unarchive.py", + "selectedTestFiles": [ + "test/units/modules/test_unarchive.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-027", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_ansible__ansible-ecea15c508f0e081525be036cf76bbb56dbcdd9d-vba6da65a0f3baefda7a058ebbd0a8dcafb8512f5", + "selectedInstanceId": "instance_ansible__ansible-ecea15c508f0e081525be036cf76bbb56dbcdd9d-vba6da65a0f3baefda7a058ebbd0a8dcafb8512f5", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "ansible/ansible", + "baseCommit": "01e7915b0a9778a934a0f0e9e9d110dbef7e31ec", + "language": "python", + "tags": [ + "python", + "core_feat", + "ui_ux_enh", + "back_end_knowledge", + "devops_knowledge", + "api_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "ansible.ansible-ansible__ansible-ecea15c508f0e081525be036cf76bbb56dbcdd9d-vba6da65a0f3baefda7a058ebbd0a8dcafb8512f5", + "beforeRepoSetCommand": "git reset --hard 01e7915b0a9778a934a0f0e9e9d110dbef7e31ec\ngit clean -fd \ngit checkout 01e7915b0a9778a934a0f0e9e9d110dbef7e31ec \ngit checkout ecea15c508f0e081525be036cf76bbb56dbcdd9d -- test/integration/targets/ansible-galaxy-collection/tasks/download.yml test/integration/targets/ansible-galaxy-collection/tasks/install.yml test/units/cli/test_galaxy.py test/units/galaxy/test_collection.py", + "selectedTestFiles": [ + "test/units/galaxy/test_collection.py", + "test/units/cli/test_galaxy.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-028", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_ansible__ansible-f02a62db509dc7463fab642c9c3458b9bc3476cc-v390e508d27db7a51eece36bb6d9698b63a5b638a", + "selectedInstanceId": "instance_ansible__ansible-f02a62db509dc7463fab642c9c3458b9bc3476cc-v390e508d27db7a51eece36bb6d9698b63a5b638a", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "ansible/ansible", + "baseCommit": "73248bf27d4c6094799512b95993382ea2139e72", + "language": "python", + "tags": [ + "python", + "core_feat", + "api_feat", + "back_end_knowledge", + "infrastructure_knowledge", + "api_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "ansible.ansible-ansible__ansible-f02a62db509dc7463fab642c9c3458b9bc3476cc-v390e508d27db7a51eece36bb6d9698b63a5b638a", + "beforeRepoSetCommand": "git reset --hard 73248bf27d4c6094799512b95993382ea2139e72\ngit clean -fd \ngit checkout 73248bf27d4c6094799512b95993382ea2139e72 \ngit checkout f02a62db509dc7463fab642c9c3458b9bc3476cc -- test/integration/targets/netapp_eseries_drive_firmware/aliases test/integration/targets/netapp_eseries_drive_firmware/tasks/main.yml test/integration/targets/netapp_eseries_drive_firmware/tasks/run.yml test/units/modules/storage/netapp/test_netapp_e_drive_firmware.py", + "selectedTestFiles": [ + "test/units/modules/storage/netapp/test_netapp_e_drive_firmware.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-029", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_ansible__ansible-f327e65d11bb905ed9f15996024f857a95592629-vba6da65a0f3baefda7a058ebbd0a8dcafb8512f5", + "selectedInstanceId": "instance_ansible__ansible-f327e65d11bb905ed9f15996024f857a95592629-vba6da65a0f3baefda7a058ebbd0a8dcafb8512f5", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "ansible/ansible", + "baseCommit": "f533d46572113655a0a698beab4b38671744a458", + "language": "python", + "tags": [ + "python", + "technical_debt_enh", + "refactoring_enh", + "compatibility_bug", + "devops_knowledge", + "infrastructure_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "ansible.ansible-ansible__ansible-f327e65d11bb905ed9f15996024f857a95592629-vba6da65a0f3baefda7a058ebbd0a8dcafb8512f5", + "beforeRepoSetCommand": "git reset --hard f533d46572113655a0a698beab4b38671744a458\ngit clean -fd \ngit checkout f533d46572113655a0a698beab4b38671744a458 \ngit checkout f327e65d11bb905ed9f15996024f857a95592629 -- test/units/cli/test_galaxy.py test/units/utils/collection_loader/test_collection_loader.py", + "selectedTestFiles": [ + "test/units/utils/collection_loader/test_collection_loader.py", + "test/units/cli/test_galaxy.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-030", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_element-hq__element-web-1216285ed2e82e62f8780b6702aa0f9abdda0b34-vnan", + "selectedInstanceId": "instance_element-hq__element-web-1216285ed2e82e62f8780b6702aa0f9abdda0b34-vnan", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "element-hq/element-web", + "baseCommit": "d7a6e3ec65cf28d2454ccb357874828bc8147eb0", + "language": "js", + "tags": [ + "js", + "ui_ux_feat", + "accessibility_feat", + "front_end_knowledge", + "web_knowledge", + "ui_ux_knowledge", + "accessibility_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "element-hq.element-element-hq__element-web-1216285ed2e82e62f8780b6702aa0f9abdda0b34", + "beforeRepoSetCommand": "git reset --hard d7a6e3ec65cf28d2454ccb357874828bc8147eb0\ngit clean -fd \ngit checkout d7a6e3ec65cf28d2454ccb357874828bc8147eb0 \ngit checkout 1216285ed2e82e62f8780b6702aa0f9abdda0b34 -- test/components/views/elements/ExternalLink-test.tsx test/components/views/elements/__snapshots__/ExternalLink-test.tsx.snap", + "selectedTestFiles": [ + "test/components/views/elements/__snapshots__/ExternalLink-test.tsx.snap", + "test/components/views/elements/ExternalLink-test.tsx", + "test/components/views/elements/ExternalLink-test.ts" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-031", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_element-hq__element-web-2760bfc8369f1bee640d6d7a7e910783143d4c5f-vnan", + "selectedInstanceId": "instance_element-hq__element-web-2760bfc8369f1bee640d6d7a7e910783143d4c5f-vnan", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "element-hq/element-web", + "baseCommit": "cdffd1ca1f7b60334a8ca3bba64d0a4e6d2b68d0", + "language": "js", + "tags": [ + "js", + "minor_bug", + "ui_ux_bug", + "ui_ux_enh", + "front_end_knowledge", + "web_knowledge", + "ui_ux_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "element-hq.element-element-hq__element-web-2760bfc8369f1bee640d6d7a7e910783143d4c5f", + "beforeRepoSetCommand": "git reset --hard cdffd1ca1f7b60334a8ca3bba64d0a4e6d2b68d0\ngit clean -fd \ngit checkout cdffd1ca1f7b60334a8ca3bba64d0a4e6d2b68d0 \ngit checkout 2760bfc8369f1bee640d6d7a7e910783143d4c5f -- test/components/views/right_panel/UserInfo-test.tsx", + "selectedTestFiles": [ + "test/components/views/right_panel/UserInfo-test.ts", + "test/audio/VoiceMessageRecording-test.ts", + "test/voice-broadcast/components/molecules/VoiceBroadcastRecordingBody-test.ts", + "test/stores/room-list/algorithms/list-ordering/NaturalAlgorithm-test.ts", + "test/components/views/right_panel/UserInfo-test.tsx", + "test/components/views/dialogs/ManualDeviceKeyVerificationDialog-test.ts", + "test/components/views/rooms/wysiwyg_composer/components/FormattingButtons-test.ts", + "test/components/views/messages/MLocationBody-test.ts" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-032", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_element-hq__element-web-33299af5c9b7a7ec5a9c31d578d4ec5b18088fb7-vnan", + "selectedInstanceId": "instance_element-hq__element-web-33299af5c9b7a7ec5a9c31d578d4ec5b18088fb7-vnan", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "element-hq/element-web", + "baseCommit": "8166306e0f8951a9554bf1437f7ef6eef54a3267", + "language": "js", + "tags": [ + "js", + "core_feat", + "ui_ux_feat", + "front_end_knowledge", + "ui_ux_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "element-hq.element-element-hq__element-web-33299af5c9b7a7ec5a9c31d578d4ec5b18088fb7", + "beforeRepoSetCommand": "git reset --hard 8166306e0f8951a9554bf1437f7ef6eef54a3267\ngit clean -fd \ngit checkout 8166306e0f8951a9554bf1437f7ef6eef54a3267 \ngit checkout 33299af5c9b7a7ec5a9c31d578d4ec5b18088fb7 -- test/components/views/rooms/RoomHeader-test.tsx test/components/views/rooms/__snapshots__/RoomHeader-test.tsx.snap", + "selectedTestFiles": [ + "test/components/views/settings/Notifications-test.ts", + "test/stores/widgets/WidgetPermissionStore-test.ts", + "test/components/views/rooms/__snapshots__/RoomHeader-test.tsx.snap", + "test/components/views/polls/pollHistory/PollListItemEnded-test.ts", + "test/components/views/rooms/RoomHeader-test.tsx", + "test/hooks/useProfileInfo-test.ts", + "test/stores/room-list/algorithms/RecentAlgorithm-test.ts", + "test/components/views/elements/AppTile-test.ts", + "test/i18n-test/languageHandler-test.ts", + "test/components/views/settings/devices/deleteDevices-test.ts", + "test/utils/DateUtils-test.ts", + "test/components/views/dialogs/CreateRoomDialog-test.ts", + "test/editor/history-test.ts", + "test/components/views/rooms/wysiwyg_composer/components/WysiwygComposer-test.ts", + "test/voice-broadcast/components/molecules/VoiceBroadcastPlaybackBody-test.ts", + "test/components/views/rooms/RoomHeader-test.ts", + "test/components/structures/RightPanel-test.ts", + "test/stores/RoomViewStore-test.ts", + "test/voice-broadcast/stores/VoiceBroadcastRecordingsStore-test.ts", + "test/components/structures/auth/ForgotPassword-test.ts" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-033", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_element-hq__element-web-404c412bcb694f04ba0c4d5479541203d701bca0-vnan", + "selectedInstanceId": "instance_element-hq__element-web-404c412bcb694f04ba0c4d5479541203d701bca0-vnan", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "element-hq/element-web", + "baseCommit": "f152613f830ec32a3de3d7f442816a63a4c732c5", + "language": "js", + "tags": [ + "js", + "ui_ux_bug", + "major_bug", + "front_end_knowledge", + "web_knowledge", + "database_knowledge", + "ui_ux_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "element-hq.element-element-hq__element-web-404c412bcb694f04ba0c4d5479541203d701bca0", + "beforeRepoSetCommand": "git reset --hard f152613f830ec32a3de3d7f442816a63a4c732c5\ngit clean -fd \ngit checkout f152613f830ec32a3de3d7f442816a63a4c732c5 \ngit checkout 404c412bcb694f04ba0c4d5479541203d701bca0 -- test/MatrixClientPeg-test.ts", + "selectedTestFiles": [ + "test/components/views/settings/tabs/user/SecurityUserSettingsTab-test.ts", + "test/utils/pillify-test.ts", + "test/utils/notifications-test.ts", + "test/components/views/settings/AddPrivilegedUsers-test.ts", + "test/SlidingSyncManager-test.ts", + "test/utils/device/parseUserAgent-test.ts", + "test/MatrixClientPeg-test.ts" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-034", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_element-hq__element-web-41dfec20bfe9b62cddbbbf621bef2e9aa9685157-vnan", + "selectedInstanceId": "instance_element-hq__element-web-41dfec20bfe9b62cddbbbf621bef2e9aa9685157-vnan", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "element-hq/element-web", + "baseCommit": "d5d1ec775caf2d3c9132122c1243898e99fdb2da", + "language": "js", + "tags": [ + "js", + "integration_feat", + "api_feat", + "security_feat", + "authentication_authorization_knowledge", + "security_knowledge", + "api_knowledge", + "back_end_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "element-hq.element-element-hq__element-web-41dfec20bfe9b62cddbbbf621bef2e9aa9685157", + "beforeRepoSetCommand": "git reset --hard d5d1ec775caf2d3c9132122c1243898e99fdb2da\ngit clean -fd \ngit checkout d5d1ec775caf2d3c9132122c1243898e99fdb2da \ngit checkout 41dfec20bfe9b62cddbbbf621bef2e9aa9685157 -- test/utils/AutoDiscoveryUtils-test.tsx", + "selectedTestFiles": [ + "test/components/views/rooms/wysiwyg_composer/EditWysiwygComposer-test.ts", + "test/utils/AutoDiscoveryUtils-test.ts", + "test/components/views/beacon/BeaconMarker-test.ts", + "test/components/views/rooms/wysiwyg_composer/SendWysiwygComposer-test.ts", + "test/utils/LruCache-test.ts", + "test/utils/beacon/bounds-test.ts", + "test/components/views/dialogs/ChangelogDialog-test.ts", + "test/utils/AutoDiscoveryUtils-test.tsx", + "test/components/views/messages/MBeaconBody-test.ts", + "test/components/views/rooms/VoiceRecordComposerTile-test.ts", + "test/components/views/rooms/BasicMessageComposer-test.ts" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-035", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_element-hq__element-web-44b98896a79ede48f5ad7ff22619a39d5f6ff03c-vnan", + "selectedInstanceId": "instance_element-hq__element-web-44b98896a79ede48f5ad7ff22619a39d5f6ff03c-vnan", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "element-hq/element-web", + "baseCommit": "19f9f9856451a8e4cce6d313d19ca8aed4b5d6b4", + "language": "js", + "tags": [ + "js", + "code_quality_enh", + "refactoring_enh", + "front_end_knowledge", + "ui_ux_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "element-hq.element-element-hq__element-web-44b98896a79ede48f5ad7ff22619a39d5f6ff03c", + "beforeRepoSetCommand": "git reset --hard 19f9f9856451a8e4cce6d313d19ca8aed4b5d6b4\ngit clean -fd \ngit checkout 19f9f9856451a8e4cce6d313d19ca8aed4b5d6b4 \ngit checkout 44b98896a79ede48f5ad7ff22619a39d5f6ff03c -- test/components/views/settings/SetIntegrationManager-test.tsx test/components/views/settings/__snapshots__/SetIntegrationManager-test.tsx.snap test/components/views/settings/tabs/user/GeneralUserSettingsTab-test.tsx test/components/views/settings/tabs/user/__snapshots__/GeneralUserSettingsTab-test.tsx.snap test/components/views/settings/tabs/user/__snapshots__/SecurityUserSettingsTab-test.tsx.snap", + "selectedTestFiles": [ + "test/components/views/elements/FilterTabGroup-test.ts", + "test/components/views/settings/SetIntegrationManager-test.tsx", + "test/components/views/context_menus/MessageContextMenu-test.ts", + "test/components/views/settings/tabs/user/GeneralUserSettingsTab-test.tsx", + "test/stores/ReleaseAnnouncementStore-test.ts", + "test/components/views/elements/Pill-test.ts", + "test/utils/StorageManager-test.ts", + "test/components/views/right_panel/PinnedMessagesCard-test.ts", + "test/components/views/user-onboarding/UserOnboardingPage-test.ts", + "test/components/views/settings/tabs/user/__snapshots__/SecurityUserSettingsTab-test.tsx.snap", + "test/components/structures/auth/ForgotPassword-test.ts", + "test/modules/ProxiedModuleApi-test.ts", + "test/components/views/settings/SetIntegrationManager-test.ts", + "test/components/views/settings/tabs/user/VoiceUserSettingsTab-test.ts", + "test/voice-broadcast/components/molecules/VoiceBroadcastRecordingPip-test.ts", + "test/components/views/dialogs/ConfirmRedactDialog-test.ts", + "test/voice-broadcast/components/molecules/VoiceBroadcastPlaybackBody-test.ts", + "test/components/views/settings/tabs/user/SecurityUserSettingsTab-test.ts", + "test/components/views/settings/tabs/user/__snapshots__/GeneralUserSettingsTab-test.tsx.snap", + "test/components/views/settings/__snapshots__/SetIntegrationManager-test.tsx.snap", + "test/components/structures/SpaceHierarchy-test.ts", + "test/utils/sets-test.ts" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-036", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_element-hq__element-web-459df4583e01e4744a52d45446e34183385442d6-vnan", + "selectedInstanceId": "instance_element-hq__element-web-459df4583e01e4744a52d45446e34183385442d6-vnan", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "element-hq/element-web", + "baseCommit": "dd91250111dcf4f398e125e14c686803315ebf5d", + "language": "js", + "tags": [ + "js", + "major_bug", + "regression_bug", + "edge_case_bug", + "integration_bug", + "front_end_knowledge", + "ui_ux_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "element-hq.element-element-hq__element-web-459df4583e01e4744a52d45446e34183385442d6", + "beforeRepoSetCommand": "git reset --hard dd91250111dcf4f398e125e14c686803315ebf5d\ngit clean -fd \ngit checkout dd91250111dcf4f398e125e14c686803315ebf5d \ngit checkout 459df4583e01e4744a52d45446e34183385442d6 -- test/components/views/voip/PipView-test.tsx test/voice-broadcast/components/molecules/VoiceBroadcastPreRecordingPip-test.tsx test/voice-broadcast/models/VoiceBroadcastPreRecording-test.ts test/voice-broadcast/stores/VoiceBroadcastPreRecordingStore-test.ts test/voice-broadcast/utils/setUpVoiceBroadcastPreRecording-test.ts test/voice-broadcast/utils/startNewVoiceBroadcastRecording-test.ts", + "selectedTestFiles": [ + "test/components/views/messages/CallEvent-test.ts", + "test/components/views/settings/devices/DeviceExpandDetailsButton-test.ts", + "test/utils/localRoom/isRoomReady-test.ts", + "test/settings/watchers/FontWatcher-test.ts", + "test/utils/device/clientInformation-test.ts", + "test/voice-broadcast/utils/setUpVoiceBroadcastPreRecording-test.ts", + "test/voice-broadcast/models/VoiceBroadcastPreRecording-test.ts", + "test/voice-broadcast/stores/VoiceBroadcastPreRecordingStore-test.ts", + "test/components/views/rooms/RoomPreviewBar-test.ts", + "test/components/views/voip/PipView-test.tsx", + "test/components/views/right_panel/UserInfo-test.ts", + "test/utils/membership-test.ts", + "test/utils/location/locationEventGeoUri-test.ts", + "test/components/views/context_menus/ThreadListContextMenu-test.ts", + "test/components/views/beacon/BeaconListItem-test.ts", + "test/voice-broadcast/utils/startNewVoiceBroadcastRecording-test.ts", + "test/components/views/right_panel/PinnedMessagesCard-test.ts", + "test/voice-broadcast/components/molecules/VoiceBroadcastPreRecordingPip-test.tsx", + "test/components/views/location/Map-test.ts", + "test/voice-broadcast/utils/findRoomLiveVoiceBroadcastFromUserAndDevice-test.ts", + "test/components/views/elements/AppTile-test.ts" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-037", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_element-hq__element-web-4fec436883b601a3cac2d4a58067e597f737b817-vnan", + "selectedInstanceId": "instance_element-hq__element-web-4fec436883b601a3cac2d4a58067e597f737b817-vnan", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "element-hq/element-web", + "baseCommit": "b8bb8f163a89cb6f2af0ac1cfc97e89deb938368", + "language": "js", + "tags": [ + "js", + "ui_ux_feat", + "core_feat", + "customization_feat", + "front_end_knowledge", + "ui_ux_knowledge", + "api_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "element-hq.element-element-hq__element-web-4fec436883b601a3cac2d4a58067e597f737b817", + "beforeRepoSetCommand": "git reset --hard b8bb8f163a89cb6f2af0ac1cfc97e89deb938368\ngit clean -fd \ngit checkout b8bb8f163a89cb6f2af0ac1cfc97e89deb938368 \ngit checkout 4fec436883b601a3cac2d4a58067e597f737b817 -- test/components/views/settings/devices/CurrentDeviceSection-test.tsx test/components/views/settings/devices/DeviceDetailHeading-test.tsx test/components/views/settings/devices/DeviceDetails-test.tsx test/components/views/settings/devices/FilteredDeviceList-test.tsx test/components/views/settings/devices/__snapshots__/CurrentDeviceSection-test.tsx.snap test/components/views/settings/devices/__snapshots__/DeviceDetailHeading-test.tsx.snap test/components/views/settings/devices/__snapshots__/DeviceDetails-test.tsx.snap test/components/views/settings/tabs/user/SessionManagerTab-test.tsx test/components/views/settings/tabs/user/__snapshots__/SessionManagerTab-test.tsx.snap", + "selectedTestFiles": [ + "test/components/views/settings/devices/DeviceDetails-test.ts", + "test/components/views/settings/tabs/user/__snapshots__/SessionManagerTab-test.tsx.snap", + "test/components/views/settings/devices/__snapshots__/DeviceDetails-test.tsx.snap", + "test/components/views/settings/devices/CurrentDeviceSection-test.tsx", + "test/components/views/settings/tabs/user/SessionManagerTab-test.ts", + "test/components/views/settings/devices/__snapshots__/DeviceDetailHeading-test.tsx.snap", + "test/components/views/settings/devices/__snapshots__/CurrentDeviceSection-test.tsx.snap", + "test/components/views/rooms/SearchResultTile-test.ts", + "test/components/structures/RightPanel-test.ts", + "test/components/views/settings/devices/DeviceDetailHeading-test.tsx", + "test/stores/widgets/StopGapWidgetDriver-test.ts", + "test/components/views/settings/devices/FilteredDeviceList-test.tsx", + "test/components/views/settings/devices/DeviceDetails-test.tsx", + "test/components/views/elements/ExternalLink-test.ts", + "test/components/views/settings/devices/CurrentDeviceSection-test.ts", + "test/settings/controllers/ThemeController-test.ts", + "test/components/views/settings/tabs/user/SessionManagerTab-test.tsx", + "test/components/views/settings/devices/DeviceDetailHeading-test.ts" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-038", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_element-hq__element-web-56c7fc1948923b4b3f3507799e725ac16bcf8018-vnan", + "selectedInstanceId": "instance_element-hq__element-web-56c7fc1948923b4b3f3507799e725ac16bcf8018-vnan", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "element-hq/element-web", + "baseCommit": "9d8efacede71e3057383684446df3bde21e7bb1a", + "language": "js", + "tags": [ + "js", + "ui_ux_bug", + "minor_bug", + "front_end_knowledge", + "ui_ux_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "element-hq.element-element-hq__element-web-56c7fc1948923b4b3f3507799e725ac16bcf8018", + "beforeRepoSetCommand": "git reset --hard 9d8efacede71e3057383684446df3bde21e7bb1a\ngit clean -fd \ngit checkout 9d8efacede71e3057383684446df3bde21e7bb1a \ngit checkout 56c7fc1948923b4b3f3507799e725ac16bcf8018 -- test/unit-tests/components/views/settings/encryption/ResetIdentityPanel-test.tsx test/unit-tests/components/views/settings/encryption/__snapshots__/ResetIdentityPanel-test.tsx.snap test/unit-tests/components/views/settings/tabs/user/__snapshots__/EncryptionUserSettingsTab-test.tsx.snap", + "selectedTestFiles": [ + "test/unit-tests/utils/exportUtils/PlainTextExport-test.ts", + "test/unit-tests/components/views/dialogs/CreateRoomDialog-test.ts", + "test/unit-tests/components/views/dialogs/security/RestoreKeyBackupDialog-test.ts", + "test/unit-tests/components/views/elements/RoomTopic-test.ts", + "test/unit-tests/components/views/settings/tabs/user/__snapshots__/EncryptionUserSettingsTab-test.tsx.snap", + "test/unit-tests/components/views/settings/SecureBackupPanel-test.ts", + "test/unit-tests/utils/crypto/deviceInfo-test.ts", + "test/unit-tests/components/views/settings/tabs/user/EncryptionUserSettingsTab-test.ts", + "test/unit-tests/utils/device/clientInformation-test.ts", + "test/unit-tests/widgets/ManagedHybrid-test.ts", + "test/unit-tests/hooks/useNotificationSettings-test.ts", + "test/unit-tests/components/views/rooms/NotificationBadge/UnreadNotificationBadge-test.ts", + "test/unit-tests/utils/direct-messages-test.ts", + "test/unit-tests/components/views/spaces/SpacePanel-test.ts", + "test/unit-tests/hooks/useUnreadNotifications-test.ts", + "test/unit-tests/utils/dm/findDMForUser-test.ts", + "test/unit-tests/utils/oidc/authorize-test.ts", + "test/unit-tests/components/views/settings/encryption/__snapshots__/ResetIdentityPanel-test.tsx.snap", + "test/unit-tests/components/views/settings/devices/FilteredDeviceList-test.ts", + "test/unit-tests/components/views/messages/DecryptionFailureBody-test.ts", + "test/unit-tests/components/views/voip/CallView-test.ts", + "test/unit-tests/stores/ReleaseAnnouncementStore-test.ts", + "test/unit-tests/hooks/useDebouncedCallback-test.ts", + "test/unit-tests/components/views/elements/RoomFacePile-test.ts", + "test/unit-tests/components/views/settings/encryption/ResetIdentityPanel-test.ts", + "test/unit-tests/utils/enums-test.ts", + "test/unit-tests/utils/exportUtils/HTMLExport-test.ts", + "test/unit-tests/components/views/beacon/OwnBeaconStatus-test.ts", + "test/unit-tests/editor/range-test.ts", + "test/unit-tests/components/views/settings/tabs/user/SidebarUserSettingsTab-test.ts", + "test/unit-tests/components/structures/RoomStatusBar-test.ts", + "test/unit-tests/components/views/dialogs/InviteDialog-test.ts", + "test/unit-tests/SecurityManager-test.ts", + "test/unit-tests/components/views/settings/encryption/ResetIdentityPanel-test.tsx", + "test/unit-tests/utils/objects-test.ts" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-039", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_element-hq__element-web-72a8f8f03b1a01bb70ef8a5bb61759416991b32c-vnan", + "selectedInstanceId": "instance_element-hq__element-web-72a8f8f03b1a01bb70ef8a5bb61759416991b32c-vnan", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "element-hq/element-web", + "baseCommit": "650b9cb0cf9bb10674057e232f4792acf83f2e46", + "language": "js", + "tags": [ + "js", + "ui_ux_enh", + "front_end_knowledge", + "ui_ux_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "element-hq.element-element-hq__element-web-72a8f8f03b1a01bb70ef8a5bb61759416991b32c", + "beforeRepoSetCommand": "git reset --hard 650b9cb0cf9bb10674057e232f4792acf83f2e46\ngit clean -fd \ngit checkout 650b9cb0cf9bb10674057e232f4792acf83f2e46 \ngit checkout 72a8f8f03b1a01bb70ef8a5bb61759416991b32c -- test/hooks/useWindowWidth-test.ts", + "selectedTestFiles": [ + "test/audio/VoiceMessageRecording-test.ts", + "test/utils/exportUtils/HTMLExport-test.ts", + "test/components/views/messages/MessageEvent-test.ts", + "test/components/views/location/LocationPicker-test.ts", + "test/components/views/dialogs/ConfirmRedactDialog-test.ts", + "test/components/views/settings/AddPrivilegedUsers-test.ts", + "test/utils/DMRoomMap-test.ts", + "test/components/views/rooms/MessageComposer-test.ts", + "test/components/views/settings/CrossSigningPanel-test.ts", + "test/components/views/dialogs/AccessSecretStorageDialog-test.ts", + "test/stores/AutoRageshakeStore-test.ts", + "test/hooks/useWindowWidth-test.ts", + "test/components/views/context_menus/ContextMenu-test.ts", + "test/utils/SearchInput-test.ts", + "test/components/views/rooms/RoomTile-test.ts", + "test/stores/room-list/algorithms/list-ordering/NaturalAlgorithm-test.ts", + "test/components/views/messages/EncryptionEvent-test.ts", + "test/notifications/PushRuleVectorState-test.ts", + "test/components/views/rooms/wysiwyg_composer/components/WysiwygComposer-test.ts", + "test/components/views/messages/MStickerBody-test.ts", + "test/stores/room-list/SpaceWatcher-test.ts" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-040", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_element-hq__element-web-75c2c1a572fa45d1ea1d1a96e9e36e303332ecaa-vnan", + "selectedInstanceId": "instance_element-hq__element-web-75c2c1a572fa45d1ea1d1a96e9e36e303332ecaa-vnan", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "element-hq/element-web", + "baseCommit": "1f8fbc819795b9c19c567b4737109183acf86162", + "language": "js", + "tags": [ + "js", + "core_feat", + "front_end_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "element-hq.element-element-hq__element-web-75c2c1a572fa45d1ea1d1a96e9e36e303332ecaa", + "beforeRepoSetCommand": "git reset --hard 1f8fbc819795b9c19c567b4737109183acf86162\ngit clean -fd \ngit checkout 1f8fbc819795b9c19c567b4737109183acf86162 \ngit checkout 75c2c1a572fa45d1ea1d1a96e9e36e303332ecaa -- test/audio/VoiceRecording-test.ts", + "selectedTestFiles": [ + "test/components/views/beacon/OwnBeaconStatus-test.ts", + "test/modules/AppModule-test.ts", + "test/voice-broadcast/stores/VoiceBroadcastPlaybacksStore-test.ts", + "test/components/views/settings/devices/CurrentDeviceSection-test.ts", + "test/components/views/rooms/RoomPreviewBar-test.ts", + "test/components/views/context_menus/EmbeddedPage-test.ts", + "test/components/views/elements/QRCode-test.ts", + "test/audio/VoiceRecording-test.ts", + "test/voice-broadcast/stores/VoiceBroadcastPreRecordingStore-test.ts", + "test/components/views/dialogs/AccessSecretStorageDialog-test.ts", + "test/components/views/right_panel/UserInfo-test.ts", + "test/voice-broadcast/utils/hasRoomLiveVoiceBroadcast-test.ts", + "test/components/views/voip/CallView-test.ts", + "test/utils/numbers-test.ts", + "test/notifications/PushRuleVectorState-test.ts", + "test/components/views/settings/devices/SecurityRecommendations-test.ts", + "test/voice-broadcast/utils/VoiceBroadcastResumer-test.ts", + "test/createRoom-test.ts", + "test/utils/maps-test.ts" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-041", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_element-hq__element-web-776ffa47641c7ec6d142ab4a47691c30ebf83c2e", + "selectedInstanceId": "instance_element-hq__element-web-776ffa47641c7ec6d142ab4a47691c30ebf83c2e", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "element-hq/element-web", + "baseCommit": "8b54be6f48631083cb853cda5def60d438daa14f", + "language": "js", + "tags": [ + "js", + "ui_ux_enh", + "code_quality_enh", + "refactoring_enh", + "front_end_knowledge", + "web_knowledge", + "ui_ux_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "element-hq.element-element-hq__element-web-776ffa47641c7ec6d142ab4a47691c30ebf83c2e", + "beforeRepoSetCommand": "git reset --hard 8b54be6f48631083cb853cda5def60d438daa14f\ngit clean -fd \ngit checkout 8b54be6f48631083cb853cda5def60d438daa14f \ngit checkout 776ffa47641c7ec6d142ab4a47691c30ebf83c2e -- test/components/views/settings/devices/__snapshots__/CurrentDeviceSection-test.tsx.snap test/components/views/settings/shared/SettingsSubsectionHeading-test.tsx test/components/views/settings/shared/__snapshots__/SettingsSubsectionHeading-test.tsx.snap test/components/views/settings/tabs/user/SessionManagerTab-test.tsx test/components/views/settings/tabs/user/__snapshots__/SessionManagerTab-test.tsx.snap", + "selectedTestFiles": [ + "test/components/views/settings/tabs/user/SessionManagerTab-test.tsx", + "test/components/views/settings/shared/__snapshots__/SettingsSubsectionHeading-test.tsx.snap", + "/app/test/components/views/settings/devices/CurrentDeviceSection-test.ts", + "test/components/views/settings/devices/__snapshots__/CurrentDeviceSection-test.tsx.snap", + "test/components/views/settings/tabs/user/__snapshots__/SessionManagerTab-test.tsx.snap", + "/app/test/components/views/settings/tabs/user/SessionManagerTab-test.ts", + "test/components/views/settings/shared/SettingsSubsectionHeading-test.tsx" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-042", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_element-hq__element-web-9a31cd0fa849da810b4fac6c6c015145e850b282-vnan", + "selectedInstanceId": "instance_element-hq__element-web-9a31cd0fa849da810b4fac6c6c015145e850b282-vnan", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "element-hq/element-web", + "baseCommit": "b03433ef8b83a4c82b9d879946fb1ab5afaca522", + "language": "js", + "tags": [ + "js", + "core_feat", + "ui_ux_feat", + "localization_feat", + "front_end_knowledge", + "web_knowledge", + "ui_ux_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "element-hq.element-element-hq__element-web-9a31cd0fa849da810b4fac6c6c015145e850b282", + "beforeRepoSetCommand": "git reset --hard b03433ef8b83a4c82b9d879946fb1ab5afaca522\ngit clean -fd \ngit checkout b03433ef8b83a4c82b9d879946fb1ab5afaca522 \ngit checkout 9a31cd0fa849da810b4fac6c6c015145e850b282 -- test/components/views/settings/JoinRuleSettings-test.tsx", + "selectedTestFiles": [ + "test/components/views/messages/MessageActionBar-test.ts", + "test/components/views/settings/EventIndexPanel-test.ts", + "test/voice-broadcast/stores/VoiceBroadcastPreRecordingStore-test.ts", + "test/components/views/elements/LabelledCheckbox-test.ts", + "test/components/views/settings/tabs/user/SessionManagerTab-test.ts", + "test/components/structures/AutocompleteInput-test.ts", + "test/components/views/settings/JoinRuleSettings-test.tsx", + "test/components/views/settings/JoinRuleSettings-test.ts", + "test/stores/SetupEncryptionStore-test.ts", + "test/components/views/context_menus/SpaceContextMenu-test.ts" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-043", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_element-hq__element-web-aec454dd6feeb93000380523cbb0b3681c0275fd-vnan", + "selectedInstanceId": "instance_element-hq__element-web-aec454dd6feeb93000380523cbb0b3681c0275fd-vnan", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "element-hq/element-web", + "baseCommit": "1c039fcd3880ef4fefa58812d375104d2d70fe6c", + "language": "js", + "tags": [ + "js", + "core_feat", + "performance_enh", + "code_quality_enh", + "front_end_knowledge", + "back_end_knowledge", + "web_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "element-hq.element-element-hq__element-web-aec454dd6feeb93000380523cbb0b3681c0275fd", + "beforeRepoSetCommand": "git reset --hard 1c039fcd3880ef4fefa58812d375104d2d70fe6c\ngit clean -fd \ngit checkout 1c039fcd3880ef4fefa58812d375104d2d70fe6c \ngit checkout aec454dd6feeb93000380523cbb0b3681c0275fd -- test/components/views/elements/Pill-test.tsx test/components/views/elements/__snapshots__/Pill-test.tsx.snap test/contexts/SdkContext-test.ts test/stores/UserProfilesStore-test.ts test/utils/LruCache-test.ts", + "selectedTestFiles": [ + "test/components/views/elements/__snapshots__/Pill-test.tsx.snap", + "test/components/views/dialogs/InviteDialog-test.ts", + "test/i18n-test/languageHandler-test.ts", + "test/components/structures/AutocompleteInput-test.ts", + "test/voice-broadcast/audio/VoiceBroadcastRecorder-test.ts", + "test/stores/UserProfilesStore-test.ts", + "test/components/views/beacon/DialogSidebar-test.ts", + "test/components/views/elements/Pill-test.tsx", + "test/components/views/messages/MImageBody-test.ts", + "test/utils/LruCache-test.ts", + "test/components/views/dialogs/ExportDialog-test.ts", + "test/contexts/SdkContext-test.ts", + "test/components/views/elements/AccessibleButton-test.ts", + "test/components/views/rooms/wysiwyg_composer/components/WysiwygComposer-test.ts" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-044", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_element-hq__element-web-ca58617cee8aa91c93553449bfdf9b3465a5119b-vnan", + "selectedInstanceId": "instance_element-hq__element-web-ca58617cee8aa91c93553449bfdf9b3465a5119b-vnan", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "element-hq/element-web", + "baseCommit": "5583d07f25071ceb4f84462150717b68a244f166", + "language": "js", + "tags": [ + "js", + "code_quality_enh", + "minor_bug", + "refactoring_enh", + "front_end_knowledge", + "web_knowledge", + "performance_knowledge", + "ui_ux_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "element-hq.element-element-hq__element-web-ca58617cee8aa91c93553449bfdf9b3465a5119b", + "beforeRepoSetCommand": "git reset --hard 5583d07f25071ceb4f84462150717b68a244f166\ngit clean -fd \ngit checkout 5583d07f25071ceb4f84462150717b68a244f166 \ngit checkout ca58617cee8aa91c93553449bfdf9b3465a5119b -- test/LegacyCallHandler-test.ts", + "selectedTestFiles": [ + "test/components/views/rooms/NotificationBadge/NotificationBadge-test.ts", + "test/LegacyCallHandler-test.ts", + "test/components/views/elements/LabelledCheckbox-test.ts", + "test/modules/ModuleRunner-test.ts", + "test/utils/beacon/geolocation-test.ts", + "test/components/views/context_menus/ContextMenu-test.ts", + "test/components/views/settings/tabs/room/VoipRoomSettingsTab-test.ts", + "test/components/views/settings/devices/SecurityRecommendations-test.ts", + "test/components/views/audio_messages/RecordingPlayback-test.ts", + "test/components/structures/auth/ForgotPassword-test.ts", + "test/components/views/spaces/SpacePanel-test.ts", + "test/components/views/settings/devices/filter-test.ts", + "test/components/views/rooms/wysiwyg_composer/utils/createMessageContent-test.ts", + "test/audio/Playback-test.ts" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-045", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_element-hq__element-web-cf3c899dd1f221aa1a1f4c5a80dffc05b9c21c85-vnan", + "selectedInstanceId": "instance_element-hq__element-web-cf3c899dd1f221aa1a1f4c5a80dffc05b9c21c85-vnan", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "element-hq/element-web", + "baseCommit": "973513cc758030917cb339ba35d6436bc2c7d5dd", + "language": "js", + "tags": [ + "js", + "ui_ux_enh", + "web_knowledge", + "front_end_knowledge", + "ui_ux_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "element-hq.element-element-hq__element-web-cf3c899dd1f221aa1a1f4c5a80dffc05b9c21c85", + "beforeRepoSetCommand": "git reset --hard 973513cc758030917cb339ba35d6436bc2c7d5dd\ngit clean -fd \ngit checkout 973513cc758030917cb339ba35d6436bc2c7d5dd \ngit checkout cf3c899dd1f221aa1a1f4c5a80dffc05b9c21c85 -- test/voice-broadcast/components/atoms/LiveBadge-test.tsx test/voice-broadcast/components/atoms/VoiceBroadcastHeader-test.tsx test/voice-broadcast/components/atoms/__snapshots__/LiveBadge-test.tsx.snap test/voice-broadcast/components/atoms/__snapshots__/VoiceBroadcastHeader-test.tsx.snap test/voice-broadcast/components/molecules/VoiceBroadcastPlaybackBody-test.tsx test/voice-broadcast/components/molecules/VoiceBroadcastRecordingBody-test.tsx test/voice-broadcast/components/molecules/__snapshots__/VoiceBroadcastPlaybackBody-test.tsx.snap test/voice-broadcast/components/molecules/__snapshots__/VoiceBroadcastRecordingBody-test.tsx.snap test/voice-broadcast/components/molecules/__snapshots__/VoiceBroadcastRecordingPip-test.tsx.snap test/voice-broadcast/models/VoiceBroadcastPlayback-test.ts", + "selectedTestFiles": [ + "test/utils/device/clientInformation-test.ts", + "test/components/views/elements/ProgressBar-test.ts", + "test/voice-broadcast/components/molecules/VoiceBroadcastRecordingBody-test.tsx", + "test/voice-broadcast/components/molecules/VoiceBroadcastRecordingPip-test.ts", + "test/voice-broadcast/components/atoms/VoiceBroadcastHeader-test.tsx", + "test/voice-broadcast/components/molecules/VoiceBroadcastRecordingBody-test.ts", + "test/voice-broadcast/components/atoms/VoiceBroadcastHeader-test.ts", + "test/voice-broadcast/components/atoms/LiveBadge-test.tsx", + "test/voice-broadcast/components/atoms/__snapshots__/VoiceBroadcastHeader-test.tsx.snap", + "test/voice-broadcast/components/atoms/__snapshots__/LiveBadge-test.tsx.snap", + "test/voice-broadcast/components/atoms/LiveBadge-test.ts", + "test/voice-broadcast/components/molecules/VoiceBroadcastPlaybackBody-test.tsx", + "test/voice-broadcast/components/molecules/__snapshots__/VoiceBroadcastPlaybackBody-test.tsx.snap", + "test/audio/Playback-test.ts", + "test/voice-broadcast/models/VoiceBroadcastPlayback-test.ts", + "test/voice-broadcast/components/molecules/VoiceBroadcastPlaybackBody-test.ts", + "test/voice-broadcast/components/molecules/__snapshots__/VoiceBroadcastRecordingBody-test.tsx.snap", + "test/utils/local-room-test.ts", + "test/components/views/dialogs/ForwardDialog-test.ts", + "test/voice-broadcast/components/molecules/__snapshots__/VoiceBroadcastRecordingPip-test.tsx.snap", + "test/events/forward/getForwardableEvent-test.ts", + "test/utils/dm/createDmLocalRoom-test.ts" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-046", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_element-hq__element-web-dae13ac8522fc6d41e64d1ac6e3174486fdcce0c-vnan", + "selectedInstanceId": "instance_element-hq__element-web-dae13ac8522fc6d41e64d1ac6e3174486fdcce0c-vnan", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "element-hq/element-web", + "baseCommit": "526645c79160ab1ad4b4c3845de27d51263a405e", + "language": "js", + "tags": [ + "js", + "code_quality_enh", + "refactoring_enh", + "ui_ux_enh", + "front_end_knowledge", + "web_knowledge", + "ui_ux_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "element-hq.element-element-hq__element-web-dae13ac8522fc6d41e64d1ac6e3174486fdcce0c", + "beforeRepoSetCommand": "git reset --hard 526645c79160ab1ad4b4c3845de27d51263a405e\ngit clean -fd \ngit checkout 526645c79160ab1ad4b4c3845de27d51263a405e \ngit checkout dae13ac8522fc6d41e64d1ac6e3174486fdcce0c -- test/Unread-test.ts test/test-utils/threads.ts", + "selectedTestFiles": [ + "test/test-utils/threads.ts", + "test/editor/position-test.ts", + "test/components/views/typography/Caption-test.ts", + "test/components/structures/UserMenu-test.ts", + "test/components/views/settings/devices/SelectableDeviceTile-test.ts", + "test/utils/localRoom/isRoomReady-test.ts", + "test/Unread-test.ts", + "test/voice-broadcast/utils/shouldDisplayAsVoiceBroadcastRecordingTile-test.ts" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-047", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_element-hq__element-web-ecfd1736e5dd9808e87911fc264e6c816653e1a9-vnan", + "selectedInstanceId": "instance_element-hq__element-web-ecfd1736e5dd9808e87911fc264e6c816653e1a9-vnan", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "element-hq/element-web", + "baseCommit": "f34c1609c3c42f095b59bc068620f342894f94ed", + "language": "js", + "tags": [ + "js", + "core_feat", + "minor_bug", + "ui_ux_bug", + "front_end_knowledge", + "ui_ux_knowledge", + "performance_knowledge", + "web_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "element-hq.element-element-hq__element-web-ecfd1736e5dd9808e87911fc264e6c816653e1a9", + "beforeRepoSetCommand": "git reset --hard f34c1609c3c42f095b59bc068620f342894f94ed\ngit clean -fd \ngit checkout f34c1609c3c42f095b59bc068620f342894f94ed \ngit checkout ecfd1736e5dd9808e87911fc264e6c816653e1a9 -- test/components/structures/RoomSearchView-test.tsx test/components/views/rooms/SearchResultTile-test.tsx", + "selectedTestFiles": [ + "test/components/views/rooms/SearchResultTile-test.ts", + "test/utils/sets-test.ts", + "test/components/structures/RoomSearchView-test.tsx", + "test/components/structures/RoomSearchView-test.ts", + "test/utils/direct-messages-test.ts", + "test/components/views/settings/devices/DeviceExpandDetailsButton-test.ts", + "test/components/views/context_menus/ThreadListContextMenu-test.ts", + "test/components/views/rooms/SearchResultTile-test.tsx" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-048", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_flipt-io__flipt-05d7234fa582df632f70a7cd10194d61bd7043b9", + "selectedInstanceId": "instance_flipt-io__flipt-05d7234fa582df632f70a7cd10194d61bd7043b9", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "flipt-io/flipt", + "baseCommit": "b64891e57df74861e89ebcfa81394e4bc096f8c7", + "language": "go", + "tags": [ + "go", + "api_feat", + "code_quality_enh", + "scalability_enh", + "back_end_knowledge", + "database_knowledge", + "api_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "flipt-io.flipt-flipt-io__flipt-05d7234fa582df632f70a7cd10194d61bd7043b9", + "beforeRepoSetCommand": "git reset --hard b64891e57df74861e89ebcfa81394e4bc096f8c7\ngit clean -fd \ngit checkout b64891e57df74861e89ebcfa81394e4bc096f8c7 \ngit checkout 05d7234fa582df632f70a7cd10194d61bd7043b9 -- internal/storage/fs/object/file_test.go internal/storage/fs/object/fileinfo_test.go internal/storage/fs/snapshot_test.go internal/storage/fs/store_test.go", + "selectedTestFiles": [ + "TestNewFile", + "TestRemapScheme", + "TestFSWithoutIndex", + "TestSupportedSchemes", + "TestFileInfoIsDir", + "TestFileInfo", + "TestGetVersion" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-049", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_flipt-io__flipt-1737085488ecdcd3299c8e61af45a8976d457b7e", + "selectedInstanceId": "instance_flipt-io__flipt-1737085488ecdcd3299c8e61af45a8976d457b7e", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "flipt-io/flipt", + "baseCommit": "1f6255dda6648ecafd94826cda3fac2486af4b0f", + "language": "go", + "tags": [ + "go", + "major_bug", + "data_bug", + "compatibility_bug", + "back_end_knowledge", + "devops_knowledge", + "cloud_knowledge", + "api_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "flipt-io.flipt-flipt-io__flipt-1737085488ecdcd3299c8e61af45a8976d457b7e", + "beforeRepoSetCommand": "git reset --hard 1f6255dda6648ecafd94826cda3fac2486af4b0f\ngit clean -fd \ngit checkout 1f6255dda6648ecafd94826cda3fac2486af4b0f \ngit checkout 1737085488ecdcd3299c8e61af45a8976d457b7e -- internal/ext/importer_test.go", + "selectedTestFiles": [ + "TestImport" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-050", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_flipt-io__flipt-21a935ad7886cc50c46852be21b37f363a926af0", + "selectedInstanceId": "instance_flipt-io__flipt-21a935ad7886cc50c46852be21b37f363a926af0", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "flipt-io/flipt", + "baseCommit": "4e1cd36398ee73acf7d9235b517f05178651c464", + "language": "go", + "tags": [ + "go", + "integration_feat", + "customization_feat", + "back_end_knowledge", + "devops_knowledge", + "infrastructure_knowledge", + "performance_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "flipt-io.flipt-flipt-io__flipt-21a935ad7886cc50c46852be21b37f363a926af0", + "beforeRepoSetCommand": "git reset --hard 4e1cd36398ee73acf7d9235b517f05178651c464\ngit clean -fd \ngit checkout 4e1cd36398ee73acf7d9235b517f05178651c464 \ngit checkout 21a935ad7886cc50c46852be21b37f363a926af0 -- config/config_test.go", + "selectedTestFiles": [ + "TestScheme", + "TestValidate", + "TestDatabaseProtocol", + "TestLoad", + "TestServeHTTP", + "TestCacheBackend", + "TestLogEncoding" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-051", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_flipt-io__flipt-2ce8a0331e8a8f63f2c1b555db8277ffe5aa2e63", + "selectedInstanceId": "instance_flipt-io__flipt-2ce8a0331e8a8f63f2c1b555db8277ffe5aa2e63", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "flipt-io/flipt", + "baseCommit": "f3421c143953d2a2e3f4373f8ec366e0904f9bdd", + "language": "go", + "tags": [ + "go", + "compatibility_bug", + "major_bug", + "back_end_knowledge", + "api_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "flipt-io.flipt-flipt-io__flipt-2ce8a0331e8a8f63f2c1b555db8277ffe5aa2e63", + "beforeRepoSetCommand": "git reset --hard f3421c143953d2a2e3f4373f8ec366e0904f9bdd\ngit clean -fd \ngit checkout f3421c143953d2a2e3f4373f8ec366e0904f9bdd \ngit checkout 2ce8a0331e8a8f63f2c1b555db8277ffe5aa2e63 -- internal/server/middleware/grpc/middleware_test.go", + "selectedTestFiles": [ + "TestCacheUnaryInterceptor_UpdateVariant", + "TestAuditUnaryInterceptor_OrderRollout", + "TestAuditUnaryInterceptor_CreateSegment", + "TestAuthMetadataAuditUnaryInterceptor", + "TestAuditUnaryInterceptor_CreateDistribution", + "TestFliptAcceptServerVersionUnaryInterceptor", + "TestAuditUnaryInterceptor_UpdateFlag", + "TestCacheUnaryInterceptor_UpdateFlag", + "TestAuditUnaryInterceptor_DeleteDistribution", + "TestAuditUnaryInterceptor_UpdateVariant", + "TestAuditUnaryInterceptor_OrderRule", + "TestAuditUnaryInterceptor_DeleteVariant", + "TestCacheUnaryInterceptor_CreateVariant", + "TestCacheUnaryInterceptor_Evaluation_Variant", + "TestAuditUnaryInterceptor_UpdateSegment", + "TestAuditUnaryInterceptor_CreateToken", + "TestAuditUnaryInterceptor_CreateRule", + "TestCacheUnaryInterceptor_DeleteVariant", + "TestEvaluationUnaryInterceptor_Noop", + "TestAuditUnaryInterceptor_CreateRollout", + "TestAuditUnaryInterceptor_DeleteRollout", + "TestAuditUnaryInterceptor_CreateConstraint", + "TestAuditUnaryInterceptor_DeleteNamespace", + "TestAuditUnaryInterceptor_CreateFlag", + "TestAuditUnaryInterceptor_CreateVariant", + "TestAuditUnaryInterceptor_UpdateRule", + "TestCacheUnaryInterceptor_DeleteFlag", + "TestAuditUnaryInterceptor_DeleteSegment", + "TestAuditUnaryInterceptor_DeleteConstraint", + "TestCacheUnaryInterceptor_Evaluate", + "TestErrorUnaryInterceptor", + "TestAuditUnaryInterceptor_UpdateConstraint", + "TestCacheUnaryInterceptor_GetFlag", + "TestValidationUnaryInterceptor", + "TestAuditUnaryInterceptor_DeleteFlag", + "TestAuditUnaryInterceptor_UpdateRollout", + "TestEvaluationUnaryInterceptor_BatchEvaluation", + "TestCacheUnaryInterceptor_Evaluation_Boolean", + "TestAuditUnaryInterceptor_CreateNamespace", + "TestAuditUnaryInterceptor_DeleteRule", + "TestAuditUnaryInterceptor_UpdateNamespace", + "TestEvaluationUnaryInterceptor_Evaluation", + "TestAuditUnaryInterceptor_UpdateDistribution" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-052", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_flipt-io__flipt-381b90f718435c4694380b5fcd0d5cf8e3b5a25a", + "selectedInstanceId": "instance_flipt-io__flipt-381b90f718435c4694380b5fcd0d5cf8e3b5a25a", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "flipt-io/flipt", + "baseCommit": "0ed96dc5d33768c4b145d68d52e80e7bce3790d0", + "language": "go", + "tags": [ + "go", + "integration_feat", + "code_quality_enh", + "back_end_knowledge", + "api_knowledge", + "devops_knowledge", + "cloud_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "flipt-io.flipt-flipt-io__flipt-381b90f718435c4694380b5fcd0d5cf8e3b5a25a", + "beforeRepoSetCommand": "git reset --hard 0ed96dc5d33768c4b145d68d52e80e7bce3790d0\ngit clean -fd \ngit checkout 0ed96dc5d33768c4b145d68d52e80e7bce3790d0 \ngit checkout 381b90f718435c4694380b5fcd0d5cf8e3b5a25a -- internal/config/config_test.go", + "selectedTestFiles": [ + "TestServeHTTP", + "TestTracingExporter", + "Test_mustBindEnv", + "TestCacheBackend", + "TestDefaultDatabaseRoot", + "TestLogEncoding", + "TestLoad", + "TestJSONSchema", + "TestScheme", + "TestMarshalYAML", + "TestDatabaseProtocol" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-053", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_flipt-io__flipt-3b2c25ee8a3ac247c3fad13ad8d64ace34ec8ee7", + "selectedInstanceId": "instance_flipt-io__flipt-3b2c25ee8a3ac247c3fad13ad8d64ace34ec8ee7", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "flipt-io/flipt", + "baseCommit": "8d72418bf67cec833da7f59beeecb5abfd48cb05", + "language": "go", + "tags": [ + "go", + "major_bug", + "integration_bug", + "compatibility_bug", + "back_end_knowledge", + "api_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "flipt-io.flipt-flipt-io__flipt-3b2c25ee8a3ac247c3fad13ad8d64ace34ec8ee7", + "beforeRepoSetCommand": "git reset --hard 8d72418bf67cec833da7f59beeecb5abfd48cb05\ngit clean -fd \ngit checkout 8d72418bf67cec833da7f59beeecb5abfd48cb05 \ngit checkout 3b2c25ee8a3ac247c3fad13ad8d64ace34ec8ee7 -- internal/server/ofrep/evaluation_test.go internal/server/ofrep/extensions_test.go internal/server/ofrep/middleware_test.go", + "selectedTestFiles": [ + "TestEvaluateFlag_Failure", + "TestEvaluateFlag_Success", + "TestGetProviderConfiguration", + "TestEvaluateBulkSuccess" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-054", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_flipt-io__flipt-3d5a345f94c2adc8a0eaa102c189c08ad4c0f8e8", + "selectedInstanceId": "instance_flipt-io__flipt-3d5a345f94c2adc8a0eaa102c189c08ad4c0f8e8", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "flipt-io/flipt", + "baseCommit": "91cc1b9fc38280a53a36e1e9543d87d7306144b2", + "language": "go", + "tags": [ + "go", + "performance_enh", + "dev_ops_enh", + "back_end_knowledge", + "devops_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "flipt-io.flipt-flipt-io__flipt-3d5a345f94c2adc8a0eaa102c189c08ad4c0f8e8", + "beforeRepoSetCommand": "git reset --hard 91cc1b9fc38280a53a36e1e9543d87d7306144b2\ngit clean -fd \ngit checkout 91cc1b9fc38280a53a36e1e9543d87d7306144b2 \ngit checkout 3d5a345f94c2adc8a0eaa102c189c08ad4c0f8e8 -- internal/config/config_test.go internal/tracing/tracing_test.go", + "selectedTestFiles": [ + "TestLoad", + "TestCacheBackend", + "TestLogEncoding", + "TestMarshalYAML", + "Test_mustBindEnv", + "TestDefaultDatabaseRoot", + "TestAnalyticsClickhouseConfiguration", + "TestJSONSchema", + "TestDatabaseProtocol", + "TestGetConfigFile", + "TestServeHTTP", + "TestScheme", + "TestTracingExporter" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-055", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_flipt-io__flipt-40007b9d97e3862bcef8c20ae6c87b22ea0627f0", + "selectedInstanceId": "instance_flipt-io__flipt-40007b9d97e3862bcef8c20ae6c87b22ea0627f0", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "flipt-io/flipt", + "baseCommit": "bbf0a917fbdf4c92017f760b63727b921eb9fc98", + "language": "go", + "tags": [ + "go", + "api_feat", + "security_feat", + "integration_feat", + "back_end_knowledge", + "api_knowledge", + "security_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "flipt-io.flipt-flipt-io__flipt-40007b9d97e3862bcef8c20ae6c87b22ea0627f0", + "beforeRepoSetCommand": "git reset --hard bbf0a917fbdf4c92017f760b63727b921eb9fc98\ngit clean -fd \ngit checkout bbf0a917fbdf4c92017f760b63727b921eb9fc98 \ngit checkout 40007b9d97e3862bcef8c20ae6c87b22ea0627f0 -- internal/config/config_test.go internal/server/authn/method/github/server_test.go", + "selectedTestFiles": [ + "TestLoad", + "TestGithubSimpleOrganizationDecode" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-056", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_flipt-io__flipt-518ec324b66a07fdd95464a5e9ca5fe7681ad8f9", + "selectedInstanceId": "instance_flipt-io__flipt-518ec324b66a07fdd95464a5e9ca5fe7681ad8f9", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "flipt-io/flipt", + "baseCommit": "0018c5df774444117b107dfe3fe503d4c7126d73", + "language": "go", + "tags": [ + "go", + "regression_bug", + "ui_ux_bug", + "data_bug", + "back_end_knowledge", + "web_knowledge", + "api_knowledge", + "devops_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "flipt-io.flipt-flipt-io__flipt-518ec324b66a07fdd95464a5e9ca5fe7681ad8f9", + "beforeRepoSetCommand": "git reset --hard 0018c5df774444117b107dfe3fe503d4c7126d73\ngit clean -fd \ngit checkout 0018c5df774444117b107dfe3fe503d4c7126d73 \ngit checkout 518ec324b66a07fdd95464a5e9ca5fe7681ad8f9 -- internal/config/config_test.go", + "selectedTestFiles": [ + "TestLoad" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-057", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_flipt-io__flipt-690672523398c2b6f6e4562f0bf9868664ab894f", + "selectedInstanceId": "instance_flipt-io__flipt-690672523398c2b6f6e4562f0bf9868664ab894f", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "flipt-io/flipt", + "baseCommit": "6da20eb7afb693a1cbee2482272e3aee2fbd43ee", + "language": "go", + "tags": [ + "go", + "refactoring_enh", + "code_quality_enh", + "back_end_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "flipt-io.flipt-flipt-io__flipt-690672523398c2b6f6e4562f0bf9868664ab894f", + "beforeRepoSetCommand": "git reset --hard 6da20eb7afb693a1cbee2482272e3aee2fbd43ee\ngit clean -fd \ngit checkout 6da20eb7afb693a1cbee2482272e3aee2fbd43ee \ngit checkout 690672523398c2b6f6e4562f0bf9868664ab894f -- internal/cmd/grpc_test.go internal/tracing/tracing_test.go", + "selectedTestFiles": [ + "TestGetTraceExporter", + "TestNewResourceDefault" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-058", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_flipt-io__flipt-6fd0f9e2587f14ac1fdd1c229f0bcae0468c8daa", + "selectedInstanceId": "instance_flipt-io__flipt-6fd0f9e2587f14ac1fdd1c229f0bcae0468c8daa", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "flipt-io/flipt", + "baseCommit": "563a8c4593610e431f0c3db55e88a679c4386991", + "language": "go", + "tags": [ + "go", + "core_feat", + "back_end_knowledge", + "infrastructure_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "flipt-io.flipt-flipt-io__flipt-6fd0f9e2587f14ac1fdd1c229f0bcae0468c8daa", + "beforeRepoSetCommand": "git reset --hard 563a8c4593610e431f0c3db55e88a679c4386991\ngit clean -fd \ngit checkout 563a8c4593610e431f0c3db55e88a679c4386991 \ngit checkout 6fd0f9e2587f14ac1fdd1c229f0bcae0468c8daa -- internal/config/config_test.go internal/oci/file_test.go", + "selectedTestFiles": [ + "TestLogEncoding", + "TestMarshalYAML", + "TestNewStore", + "TestStore_Fetch", + "TestLoad", + "TestServeHTTP", + "TestTracingExporter", + "Test_mustBindEnv", + "TestDatabaseProtocol", + "TestScheme", + "TestJSONSchema", + "TestStore_Fetch_InvalidMediaType", + "TestCacheBackend" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-059", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_flipt-io__flipt-756f00f79ba8abf9fe53f3c6c818123b42eb7355", + "selectedInstanceId": "instance_flipt-io__flipt-756f00f79ba8abf9fe53f3c6c818123b42eb7355", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "flipt-io/flipt", + "baseCommit": "266e5e143e87519047b9be3202a0aba273b83de3", + "language": "go", + "tags": [ + "go", + "refactoring_enh", + "code_quality_enh", + "scalability_enh", + "technical_debt_enh", + "back_end_knowledge", + "api_knowledge", + "infrastructure_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "flipt-io.flipt-flipt-io__flipt-756f00f79ba8abf9fe53f3c6c818123b42eb7355", + "beforeRepoSetCommand": "git reset --hard 266e5e143e87519047b9be3202a0aba273b83de3\ngit clean -fd \ngit checkout 266e5e143e87519047b9be3202a0aba273b83de3 \ngit checkout 756f00f79ba8abf9fe53f3c6c818123b42eb7355 -- internal/config/config_test.go", + "selectedTestFiles": [ + "TestLogEncoding", + "TestServeHTTP", + "TestLoad", + "TestDatabaseProtocol", + "TestScheme", + "TestJSONSchema", + "TestCacheBackend" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-060", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_flipt-io__flipt-8bd3604dc54b681f1f0f7dd52cbc70b3024184b6", + "selectedInstanceId": "instance_flipt-io__flipt-8bd3604dc54b681f1f0f7dd52cbc70b3024184b6", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "flipt-io/flipt", + "baseCommit": "25a5f278e1116ca22f86d86b4a5259ca05ef2623", + "language": "go", + "tags": [ + "go", + "critical_bug", + "back_end_knowledge", + "devops_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "flipt-io.flipt-flipt-io__flipt-8bd3604dc54b681f1f0f7dd52cbc70b3024184b6", + "beforeRepoSetCommand": "git reset --hard 25a5f278e1116ca22f86d86b4a5259ca05ef2623\ngit clean -fd \ngit checkout 25a5f278e1116ca22f86d86b4a5259ca05ef2623 \ngit checkout 8bd3604dc54b681f1f0f7dd52cbc70b3024184b6 -- internal/server/audit/template/executer_test.go internal/server/audit/template/leveled_logger_test.go internal/server/audit/webhook/client_test.go", + "selectedTestFiles": [ + "TestExecuter_JSON_Failure", + "TestExecuter_Execute", + "TestConstructorWebhookTemplate", + "TestWebhookClient", + "TestExecuter_Execute_toJson_valid_Json", + "TestLeveledLogger", + "TestConstructorWebhookClient" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-061", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_flipt-io__flipt-96820c3ad10b0b2305e8877b6b303f7fafdf815f", + "selectedInstanceId": "instance_flipt-io__flipt-96820c3ad10b0b2305e8877b6b303f7fafdf815f", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "flipt-io/flipt", + "baseCommit": "8dd44097778951eaa6976631d35bc418590d1555", + "language": "go", + "tags": [ + "go", + "major_bug", + "data_bug", + "compatibility_bug", + "back_end_knowledge", + "api_knowledge", + "authentication_authorization_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "flipt-io.flipt-flipt-io__flipt-96820c3ad10b0b2305e8877b6b303f7fafdf815f", + "beforeRepoSetCommand": "git reset --hard 8dd44097778951eaa6976631d35bc418590d1555\ngit clean -fd \ngit checkout 8dd44097778951eaa6976631d35bc418590d1555 \ngit checkout 96820c3ad10b0b2305e8877b6b303f7fafdf815f -- internal/oci/ecr/credentials_store_test.go internal/oci/ecr/ecr_test.go internal/oci/ecr/mock_Client_test.go internal/oci/ecr/mock_PrivateClient_test.go internal/oci/ecr/mock_PublicClient_test.go internal/oci/file_test.go", + "selectedTestFiles": [ + "TestStore_List", + "TestFile", + "TestWithCredentials", + "TestStore_Fetch_InvalidMediaType", + "TestStore_FetchWithECR", + "TestAuthenicationTypeIsValid", + "TestParseReference", + "TestStore_Fetch", + "TestECRCredential", + "TestStore_Copy", + "TestPrivateClient", + "TestDefaultClientFunc", + "TestWithManifestVersion", + "TestCredential", + "TestPublicClient", + "TestStore_Build" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-062", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_flipt-io__flipt-9d25c18b79bc7829a6fb08ec9e8793d5d17e2868", + "selectedInstanceId": "instance_flipt-io__flipt-9d25c18b79bc7829a6fb08ec9e8793d5d17e2868", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "flipt-io/flipt", + "baseCommit": "fa8f302adcde48b4a94bf45726fe4da73de5e5ab", + "language": "go", + "tags": [ + "go", + "core_feat", + "api_feat", + "integration_feat", + "back_end_knowledge", + "api_knowledge", + "devops_knowledge", + "infrastructure_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "flipt-io.flipt-flipt-io__flipt-9d25c18b79bc7829a6fb08ec9e8793d5d17e2868", + "beforeRepoSetCommand": "git reset --hard fa8f302adcde48b4a94bf45726fe4da73de5e5ab\ngit clean -fd \ngit checkout fa8f302adcde48b4a94bf45726fe4da73de5e5ab \ngit checkout 9d25c18b79bc7829a6fb08ec9e8793d5d17e2868 -- internal/server/evaluation/ofrep_bridge_test.go internal/server/ofrep/evaluation_test.go internal/server/ofrep/extensions_test.go", + "selectedTestFiles": [ + "TestBoolean_SegmentMatch_Constraint_EntityId", + "TestEvaluator_ErrorParsingDateTime", + "TestVariant_FlagNotFound", + "TestEvaluator_DistributionNotMatched", + "TestEvaluator_MatchAll_RolloutDistribution", + "TestEvaluator_MatchAny_NoConstraints", + "TestBoolean_DefaultRule_NoRollouts", + "TestEvaluator_MatchAll_RolloutDistribution_MultiRule", + "TestEvaluateFlag_Failure", + "TestBoolean_DefaultRuleFallthrough_WithPercentageRollout", + "TestBoolean_PercentageRuleMatch", + "TestEvaluator_RulesOutOfOrder", + "TestEvaluator_ErrorGettingRules", + "TestEvaluator_FlagDisabled_DefaultVariant", + "TestVariant_Success", + "TestOFREPEvaluationBridge_Boolean", + "TestEvaluateFlag_Success", + "Test_Server_AllowsNamespaceScopedAuthentication", + "TestBoolean_NonBooleanFlagError", + "TestVariant_EvaluateFailure_OnGetEvaluationRules", + "TestBoolean_SegmentMatch_MultipleSegments_WithAnd", + "Test_matchesNumber", + "TestEvaluator_MatchAll_NoDistributions_DefaultVariant", + "TestEvaluator_DistributionNotMatched_DefaultVariant", + "TestEvaluator_MatchAny_NoDistributions_DefaultVariant", + "TestBoolean_FlagNotFoundError", + "TestEvaluator_ErrorGettingDistributions", + "TestBatch_Success", + "TestGetProviderConfiguration", + "TestBatch_UnknownFlagType", + "Test_matchesDateTime", + "TestEvaluator_MatchAny_SingleVariantDistribution", + "TestEvaluator_FlagNoRules_DefaultVariant", + "TestBatch_InternalError_GetFlag", + "TestEvaluator_MultipleZeroRolloutDistributions", + "TestEvaluator_MatchEntityId", + "Test_matchesBool", + "TestEvaluator_NonVariantFlag", + "TestOFREPEvaluationBridge_Variant", + "TestEvaluator_MatchAll_MultipleSegments", + "TestEvaluator_MatchAny_RolloutDistribution", + "TestBoolean_SegmentMatch_MultipleConstraints", + "TestEvaluator_MatchAny_RolloutDistribution_MultiRule", + "TestVariant_FlagDisabled", + "TestEvaluator_FlagNoRules", + "TestVariant_NonVariantFlag", + "TestBoolean_RulesOutOfOrder", + "TestBoolean_PercentageRuleFallthrough_SegmentMatch", + "Test_matchesString", + "TestEvaluator_FlagDisabled", + "TestEvaluator_MatchAll_NoConstraints", + "TestEvaluator_FirstRolloutRuleIsZero", + "TestEvaluator_MatchAll_NoVariants_NoDistributions", + "TestEvaluator_MatchAll_SingleVariantDistribution", + "TestEvaluator_MatchAny_NoVariants_NoDistributions", + "TestEvaluator_ErrorParsingNumber" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-063", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_flipt-io__flipt-9f8127f225a86245fa35dca4885c2daef824ee55", + "selectedInstanceId": "instance_flipt-io__flipt-9f8127f225a86245fa35dca4885c2daef824ee55", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "flipt-io/flipt", + "baseCommit": "2d0ff0c91a63a1165f5ca528faa1f0785b1f730c", + "language": "go", + "tags": [ + "go", + "core_feat", + "integration_feat", + "back_end_knowledge", + "database_knowledge", + "infrastructure_knowledge", + "api_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "flipt-io.flipt-flipt-io__flipt-9f8127f225a86245fa35dca4885c2daef824ee55", + "beforeRepoSetCommand": "git reset --hard 2d0ff0c91a63a1165f5ca528faa1f0785b1f730c\ngit clean -fd \ngit checkout 2d0ff0c91a63a1165f5ca528faa1f0785b1f730c \ngit checkout 9f8127f225a86245fa35dca4885c2daef824ee55 -- .github/workflows/test.yml internal/storage/sql/db_test.go", + "selectedTestFiles": [ + "TestMigratorExpectedVersions", + "TestOpen", + "TestMigratorRun", + "TestDBTestSuite", + "TestParse", + "TestMigratorRun_NoChange" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-064", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_flipt-io__flipt-a0cbc0cb65ae601270bdbe3f5313e2dfd49c80e4", + "selectedInstanceId": "instance_flipt-io__flipt-a0cbc0cb65ae601270bdbe3f5313e2dfd49c80e4", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "flipt-io/flipt", + "baseCommit": "fee220d0a20adfb21686685bef2a1d6c2ff6fc17", + "language": "go", + "tags": [ + "go", + "core_feat", + "customization_feat", + "back_end_knowledge", + "devops_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "flipt-io.flipt-flipt-io__flipt-a0cbc0cb65ae601270bdbe3f5313e2dfd49c80e4", + "beforeRepoSetCommand": "git reset --hard fee220d0a20adfb21686685bef2a1d6c2ff6fc17\ngit clean -fd \ngit checkout fee220d0a20adfb21686685bef2a1d6c2ff6fc17 \ngit checkout a0cbc0cb65ae601270bdbe3f5313e2dfd49c80e4 -- internal/config/config_test.go", + "selectedTestFiles": [ + "TestLoad" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-065", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_flipt-io__flipt-a42d38a1bb1df267c53d9d4a706cf34825ae3da9", + "selectedInstanceId": "instance_flipt-io__flipt-a42d38a1bb1df267c53d9d4a706cf34825ae3da9", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "flipt-io/flipt", + "baseCommit": "ee02b164f6728d3227c42671028c67a4afd36918", + "language": "go", + "tags": [ + "go", + "api_feat", + "security_feat", + "security_enh", + "authentication_authorization_knowledge", + "security_knowledge", + "web_knowledge", + "full_stack_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "flipt-io.flipt-flipt-io__flipt-a42d38a1bb1df267c53d9d4a706cf34825ae3da9", + "beforeRepoSetCommand": "git reset --hard ee02b164f6728d3227c42671028c67a4afd36918\ngit clean -fd \ngit checkout ee02b164f6728d3227c42671028c67a4afd36918 \ngit checkout a42d38a1bb1df267c53d9d4a706cf34825ae3da9 -- internal/config/config_test.go test/api.sh test/config/test-with-auth.yml", + "selectedTestFiles": [ + "TestLoad", + "TestLogEncoding", + "TestServeHTTP", + "Test_mustBindEnv", + "TestDatabaseProtocol", + "TestScheme", + "TestCacheBackend", + "TestJSONSchema" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-066", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_flipt-io__flipt-af7a0be46d15f0b63f16a868d13f3b48a838e7ce", + "selectedInstanceId": "instance_flipt-io__flipt-af7a0be46d15f0b63f16a868d13f3b48a838e7ce", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "flipt-io/flipt", + "baseCommit": "165ba79a44732208147f516fa6fa4d1dc72b7008", + "language": "go", + "tags": [ + "go", + "dev_ops_enh", + "infrastructure_knowledge", + "devops_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "flipt-io.flipt-flipt-io__flipt-af7a0be46d15f0b63f16a868d13f3b48a838e7ce", + "beforeRepoSetCommand": "git reset --hard 165ba79a44732208147f516fa6fa4d1dc72b7008\ngit clean -fd \ngit checkout 165ba79a44732208147f516fa6fa4d1dc72b7008 \ngit checkout af7a0be46d15f0b63f16a868d13f3b48a838e7ce -- internal/config/config_test.go", + "selectedTestFiles": [ + "Test_mustBindEnv", + "TestJSONSchema", + "TestServeHTTP", + "TestLoad", + "TestScheme", + "TestCacheBackend", + "TestDatabaseProtocol", + "TestLogEncoding" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-067", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_flipt-io__flipt-b3cd920bbb25e01fdb2dab66a5a913363bc62f6c", + "selectedInstanceId": "instance_flipt-io__flipt-b3cd920bbb25e01fdb2dab66a5a913363bc62f6c", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "flipt-io/flipt", + "baseCommit": "490cc12996575ec353e9755ca1c21e222a4e5191", + "language": "go", + "tags": [ + "go", + "core_feat", + "performance_feat", + "customization_feat", + "back_end_knowledge", + "api_knowledge", + "devops_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "flipt-io.flipt-flipt-io__flipt-b3cd920bbb25e01fdb2dab66a5a913363bc62f6c", + "beforeRepoSetCommand": "git reset --hard 490cc12996575ec353e9755ca1c21e222a4e5191\ngit clean -fd \ngit checkout 490cc12996575ec353e9755ca1c21e222a4e5191 \ngit checkout b3cd920bbb25e01fdb2dab66a5a913363bc62f6c -- internal/ext/exporter_test.go", + "selectedTestFiles": [ + "TestImport", + "TestImport_Namespaces_Mix_And_Match", + "TestExport", + "TestImport_InvalidVersion", + "TestImport_Export", + "TestImport_FlagType_LTVersion1_1", + "FuzzImport", + "TestImport_Rollouts_LTVersion1_1" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-068", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_flipt-io__flipt-c1fd7a81ef9f23e742501bfb26d914eb683262aa", + "selectedInstanceId": "instance_flipt-io__flipt-c1fd7a81ef9f23e742501bfb26d914eb683262aa", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "flipt-io/flipt", + "baseCommit": "dbe263961b187e1c5d7fe34c65b000985a2da5a0", + "language": "go", + "tags": [ + "go", + "major_bug", + "security_bug", + "data_bug", + "back_end_knowledge", + "authentication_authorization_knowledge", + "security_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "flipt-io.flipt-flipt-io__flipt-c1fd7a81ef9f23e742501bfb26d914eb683262aa", + "beforeRepoSetCommand": "git reset --hard dbe263961b187e1c5d7fe34c65b000985a2da5a0\ngit clean -fd \ngit checkout dbe263961b187e1c5d7fe34c65b000985a2da5a0 \ngit checkout c1fd7a81ef9f23e742501bfb26d914eb683262aa -- internal/config/config_test.go", + "selectedTestFiles": [ + "TestLoad" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-069", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_flipt-io__flipt-c8d71ad7ea98d97546f01cce4ccb451dbcf37d3b", + "selectedInstanceId": "instance_flipt-io__flipt-c8d71ad7ea98d97546f01cce4ccb451dbcf37d3b", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "flipt-io/flipt", + "baseCommit": "29d3f9db40c83434d0e3cc082af8baec64c391a9", + "language": "go", + "tags": [ + "go", + "major_bug", + "data_bug", + "dev_ops_enh", + "back_end_knowledge", + "database_knowledge", + "api_knowledge", + "devops_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "flipt-io.flipt-flipt-io__flipt-c8d71ad7ea98d97546f01cce4ccb451dbcf37d3b", + "beforeRepoSetCommand": "git reset --hard 29d3f9db40c83434d0e3cc082af8baec64c391a9\ngit clean -fd \ngit checkout 29d3f9db40c83434d0e3cc082af8baec64c391a9 \ngit checkout c8d71ad7ea98d97546f01cce4ccb451dbcf37d3b -- internal/cue/validate_fuzz_test.go internal/cue/validate_test.go internal/storage/fs/snapshot_test.go", + "selectedTestFiles": [ + "FuzzValidate", + "TestFSWithIndex", + "TestFS_Invalid_VariantFlag_Distribution", + "TestFSWithoutIndex", + "TestValidate_Failure", + "TestFS_Invalid_VariantFlag_Segment", + "Test_Store", + "TestFS_Invalid_BooleanFlag_Segment", + "TestValidate_Latest_Segments_V2", + "TestValidate_V1_Success", + "TestValidate_Latest_Success" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-070", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_flipt-io__flipt-cd18e54a0371fa222304742c6312e9ac37ea86c1", + "selectedInstanceId": "instance_flipt-io__flipt-cd18e54a0371fa222304742c6312e9ac37ea86c1", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "flipt-io/flipt", + "baseCommit": "9e469bf851c6519616c2b220f946138b71fab047", + "language": "go", + "tags": [ + "go", + "major_bug", + "data_bug", + "back_end_knowledge", + "devops_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "flipt-io.flipt-flipt-io__flipt-cd18e54a0371fa222304742c6312e9ac37ea86c1", + "beforeRepoSetCommand": "git reset --hard 9e469bf851c6519616c2b220f946138b71fab047\ngit clean -fd \ngit checkout 9e469bf851c6519616c2b220f946138b71fab047 \ngit checkout cd18e54a0371fa222304742c6312e9ac37ea86c1 -- config/schema_test.go internal/config/config_test.go", + "selectedTestFiles": [ + "TestScheme", + "TestJSONSchema", + "Test_mustBindEnv", + "TestLoad", + "TestCacheBackend", + "TestTracingExporter", + "TestLogEncoding", + "Test_CUE", + "TestServeHTTP", + "TestDatabaseProtocol" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-071", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_flipt-io__flipt-cd2f3b0a9d4d8b8a6d3d56afab65851ecdc408e8", + "selectedInstanceId": "instance_flipt-io__flipt-cd2f3b0a9d4d8b8a6d3d56afab65851ecdc408e8", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "flipt-io/flipt", + "baseCommit": "a91a0258e72c0f0aac3d33ae5c226a85c80ecdf8", + "language": "go", + "tags": [ + "go", + "core_feat", + "back_end_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "flipt-io.flipt-flipt-io__flipt-cd2f3b0a9d4d8b8a6d3d56afab65851ecdc408e8", + "beforeRepoSetCommand": "git reset --hard a91a0258e72c0f0aac3d33ae5c226a85c80ecdf8\ngit clean -fd \ngit checkout a91a0258e72c0f0aac3d33ae5c226a85c80ecdf8 \ngit checkout cd2f3b0a9d4d8b8a6d3d56afab65851ecdc408e8 -- internal/server/evaluation/legacy_evaluator_test.go rpc/flipt/validation_fuzz_test.go rpc/flipt/validation_test.go", + "selectedTestFiles": [ + "Test_matchesNumber", + "Test_matchesString" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-072", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_flipt-io__flipt-e2bd19dafa7166c96b082fb2a59eb54b4be0d778", + "selectedInstanceId": "instance_flipt-io__flipt-e2bd19dafa7166c96b082fb2a59eb54b4be0d778", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "flipt-io/flipt", + "baseCommit": "0eaf98f050d86247bfe9bd7e41178865cb585060", + "language": "go", + "tags": [ + "go", + "major_bug", + "performance_feat", + "performance_enh", + "back_end_knowledge", + "api_knowledge", + "database_knowledge", + "devops_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "flipt-io.flipt-flipt-io__flipt-e2bd19dafa7166c96b082fb2a59eb54b4be0d778", + "beforeRepoSetCommand": "git reset --hard 0eaf98f050d86247bfe9bd7e41178865cb585060\ngit clean -fd \ngit checkout 0eaf98f050d86247bfe9bd7e41178865cb585060 \ngit checkout e2bd19dafa7166c96b082fb2a59eb54b4be0d778 -- internal/cache/cache_test.go internal/server/middleware/grpc/middleware_test.go internal/storage/cache/cache_test.go", + "selectedTestFiles": [ + "TestAuditUnaryInterceptor_UpdateVariant", + "TestAuditUnaryInterceptor_CreateSegment", + "TestGetJSONHandleGetError", + "TestAuditUnaryInterceptor_CreateToken", + "TestAuditUnaryInterceptor_UpdateRollout", + "TestEvaluationCacheUnaryInterceptor_Evaluate", + "TestCacheControlUnaryInterceptor", + "TestAuditUnaryInterceptor_DeleteVariant", + "TestGetFlagCached", + "TestGetEvaluationRulesCached", + "TestAuditUnaryInterceptor_CreateDistribution", + "TestEvaluationUnaryInterceptor_Noop", + "TestEvaluationUnaryInterceptor_Evaluation", + "TestAuditUnaryInterceptor_UpdateRule", + "TestAuditUnaryInterceptor_CreateNamespace", + "TestGetEvaluationRules", + "TestAuditUnaryInterceptor_DeleteDistribution", + "TestValidationUnaryInterceptor", + "TestAuditUnaryInterceptor_DeleteNamespace", + "TestWithDoNotStore", + "TestAuditUnaryInterceptor_UpdateConstraint", + "TestIsDoNotStore", + "TestAuditUnaryInterceptor_CreateVariant", + "TestEvaluationCacheUnaryInterceptor_Evaluation_Variant", + "TestAuditUnaryInterceptor_DeleteRollout", + "TestEvaluationUnaryInterceptor_BatchEvaluation", + "TestAuditUnaryInterceptor_DeleteSegment", + "TestAuditUnaryInterceptor_DeleteConstraint", + "TestAuditUnaryInterceptor_DeleteFlag", + "TestAuditUnaryInterceptor_UpdateNamespace", + "TestAuditUnaryInterceptor_CreateRule", + "TestAuthMetadataAuditUnaryInterceptor", + "TestAuditUnaryInterceptor_UpdateFlag", + "TestAuditUnaryInterceptor_CreateConstraint", + "TestErrorUnaryInterceptor", + "TestEvaluationCacheUnaryInterceptor_Evaluation_Boolean", + "TestAuditUnaryInterceptor_UpdateSegment", + "TestAuditUnaryInterceptor_DeleteRule", + "TestGetJSONHandleUnmarshalError", + "TestSetJSONHandleMarshalError", + "TestAuditUnaryInterceptor_CreateFlag", + "TestAuditUnaryInterceptor_UpdateDistribution", + "TestAuditUnaryInterceptor_CreateRollout" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-073", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_flipt-io__flipt-e50808c03e4b9d25a6a78af9c61a3b1616ea356b", + "selectedInstanceId": "instance_flipt-io__flipt-e50808c03e4b9d25a6a78af9c61a3b1616ea356b", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "flipt-io/flipt", + "baseCommit": "5069ba6fa22fbbf208352ff341ea7a85d6eca29f", + "language": "go", + "tags": [ + "go", + "api_feat", + "integration_feat", + "core_feat", + "back_end_knowledge", + "api_knowledge", + "infrastructure_knowledge", + "devops_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "flipt-io.flipt-flipt-io__flipt-e50808c03e4b9d25a6a78af9c61a3b1616ea356b", + "beforeRepoSetCommand": "git reset --hard 5069ba6fa22fbbf208352ff341ea7a85d6eca29f\ngit clean -fd \ngit checkout 5069ba6fa22fbbf208352ff341ea7a85d6eca29f \ngit checkout e50808c03e4b9d25a6a78af9c61a3b1616ea356b -- internal/config/config_test.go internal/server/audit/audit_test.go internal/server/auth/server_test.go internal/server/middleware/grpc/middleware_test.go internal/server/middleware/grpc/support_test.go", + "selectedTestFiles": [ + "TestAuditUnaryInterceptor_CreateFlag", + "TestCacheUnaryInterceptor_UpdateVariant", + "TestValidationUnaryInterceptor", + "TestTracingExporter", + "TestAuditUnaryInterceptor_DeleteConstraint", + "TestJSONSchema", + "TestAuditUnaryInterceptor_UpdateConstraint", + "TestAuditUnaryInterceptor_UpdateRule", + "TestAuditUnaryInterceptor_UpdateNamespace", + "TestAuditUnaryInterceptor_CreateNamespace", + "TestAuditUnaryInterceptor_DeleteVariant", + "TestAuditUnaryInterceptor_UpdateSegment", + "TestCacheUnaryInterceptor_GetFlag", + "TestAuditUnaryInterceptor_CreateVariant", + "TestAuditUnaryInterceptor_CreateConstraint", + "TestErrorUnaryInterceptor", + "TestSinkSpanExporter", + "TestServeHTTP", + "Test_mustBindEnv", + "TestEvaluationUnaryInterceptor_BatchEvaluation", + "TestScheme", + "TestAuditUnaryInterceptor_DeleteFlag", + "TestAuditUnaryInterceptor_DeleteNamespace", + "TestLoad", + "TestCacheUnaryInterceptor_DeleteFlag", + "TestAuditUnaryInterceptor_DeleteDistribution", + "TestAuditUnaryInterceptor_CreateSegment", + "TestEvaluationUnaryInterceptor_Noop", + "TestAuditUnaryInterceptor_DeleteRule", + "TestCacheUnaryInterceptor_DeleteVariant", + "TestAuditUnaryInterceptor_DeleteSegment", + "TestEvaluationUnaryInterceptor_Evaluation", + "TestAuditUnaryInterceptor_CreateRule", + "TestAuditUnaryInterceptor_UpdateDistribution", + "TestLogEncoding", + "TestAuditUnaryInterceptor_CreateDistribution", + "TestCacheBackend", + "TestCacheUnaryInterceptor_UpdateFlag", + "TestCacheUnaryInterceptor_Evaluate", + "TestAuditUnaryInterceptor_UpdateVariant", + "TestDatabaseProtocol", + "TestCacheUnaryInterceptor_CreateVariant", + "TestAuditUnaryInterceptor_UpdateFlag" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-074", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_flipt-io__flipt-ea9a2663b176da329b3f574da2ce2a664fc5b4a1", + "selectedInstanceId": "instance_flipt-io__flipt-ea9a2663b176da329b3f574da2ce2a664fc5b4a1", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "flipt-io/flipt", + "baseCommit": "866ba43dd49c238c97831362cdab50630b0b9aa7", + "language": "go", + "tags": [ + "go", + "major_bug", + "authentication_authorization_knowledge", + "ui_ux_knowledge", + "devops_knowledge", + "api_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "flipt-io.flipt-flipt-io__flipt-ea9a2663b176da329b3f574da2ce2a664fc5b4a1", + "beforeRepoSetCommand": "git reset --hard 866ba43dd49c238c97831362cdab50630b0b9aa7\ngit clean -fd \ngit checkout 866ba43dd49c238c97831362cdab50630b0b9aa7 \ngit checkout ea9a2663b176da329b3f574da2ce2a664fc5b4a1 -- internal/server/authz/engine/bundle/engine_test.go internal/server/authz/engine/rego/engine_test.go internal/server/authz/middleware/grpc/middleware_test.go internal/server/namespace_test.go", + "selectedTestFiles": [ + "TestEngine_IsAuthMethod", + "TestUpdateRule", + "TestUpdateConstraint", + "TestListRollouts_PaginationPageToken", + "TestOrderRules", + "TestListSegments_PaginationOffset", + "TestDeleteSegment", + "TestBatchEvaluate_FlagNotFound", + "TestDeleteDistribution", + "TestAuthorizationRequiredInterceptor", + "TestListNamespaces_WithAuthz", + "TestCreateSegment", + "TestUpdateDistribution", + "TestDeleteRollout", + "TestDeleteNamespace_HasFlagsWithForce", + "TestDeleteNamespace", + "TestDeleteRule", + "TestBatchEvaluate_FlagNotFoundExcluded", + "TestBatchEvaluate", + "TestCreateRule_MultipleSegments", + "TestEngine_IsAllowed", + "TestListSegments_PaginationPageToken", + "TestCreateRollout", + "TestDeleteConstraint", + "TestListFlags_PaginationOffset", + "TestDeleteNamespace_NonExistent", + "TestListNamespaces_PaginationOffset", + "TestListRules_PaginationOffset", + "TestEngine_NewEngine", + "TestCreateDistribution", + "TestListRules_PaginationPageToken", + "TestDeleteNamespace_HasFlags", + "TestUpdateRollout", + "TestOrderRollouts", + "TestUpdateNamespace", + "TestDeleteNamespace_ProtectedWithForce", + "TestListFlags_PaginationPageToken", + "TestUpdateSegment", + "TestCreateRule", + "TestListNamespaces_PaginationPageToken", + "TestCreateNamespace", + "TestCreateConstraint", + "TestViewableNamespaces", + "TestBatchEvaluate_NamespaceMismatch", + "TestDeleteNamespace_Protected" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-075", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_flipt-io__flipt-ee02b164f6728d3227c42671028c67a4afd36918", + "selectedInstanceId": "instance_flipt-io__flipt-ee02b164f6728d3227c42671028c67a4afd36918", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "flipt-io/flipt", + "baseCommit": "e38e41543f08b762904ed8a08969d0b6aba67166", + "language": "go", + "tags": [ + "go", + "code_quality_enh", + "refactoring_enh", + "technical_debt_enh", + "dev_ops_enh", + "back_end_knowledge", + "devops_knowledge", + "api_knowledge", + "security_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "flipt-io.flipt-flipt-io__flipt-ee02b164f6728d3227c42671028c67a4afd36918", + "beforeRepoSetCommand": "git reset --hard e38e41543f08b762904ed8a08969d0b6aba67166\ngit clean -fd \ngit checkout e38e41543f08b762904ed8a08969d0b6aba67166 \ngit checkout ee02b164f6728d3227c42671028c67a4afd36918 -- internal/release/check_test.go", + "selectedTestFiles": [ + "TestIs", + "TestCheck" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-076", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_flipt-io__flipt-f1bc91a1b999656dbdb2495ccb57bf2105b84920", + "selectedInstanceId": "instance_flipt-io__flipt-f1bc91a1b999656dbdb2495ccb57bf2105b84920", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "flipt-io/flipt", + "baseCommit": "56d261e7c28e47faf537061e6ad3b306459a3e93", + "language": "go", + "tags": [ + "go", + "code_quality_enh", + "scalability_enh", + "refactoring_enh", + "back_end_knowledge", + "api_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "flipt-io.flipt-flipt-io__flipt-f1bc91a1b999656dbdb2495ccb57bf2105b84920", + "beforeRepoSetCommand": "git reset --hard 56d261e7c28e47faf537061e6ad3b306459a3e93\ngit clean -fd \ngit checkout 56d261e7c28e47faf537061e6ad3b306459a3e93 \ngit checkout f1bc91a1b999656dbdb2495ccb57bf2105b84920 -- server/evaluator_test.go server/rule_test.go storage/db_test.go storage/evaluator_test.go storage/rule_test.go", + "selectedTestFiles": [ + "TestFlagsPagination", + "TestUpdateFlag_NotFound", + "TestUpdateRule", + "TestListSegments", + "TestCreateVariant_FlagNotFound", + "TestUpdateVariant_NotFound", + "TestUpdateVariant_DuplicateName", + "TestUpdateRuleAndDistribution", + "TestNew", + "TestEvaluate_RolloutDistribution", + "Test_matchesString", + "TestListRulesPagination", + "TestCreateRule", + "TestUpdateConstraint_NotFound", + "TestCreateConstraint", + "TestCreateDistribution", + "TestCreateVariant_DuplicateName", + "TestCreateSegment", + "TestEvaluate_NoConstraints", + "TestUpdateSegment_NotFound", + "TestCreateConstraint_ErrInvalid", + "TestCreateDistribution_NoRule", + "TestDeleteConstraint_NotFound", + "TestParse", + "TestGetRule", + "TestUpdateConstraint_ErrInvalid", + "TestCreateConstraint_SegmentNotFound", + "TestDeleteConstraint", + "Test_evaluate", + "TestGetSegment", + "TestCreateVariant_DuplicateName_DifferentFlag", + "TestUpdateSegment", + "TestCreateRule_FlagNotFound", + "TestErrorUnaryInterceptor", + "Test_matchesNumber", + "TestCreateRuleAndDistribution", + "TestEvaluate_FlagDisabled", + "TestDeleteSegment_NotFound", + "TestDeleteDistribution", + "Test_validate", + "TestDeleteSegment", + "TestCreateSegment_DuplicateKey", + "TestDeleteRule", + "TestWithCache", + "TestCreateRule_SegmentNotFound", + "TestDeleteRule_NotFound", + "TestGetSegmentNotFound", + "TestUpdateConstraint", + "TestEvaluate_NoVariants_NoDistributions", + "TestEvaluate_SingleVariantDistribution", + "TestDeleteVariant_NotFound", + "TestListSegmentsPagination", + "TestListRules", + "TestEvaluate_FlagNotFound", + "Test_matchesBool", + "TestDeleteFlag_NotFound", + "TestEvaluate", + "TestUpdateRule_NotFound", + "TestCreateFlag_DuplicateKey", + "TestOrderRules", + "TestEvaluate_FlagNoRules", + "TestUpdateDistribution" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-077", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_flipt-io__flipt-f36bd61fb1cee4669de1f00e59da462bfeae8765", + "selectedInstanceId": "instance_flipt-io__flipt-f36bd61fb1cee4669de1f00e59da462bfeae8765", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "flipt-io/flipt", + "baseCommit": "54e188b64f0dda5a1ab9caf8425f94dac3d08f40", + "language": "go", + "tags": [ + "go", + "major_bug", + "ui_ux_bug", + "back_end_knowledge", + "ui_ux_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "flipt-io.flipt-flipt-io__flipt-f36bd61fb1cee4669de1f00e59da462bfeae8765", + "beforeRepoSetCommand": "git reset --hard 54e188b64f0dda5a1ab9caf8425f94dac3d08f40\ngit clean -fd \ngit checkout 54e188b64f0dda5a1ab9caf8425f94dac3d08f40 \ngit checkout f36bd61fb1cee4669de1f00e59da462bfeae8765 -- internal/cue/validate_test.go", + "selectedTestFiles": [ + "TestValidate_Success", + "TestValidate_Failure" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-078", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_future-architect__vuls-0ec945d0510cdebf92cdd8999f94610772689f14", + "selectedInstanceId": "instance_future-architect__vuls-0ec945d0510cdebf92cdd8999f94610772689f14", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "future-architect/vuls", + "baseCommit": "d3bf2a6f26e8e549c0732c26fdcc82725d3c6633", + "language": "go", + "tags": [ + "go", + "data_bug", + "back_end_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "future-architect.vuls-future-architect__vuls-0ec945d0510cdebf92cdd8999f94610772689f14", + "beforeRepoSetCommand": "git reset --hard d3bf2a6f26e8e549c0732c26fdcc82725d3c6633\ngit clean -fd \ngit checkout d3bf2a6f26e8e549c0732c26fdcc82725d3c6633 \ngit checkout 0ec945d0510cdebf92cdd8999f94610772689f14 -- scanner/redhatbase_test.go", + "selectedTestFiles": [ + "Test_redhatBase_parseInstalledPackagesLine/not_standard_rpm_style_source_package", + "Test_redhatBase_parseInstalledPackagesLine/release_is_empty", + "Test_redhatBase_parseInstalledPackagesLine/not_standard_rpm_style_source_package_2", + "Test_redhatBase_parseInstalledPackagesLine", + "Test_redhatBase_parseInstalledPackagesLine/not_standard_rpm_style_source_package_3", + "Test_redhatBase_parseInstalledPackagesLine/release_is_empty_2" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-079", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_future-architect__vuls-1832b4ee3a20177ad313d806983127cb6e53f5cf", + "selectedInstanceId": "instance_future-architect__vuls-1832b4ee3a20177ad313d806983127cb6e53f5cf", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "future-architect/vuls", + "baseCommit": "78b52d6a7f480bd610b692de9bf0c86f57332f23", + "language": "go", + "tags": [ + "go", + "core_feat", + "integration_feat", + "back_end_knowledge", + "infrastructure_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "future-architect.vuls-future-architect__vuls-1832b4ee3a20177ad313d806983127cb6e53f5cf", + "beforeRepoSetCommand": "git reset --hard 78b52d6a7f480bd610b692de9bf0c86f57332f23\ngit clean -fd \ngit checkout 78b52d6a7f480bd610b692de9bf0c86f57332f23 \ngit checkout 1832b4ee3a20177ad313d806983127cb6e53f5cf -- config/os_test.go scanner/base_test.go scanner/freebsd_test.go scanner/macos_test.go", + "selectedTestFiles": [ + "Test_getAmazonLinuxVersion/1", + "TestParseDockerPs", + "Test_updatePortStatus/update_match_multi_address", + "TestEOL_IsStandardSupportEnded/Debian_9_supported", + "TestEOL_IsStandardSupportEnded/macOS_13.4.1_supported", + "TestEOL_IsStandardSupportEnded/Alpine_3.15_supported", + "TestPortScanConf_IsZero/not_zero", + "TestParseOSRelease", + "Test_findPortScanSuccessOn/asterisk_match", + "TestEOL_IsStandardSupportEnded/Alma_Linux_10_Not_Found", + "Test_parseSWVers/MacOS", + "Test_debian_parseGetPkgName/success", + "Test_windows_detectKBsFromKernelVersion/10.0.19045.2129", + "TestParseNeedsRestarting", + "Test_redhatBase_parseRpmQfLine/permission_denied_will_be_ignored", + "TestEOL_IsStandardSupportEnded/Ubuntu_14.04_eol", + "Test_redhatBase_parseRpmQfLine/err", + "TestEOL_IsStandardSupportEnded/Rocky_Linux_8_supported", + "TestScanModule_validate/err", + "TestEOL_IsStandardSupportEnded/Oracle_Linux_6_eol", + "TestEOL_IsStandardSupportEnded/Ubuntu_14.10_eol", + "TestPortScanConf_getScanTechniques/unknown", + "TestEOL_IsStandardSupportEnded/Alma_Linux_8_supported", + "TestEOL_IsStandardSupportEnded/Fedora_37_eol_since_2023-12-16", + "Test_detectScanDest/asterisk", + "Test_windows_detectKBsFromKernelVersion", + "TestSplitIntoBlocks", + "Test_base_parseLsProcExe", + "TestParseSSHConfiguration", + "TestEOL_IsStandardSupportEnded/amazon_linux_1_supported", + "Test_findPortScanSuccessOn/no_match_port", + "TestEOL_IsStandardSupportEnded/Ubuntu_22.04_supported", + "TestEOL_IsStandardSupportEnded/Fedora_38_eol_since_2024-05-15", + "Test_getAmazonLinuxVersion/2025", + "TestParseIp", + "TestParseChangelog", + "Test_getAmazonLinuxVersion/2", + "Test_detectScanDest/dup-addr-port", + "TestParseYumCheckUpdateLinesAmazon", + "TestParseAptCachePolicy", + "Test_findPortScanSuccessOn/no_match_address", + "TestParseApkInfo", + "Test_windows_detectKBsFromKernelVersion/10.0.19045.2130", + "TestEOL_IsStandardSupportEnded/Ubuntu_18.04_ext_supported", + "TestEOL_IsStandardSupportEnded/Ubuntu_20.04_supported", + "TestEOL_IsStandardSupportEnded/Debian_10_supported", + "TestEOL_IsStandardSupportEnded/Fedora_37_supported", + "Test_base_parseGrepProcMap/systemd", + "TestEOL_IsStandardSupportEnded/Ubuntu_20.10_supported", + "TestEOL_IsStandardSupportEnded/Fedora_39_not_found", + "TestParseInstalledPackagesLinesRedhat", + "Test_parseSystemInfo", + "Test_detectOSName/Windows_10_Version_21H2_for_x64-based_Systems", + "Test_detectOSName/err", + "Test_formatKernelVersion/major.minor.build.revision", + "TestEOL_IsStandardSupportEnded/Ubuntu_22.10_supported", + "Test_getAmazonLinuxVersion/2029", + "Test_redhatBase_parseDnfModuleList", + "TestHosts", + "TestDistro_MajorVersion", + "Test_parseWmiObject/happy", + "Test_macos_parseInstalledPackages/happy", + "TestEOL_IsStandardSupportEnded/Alpine_3.9_eol", + "Test_base_parseLsProcExe/systemd", + "Test_detectScanDest/single-addr", + "TestEOL_IsStandardSupportEnded/Debian_11_supported", + "TestEOL_IsStandardSupportEnded/Mac_OS_X_10.15_EOL", + "Test_updatePortStatus/update_multi_packages", + "Test_redhatBase_parseRpmQfLine/is_not_owned_by_any_package", + "Test_parseGetPackageMSU/happy", + "Test_detectScanDest", + "Test_parseSWVers/Mac_OS_X_Server", + "TestEOL_IsStandardSupportEnded/CentOS_stream10_Not_Found", + "Test_majorDotMinor", + "TestEOL_IsStandardSupportEnded/Alpine_3.18_not_found", + "Test_majorDotMinor/major_dot_minor", + "TestParseYumCheckUpdateLine", + "TestScanModule_validate", + "Test_parseInstalledPackages", + "TestScanModule_IsZero/zero", + "TestEOL_IsStandardSupportEnded/freebsd_13_supported", + "TestParseApkVersion", + "TestScanModule_IsZero", + "TestEOL_IsStandardSupportEnded/Debian_13_is_not_supported_yet", + "TestEOL_IsStandardSupportEnded/CentOS_6_eol", + "Test_detectOSName/Windows_Server_2022", + "Test_parseWindowsUpdaterSearch", + "TestParseChangelog/vlc", + "Test_parseSystemInfo/Workstation", + "Test_parseGetHotfix/happy", + "TestEOL_IsStandardSupportEnded/CentOS_7_supported", + "TestEOL_IsStandardSupportEnded/Rocky_Linux_9_supported", + "TestEOL_IsStandardSupportEnded/RHEL10_not_found", + "Test_detectScanDest/empty", + "TestEOL_IsStandardSupportEnded/Ubuntu_5.10_not_found", + "TestEOL_IsStandardSupportEnded/Fedora_32_supported", + "Test_formatKernelVersion/major.minor.build", + "Test_parseInstalledPackages/happy", + "Test_getAmazonLinuxVersion", + "TestEOL_IsStandardSupportEnded/freebsd_12_supported", + "Test_base_parseLsOf/lsof", + "TestEOL_IsStandardSupportEnded/Alpine_3.11_supported", + "TestEOL_IsStandardSupportEnded/Rocky_Linux_10_Not_Found", + "Test_parseSWVers", + "TestParseCheckRestart", + "TestEOL_IsStandardSupportEnded/Fedora_33_supported", + "Test_getAmazonLinuxVersion/2022", + "TestViaHTTP", + "TestScanUpdatablePackage", + "Test_parseWmiObject", + "TestEOL_IsStandardSupportEnded/Ubuntu_18.04_supported", + "TestEOL_IsStandardSupportEnded/alpine_3.10_supported", + "TestEOL_IsStandardSupportEnded/Ubuntu_21.04_supported", + "Test_getAmazonLinuxVersion/2023", + "TestIsRunningKernelRedHatLikeLinux", + "TestEOL_IsStandardSupportEnded/CentOS_8_supported", + "TestEOL_IsStandardSupportEnded/Windows_10_EOL", + "TestEOL_IsStandardSupportEnded/Oracle_Linux_10_not_found", + "TestParseIfconfig", + "TestEOL_IsStandardSupportEnded/Alpine_3.17_supported", + "Test_base_parseLsOf/lsof-duplicate-port", + "Test_macos_parseInstalledPackages", + "Test_redhatBase_rebootRequired/kerne_needs-reboot", + "TestIsRunningKernelSUSE", + "TestEOL_IsStandardSupportEnded/Debian_8_supported", + "TestEOL_IsStandardSupportEnded/Alpine_3.16_supported", + "TestEOL_IsStandardSupportEnded/freebsd_11_supported", + "Test_majorDotMinor/major", + "TestDecorateCmd", + "Test_parseSystemInfo/Domain_Controller", + "TestEOL_IsStandardSupportEnded", + "Test_updatePortStatus/nil_listen_ports", + "TestEOL_IsStandardSupportEnded/Alma_Linux_9_supported", + "TestEOL_IsStandardSupportEnded/Oracle_Linux_8_supported", + "TestEOL_IsStandardSupportEnded/Alpine_3.12_supported", + "TestParseInstalledPackagesLineFromRepoquery", + "TestEOL_IsStandardSupportEnded/Fedora_36_supported", + "Test_redhatBase_parseRpmQfLine/valid_line", + "TestEOL_IsStandardSupportEnded/Ubuntu_23.04_supported", + "TestSyslogConfValidate", + "Test_updatePortStatus/update_match_single_address", + "TestParseChangelog/realvnc-vnc-server", + "Test_windows_detectKBsFromKernelVersion/10.0.20348.9999", + "TestEOL_IsStandardSupportEnded/RHEL6_eol", + "Test_parseWindowsUpdateHistory", + "TestPortScanConf_IsZero", + "Test_redhatBase_parseRpmQfLine", + "Test_findPortScanSuccessOn", + "TestEOL_IsStandardSupportEnded/amazon_linux_2022_supported", + "TestGetUpdatablePackNames", + "Test_findPortScanSuccessOn/open_empty", + "TestEOL_IsStandardSupportEnded/Fedora_38_supported", + "Test_redhatBase_rebootRequired/uek_kernel_no-reboot", + "TestPortScanConf_getScanTechniques/nil", + "TestParseSystemctlStatus", + "Test_redhatBase_rebootRequired/uek_kernel_needs-reboot", + "TestPortScanConf_getScanTechniques/single", + "Test_detectScanDest/multi-addr", + "Test_updatePortStatus/update_match_asterisk", + "Test_detectOSName", + "Test_redhatBase_rebootRequired", + "Test_detectOSName/Windows_Server_2019", + "TestScanUpdatablePackages", + "TestPortScanConf_getScanTechniques", + "TestScanModule_IsZero/not_zero", + "TestParseSSHScan", + "Test_parseWindowsUpdaterSearch/happy", + "Test_redhatBase_rebootRequired/kerne_no-reboot", + "TestEOL_IsStandardSupportEnded/freebsd_10_eol", + "Test_parseRegistry/happy", + "Test_getAmazonLinuxVersion/2031", + "Test_parseWindowsUpdateHistory/happy", + "TestEOL_IsStandardSupportEnded/amazon_linux_2_supported", + "Test_detectOSName/Windows_10_for_x64-based_Systems", + "TestParseBlock", + "Test_parseRegistry", + "Test_redhatBase_parseRpmQfLine/No_such_file_or_directory_will_be_ignored", + "TestEOL_IsStandardSupportEnded/Ubuntu_20.04_ext_supported", + "Test_parseSystemInfo/Server", + "TestEOL_IsStandardSupportEnded/Ubuntu_21.10_supported", + "Test_updatePortStatus/nil_affected_procs", + "Test_majorDotMinor/empty", + "Test_findPortScanSuccessOn/port_empty", + "Test_parseSWVers/MacOS_Server", + "TestEOL_IsStandardSupportEnded/Fedora_35_supported", + "TestEOL_IsStandardSupportEnded/RHEL7_supported", + "Test_windows_parseIP/en", + "TestEOL_IsStandardSupportEnded/Windows_10_Version_22H2_supported", + "Test_base_parseGrepProcMap", + "Test_redhatBase_parseDnfModuleList/Success", + "TestEOL_IsStandardSupportEnded/amazon_linux_1_eol_on_2023-6-30", + "TestEOL_IsStandardSupportEnded/RHEL9_supported", + "Test_windows_parseIP", + "TestScanModule_validate/valid", + "TestEOL_IsStandardSupportEnded/Fedora_32_eol_since_2021-5-25", + "TestParsePkgInfo", + "Test_parseGetPackageMSU", + "TestPortScanConf_IsZero/zero", + "Test_parseSWVers/ProductName_error", + "TestToCpeURI", + "Test_findPortScanSuccessOn/single_match", + "TestEOL_IsStandardSupportEnded/Fedora_34_eol_since_2022-6-7", + "TestParseYumCheckUpdateLines", + "TestEOL_IsStandardSupportEnded/RHEL8_supported", + "TestEOL_IsStandardSupportEnded/Fedora_36_eol_since_2023-05-17", + "Test_windows_parseIP/ja", + "Test_parseGetComputerInfo", + "TestEOL_IsStandardSupportEnded/Fedora_34_supported", + "TestEOL_IsStandardSupportEnded/Oracle_Linux_9_supported", + "TestEOL_IsStandardSupportEnded/Debian_12_supported", + "TestNormalizedForWindows", + "TestEOL_IsStandardSupportEnded/amazon_linux_2023_supported", + "Test_parseGetHotfix", + "Test_getAmazonLinuxVersion/2018.03", + "TestEOL_IsStandardSupportEnded/CentOS_stream9_supported", + "TestEOL_IsStandardSupportEnded/Alpine_3.14_supported", + "TestEOL_IsStandardSupportEnded/Fedora_33_eol_since_2021-11-30", + "Test_updatePortStatus", + "TestParseInstalledPackagesLine", + "Test_getAmazonLinuxVersion/2017.09", + "TestParseLxdPs", + "TestEOL_IsStandardSupportEnded/CentOS_stream8_supported", + "TestEOL_IsStandardSupportEnded/Fedora_35_eol_since_2022-12-13", + "Test_debian_parseGetPkgName", + "TestParsePkgVersion", + "Test_parseSWVers/ProductVersion_error", + "Test_formatKernelVersion", + "Test_base_parseLsOf", + "TestIsAwsInstanceID", + "Test_windows_detectKBsFromKernelVersion/err", + "Test_majorDotMinor/major_dot_minor_dot_release", + "Test_windows_detectKBsFromKernelVersion/10.0.22621.1105", + "Test_getAmazonLinuxVersion/2027", + "Test_parseGetComputerInfo/happy", + "TestGetChangelogCache", + "TestPortScanConf_getScanTechniques/multiple", + "TestSplitAptCachePolicy", + "TestParseSSHKeygen", + "Test_parseSWVers/Mac_OS_X", + "TestEOL_IsStandardSupportEnded/Oracle_Linux_7_supported", + "TestEOL_IsStandardSupportEnded/freebsd_11_eol_on_2021-9-30", + "TestEOL_IsStandardSupportEnded/Ubuntu_16.04_supported", + "TestGetCveIDsFromChangelog", + "Test_windows_detectKBsFromKernelVersion/10.0.20348.1547", + "TestEOL_IsStandardSupportEnded/amazon_linux_2031_not_found" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-080", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_future-architect__vuls-50580f6e98eeb36f53f27222f7f4fdfea0b21e8d", + "selectedInstanceId": "instance_future-architect__vuls-50580f6e98eeb36f53f27222f7f4fdfea0b21e8d", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "future-architect/vuls", + "baseCommit": "472df0e1b6ab9b50f8605af957ad054c7a732938", + "language": "go", + "tags": [ + "go", + "integration_feat", + "core_feat", + "code_quality_enh", + "back_end_knowledge", + "security_knowledge", + "api_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "future-architect.vuls-future-architect__vuls-50580f6e98eeb36f53f27222f7f4fdfea0b21e8d", + "beforeRepoSetCommand": "git reset --hard 472df0e1b6ab9b50f8605af957ad054c7a732938\ngit clean -fd \ngit checkout 472df0e1b6ab9b50f8605af957ad054c7a732938 \ngit checkout 50580f6e98eeb36f53f27222f7f4fdfea0b21e8d -- detector/wordpress_test.go", + "selectedTestFiles": [ + "Test_convertToVinfos", + "Test_convertToVinfos/WordPress_vulnerabilities_Enterprise", + "Test_convertToVinfos/WordPress_vulnerabilities_Researcher" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-081", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_future-architect__vuls-5af1a227339e46c7abf3f2815e4c636a0c01098e", + "selectedInstanceId": "instance_future-architect__vuls-5af1a227339e46c7abf3f2815e4c636a0c01098e", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "future-architect/vuls", + "baseCommit": "053306944695e6616f320393f865f667e984481d", + "language": "go", + "tags": [ + "go", + "compatibility_bug", + "edge_case_bug", + "back_end_knowledge", + "infrastructure_knowledge", + "devops_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "future-architect.vuls-future-architect__vuls-5af1a227339e46c7abf3f2815e4c636a0c01098e", + "beforeRepoSetCommand": "git reset --hard 053306944695e6616f320393f865f667e984481d\ngit clean -fd \ngit checkout 053306944695e6616f320393f865f667e984481d \ngit checkout 5af1a227339e46c7abf3f2815e4c636a0c01098e -- scanner/redhatbase_test.go scanner/utils_test.go", + "selectedTestFiles": [ + "Test_base_parseGrepProcMap/systemd", + "Test_isRunningKernel/SUES_not_kernel", + "Test_parseGetComputerInfo", + "Test_detectOSName/Windows_10_Version_21H2_for_x64-based_Systems", + "Test_parseWmiObject", + "Test_parseSWVers/ProductName_error", + "Test_windows_parseIP/ja", + "TestNormalizedForWindows", + "Test_detectScanDest/single-addr", + "Test_isRunningKernel/SUSE_kernel_and_running", + "TestParseYumCheckUpdateLines", + "Test_windows_parseIP/en", + "Test_parseRegistry/happy", + "Test_redhatBase_parseRpmQfLine/is_not_owned_by_any_package", + "Test_detectScanDest/asterisk", + "Test_updatePortStatus/update_match_multi_address", + "TestParseAptCachePolicy", + "TestSplitIntoBlocks", + "TestParseInstalledPackagesLinesRedhat", + "TestDecorateCmd", + "TestParseSSHConfiguration", + "Test_detectOSName/Windows_10_for_x64-based_Systems", + "TestParseInstalledPackagesLineFromRepoquery", + "TestParseChangelog/vlc", + "Test_findPortScanSuccessOn/asterisk_match", + "TestParseYumCheckUpdateLine", + "Test_isRunningKernel/Amazon_kernel_and_running", + "TestParseYumCheckUpdateLinesAmazon", + "Test_base_parseLsOf", + "Test_redhatBase_rebootRequired/kerne_no-reboot", + "Test_formatKernelVersion/major.minor.build", + "Test_windows_detectKBsFromKernelVersion/10.0.22621.1105", + "Test_detectScanDest/multi-addr", + "TestParseSSHKeygen", + "TestGetChangelogCache", + "Test_updatePortStatus/update_match_asterisk", + "TestParseApkInfo", + "TestIsAwsInstanceID", + "Test_parseSWVers/Mac_OS_X_Server", + "Test_redhatBase_parseRpmQfLine/valid_line", + "Test_updatePortStatus/nil_affected_procs", + "Test_redhatBase_rebootRequired/kerne_needs-reboot", + "Test_macos_parseInstalledPackages/happy", + "TestScanUpdatablePackages", + "Test_parseRegistry", + "Test_windows_detectKBsFromKernelVersion/err", + "TestScanUpdatablePackage", + "Test_redhatBase_parseRpmQfLine", + "Test_parseGetPackageMSU/happy", + "Test_isRunningKernel", + "Test_isRunningKernel/Amazon_not_kernel", + "Test_macos_parseInstalledPackages", + "TestParseBlock", + "Test_parseGetHotfix", + "Test_isRunningKernel/old_redhat_kernel_release_style", + "Test_windows_detectKBsFromKernelVersion/10.0.20348.1547", + "Test_windows_parseIP", + "Test_detectScanDest", + "Test_parseInstalledPackages", + "Test_debian_parseGetPkgName/success", + "Test_redhatBase_parseRpmQfLine/No_such_file_or_directory_will_be_ignored", + "Test_isRunningKernel/Amazon_kernel_but_not_running", + "Test_base_parseLsProcExe/systemd", + "TestParseChangelog", + "Test_debian_parseGetPkgName", + "Test_updatePortStatus", + "Test_parseWmiObject/happy", + "Test_updatePortStatus/nil_listen_ports", + "TestGetCveIDsFromChangelog", + "Test_formatKernelVersion/major.minor.build.revision", + "Test_parseSystemInfo/Server", + "Test_detectOSName/err", + "Test_updatePortStatus/update_multi_packages", + "Test_parseWindowsUpdateHistory/happy", + "Test_windows_detectKBsFromKernelVersion/10.0.20348.9999", + "TestParseOSRelease", + "Test_isRunningKernel/SUES_kernel_but_not_running", + "Test_detectScanDest/empty", + "Test_parseInstalledPackages/happy", + "TestParsePkgInfo", + "Test_detectScanDest/dup-addr-port", + "Test_parseGetPackageMSU", + "Test_base_parseLsProcExe", + "Test_parseWindowsUpdateHistory", + "Test_findPortScanSuccessOn/open_empty", + "TestParseLxdPs", + "Test_findPortScanSuccessOn/no_match_port", + "TestParseSystemctlStatus", + "Test_findPortScanSuccessOn/port_empty", + "Test_parseWindowsUpdaterSearch", + "Test_findPortScanSuccessOn/no_match_address", + "Test_parseSWVers/MacOS_Server", + "Test_updatePortStatus/update_match_single_address", + "Test_redhatBase_parseRpmQfLine/permission_denied_will_be_ignored", + "TestParseChangelog/realvnc-vnc-server", + "Test_formatKernelVersion", + "Test_redhatBase_parseRpmQfLine/err", + "Test_windows_detectKBsFromKernelVersion", + "Test_base_parseLsOf/lsof-duplicate-port", + "Test_redhatBase_rebootRequired", + "Test_base_parseLsOf/lsof", + "Test_parseSystemInfo", + "Test_parseSWVers/Mac_OS_X", + "Test_detectOSName", + "Test_redhatBase_rebootRequired/uek_kernel_needs-reboot", + "Test_parseSWVers/ProductVersion_error", + "TestParsePkgVersion", + "Test_parseSystemInfo/Domain_Controller", + "Test_detectOSName/Windows_Server_2019", + "TestParseIp", + "TestGetUpdatablePackNames", + "TestSplitAptCachePolicy", + "Test_parseSystemInfo/Workstation", + "Test_windows_detectKBsFromKernelVersion/10.0.19045.2130", + "TestParseInstalledPackagesLine", + "Test_parseGetHotfix/happy", + "TestParseIfconfig", + "TestParseApkVersion", + "Test_isRunningKernel/kernel_is_kernel-debug,_but_pack_is_kernel", + "TestViaHTTP", + "Test_detectOSName/Windows_Server_2022", + "TestParseSSHScan", + "Test_parseWindowsUpdaterSearch/happy", + "TestParseCheckRestart", + "Test_parseSWVers/MacOS", + "TestParseDockerPs", + "Test_findPortScanSuccessOn/single_match", + "Test_windows_detectKBsFromKernelVersion/10.0.19045.2129", + "Test_base_parseGrepProcMap", + "TestParseNeedsRestarting", + "Test_parseGetComputerInfo/happy", + "Test_findPortScanSuccessOn", + "Test_parseSWVers", + "Test_redhatBase_rebootRequired/uek_kernel_no-reboot" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-082", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_future-architect__vuls-83bcca6e669ba2e4102f26c4a2b52f78c7861f1a", + "selectedInstanceId": "instance_future-architect__vuls-83bcca6e669ba2e4102f26c4a2b52f78c7861f1a", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "future-architect/vuls", + "baseCommit": "a124518d78779cd9daefd92bb66b25da37516363", + "language": "go", + "tags": [ + "go", + "core_feat", + "security_feat", + "integration_feat", + "performance_feat", + "security_knowledge", + "infrastructure_knowledge", + "networking_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "future-architect.vuls-future-architect__vuls-83bcca6e669ba2e4102f26c4a2b52f78c7861f1a", + "beforeRepoSetCommand": "git reset --hard a124518d78779cd9daefd92bb66b25da37516363\ngit clean -fd \ngit checkout a124518d78779cd9daefd92bb66b25da37516363 \ngit checkout 83bcca6e669ba2e4102f26c4a2b52f78c7861f1a -- scan/base_test.go", + "selectedTestFiles": [ + "TestSplitIntoBlocks", + "Test_detectScanDest/asterisk", + "Test_base_parseListenPorts/empty", + "Test_base_parseLsProcExe/systemd", + "Test_base_parseLsOf/lsof", + "Test_updatePortStatus/nil_affected_procs", + "Test_matchListenPorts", + "Test_detectScanDest/dup-addr", + "TestGetUpdatablePackNames", + "Test_updatePortStatus/nil_listen_ports", + "TestParseYumCheckUpdateLinesAmazon", + "TestParseSystemctlStatus", + "Test_detectScanDest", + "Test_base_parseGrepProcMap", + "Test_detectScanDest/single-addr", + "TestParseChangelog/vlc", + "TestParseDockerPs", + "TestParseYumCheckUpdateLine", + "Test_base_parseListenPorts/normal", + "Test_debian_parseGetPkgName", + "Test_debian_parseGetPkgName/success", + "Test_base_parseLsProcExe", + "TestParseIp", + "Test_detectScanDest/empty", + "Test_matchListenPorts/open_empty", + "TestGetChangelogCache", + "TestParseYumCheckUpdateLines", + "TestDecorateCmd", + "TestParseBlock", + "Test_matchListenPorts/single_match", + "Test_matchListenPorts/asterisk_match", + "TestParsePkgInfo", + "TestScanUpdatablePackages", + "Test_updatePortStatus/update_multi_packages", + "TestIsRunningKernelSUSE", + "TestParseIfconfig", + "TestParseInstalledPackagesLinesRedhat", + "Test_updatePortStatus/update_match_single_address", + "Test_updatePortStatus/update_match_asterisk", + "Test_base_parseListenPorts", + "TestParseChangelog/realvnc-vnc-server", + "Test_updatePortStatus", + "TestScanUpdatablePackage", + "Test_base_parseLsOf", + "Test_updatePortStatus/update_match_multi_address", + "TestParseAptCachePolicy", + "TestIsRunningKernelRedHatLikeLinux", + "TestGetCveIDsFromChangelog", + "TestSplitAptCachePolicy", + "Test_base_parseListenPorts/ipv6_loopback", + "TestIsAwsInstanceID", + "TestParseLxdPs", + "Test_detectScanDest/multi-addr", + "Test_matchListenPorts/no_match_address", + "Test_matchListenPorts/no_match_port", + "TestParseScanedPackagesLineRedhat", + "Test_matchListenPorts/port_empty", + "TestParseChangelog", + "TestParseNeedsRestarting", + "Test_base_parseListenPorts/asterisk", + "TestViaHTTP", + "Test_base_parseGrepProcMap/systemd", + "TestParseApkVersion", + "TestParseOSRelease", + "TestParseApkInfo", + "TestParsePkgVersion", + "TestParseCheckRestart" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-083", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_future-architect__vuls-86b60e1478e44d28b1aff6b9ac7e95ceb05bc5fc", + "selectedInstanceId": "instance_future-architect__vuls-86b60e1478e44d28b1aff6b9ac7e95ceb05bc5fc", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "future-architect/vuls", + "baseCommit": "42fdc08933d2b60adc555972b32ae599386b0b9c", + "language": "go", + "tags": [ + "go", + "core_feat", + "back_end_knowledge", + "infrastructure_knowledge", + "devops_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "future-architect.vuls-future-architect__vuls-86b60e1478e44d28b1aff6b9ac7e95ceb05bc5fc", + "beforeRepoSetCommand": "git reset --hard 42fdc08933d2b60adc555972b32ae599386b0b9c\ngit clean -fd \ngit checkout 42fdc08933d2b60adc555972b32ae599386b0b9c \ngit checkout 86b60e1478e44d28b1aff6b9ac7e95ceb05bc5fc -- config/tomlloader_test.go", + "selectedTestFiles": [ + "TestPortScanConf_getScanTechniques/multiple", + "TestEOL_IsStandardSupportEnded/freebsd_11_supported", + "TestEOL_IsStandardSupportEnded/Oracle_Linux_8_supported", + "TestEOL_IsStandardSupportEnded/Fedora_35_supported", + "TestEOL_IsStandardSupportEnded/Ubuntu_21.10_supported", + "TestScanModule_IsZero/not_zero", + "TestEOL_IsStandardSupportEnded/Oracle_Linux_7_supported", + "TestEOL_IsStandardSupportEnded/Fedora_32_eol_on_2022-5-17", + "TestPortScanConf_IsZero/not_zero", + "TestEOL_IsStandardSupportEnded/Oracle_Linux_9_not_found", + "TestEOL_IsStandardSupportEnded/RHEL6_eol", + "TestEOL_IsStandardSupportEnded/Fedora_35_eol_on_2022-12-7", + "TestEOL_IsStandardSupportEnded/CentOS_stream8_supported", + "TestEOL_IsStandardSupportEnded/Fedora_33_supported", + "TestEOL_IsStandardSupportEnded/Ubuntu_18.04_ext_supported", + "Test_majorDotMinor/major_dot_minor_dot_release", + "TestEOL_IsStandardSupportEnded/Alpine_3.15_supported", + "TestEOL_IsStandardSupportEnded/Ubuntu_12.10_not_found", + "TestEOL_IsStandardSupportEnded/amazon_linux_2_supported", + "TestEOL_IsStandardSupportEnded/Fedora_32_supported", + "TestEOL_IsStandardSupportEnded/Ubuntu_18.04_supported", + "Test_majorDotMinor/empty", + "TestScanModule_validate", + "TestPortScanConf_getScanTechniques", + "TestToCpeURI", + "TestScanModule_validate/err", + "TestEOL_IsStandardSupportEnded/freebsd_12_supported", + "TestEOL_IsStandardSupportEnded/Ubuntu_22.04_supported", + "Test_majorDotMinor/major", + "TestEOL_IsStandardSupportEnded/Rocky_Linux_9_Not_Found", + "TestEOL_IsStandardSupportEnded/Alpine_3.9_eol", + "TestEOL_IsStandardSupportEnded/Ubuntu_20.04_ext_supported", + "TestPortScanConf_getScanTechniques/unknown", + "Test_majorDotMinor", + "TestEOL_IsStandardSupportEnded/Ubuntu_20.10_supported", + "TestEOL_IsStandardSupportEnded/Alpine_3.14_supported", + "TestEOL_IsStandardSupportEnded/Debian_10_supported", + "TestPortScanConf_IsZero/zero", + "TestEOL_IsStandardSupportEnded", + "TestEOL_IsStandardSupportEnded/Alpine_3.12_supported", + "TestEOL_IsStandardSupportEnded/Ubuntu_16.04_supported", + "TestEOL_IsStandardSupportEnded/Alpine_3.11_supported", + "TestEOL_IsStandardSupportEnded/amazon_linux_2022_supported", + "TestEOL_IsStandardSupportEnded/Ubuntu_20.04_supported", + "TestEOL_IsStandardSupportEnded/CentOS_6_eol", + "TestEOL_IsStandardSupportEnded/Debian_11_supported", + "TestEOL_IsStandardSupportEnded/Rocky_Linux_8_supported", + "TestScanModule_IsZero/zero", + "TestHosts", + "TestEOL_IsStandardSupportEnded/CentOS_8_supported", + "TestEOL_IsStandardSupportEnded/Debian_9_supported", + "TestEOL_IsStandardSupportEnded/Alma_Linux_8_supported", + "TestEOL_IsStandardSupportEnded/Alpine_3.16_not_found", + "TestPortScanConf_IsZero", + "TestEOL_IsStandardSupportEnded/Alma_Linux_10_Not_Found", + "TestPortScanConf_getScanTechniques/nil", + "TestDistro_MajorVersion", + "TestEOL_IsStandardSupportEnded/freebsd_10_eol", + "TestEOL_IsStandardSupportEnded/RHEL8_supported", + "TestEOL_IsStandardSupportEnded/CentOS_stream10_Not_Found", + "TestEOL_IsStandardSupportEnded/Ubuntu_14.04_eol", + "TestEOL_IsStandardSupportEnded/Debian_8_supported", + "TestEOL_IsStandardSupportEnded/RHEL9_supported", + "TestEOL_IsStandardSupportEnded/Ubuntu_14.10_eol", + "TestEOL_IsStandardSupportEnded/alpine_3.10_supported", + "TestEOL_IsStandardSupportEnded/amazon_linux_1_eol_on_2023-6-30", + "TestEOL_IsStandardSupportEnded/Alma_Linux_9_supported", + "TestEOL_IsStandardSupportEnded/Ubuntu_21.04_supported", + "TestScanModule_validate/valid", + "TestEOL_IsStandardSupportEnded/RHEL7_supported", + "Test_majorDotMinor/major_dot_minor", + "TestEOL_IsStandardSupportEnded/Fedora_32_eol_on_2021-5-25", + "TestPortScanConf_getScanTechniques/single", + "TestEOL_IsStandardSupportEnded/RHEL10_not_found", + "TestSyslogConfValidate", + "TestEOL_IsStandardSupportEnded/Fedora_33_eol_on_2021-5-26", + "TestEOL_IsStandardSupportEnded/Oracle_Linux_6_eol", + "TestEOL_IsStandardSupportEnded/freebsd_13_supported", + "TestEOL_IsStandardSupportEnded/Fedora_34_supported", + "TestScanModule_IsZero", + "TestEOL_IsStandardSupportEnded/freebsd_11_eol_on_2021-9-30", + "TestEOL_IsStandardSupportEnded/Debian_12_is_not_supported_yet", + "TestEOL_IsStandardSupportEnded/amazon_linux_1_supported", + "TestEOL_IsStandardSupportEnded/CentOS_stream9_supported", + "TestEOL_IsStandardSupportEnded/CentOS_7_supported" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-084", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_future-architect__vuls-c11ba27509f733d7d280bdf661cbbe2e7a99df4c", + "selectedInstanceId": "instance_future-architect__vuls-c11ba27509f733d7d280bdf661cbbe2e7a99df4c", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "future-architect/vuls", + "baseCommit": "8a611f9ba6d470cbaa3b781c92c96f5fc9f0b3ba", + "language": "go", + "tags": [ + "go", + "data_bug", + "ui_ux_bug", + "back_end_knowledge", + "security_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "future-architect.vuls-future-architect__vuls-c11ba27509f733d7d280bdf661cbbe2e7a99df4c", + "beforeRepoSetCommand": "git reset --hard 8a611f9ba6d470cbaa3b781c92c96f5fc9f0b3ba\ngit clean -fd \ngit checkout 8a611f9ba6d470cbaa3b781c92c96f5fc9f0b3ba \ngit checkout c11ba27509f733d7d280bdf661cbbe2e7a99df4c -- models/library_test.go", + "selectedTestFiles": [ + "TestFormatMaxCvssScore", + "TestSummaries", + "TestExcept", + "TestFilterIgnorePkgsContainer", + "TestLibraryScanners_Find/miss", + "TestCvss3Scores", + "TestDistroAdvisories_AppendIfMissing/duplicate_no_append", + "TestStorePackageStatueses", + "TestVulnInfo_AttackVector/3.1:N", + "TestPackage_FormatVersionFromTo/nfy2", + "TestToSortedSlice", + "TestCvss2Scores", + "TestFilterByCvssOver", + "TestVulnInfo_AttackVector/2.0:L", + "TestPackage_FormatVersionFromTo", + "TestMerge", + "TestDistroAdvisories_AppendIfMissing/append", + "TestVendorLink", + "TestPackage_FormatVersionFromTo/nfy3", + "TestTitles", + "TestPackage_FormatVersionFromTo/nfy", + "TestCountGroupBySeverity", + "TestPackage_FormatVersionFromTo/fixed", + "TestVulnInfo_AttackVector/3.0:N", + "TestLibraryScanners_Find", + "TestVulnInfo_AttackVector", + "TestSortPackageStatues", + "TestPackage_FormatVersionFromTo/nfy#01", + "TestFilterIgnoreCveIDsContainer", + "TestFilterIgnoreCveIDs", + "TestVulnInfo_AttackVector/2.0:N", + "TestAppendIfMissing", + "TestMergeNewVersion", + "TestFilterUnfixed", + "TestIsDisplayUpdatableNum", + "TestSourceLinks", + "TestLibraryScanners_Find/multi_file", + "TestAddBinaryName", + "TestMaxCvssScores", + "TestFindByBinName", + "TestSortByConfiden", + "TestFilterIgnorePkgs", + "TestMaxCvss2Scores", + "TestDistroAdvisories_AppendIfMissing", + "TestVulnInfo_AttackVector/2.0:A", + "TestMaxCvss3Scores", + "TestLibraryScanners_Find/single_file" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-085", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_gravitational__teleport-0ac7334939981cf85b9591ac295c3816954e287e", + "selectedInstanceId": "instance_gravitational__teleport-0ac7334939981cf85b9591ac295c3816954e287e", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "gravitational/teleport", + "baseCommit": "42beccb27e0e5797a10db05bf126e529273d2d95", + "language": "go", + "tags": [ + "go", + "performance_enh", + "code_quality_enh", + "back_end_knowledge", + "database_knowledge", + "performance_knowledge", + "authentication_authorization_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "gravitational.teleport-gravitational__teleport-0ac7334939981cf85b9591ac295c3816954e287e", + "beforeRepoSetCommand": "git reset --hard 42beccb27e0e5797a10db05bf126e529273d2d95\ngit clean -fd \ngit checkout 42beccb27e0e5797a10db05bf126e529273d2d95 \ngit checkout 0ac7334939981cf85b9591ac295c3816954e287e -- lib/srv/db/access_test.go lib/srv/db/ha_test.go", + "selectedTestFiles": [ + "TestAccessMySQL/has_access_to_nothing", + "TestProxyProtocolPostgres", + "TestAccessPostgres/has_access_to_nothing", + "TestAuthTokens/correct_Postgres_Cloud_SQL_IAM_auth_token", + "TestAuthTokens/correct_Postgres_Redshift_IAM_auth_token", + "TestAccessPostgres", + "TestAuthTokens/incorrect_MySQL_RDS_IAM_auth_token", + "TestAccessMySQL/access_allowed_to_specific_user", + "TestHA", + "TestAuthTokens/incorrect_Postgres_Redshift_IAM_auth_token", + "TestAuditMySQL", + "TestAccessDisabled", + "TestAccessMySQL", + "TestAccessMySQL/has_access_to_all_database_users", + "TestAccessPostgres/access_allowed_to_specific_user/database", + "TestAuthTokens/incorrect_Postgres_RDS_IAM_auth_token", + "TestAuthTokens/correct_MySQL_RDS_IAM_auth_token", + "TestAccessPostgres/no_access_to_users", + "TestProxyClientDisconnectDueToIdleConnection", + "TestAuthTokens/incorrect_Postgres_Cloud_SQL_IAM_auth_token", + "TestDatabaseServerStart", + "TestProxyClientDisconnectDueToCertExpiration", + "TestAccessMySQL/access_denied_to_specific_user", + "TestAccessPostgres/has_access_to_all_database_names_and_users", + "TestAuthTokens", + "TestAccessPostgres/no_access_to_databases", + "TestAuditPostgres", + "TestAccessPostgres/access_denied_to_specific_user/database", + "TestProxyProtocolMySQL", + "TestAuthTokens/correct_Postgres_RDS_IAM_auth_token" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-086", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_gravitational__teleport-3fa6904377c006497169945428e8197158667910-v626ec2a48416b10a88641359a169d99e935ff037", + "selectedInstanceId": "instance_gravitational__teleport-3fa6904377c006497169945428e8197158667910-v626ec2a48416b10a88641359a169d99e935ff037", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "gravitational/teleport", + "baseCommit": "481158d6310e36e3c1115e25ab3fdf1c1ed45e60", + "language": "go", + "tags": [ + "go", + "major_bug", + "code_quality_enh", + "back_end_knowledge", + "infrastructure_knowledge", + "devops_knowledge", + "security_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "gravitational.teleport-gravitational__teleport-3fa6904377c006497169945428e8197158667910-v626ec2a48416b10a88641359a169d99e935ff03", + "beforeRepoSetCommand": "git reset --hard 481158d6310e36e3c1115e25ab3fdf1c1ed45e60\ngit clean -fd \ngit checkout 481158d6310e36e3c1115e25ab3fdf1c1ed45e60 \ngit checkout 3fa6904377c006497169945428e8197158667910 -- lib/kube/proxy/forwarder_test.go", + "selectedTestFiles": [ + "TestParseResourcePath//api/v1/watch/namespaces/kube-system/pods/foo", + "TestParseResourcePath", + "TestParseResourcePath//apis", + "TestParseResourcePath//api/v1/pods", + "TestParseResourcePath/#00", + "TestParseResourcePath//api/v1/namespaces/kube-system/pods", + "TestAuthenticate/unsupported_user_type", + "TestParseResourcePath//apis/apps/v1/", + "TestAuthenticate/local_user_and_cluster,_no_tunnel", + "TestParseResourcePath//apis/apps", + "TestParseResourcePath//api/v1/", + "TestParseResourcePath//api/v1/watch/pods", + "TestParseResourcePath//api/v1/nodes/foo/proxy/bar", + "TestParseResourcePath//apis/apiregistration.k8s.io/v1/apiservices/foo/status", + "TestAuthenticate/remote_user_and_local_cluster", + "TestParseResourcePath//apis/", + "TestAuthenticate/unknown_kubernetes_cluster_in_local_cluster", + "TestAuthenticate/custom_kubernetes_cluster_in_remote_cluster", + "TestGetKubeCreds/proxy_service,_no_kube_creds", + "TestParseResourcePath//apis/apps/v1", + "TestGetKubeCreds/kubernetes_service,_no_kube_creds", + "TestGetKubeCreds/proxy_service,_with_kube_creds", + "TestAuthenticate/local_user_and_cluster,_no_kubeconfig", + "TestParseResourcePath//api/v1/namespaces/kube-system/pods/foo/exec", + "TestParseResourcePath//", + "TestAuthenticate/local_user_and_remote_cluster", + "TestParseResourcePath//apis/apps/", + "TestParseResourcePath//api/v1/namespaces/kube-system", + "TestAuthenticate/remote_user_and_remote_cluster", + "TestParseResourcePath//api/", + "TestParseResourcePath//api", + "TestAuthenticate/authorization_failure", + "TestAuthenticate/custom_kubernetes_cluster_in_local_cluster", + "TestAuthenticate/local_user_and_remote_cluster,_no_tunnel", + "TestParseResourcePath//apis/rbac.authorization.k8s.io/v1/watch/clusterroles", + "TestAuthenticate/local_user_and_cluster", + "TestParseResourcePath//apis/rbac.authorization.k8s.io/v1/clusterroles/foo", + "TestParseResourcePath//api/v1/watch/namespaces/kube-system/pods", + "TestAuthenticate/kube_users_passed_in_request", + "TestParseResourcePath//api/v1", + "TestAuthenticate/local_user_and_remote_cluster,_no_kubeconfig", + "TestParseResourcePath//apis/rbac.authorization.k8s.io/v1/watch/clusterroles/foo", + "TestAuthenticate", + "TestGetKubeCreds", + "TestParseResourcePath//api/v1/namespaces/kube-system/pods/foo", + "TestGetKubeCreds/kubernetes_service,_with_kube_creds", + "TestParseResourcePath//apis/rbac.authorization.k8s.io/v1/clusterroles", + "TestParseResourcePath//api/v1/watch/namespaces/kube-system" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-087", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_gravitational__teleport-4d0117b50dc8cdb91c94b537a4844776b224cd3d", + "selectedInstanceId": "instance_gravitational__teleport-4d0117b50dc8cdb91c94b537a4844776b224cd3d", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "gravitational/teleport", + "baseCommit": "e2412a7c37314c9482d856f407f737c9e5b34bce", + "language": "go", + "tags": [ + "go", + "integration_feat", + "security_feat", + "performance_feat", + "back_end_knowledge", + "database_knowledge", + "infrastructure_knowledge", + "performance_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "gravitational.teleport-gravitational__teleport-4d0117b50dc8cdb91c94b537a4844776b224cd3d", + "beforeRepoSetCommand": "git reset --hard e2412a7c37314c9482d856f407f737c9e5b34bce\ngit clean -fd \ngit checkout e2412a7c37314c9482d856f407f737c9e5b34bce \ngit checkout 4d0117b50dc8cdb91c94b537a4844776b224cd3d -- lib/events/dynamoevents/dynamoevents_test.go", + "selectedTestFiles": [ + "TestDateRangeGenerator", + "TestDynamoevents" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-088", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_gravitational__teleport-5dca072bb4301f4579a15364fcf37cc0c39f7f6c", + "selectedInstanceId": "instance_gravitational__teleport-5dca072bb4301f4579a15364fcf37cc0c39f7f6c", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "gravitational/teleport", + "baseCommit": "d45e26cec6dc799afbb9eac4381d70f95c21c41f", + "language": "go", + "tags": [ + "go", + "performance_bug", + "edge_case_bug", + "integration_bug", + "security_bug", + "security_knowledge", + "authentication_authorization_knowledge", + "back_end_knowledge", + "networking_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "gravitational.teleport-gravitational__teleport-5dca072bb4301f4579a15364fcf37cc0c39f7f6c", + "beforeRepoSetCommand": "git reset --hard d45e26cec6dc799afbb9eac4381d70f95c21c41f\ngit clean -fd \ngit checkout d45e26cec6dc799afbb9eac4381d70f95c21c41f \ngit checkout 5dca072bb4301f4579a15364fcf37cc0c39f7f6c -- lib/kube/proxy/forwarder_test.go lib/kube/proxy/server_test.go", + "selectedTestFiles": [ + "TestParseResourcePath//api/v1/watch/namespaces/kube-system/pods/foo", + "TestAuthenticate/local_user_and_remote_cluster,_no_local_kube_users_or_groups", + "TestAuthenticate/unsupported_user_type", + "TestParseResourcePath//api/v1/watch/pods", + "TestParseResourcePath//apis/rbac.authorization.k8s.io/v1/watch/clusterroles/foo", + "TestParseResourcePath//api/v1/namespaces/kube-system/pods/foo", + "TestParseResourcePath//api/", + "TestAuthenticate/local_user_and_remote_cluster,_no_tunnel", + "TestParseResourcePath//apis/apps/v1", + "TestAuthenticate/custom_kubernetes_cluster_in_remote_cluster", + "TestAuthenticate/authorization_failure", + "TestParseResourcePath//api/v1/namespaces/kube-system", + "TestParseResourcePath//apis", + "TestParseResourcePath//api/v1/watch/namespaces/kube-system/pods", + "TestParseResourcePath//apis/apps", + "TestParseResourcePath//apis/apps/v1/", + "TestMTLSClientCAs/1000_CAs", + "TestParseResourcePath//api/v1/namespaces/kube-system/pods", + "TestParseResourcePath//api", + "TestParseResourcePath//apis/rbac.authorization.k8s.io/v1/clusterroles", + "TestParseResourcePath//apis/apiregistration.k8s.io/v1/apiservices/foo/status", + "TestParseResourcePath/#00", + "TestMTLSClientCAs/100_CAs", + "TestParseResourcePath//", + "TestAuthenticate/remote_user_and_local_cluster", + "TestAuthenticate/local_user_and_cluster,_no_kubeconfig", + "TestAuthenticate/local_user_and_cluster", + "TestAuthenticate", + "TestParseResourcePath//api/v1/pods", + "TestAuthenticate/local_user_and_remote_cluster,_no_kubeconfig", + "TestAuthenticate/remote_user_and_remote_cluster", + "TestAuthenticate/custom_kubernetes_cluster_in_local_cluster", + "TestParseResourcePath//api/v1/watch/namespaces/kube-system", + "TestParseResourcePath//apis/rbac.authorization.k8s.io/v1/clusterroles/foo", + "TestParseResourcePath", + "TestParseResourcePath//apis/rbac.authorization.k8s.io/v1/watch/clusterroles", + "TestParseResourcePath//apis/", + "TestMTLSClientCAs/1_CA", + "TestParseResourcePath//api/v1/namespaces/kube-system/pods/foo/exec", + "TestParseResourcePath//api/v1", + "TestParseResourcePath//api/v1/nodes/foo/proxy/bar", + "TestAuthenticate/unknown_kubernetes_cluster_in_local_cluster", + "TestAuthenticate/kube_users_passed_in_request", + "TestParseResourcePath//apis/apps/", + "TestAuthenticate/local_user_and_cluster,_no_tunnel", + "TestMTLSClientCAs", + "TestAuthenticate/local_user_and_remote_cluster", + "TestParseResourcePath//api/v1/" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-089", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_gravitational__teleport-629dc432eb191ca479588a8c49205debb83e80e2", + "selectedInstanceId": "instance_gravitational__teleport-629dc432eb191ca479588a8c49205debb83e80e2", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "gravitational/teleport", + "baseCommit": "c12ce90636425daef82da2a7fe2a495e9064d1f8", + "language": "go", + "tags": [ + "go", + "core_feat", + "code_quality_enh", + "back_end_knowledge", + "infrastructure_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "gravitational.teleport-gravitational__teleport-629dc432eb191ca479588a8c49205debb83e80e2", + "beforeRepoSetCommand": "git reset --hard c12ce90636425daef82da2a7fe2a495e9064d1f8\ngit clean -fd \ngit checkout c12ce90636425daef82da2a7fe2a495e9064d1f8 \ngit checkout 629dc432eb191ca479588a8c49205debb83e80e2 -- lib/utils/concurrentqueue/queue_test.go", + "selectedTestFiles": [ + "TestBackpressure", + "TestOrdering" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-090", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_gravitational__teleport-6a14edcf1ff010172fdbac622d0a474ed6af46de", + "selectedInstanceId": "instance_gravitational__teleport-6a14edcf1ff010172fdbac622d0a474ed6af46de", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "gravitational/teleport", + "baseCommit": "9c041361f9ff3270d252609d5aa92eb1251cd8d7", + "language": "go", + "tags": [ + "go", + "major_bug", + "regression_bug", + "back_end_knowledge", + "networking_knowledge", + "authentication_authorization_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "gravitational.teleport-gravitational__teleport-6a14edcf1ff010172fdbac622d0a474ed6af46de", + "beforeRepoSetCommand": "git reset --hard 9c041361f9ff3270d252609d5aa92eb1251cd8d7\ngit clean -fd \ngit checkout 9c041361f9ff3270d252609d5aa92eb1251cd8d7 \ngit checkout 6a14edcf1ff010172fdbac622d0a474ed6af46de -- lib/auth/trustedcluster_test.go", + "selectedTestFiles": [ + "TestRemoteClusterStatus" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-091", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_gravitational__teleport-6eaaf3a27e64f4ef4ef855bd35d7ec338cf17460-v626ec2a48416b10a88641359a169d99e935ff037", + "selectedInstanceId": "instance_gravitational__teleport-6eaaf3a27e64f4ef4ef855bd35d7ec338cf17460-v626ec2a48416b10a88641359a169d99e935ff037", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "gravitational/teleport", + "baseCommit": "a51596d8d779935e1dfa8d0fabce39d9edd91457", + "language": "go", + "tags": [ + "go", + "core_feat", + "performance_feat", + "api_feat", + "back_end_knowledge", + "api_knowledge", + "performance_knowledge", + "devops_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "gravitational.teleport-gravitational__teleport-6eaaf3a27e64f4ef4ef855bd35d7ec338cf17460-v626ec2a48416b10a88641359a169d99e935ff03", + "beforeRepoSetCommand": "git reset --hard a51596d8d779935e1dfa8d0fabce39d9edd91457\ngit clean -fd \ngit checkout a51596d8d779935e1dfa8d0fabce39d9edd91457 \ngit checkout 6eaaf3a27e64f4ef4ef855bd35d7ec338cf17460 -- lib/benchmark/linear_test.go", + "selectedTestFiles": [ + "TestGetBenchmarkNotEvenMultiple", + "TestGetBenchmark", + "TestValidateConfig" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-092", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_gravitational__teleport-769b4b5eec7286b7b14e179f2cc52e6b15d2d9f3-v626ec2a48416b10a88641359a169d99e935ff037", + "selectedInstanceId": "instance_gravitational__teleport-769b4b5eec7286b7b14e179f2cc52e6b15d2d9f3-v626ec2a48416b10a88641359a169d99e935ff037", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "gravitational/teleport", + "baseCommit": "1879807d3c30626ec02a730133ac5592cece310c", + "language": "go", + "tags": [ + "go", + "core_feat", + "integration_feat", + "customization_feat", + "security_feat", + "back_end_knowledge", + "security_knowledge", + "authentication_authorization_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "gravitational.teleport-gravitational__teleport-769b4b5eec7286b7b14e179f2cc52e6b15d2d9f3-v626ec2a48416b10a88641359a169d99e935ff03", + "beforeRepoSetCommand": "git reset --hard 1879807d3c30626ec02a730133ac5592cece310c\ngit clean -fd \ngit checkout 1879807d3c30626ec02a730133ac5592cece310c \ngit checkout 769b4b5eec7286b7b14e179f2cc52e6b15d2d9f3 -- tool/tsh/tsh_test.go", + "selectedTestFiles": [ + "TestMakeClient", + "TestReadClusterFlag", + "TestOptions/Incomplete_option", + "TestReadClusterFlag/nothing_set", + "TestFormatConnectCommand/no_default_user/database_are_specified", + "TestOptions/Forward_Agent_Local", + "TestFormatConnectCommand/unsupported_database_protocol", + "TestReadClusterFlag/TELEPORT_SITE_and_TELEPORT_CLUSTER_and_CLI_flag_is_set,_prefer_CLI", + "TestFetchDatabaseCreds", + "TestFormatConnectCommand/default_user_is_specified", + "TestOptions/Forward_Agent_No", + "TestOptions/Forward_Agent_InvalidValue", + "TestFormatConnectCommand/default_database_is_specified", + "TestReadClusterFlag/TELEPORT_SITE_set", + "TestOIDCLogin", + "TestRelogin", + "TestOptions/AddKeysToAgent_Invalid_Value", + "TestFormatConnectCommand/default_user/database_are_specified", + "TestOptions/Forward_Agent_Yes", + "TestFormatConnectCommand", + "TestOptions/Equals_Sign_Delimited", + "TestReadClusterFlag/TELEPORT_SITE_and_TELEPORT_CLUSTER_set,_prefer_TELEPORT_CLUSTER", + "TestOptions/Invalid_key", + "TestIdentityRead", + "TestOptions/Space_Delimited", + "TestReadClusterFlag/TELEPORT_CLUSTER_set", + "TestFailedLogin", + "TestOptions" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-093", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_gravitational__teleport-78b0d8c72637df1129fb6ff84fc49ef4b5ab1288", + "selectedInstanceId": "instance_gravitational__teleport-78b0d8c72637df1129fb6ff84fc49ef4b5ab1288", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "gravitational/teleport", + "baseCommit": "19c5768873a50f1f84f0906add418c29c68f4bc2", + "language": "go", + "tags": [ + "go", + "performance_feat", + "performance_enh", + "code_quality_enh", + "back_end_knowledge", + "infrastructure_knowledge", + "performance_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "gravitational.teleport-gravitational__teleport-78b0d8c72637df1129fb6ff84fc49ef4b5ab1288", + "beforeRepoSetCommand": "git reset --hard 19c5768873a50f1f84f0906add418c29c68f4bc2\ngit clean -fd \ngit checkout 19c5768873a50f1f84f0906add418c29c68f4bc2 \ngit checkout 78b0d8c72637df1129fb6ff84fc49ef4b5ab1288 -- lib/cache/fncache_test.go", + "selectedTestFiles": [ + "TestFnCacheCancellation", + "TestApplicationServers", + "TestDatabases", + "TestDatabaseServers", + "TestFnCacheSanity", + "TestState" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-094", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_gravitational__teleport-87a593518b6ce94624f6c28516ce38cc30cbea5a", + "selectedInstanceId": "instance_gravitational__teleport-87a593518b6ce94624f6c28516ce38cc30cbea5a", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "gravitational/teleport", + "baseCommit": "49ab2a7bfd935317818b20a5bd0b59ac4d5289c9", + "language": "go", + "tags": [ + "go", + "core_feat", + "api_feat", + "integration_feat", + "database_knowledge", + "cloud_knowledge", + "infrastructure_knowledge", + "authentication_authorization_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "gravitational.teleport-gravitational__teleport-87a593518b6ce94624f6c28516ce38cc30cbea5a", + "beforeRepoSetCommand": "git reset --hard 49ab2a7bfd935317818b20a5bd0b59ac4d5289c9\ngit clean -fd \ngit checkout 49ab2a7bfd935317818b20a5bd0b59ac4d5289c9 \ngit checkout 87a593518b6ce94624f6c28516ce38cc30cbea5a -- lib/client/conntest/database/sqlserver_test.go", + "selectedTestFiles": [ + "TestMySQLErrors/invalid_database_user", + "TestSQLServerErrors", + "TestPostgresErrors", + "TestPostgresErrors/invalid_database_error", + "TestMySQLErrors/invalid_database_user_access_denied", + "TestMySQLPing", + "TestMySQLErrors/connection_refused_string", + "TestMySQLErrors/connection_refused_host_not_allowed", + "TestMySQLErrors", + "TestSQLServerErrors/ConnectionRefusedError", + "TestPostgresErrors/invalid_user_error", + "TestSQLServerErrors/InvalidDatabaseNameError", + "TestMySQLErrors/invalid_database_name_access_denied", + "TestMySQLErrors/invalid_database_name", + "TestMySQLErrors/connection_refused_host_blocked", + "TestPostgresErrors/connection_refused_error", + "TestPostgresPing", + "TestSQLServerPing", + "TestSQLServerErrors/InvalidDatabaseUserError" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-095", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_gravitational__teleport-96019ce0be7a2c8e36363f359eb7c943b41dde70", + "selectedInstanceId": "instance_gravitational__teleport-96019ce0be7a2c8e36363f359eb7c943b41dde70", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "gravitational/teleport", + "baseCommit": "d05df372ce37abd7c190f9fbb68192a773330e63", + "language": "go", + "tags": [ + "go", + "major_bug", + "compatibility_bug", + "back_end_knowledge", + "api_knowledge", + "infrastructure_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "gravitational.teleport-gravitational__teleport-96019ce0be7a2c8e36363f359eb7c943b41dde70", + "beforeRepoSetCommand": "git reset --hard d05df372ce37abd7c190f9fbb68192a773330e63\ngit clean -fd \ngit checkout d05df372ce37abd7c190f9fbb68192a773330e63 \ngit checkout 96019ce0be7a2c8e36363f359eb7c943b41dde70 -- lib/kube/proxy/forwarder_test.go", + "selectedTestFiles": [ + "TestAuthenticate/local_user_and_remote_cluster,_no_tunnel", + "TestAuthenticate/unknown_kubernetes_cluster_in_local_cluster", + "TestAuthenticate" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-096", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_gravitational__teleport-b4e7cd3a5e246736d3fe8d6886af55030b232277", + "selectedInstanceId": "instance_gravitational__teleport-b4e7cd3a5e246736d3fe8d6886af55030b232277", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "gravitational/teleport", + "baseCommit": "85addfbd36943a4b655e1a4241979789e8b4ff22", + "language": "go", + "tags": [ + "go", + "critical_bug", + "security_bug", + "back_end_knowledge", + "security_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "gravitational.teleport-gravitational__teleport-b4e7cd3a5e246736d3fe8d6886af55030b232277", + "beforeRepoSetCommand": "git reset --hard 85addfbd36943a4b655e1a4241979789e8b4ff22\ngit clean -fd \ngit checkout 85addfbd36943a4b655e1a4241979789e8b4ff22 \ngit checkout b4e7cd3a5e246736d3fe8d6886af55030b232277 -- lib/backend/report_test.go", + "selectedTestFiles": [ + "TestReporterTopRequestsLimit", + "TestBuildKeyLabel", + "TestInit", + "TestParams" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-097", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_gravitational__teleport-b5d8169fc0a5e43fee2616c905c6d32164654dc6", + "selectedInstanceId": "instance_gravitational__teleport-b5d8169fc0a5e43fee2616c905c6d32164654dc6", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "gravitational/teleport", + "baseCommit": "103f3de22f3986d4f73d931b1e433e011b26a488", + "language": "go", + "tags": [ + "go", + "code_quality_enh", + "refactoring_enh", + "security_enh", + "back_end_knowledge", + "security_knowledge", + "api_knowledge", + "authentication_authorization_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "gravitational.teleport-gravitational__teleport-b5d8169fc0a5e43fee2616c905c6d32164654dc6", + "beforeRepoSetCommand": "git reset --hard 103f3de22f3986d4f73d931b1e433e011b26a488\ngit clean -fd \ngit checkout 103f3de22f3986d4f73d931b1e433e011b26a488 \ngit checkout b5d8169fc0a5e43fee2616c905c6d32164654dc6 -- lib/auth/init_test.go", + "selectedTestFiles": [ + "TestMigrateOSS/User", + "TestMigrateOSS", + "TestMigrateOSS/EmptyCluster", + "TestMigrateOSS/TrustedCluster" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-098", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_gravitational__teleport-ba6c4a135412c4296dd5551bd94042f0dc024504-v626ec2a48416b10a88641359a169d99e935ff037", + "selectedInstanceId": "instance_gravitational__teleport-ba6c4a135412c4296dd5551bd94042f0dc024504-v626ec2a48416b10a88641359a169d99e935ff037", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "gravitational/teleport", + "baseCommit": "396812cebff3ebfd075bbed04acefc65e787a537", + "language": "go", + "tags": [ + "go", + "dev_ops_enh", + "performance_enh", + "back_end_knowledge", + "infrastructure_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "gravitational.teleport-gravitational__teleport-ba6c4a135412c4296dd5551bd94042f0dc024504-v626ec2a48416b10a88641359a169d99e935ff03", + "beforeRepoSetCommand": "git reset --hard 396812cebff3ebfd075bbed04acefc65e787a537\ngit clean -fd \ngit checkout 396812cebff3ebfd075bbed04acefc65e787a537 \ngit checkout ba6c4a135412c4296dd5551bd94042f0dc024504 -- lib/service/service_test.go lib/service/state_test.go", + "selectedTestFiles": [ + "TestMonitor/ok_event_remains_in_recovering_state_because_not_enough_time_passed", + "TestProcessStateGetState/multiple_components,_one_is_recovering", + "TestProcessStateGetState/multiple_components,_one_is_degraded", + "TestMonitor/ok_event_in_new_component_causes_overall_OK_state", + "TestMonitor/ok_event_after_enough_time_causes_OK_state", + "TestMonitor", + "TestProcessStateGetState/one_component_in_stateOK", + "TestMonitor/ok_event_in_one_component_keeps_overall_status_degraded_due_to_other_component", + "TestMonitor/degraded_event_in_a_new_component_causes_degraded_state", + "TestMonitor/degraded_event_causes_degraded_state", + "TestProcessStateGetState/multiple_components,_one_is_starting", + "TestProcessStateGetState/no_components", + "TestMonitor/ok_event_in_new_component_causes_overall_recovering_state", + "TestProcessStateGetState/multiple_components_in_stateOK", + "TestProcessStateGetState", + "TestMonitor/ok_event_causes_recovering_state" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-099", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_gravitational__teleport-e6681abe6a7113cfd2da507f05581b7bdf398540-v626ec2a48416b10a88641359a169d99e935ff037", + "selectedInstanceId": "instance_gravitational__teleport-e6681abe6a7113cfd2da507f05581b7bdf398540-v626ec2a48416b10a88641359a169d99e935ff037", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "gravitational/teleport", + "baseCommit": "be52cd325af38f53fa6b6f869bc10b88160e06e2", + "language": "go", + "tags": [ + "go", + "core_feat", + "back_end_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "gravitational.teleport-gravitational__teleport-e6681abe6a7113cfd2da507f05581b7bdf398540-v626ec2a48416b10a88641359a169d99e935ff03", + "beforeRepoSetCommand": "git reset --hard be52cd325af38f53fa6b6f869bc10b88160e06e2\ngit clean -fd \ngit checkout be52cd325af38f53fa6b6f869bc10b88160e06e2 \ngit checkout e6681abe6a7113cfd2da507f05581b7bdf398540 -- lib/events/auditwriter_test.go lib/events/emitter_test.go lib/events/stream_test.go", + "selectedTestFiles": [ + "TestStreamerCompleteEmpty", + "TestAsyncEmitter/Receive", + "TestProtoStreamer/small_load_test_with_some_uneven_numbers", + "TestProtoStreamer/one_event_using_the_whole_part", + "TestWriterEmitter", + "TestAsyncEmitter/Slow", + "TestProtoStreamer/no_events", + "TestExport", + "TestAuditWriter", + "TestAsyncEmitter/Close", + "TestProtoStreamer/5MB_similar_to_S3_min_size_in_bytes", + "TestProtoStreamer", + "TestAuditWriter/ResumeMiddle", + "TestAsyncEmitter", + "TestAuditWriter/Backoff", + "TestAuditWriter/Session", + "TestAuditLog", + "TestAuditWriter/ResumeStart", + "TestProtoStreamer/get_a_part_per_message" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-100", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_gravitational__teleport-eda668c30d9d3b56d9c69197b120b01013611186", + "selectedInstanceId": "instance_gravitational__teleport-eda668c30d9d3b56d9c69197b120b01013611186", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "gravitational/teleport", + "baseCommit": "4f6f52f86d65f506d1884a9f56bcd919a8744734", + "language": "go", + "tags": [ + "go", + "major_bug", + "regression_bug", + "integration_bug", + "back_end_knowledge", + "cloud_knowledge", + "devops_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "gravitational.teleport-gravitational__teleport-eda668c30d9d3b56d9c69197b120b01013611186", + "beforeRepoSetCommand": "git reset --hard 4f6f52f86d65f506d1884a9f56bcd919a8744734\ngit clean -fd \ngit checkout 4f6f52f86d65f506d1884a9f56bcd919a8744734 \ngit checkout eda668c30d9d3b56d9c69197b120b01013611186 -- lib/kube/proxy/forwarder_test.go", + "selectedTestFiles": [ + "TestAuthenticate/local_user_and_cluster,_no_tunnel", + "TestParseResourcePath//", + "TestGetKubeCreds/proxy_service,_no_kube_creds", + "TestAuthenticate/local_user_and_remote_cluster,_no_local_kube_users_or_groups", + "TestGetServerInfo/GetServerInfo_gets_correct_public_addr_with_PublicAddr_set", + "TestParseResourcePath//apis/", + "TestMTLSClientCAs", + "TestAuthenticate/custom_kubernetes_cluster_in_local_cluster", + "TestParseResourcePath", + "TestParseResourcePath//api/v1/namespaces/kube-system/pods/foo/exec", + "TestParseResourcePath//apis", + "TestGetKubeCreds/kubernetes_service,_with_kube_creds", + "TestParseResourcePath//api/v1/namespaces/kube-system", + "TestAuthenticate/remote_user_and_local_cluster", + "TestParseResourcePath//apis/apps/v1", + "TestParseResourcePath//apis/rbac.authorization.k8s.io/v1/clusterroles/foo", + "TestAuthenticate/local_user_and_remote_cluster,_no_kubeconfig", + "TestGetKubeCreds/legacy_proxy_service,_with_kube_creds", + "TestParseResourcePath//api/v1/watch/namespaces/kube-system/pods/foo", + "TestAuthenticate/unsupported_user_type", + "TestNewClusterSessionLocal", + "TestClusterSessionDial", + "TestParseResourcePath//api/v1", + "TestParseResourcePath//apis/rbac.authorization.k8s.io/v1/clusterroles", + "TestParseResourcePath//apis/rbac.authorization.k8s.io/v1/watch/clusterroles/foo", + "TestParseResourcePath//apis/apps/", + "TestGetServerInfo/GetServerInfo_gets_listener_addr_with_PublicAddr_unset", + "TestParseResourcePath//api/v1/namespaces/kube-system/pods", + "TestParseResourcePath//apis/rbac.authorization.k8s.io/v1/watch/clusterroles", + "TestAuthenticate/remote_user_and_remote_cluster", + "TestNewClusterSessionRemote", + "TestParseResourcePath//api/v1/watch/namespaces/kube-system", + "TestParseResourcePath//api/v1/", + "TestParseResourcePath//api/v1/namespaces/kube-system/pods/foo", + "TestParseResourcePath//api/", + "TestParseResourcePath//api/v1/pods", + "TestGetKubeCreds/legacy_proxy_service,_no_kube_creds", + "TestParseResourcePath//api/v1/watch/namespaces/kube-system/pods", + "TestAuthenticate/local_user_and_remote_cluster", + "TestParseResourcePath//apis/apiregistration.k8s.io/v1/apiservices/foo/status", + "TestGetKubeCreds/Missing_cluster_does_not_fail_operation", + "TestParseResourcePath//apis/apps/v1/", + "TestParseResourcePath//apis/apps", + "TestGetKubeCreds/proxy_service,_with_kube_creds", + "TestGetKubeCreds", + "TestAuthenticate/authorization_failure", + "TestParseResourcePath/#00", + "TestMTLSClientCAs/1_CA", + "TestNewClusterSessionDirect", + "TestParseResourcePath//api", + "TestGetKubeCreds/kubernetes_service,_no_kube_creds", + "TestAuthenticate/local_user_and_cluster", + "TestAuthenticate/kube_users_passed_in_request", + "TestAuthenticate", + "TestParseResourcePath//api/v1/watch/pods", + "TestAuthenticate/local_user_and_cluster,_no_kubeconfig", + "TestAuthenticate/local_user_and_remote_cluster,_no_tunnel", + "TestGetServerInfo", + "TestMTLSClientCAs/100_CAs", + "TestParseResourcePath//api/v1/nodes/foo/proxy/bar", + "TestAuthenticate/custom_kubernetes_cluster_in_remote_cluster", + "TestMTLSClientCAs/1000_CAs", + "TestAuthenticate/unknown_kubernetes_cluster_in_local_cluster" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-101", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_gravitational__teleport-f432a71a13e698b6e1c4672a2e9e9c1f32d35c12", + "selectedInstanceId": "instance_gravitational__teleport-f432a71a13e698b6e1c4672a2e9e9c1f32d35c12", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "gravitational/teleport", + "baseCommit": "a4a6a3e42d90918341224dd7f2ba45856b1b6c70", + "language": "go", + "tags": [ + "go", + "code_quality_enh", + "refactoring_enh", + "dev_ops_enh", + "devops_knowledge", + "back_end_knowledge", + "infrastructure_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "gravitational.teleport-gravitational__teleport-f432a71a13e698b6e1c4672a2e9e9c1f32d35c12", + "beforeRepoSetCommand": "git reset --hard a4a6a3e42d90918341224dd7f2ba45856b1b6c70\ngit clean -fd \ngit checkout a4a6a3e42d90918341224dd7f2ba45856b1b6c70 \ngit checkout f432a71a13e698b6e1c4672a2e9e9c1f32d35c12 -- lib/auth/keystore/keystore_test.go", + "selectedTestFiles": [ + "TestKeyStore/raw_keystore", + "TestKeyStore" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-102", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_gravitational__teleport-fb0ab2b9b771377a689fd0d0374777c251e58bbf", + "selectedInstanceId": "instance_gravitational__teleport-fb0ab2b9b771377a689fd0d0374777c251e58bbf", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "gravitational/teleport", + "baseCommit": "b58ad484649e51b439ba11df387e25e23e8296d1", + "language": "go", + "tags": [ + "go", + "performance_feat", + "analytics_feat", + "code_quality_enh", + "back_end_knowledge", + "devops_knowledge", + "infrastructure_knowledge", + "performance_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "gravitational.teleport-gravitational__teleport-fb0ab2b9b771377a689fd0d0374777c251e58bbf", + "beforeRepoSetCommand": "git reset --hard b58ad484649e51b439ba11df387e25e23e8296d1\ngit clean -fd \ngit checkout b58ad484649e51b439ba11df387e25e23e8296d1 \ngit checkout fb0ab2b9b771377a689fd0d0374777c251e58bbf -- lib/utils/circular_buffer_test.go", + "selectedTestFiles": [ + "TestChConn", + "TestReadAtMost", + "TestNewCircularBuffer", + "TestEscapeControl", + "TestSlice", + "TestCircularBuffer_Data", + "TestUtils", + "TestFilterAWSRoles", + "TestConsolefLongComponent", + "TestAllowNewlines" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-103", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_gravitational__teleport-fd2959260ef56463ad8afa4c973f47a50306edd4", + "selectedInstanceId": "instance_gravitational__teleport-fd2959260ef56463ad8afa4c973f47a50306edd4", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "gravitational/teleport", + "baseCommit": "025143d85654c604656571c363d0c7b9a6579f62", + "language": "go", + "tags": [ + "go", + "api_feat", + "customization_feat", + "ui_ux_feat", + "cloud_knowledge", + "infrastructure_knowledge", + "devops_knowledge", + "api_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "gravitational.teleport-gravitational__teleport-fd2959260ef56463ad8afa4c973f47a50306edd4", + "beforeRepoSetCommand": "git reset --hard 025143d85654c604656571c363d0c7b9a6579f62\ngit clean -fd \ngit checkout 025143d85654c604656571c363d0c7b9a6579f62 \ngit checkout fd2959260ef56463ad8afa4c973f47a50306edd4 -- lib/config/configuration_test.go", + "selectedTestFiles": [ + "TestProxyKube/new_and_old_formats", + "TestProxyKube/legacy_format,_no_local_cluster", + "TestProxyKube/not_configured", + "TestProxyKube/legacy_format,_with_local_cluster", + "TestProxyKube", + "TestProxyKube/new_format_and_old_explicitly_disabled", + "TestProxyKube/new_format" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-104", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_internetarchive__openlibrary-08ac40d050a64e1d2646ece4959af0c42bf6b7b5-v0f5aece3601a5b4419f7ccec1dbda2071be28ee4", + "selectedInstanceId": "instance_internetarchive__openlibrary-08ac40d050a64e1d2646ece4959af0c42bf6b7b5-v0f5aece3601a5b4419f7ccec1dbda2071be28ee4", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "internetarchive/openlibrary", + "baseCommit": "1e32ae3ec085dc3ab8611b88b56baf6354fa0762", + "language": "python", + "tags": [ + "python", + "core_feat", + "code_quality_enh", + "integration_feat", + "back_end_knowledge", + "database_knowledge", + "api_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "internetarchive.openlibrary-internetarchive__openlibrary-08ac40d050a64e1d2646ece4959af0c42bf6b7b5-v0f5aece3601a5b4419f7ccec1dbda", + "beforeRepoSetCommand": "git reset --hard 1e32ae3ec085dc3ab8611b88b56baf6354fa0762\ngit clean -fd \ngit checkout 1e32ae3ec085dc3ab8611b88b56baf6354fa0762 \ngit checkout 08ac40d050a64e1d2646ece4959af0c42bf6b7b5 -- openlibrary/catalog/add_book/tests/test_add_book.py openlibrary/catalog/marc/tests/test_data/bin_expect/ithaca_college_75002321.json openlibrary/catalog/marc/tests/test_data/bin_expect/lesnoirsetlesrou0000garl_meta.json openlibrary/catalog/marc/tests/test_data/bin_expect/memoirsofjosephf00fouc_meta.json openlibrary/catalog/marc/tests/test_data/bin_expect/warofrebellionco1473unit_meta.json openlibrary/catalog/marc/tests/test_data/xml_expect/00schlgoog.json openlibrary/catalog/marc/tests/test_data/xml_expect/warofrebellionco1473unit.json", + "selectedTestFiles": [ + "openlibrary/catalog/marc/tests/test_parse.py", + "openlibrary/catalog/add_book/tests/test_add_book.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-105", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_internetarchive__openlibrary-0a90f9f0256e4f933523e9842799e39f95ae29ce-v76304ecdb3a5954fcf13feb710e8c40fcf24b73c", + "selectedInstanceId": "instance_internetarchive__openlibrary-0a90f9f0256e4f933523e9842799e39f95ae29ce-v76304ecdb3a5954fcf13feb710e8c40fcf24b73c", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "internetarchive/openlibrary", + "baseCommit": "5c6c22f3d2edf2f1b10f5dc335e32cb6a5f40341", + "language": "python", + "tags": [ + "python", + "major_bug", + "data_bug", + "integration_bug", + "edge_case_bug", + "back_end_knowledge", + "api_knowledge", + "database_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "internetarchive.openlibrary-internetarchive__openlibrary-0a90f9f0256e4f933523e9842799e39f95ae29ce-v76304ecdb3a5954fcf13feb710e8c", + "beforeRepoSetCommand": "git reset --hard 5c6c22f3d2edf2f1b10f5dc335e32cb6a5f40341\ngit clean -fd \ngit checkout 5c6c22f3d2edf2f1b10f5dc335e32cb6a5f40341 \ngit checkout 0a90f9f0256e4f933523e9842799e39f95ae29ce -- openlibrary/plugins/importapi/tests/test_code.py openlibrary/plugins/upstream/tests/test_utils.py openlibrary/utils/tests/test_isbn.py", + "selectedTestFiles": [ + "openlibrary/plugins/importapi/tests/test_code.py", + "openlibrary/utils/tests/test_isbn.py", + "openlibrary/plugins/upstream/tests/test_utils.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-106", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_internetarchive__openlibrary-0dc5b20fa186f9714f8a838178597e69f549d026-v2d9a6c849c60ed19fd0858ce9e40b7cc8e097e59", + "selectedInstanceId": "instance_internetarchive__openlibrary-0dc5b20fa186f9714f8a838178597e69f549d026-v2d9a6c849c60ed19fd0858ce9e40b7cc8e097e59", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "internetarchive/openlibrary", + "baseCommit": "bd9d2a04efbbaec1575faa02a02eea995badf7f0", + "language": "python", + "tags": [ + "python", + "core_feat", + "refactoring_enh", + "back_end_knowledge", + "ml_ai_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "internetarchive.openlibrary-internetarchive__openlibrary-0dc5b20fa186f9714f8a838178597e69f549d026-v2d9a6c849c60ed19fd0858ce9e40b", + "beforeRepoSetCommand": "git reset --hard bd9d2a04efbbaec1575faa02a02eea995badf7f0\ngit clean -fd \ngit checkout bd9d2a04efbbaec1575faa02a02eea995badf7f0 \ngit checkout 0dc5b20fa186f9714f8a838178597e69f549d026 -- openlibrary/catalog/marc/tests/test_data/bin_expect/880_Nihon_no_chasho.json openlibrary/catalog/marc/tests/test_data/bin_expect/880_arabic_french_many_linkages.json", + "selectedTestFiles": [ + "openlibrary/catalog/marc/tests/test_parse.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-107", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_internetarchive__openlibrary-30bc73a1395fba2300087c7f307e54bb5372b60a-v76304ecdb3a5954fcf13feb710e8c40fcf24b73c", + "selectedInstanceId": "instance_internetarchive__openlibrary-30bc73a1395fba2300087c7f307e54bb5372b60a-v76304ecdb3a5954fcf13feb710e8c40fcf24b73c", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "internetarchive/openlibrary", + "baseCommit": "540853735859789920caf9ee78a762ebe14f6103", + "language": "python", + "tags": [ + "python", + "performance_feat", + "performance_enh", + "code_quality_enh", + "back_end_knowledge", + "api_knowledge", + "database_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "internetarchive.openlibrary-internetarchive__openlibrary-30bc73a1395fba2300087c7f307e54bb5372b60a-v76304ecdb3a5954fcf13feb710e8c", + "beforeRepoSetCommand": "git reset --hard 540853735859789920caf9ee78a762ebe14f6103\ngit clean -fd \ngit checkout 540853735859789920caf9ee78a762ebe14f6103 \ngit checkout 30bc73a1395fba2300087c7f307e54bb5372b60a -- openlibrary/coverstore/tests/test_archive.py", + "selectedTestFiles": [ + "openlibrary/coverstore/tests/test_archive.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-108", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_internetarchive__openlibrary-4b7ea2977be2747496ba792a678940baa985f7ea-v0f5aece3601a5b4419f7ccec1dbda2071be28ee4", + "selectedInstanceId": "instance_internetarchive__openlibrary-4b7ea2977be2747496ba792a678940baa985f7ea-v0f5aece3601a5b4419f7ccec1dbda2071be28ee4", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "internetarchive/openlibrary", + "baseCommit": "86d9b8a203cab5bc446614f3cbffd77b4751084a", + "language": "python", + "tags": [ + "python", + "api_feat", + "core_feat", + "integration_feat", + "api_knowledge", + "back_end_knowledge", + "database_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "internetarchive.openlibrary-internetarchive__openlibrary-4b7ea2977be2747496ba792a678940baa985f7ea-v0f5aece3601a5b4419f7ccec1dbda", + "beforeRepoSetCommand": "git reset --hard 86d9b8a203cab5bc446614f3cbffd77b4751084a\ngit clean -fd \ngit checkout 86d9b8a203cab5bc446614f3cbffd77b4751084a \ngit checkout 4b7ea2977be2747496ba792a678940baa985f7ea -- openlibrary/catalog/add_book/tests/test_load_book.py", + "selectedTestFiles": [ + "openlibrary/catalog/add_book/tests/test_load_book.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-109", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_internetarchive__openlibrary-53e02a22972e9253aeded0e1981e6845e1e521fe-vfa6ff903cb27f336e17654595dd900fa943dcd91", + "selectedInstanceId": "instance_internetarchive__openlibrary-53e02a22972e9253aeded0e1981e6845e1e521fe-vfa6ff903cb27f336e17654595dd900fa943dcd91", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "internetarchive/openlibrary", + "baseCommit": "2e2140e23dde91a233a91e9a7d04648450387721", + "language": "python", + "tags": [ + "python", + "refactoring_enh", + "back_end_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "internetarchive.openlibrary-internetarchive__openlibrary-53e02a22972e9253aeded0e1981e6845e1e521fe-vfa6ff903cb27f336e17654595dd90", + "beforeRepoSetCommand": "git reset --hard 2e2140e23dde91a233a91e9a7d04648450387721\ngit clean -fd \ngit checkout 2e2140e23dde91a233a91e9a7d04648450387721 \ngit checkout 53e02a22972e9253aeded0e1981e6845e1e521fe -- openlibrary/tests/solr/test_update_work.py", + "selectedTestFiles": [ + "openlibrary/tests/solr/test_update_work.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-110", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_internetarchive__openlibrary-5c6c22f3d2edf2f1b10f5dc335e32cb6a5f40341-v76304ecdb3a5954fcf13feb710e8c40fcf24b73c", + "selectedInstanceId": "instance_internetarchive__openlibrary-5c6c22f3d2edf2f1b10f5dc335e32cb6a5f40341-v76304ecdb3a5954fcf13feb710e8c40fcf24b73c", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "internetarchive/openlibrary", + "baseCommit": "31d6ecf3c04c8e365e557f2a9a6a65df2b8d01a3", + "language": "python", + "tags": [ + "python", + "code_quality_enh", + "back_end_knowledge", + "devops_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "internetarchive.openlibrary-internetarchive__openlibrary-5c6c22f3d2edf2f1b10f5dc335e32cb6a5f40341-v76304ecdb3a5954fcf13feb710e8c", + "beforeRepoSetCommand": "git reset --hard 31d6ecf3c04c8e365e557f2a9a6a65df2b8d01a3\ngit clean -fd \ngit checkout 31d6ecf3c04c8e365e557f2a9a6a65df2b8d01a3 \ngit checkout 5c6c22f3d2edf2f1b10f5dc335e32cb6a5f40341 -- openlibrary/plugins/importapi/tests/test_code.py openlibrary/plugins/upstream/tests/test_utils.py", + "selectedTestFiles": [ + "openlibrary/plugins/importapi/tests/test_code.py", + "openlibrary/plugins/upstream/tests/test_utils.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-111", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_internetarchive__openlibrary-5de7de19211e71b29b2f2ba3b1dff2fe065d660f-v08d8e8889ec945ab821fb156c04c7d2e2810debb", + "selectedInstanceId": "instance_internetarchive__openlibrary-5de7de19211e71b29b2f2ba3b1dff2fe065d660f-v08d8e8889ec945ab821fb156c04c7d2e2810debb", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "internetarchive/openlibrary", + "baseCommit": "5f7d8d190e2f0d837545e582fd5db99aae51a979", + "language": "python", + "tags": [ + "python", + "refactoring_enh", + "back_end_knowledge", + "api_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "internetarchive.openlibrary-internetarchive__openlibrary-5de7de19211e71b29b2f2ba3b1dff2fe065d660f-v08d8e8889ec945ab821fb156c04c7", + "beforeRepoSetCommand": "git reset --hard 5f7d8d190e2f0d837545e582fd5db99aae51a979\ngit clean -fd \ngit checkout 5f7d8d190e2f0d837545e582fd5db99aae51a979 \ngit checkout 5de7de19211e71b29b2f2ba3b1dff2fe065d660f -- openlibrary/tests/core/test_models.py", + "selectedTestFiles": [ + "openlibrary/tests/core/test_models.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-112", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_internetarchive__openlibrary-9bdfd29fac883e77dcbc4208cab28c06fd963ab2-v76304ecdb3a5954fcf13feb710e8c40fcf24b73c", + "selectedInstanceId": "instance_internetarchive__openlibrary-9bdfd29fac883e77dcbc4208cab28c06fd963ab2-v76304ecdb3a5954fcf13feb710e8c40fcf24b73c", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "internetarchive/openlibrary", + "baseCommit": "b2086f9bf54a3a8289e558a8f542673e3d01b376", + "language": "python", + "tags": [ + "python", + "edge_case_bug", + "back_end_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "internetarchive.openlibrary-internetarchive__openlibrary-9bdfd29fac883e77dcbc4208cab28c06fd963ab2-v76304ecdb3a5954fcf13feb710e8c", + "beforeRepoSetCommand": "git reset --hard b2086f9bf54a3a8289e558a8f542673e3d01b376\ngit clean -fd \ngit checkout b2086f9bf54a3a8289e558a8f542673e3d01b376 \ngit checkout 9bdfd29fac883e77dcbc4208cab28c06fd963ab2 -- openlibrary/plugins/worksearch/tests/test_worksearch.py", + "selectedTestFiles": [ + "openlibrary/plugins/worksearch/tests/test_worksearch.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-113", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_internetarchive__openlibrary-9cd47f4dc21e273320d9e30d889c864f8cb20ccf-v0f5aece3601a5b4419f7ccec1dbda2071be28ee4", + "selectedInstanceId": "instance_internetarchive__openlibrary-9cd47f4dc21e273320d9e30d889c864f8cb20ccf-v0f5aece3601a5b4419f7ccec1dbda2071be28ee4", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "internetarchive/openlibrary", + "baseCommit": "7fab050b6a99923d9d2efcc2f79311580b88f8a9", + "language": "python", + "tags": [ + "python", + "major_bug", + "performance_bug", + "integration_bug", + "back_end_knowledge", + "api_knowledge", + "infrastructure_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "internetarchive.openlibrary-internetarchive__openlibrary-9cd47f4dc21e273320d9e30d889c864f8cb20ccf-v0f5aece3601a5b4419f7ccec1dbda", + "beforeRepoSetCommand": "git reset --hard 7fab050b6a99923d9d2efcc2f79311580b88f8a9\ngit clean -fd \ngit checkout 7fab050b6a99923d9d2efcc2f79311580b88f8a9 \ngit checkout 9cd47f4dc21e273320d9e30d889c864f8cb20ccf -- openlibrary/catalog/add_book/tests/test_add_book.py", + "selectedTestFiles": [ + "openlibrary/catalog/add_book/tests/test_add_book.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-114", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_internetarchive__openlibrary-c8996ecc40803b9155935fd7ff3b8e7be6c1437c-ve8fc82d8aae8463b752a211156c5b7b59f349237", + "selectedInstanceId": "instance_internetarchive__openlibrary-c8996ecc40803b9155935fd7ff3b8e7be6c1437c-ve8fc82d8aae8463b752a211156c5b7b59f349237", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "internetarchive/openlibrary", + "baseCommit": "f0341c0ba81c790241b782f5103ce5c9a6edf8e3", + "language": "python", + "tags": [ + "python", + "data_bug", + "back_end_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "internetarchive.openlibrary-internetarchive__openlibrary-c8996ecc40803b9155935fd7ff3b8e7be6c1437c-ve8fc82d8aae8463b752a211156c5b", + "beforeRepoSetCommand": "git reset --hard f0341c0ba81c790241b782f5103ce5c9a6edf8e3\ngit clean -fd \ngit checkout f0341c0ba81c790241b782f5103ce5c9a6edf8e3 \ngit checkout c8996ecc40803b9155935fd7ff3b8e7be6c1437c -- openlibrary/catalog/add_book/tests/test_add_book.py openlibrary/tests/catalog/test_utils.py", + "selectedTestFiles": [ + "openlibrary/catalog/add_book/tests/test_add_book.py", + "openlibrary/tests/catalog/test_utils.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-115", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_internetarchive__openlibrary-e1e502986a3b003899a8347ac8a7ff7b08cbfc39-v08d8e8889ec945ab821fb156c04c7d2e2810debb", + "selectedInstanceId": "instance_internetarchive__openlibrary-e1e502986a3b003899a8347ac8a7ff7b08cbfc39-v08d8e8889ec945ab821fb156c04c7d2e2810debb", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "internetarchive/openlibrary", + "baseCommit": "77c16d530b4d5c0f33d68bead2c6b329aee9b996", + "language": "python", + "tags": [ + "python", + "ui_ux_feat", + "core_feat", + "code_quality_enh", + "back_end_knowledge", + "web_knowledge", + "ui_ux_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "internetarchive.openlibrary-internetarchive__openlibrary-e1e502986a3b003899a8347ac8a7ff7b08cbfc39-v08d8e8889ec945ab821fb156c04c7", + "beforeRepoSetCommand": "git reset --hard 77c16d530b4d5c0f33d68bead2c6b329aee9b996\ngit clean -fd \ngit checkout 77c16d530b4d5c0f33d68bead2c6b329aee9b996 \ngit checkout e1e502986a3b003899a8347ac8a7ff7b08cbfc39 -- openlibrary/plugins/upstream/tests/test_table_of_contents.py", + "selectedTestFiles": [ + "openlibrary/plugins/upstream/tests/test_table_of_contents.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-116", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_internetarchive__openlibrary-f343c08f89c772f7ba6c0246f384b9e6c3dc0add-v08d8e8889ec945ab821fb156c04c7d2e2810debb", + "selectedInstanceId": "instance_internetarchive__openlibrary-f343c08f89c772f7ba6c0246f384b9e6c3dc0add-v08d8e8889ec945ab821fb156c04c7d2e2810debb", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "internetarchive/openlibrary", + "baseCommit": "53d376b148897466bb86d5accb51912bbbe9a8ed", + "language": "python", + "tags": [ + "python", + "core_feat", + "customization_feat", + "back_end_knowledge", + "database_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "internetarchive.openlibrary-internetarchive__openlibrary-f343c08f89c772f7ba6c0246f384b9e6c3dc0add-v08d8e8889ec945ab821fb156c04c7", + "beforeRepoSetCommand": "git reset --hard 53d376b148897466bb86d5accb51912bbbe9a8ed\ngit clean -fd \ngit checkout 53d376b148897466bb86d5accb51912bbbe9a8ed \ngit checkout f343c08f89c772f7ba6c0246f384b9e6c3dc0add -- openlibrary/catalog/add_book/tests/test_load_book.py", + "selectedTestFiles": [ + "openlibrary/catalog/add_book/tests/test_load_book.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-117", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_navidrome__navidrome-0488fb92cb02a82924fb1181bf1642f2e87096db", + "selectedInstanceId": "instance_navidrome__navidrome-0488fb92cb02a82924fb1181bf1642f2e87096db", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "navidrome/navidrome", + "baseCommit": "61903facdf5d56277bf57c7aa83bce7fb35b597a", + "language": "go", + "tags": [ + "go", + "performance_bug", + "ui_ux_bug", + "minor_bug", + "web_knowledge", + "api_knowledge", + "ui_ux_knowledge", + "full_stack_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "navidrome.navidrome-navidrome__navidrome-0488fb92cb02a82924fb1181bf1642f2e87096db", + "beforeRepoSetCommand": "git reset --hard 61903facdf5d56277bf57c7aa83bce7fb35b597a\ngit clean -fd \ngit checkout 61903facdf5d56277bf57c7aa83bce7fb35b597a \ngit checkout 0488fb92cb02a82924fb1181bf1642f2e87096db -- core/artwork/artwork_internal_test.go core/artwork/artwork_test.go server/subsonic/media_retrieval_test.go", + "selectedTestFiles": [ + "TestArtwork", + "TestSubsonicApi" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-118", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_navidrome__navidrome-10108c63c9b5bdf2966ffb3239bbfd89683e37b7", + "selectedInstanceId": "instance_navidrome__navidrome-10108c63c9b5bdf2966ffb3239bbfd89683e37b7", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "navidrome/navidrome", + "baseCommit": "aac6e2cb0774aa256c00098b2d88bf8af288da79", + "language": "go", + "tags": [ + "go", + "minor_bug", + "customization_feat", + "back_end_knowledge", + "web_knowledge", + "api_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "navidrome.navidrome-navidrome__navidrome-10108c63c9b5bdf2966ffb3239bbfd89683e37b7", + "beforeRepoSetCommand": "git reset --hard aac6e2cb0774aa256c00098b2d88bf8af288da79\ngit clean -fd \ngit checkout aac6e2cb0774aa256c00098b2d88bf8af288da79 \ngit checkout 10108c63c9b5bdf2966ffb3239bbfd89683e37b7 -- server/serve_index_test.go server/serve_test.go", + "selectedTestFiles": [ + "TestServer" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-119", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_navidrome__navidrome-27875ba2dd1673ddf8affca526b0664c12c3b98b", + "selectedInstanceId": "instance_navidrome__navidrome-27875ba2dd1673ddf8affca526b0664c12c3b98b", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "navidrome/navidrome", + "baseCommit": "28f7ef43c1083fab4f7eabbade6a5bab83615e73", + "language": "go", + "tags": [ + "go", + "core_feat", + "customization_feat", + "integration_feat", + "back_end_knowledge", + "infrastructure_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "navidrome.navidrome-navidrome__navidrome-27875ba2dd1673ddf8affca526b0664c12c3b98b", + "beforeRepoSetCommand": "git reset --hard 28f7ef43c1083fab4f7eabbade6a5bab83615e73\ngit clean -fd \ngit checkout 28f7ef43c1083fab4f7eabbade6a5bab83615e73 \ngit checkout 27875ba2dd1673ddf8affca526b0664c12c3b98b -- server/serve_index_test.go", + "selectedTestFiles": [ + "TestServer" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-120", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_navidrome__navidrome-28389fb05e1523564dfc61fa43ed8eb8a10f938c", + "selectedInstanceId": "instance_navidrome__navidrome-28389fb05e1523564dfc61fa43ed8eb8a10f938c", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "navidrome/navidrome", + "baseCommit": "5d8318f7b362c5b044f892f775d264a5d40e4e24", + "language": "go", + "tags": [ + "go", + "core_feat", + "api_feat", + "code_quality_enh", + "refactoring_enh", + "back_end_knowledge", + "database_knowledge", + "api_knowledge", + "authentication_authorization_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "navidrome.navidrome-navidrome__navidrome-28389fb05e1523564dfc61fa43ed8eb8a10f938c", + "beforeRepoSetCommand": "git reset --hard 5d8318f7b362c5b044f892f775d264a5d40e4e24\ngit clean -fd \ngit checkout 5d8318f7b362c5b044f892f775d264a5d40e4e24 \ngit checkout 28389fb05e1523564dfc61fa43ed8eb8a10f938c -- core/playlists_test.go model/playlists_test.go", + "selectedTestFiles": [ + "TestModel" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-121", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_navidrome__navidrome-3972616585e82305eaf26aa25697b3f5f3082288", + "selectedInstanceId": "instance_navidrome__navidrome-3972616585e82305eaf26aa25697b3f5f3082288", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "navidrome/navidrome", + "baseCommit": "d0ce0303864d6859ee683214baab9c647f7467fe", + "language": "go", + "tags": [ + "go", + "api_feat", + "core_feat", + "back_end_knowledge", + "api_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "navidrome.navidrome-navidrome__navidrome-3972616585e82305eaf26aa25697b3f5f3082288", + "beforeRepoSetCommand": "git reset --hard d0ce0303864d6859ee683214baab9c647f7467fe\ngit clean -fd \ngit checkout d0ce0303864d6859ee683214baab9c647f7467fe \ngit checkout 3972616585e82305eaf26aa25697b3f5f3082288 -- model/criteria/criteria_suite_test.go model/criteria/criteria_test.go model/criteria/operators_test.go", + "selectedTestFiles": [ + "TestCriteria" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-122", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_navidrome__navidrome-7073d18b54da7e53274d11c9e2baef1242e8769e", + "selectedInstanceId": "instance_navidrome__navidrome-7073d18b54da7e53274d11c9e2baef1242e8769e", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "navidrome/navidrome", + "baseCommit": "7fc964aec5d432480d76bb90eee14a09aca5125f", + "language": "go", + "tags": [ + "go", + "code_quality_enh", + "refactoring_enh", + "back_end_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "navidrome.navidrome-navidrome__navidrome-7073d18b54da7e53274d11c9e2baef1242e8769e", + "beforeRepoSetCommand": "git reset --hard 7fc964aec5d432480d76bb90eee14a09aca5125f\ngit clean -fd \ngit checkout 7fc964aec5d432480d76bb90eee14a09aca5125f \ngit checkout 7073d18b54da7e53274d11c9e2baef1242e8769e -- core/agents/lastfm/agent_test.go core/agents/lastfm/client_test.go core/agents/listenbrainz/agent_test.go core/agents/listenbrainz/auth_router_test.go core/agents/listenbrainz/client_test.go core/agents/spotify/client_test.go", + "selectedTestFiles": [ + "TestListenBrainz", + "TestSpotify", + "TestLastFM" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-123", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_navidrome__navidrome-87d4db7638b37eeb754b217440ab7a372f669205", + "selectedInstanceId": "instance_navidrome__navidrome-87d4db7638b37eeb754b217440ab7a372f669205", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "navidrome/navidrome", + "baseCommit": "213ceeca7893d3c85eb688e6e99c09dd6cd7e453", + "language": "go", + "tags": [ + "go", + "core_feat", + "minor_bug", + "back_end_knowledge", + "ui_ux_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "navidrome.navidrome-navidrome__navidrome-87d4db7638b37eeb754b217440ab7a372f669205", + "beforeRepoSetCommand": "git reset --hard 213ceeca7893d3c85eb688e6e99c09dd6cd7e453\ngit clean -fd \ngit checkout 213ceeca7893d3c85eb688e6e99c09dd6cd7e453 \ngit checkout 87d4db7638b37eeb754b217440ab7a372f669205 -- core/artwork_internal_test.go", + "selectedTestFiles": [ + "TestCore" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-124", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_navidrome__navidrome-89b12b34bea5687c70e4de2109fd1e7330bb2ba2", + "selectedInstanceId": "instance_navidrome__navidrome-89b12b34bea5687c70e4de2109fd1e7330bb2ba2", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "navidrome/navidrome", + "baseCommit": "4e0177ee5340b888126092d1146c1c7b6a92fed8", + "language": "go", + "tags": [ + "go", + "edge_case_bug", + "back_end_knowledge", + "api_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "navidrome.navidrome-navidrome__navidrome-89b12b34bea5687c70e4de2109fd1e7330bb2ba2", + "beforeRepoSetCommand": "git reset --hard 4e0177ee5340b888126092d1146c1c7b6a92fed8\ngit clean -fd \ngit checkout 4e0177ee5340b888126092d1146c1c7b6a92fed8 \ngit checkout 89b12b34bea5687c70e4de2109fd1e7330bb2ba2 -- core/agents/lastfm_test.go tests/fake_http_client.go tests/fixtures/lastfm.artist.error3.json tests/fixtures/lastfm.artist.error6.json tests/fixtures/lastfm.artist.getinfo.unknown.json tests/fixtures/lastfm.artist.getsimilar.unknown.json tests/fixtures/lastfm.artist.gettoptracks.unknown.json utils/lastfm/client_test.go utils/lastfm/responses_test.go", + "selectedTestFiles": [ + "TestAgents", + "TestLastFM" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-125", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_navidrome__navidrome-8d56ec898e776e7e53e352cb9b25677975787ffc", + "selectedInstanceId": "instance_navidrome__navidrome-8d56ec898e776e7e53e352cb9b25677975787ffc", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "navidrome/navidrome", + "baseCommit": "5064cb2a46e4b7ca6a834b3c48a409eec8ca1830", + "language": "go", + "tags": [ + "go", + "minor_bug", + "back_end_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "navidrome.navidrome-navidrome__navidrome-8d56ec898e776e7e53e352cb9b25677975787ffc", + "beforeRepoSetCommand": "git reset --hard 5064cb2a46e4b7ca6a834b3c48a409eec8ca1830\ngit clean -fd \ngit checkout 5064cb2a46e4b7ca6a834b3c48a409eec8ca1830 \ngit checkout 8d56ec898e776e7e53e352cb9b25677975787ffc -- persistence/album_repository_test.go", + "selectedTestFiles": [ + "TestPersistence" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-126", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_navidrome__navidrome-97434c1789a6444b30aae5ff5aa124a96a88f504", + "selectedInstanceId": "instance_navidrome__navidrome-97434c1789a6444b30aae5ff5aa124a96a88f504", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "navidrome/navidrome", + "baseCommit": "f8ee6db72a015f33b41879d1ba253eeff8884bfd", + "language": "go", + "tags": [ + "go", + "core_feat", + "api_feat", + "code_quality_enh", + "back_end_knowledge", + "database_knowledge", + "api_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "navidrome.navidrome-navidrome__navidrome-97434c1789a6444b30aae5ff5aa124a96a88f504", + "beforeRepoSetCommand": "git reset --hard f8ee6db72a015f33b41879d1ba253eeff8884bfd\ngit clean -fd \ngit checkout f8ee6db72a015f33b41879d1ba253eeff8884bfd \ngit checkout 97434c1789a6444b30aae5ff5aa124a96a88f504 -- core/players_test.go", + "selectedTestFiles": [ + "TestCore" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-127", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_navidrome__navidrome-9c3b4561652a15846993d477003e111f0df0c585", + "selectedInstanceId": "instance_navidrome__navidrome-9c3b4561652a15846993d477003e111f0df0c585", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "navidrome/navidrome", + "baseCommit": "23bebe4e06124becf1000e88472ae71a6ca7de4c", + "language": "go", + "tags": [ + "go", + "core_feat", + "dev_ops_enh", + "back_end_knowledge", + "devops_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "navidrome.navidrome-navidrome__navidrome-9c3b4561652a15846993d477003e111f0df0c585", + "beforeRepoSetCommand": "git reset --hard 23bebe4e06124becf1000e88472ae71a6ca7de4c\ngit clean -fd \ngit checkout 23bebe4e06124becf1000e88472ae71a6ca7de4c \ngit checkout 9c3b4561652a15846993d477003e111f0df0c585 -- log/formatters_test.go", + "selectedTestFiles": [ + "TestLevelThreshold/errorLogLevel", + "TestEntryMessage", + "TestLevelThreshold", + "TestLevels", + "TestEntryDataValues", + "TestLevels/undefinedAcceptedLevels", + "TestLevelThreshold/unknownLogLevel", + "TestEntryDataValues/map_value", + "TestLevels/definedAcceptedLevels", + "TestInvalidRegex", + "TestEntryDataValues/match_on_key", + "TestEntryDataValues/string_value", + "TestLog" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-128", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_navidrome__navidrome-b65e76293a917ee2dfc5d4b373b1c62e054d0dca", + "selectedInstanceId": "instance_navidrome__navidrome-b65e76293a917ee2dfc5d4b373b1c62e054d0dca", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "navidrome/navidrome", + "baseCommit": "5f6f74ff2dde934aae2f2def911506d8c4ba66e1", + "language": "go", + "tags": [ + "go", + "core_feat", + "code_quality_enh", + "api_feat", + "web_knowledge", + "api_knowledge", + "full_stack_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "navidrome.navidrome-navidrome__navidrome-b65e76293a917ee2dfc5d4b373b1c62e054d0dca", + "beforeRepoSetCommand": "git reset --hard 5f6f74ff2dde934aae2f2def911506d8c4ba66e1\ngit clean -fd \ngit checkout 5f6f74ff2dde934aae2f2def911506d8c4ba66e1 \ngit checkout b65e76293a917ee2dfc5d4b373b1c62e054d0dca -- server/events/diode_test.go server/events/sse_test.go server/subsonic/middlewares_test.go", + "selectedTestFiles": [ + "TestSubsonicApi", + "TestEvents" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-129", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_navidrome__navidrome-d21932bd1b2379b0ebca2d19e5d8bae91040268a", + "selectedInstanceId": "instance_navidrome__navidrome-d21932bd1b2379b0ebca2d19e5d8bae91040268a", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "navidrome/navidrome", + "baseCommit": "c72add516a0f260e83a289c2355b2e74071311e0", + "language": "go", + "tags": [ + "go", + "core_feat", + "code_quality_enh", + "refactoring_enh", + "database_knowledge", + "back_end_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "navidrome.navidrome-navidrome__navidrome-d21932bd1b2379b0ebca2d19e5d8bae91040268a", + "beforeRepoSetCommand": "git reset --hard c72add516a0f260e83a289c2355b2e74071311e0\ngit clean -fd \ngit checkout c72add516a0f260e83a289c2355b2e74071311e0 \ngit checkout d21932bd1b2379b0ebca2d19e5d8bae91040268a -- persistence/sql_smartplaylist_test.go", + "selectedTestFiles": [ + "TestPersistence" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-130", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_navidrome__navidrome-d5df102f9f97c21715c756069c9e141da2a422dc", + "selectedInstanceId": "instance_navidrome__navidrome-d5df102f9f97c21715c756069c9e141da2a422dc", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "navidrome/navidrome", + "baseCommit": "20271df4fb0b94e201ed5e4b6501d591aa8cd813", + "language": "go", + "tags": [ + "go", + "api_feat", + "core_feat", + "ui_ux_feat", + "back_end_knowledge", + "api_knowledge", + "security_knowledge", + "ui_ux_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "navidrome.navidrome-navidrome__navidrome-d5df102f9f97c21715c756069c9e141da2a422dc", + "beforeRepoSetCommand": "git reset --hard 20271df4fb0b94e201ed5e4b6501d591aa8cd813\ngit clean -fd \ngit checkout 20271df4fb0b94e201ed5e4b6501d591aa8cd813 \ngit checkout d5df102f9f97c21715c756069c9e141da2a422dc -- core/share_test.go", + "selectedTestFiles": [ + "TestCore" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-131", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_navidrome__navidrome-e12a14a87d392ac70ee4cc8079e3c3e0103dbcb2", + "selectedInstanceId": "instance_navidrome__navidrome-e12a14a87d392ac70ee4cc8079e3c3e0103dbcb2", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "navidrome/navidrome", + "baseCommit": "0079a9b93873621fa0bed5439a0f753f48e081b5", + "language": "go", + "tags": [ + "go", + "core_feat", + "performance_enh", + "ui_ux_enh", + "ui_ux_knowledge", + "full_stack_knowledge", + "database_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "navidrome.navidrome-navidrome__navidrome-e12a14a87d392ac70ee4cc8079e3c3e0103dbcb2", + "beforeRepoSetCommand": "git reset --hard 0079a9b93873621fa0bed5439a0f753f48e081b5\ngit clean -fd \ngit checkout 0079a9b93873621fa0bed5439a0f753f48e081b5 \ngit checkout e12a14a87d392ac70ee4cc8079e3c3e0103dbcb2 -- scanner/metadata/ffmpeg/ffmpeg_test.go scanner/metadata/metadata_test.go scanner/metadata/taglib/taglib_test.go", + "selectedTestFiles": [ + "TestFFMpeg", + "TestMetadata", + "TestTagLib" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-132", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_navidrome__navidrome-f78257235ec3429ef42af6687738cd327ec77ce8", + "selectedInstanceId": "instance_navidrome__navidrome-f78257235ec3429ef42af6687738cd327ec77ce8", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "navidrome/navidrome", + "baseCommit": "1a6a284bc124d579c44053a6b0435cd20ead715c", + "language": "go", + "tags": [ + "go", + "core_feat", + "customization_feat", + "back_end_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "navidrome.navidrome-navidrome__navidrome-f78257235ec3429ef42af6687738cd327ec77ce8", + "beforeRepoSetCommand": "git reset --hard 1a6a284bc124d579c44053a6b0435cd20ead715c\ngit clean -fd \ngit checkout 1a6a284bc124d579c44053a6b0435cd20ead715c \ngit checkout f78257235ec3429ef42af6687738cd327ec77ce8 -- log/log_test.go", + "selectedTestFiles": [ + "TestLevelThreshold", + "TestInvalidRegex", + "TestLevelThreshold/errorLogLevel", + "TestEntryDataValues/map_value", + "TestEntryMessage", + "TestLevels/undefinedAcceptedLevels", + "TestEntryDataValues/match_on_key", + "TestLevels", + "TestEntryDataValues", + "TestEntryDataValues/string_value", + "TestLevels/definedAcceptedLevels", + "TestLog", + "TestLevelThreshold/unknownLogLevel" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-133", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_navidrome__navidrome-fa85e2a7816a6fe3829a4c0d8e893e982b0985da", + "selectedInstanceId": "instance_navidrome__navidrome-fa85e2a7816a6fe3829a4c0d8e893e982b0985da", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "navidrome/navidrome", + "baseCommit": "5360283bb0368c5226b301f99b7095a54407b053", + "language": "go", + "tags": [ + "go", + "data_bug", + "edge_case_bug", + "integration_bug", + "back_end_knowledge", + "database_knowledge", + "authentication_authorization_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "navidrome.navidrome-navidrome__navidrome-fa85e2a7816a6fe3829a4c0d8e893e982b0985da", + "beforeRepoSetCommand": "git reset --hard 5360283bb0368c5226b301f99b7095a54407b053\ngit clean -fd \ngit checkout 5360283bb0368c5226b301f99b7095a54407b053 \ngit checkout fa85e2a7816a6fe3829a4c0d8e893e982b0985da -- core/players_test.go persistence/persistence_suite_test.go persistence/persistence_test.go persistence/player_repository_test.go", + "selectedTestFiles": [ + "TestCore", + "TestPersistence" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-134", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_NodeBB__NodeBB-6489e9fd9ed16ea743cc5627f4d86c72fbdb3a8a-v2c59007b1005cd5cd14cbb523ca5229db1fd2dd8", + "selectedInstanceId": "instance_nodebb__nodebb-6489e9fd9ed16ea743cc5627f4d86c72fbdb3a8a-v2c59007b1005cd5cd14cbb523ca5229db1fd2dd8", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "NodeBB/NodeBB", + "baseCommit": "84dfda59e6a0e8a77240f939a7cb8757e6eaf945", + "language": "js", + "tags": [ + "js", + "refactoring_enh", + "major_bug", + "data_bug", + "database_knowledge", + "back_end_knowledge", + "infrastructure_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "nodebb.nodebb-NodeBB__NodeBB-6489e9fd9ed16ea743cc5627f4d86c72fbdb3a8a-v2c59007b1005cd5cd14cbb523ca5229db1fd2dd8", + "beforeRepoSetCommand": "git reset --hard 84dfda59e6a0e8a77240f939a7cb8757e6eaf945\ngit clean -fd \ngit checkout 84dfda59e6a0e8a77240f939a7cb8757e6eaf945 \ngit checkout 6489e9fd9ed16ea743cc5627f4d86c72fbdb3a8a -- test/posts/uploads.js", + "selectedTestFiles": [ + "test/posts/uploads.js" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-135", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_NodeBB__NodeBB-9c576a0758690f45a6ca03b5884c601e473bf2c1-vd59a5728dfc977f44533186ace531248c2917516", + "selectedInstanceId": "instance_nodebb__nodebb-9c576a0758690f45a6ca03b5884c601e473bf2c1-vd59a5728dfc977f44533186ace531248c2917516", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "NodeBB/NodeBB", + "baseCommit": "09f3ac6574b3192497df0403306aff8d6f20448b", + "language": "js", + "tags": [ + "js", + "code_quality_enh", + "security_enh", + "ui_ux_enh", + "refactoring_enh", + "back_end_knowledge", + "authentication_authorization_knowledge", + "security_knowledge", + "ui_ux_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "nodebb.nodebb-NodeBB__NodeBB-9c576a0758690f45a6ca03b5884c601e473bf2c1-vd59a5728dfc977f44533186ace531248c2917516", + "beforeRepoSetCommand": "git reset --hard 09f3ac6574b3192497df0403306aff8d6f20448b\ngit clean -fd \ngit checkout 09f3ac6574b3192497df0403306aff8d6f20448b \ngit checkout 9c576a0758690f45a6ca03b5884c601e473bf2c1 -- test/user.js test/user/emails.js", + "selectedTestFiles": [ + "test/user.js", + "test/user/emails.js" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-136", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_NodeBB__NodeBB-a5afad27e52fd336163063ba40dcadc80233ae10-vd59a5728dfc977f44533186ace531248c2917516", + "selectedInstanceId": "instance_nodebb__nodebb-a5afad27e52fd336163063ba40dcadc80233ae10-vd59a5728dfc977f44533186ace531248c2917516", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "NodeBB/NodeBB", + "baseCommit": "7800016f2f1b89d2d3cfea6a7da7c77096b7b927", + "language": "js", + "tags": [ + "js", + "api_feat", + "code_quality_enh", + "back_end_knowledge", + "web_knowledge", + "api_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "nodebb.nodebb-NodeBB__NodeBB-a5afad27e52fd336163063ba40dcadc80233ae10-vd59a5728dfc977f44533186ace531248c2917516", + "beforeRepoSetCommand": "git reset --hard 7800016f2f1b89d2d3cfea6a7da7c77096b7b927\ngit clean -fd \ngit checkout 7800016f2f1b89d2d3cfea6a7da7c77096b7b927 \ngit checkout a5afad27e52fd336163063ba40dcadc80233ae10 -- test/messaging.js test/user.js", + "selectedTestFiles": [ + "test/i18n.js", + "test/user.js", + "test/messaging.js" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-137", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_NodeBB__NodeBB-be43cd25974681c9743d424238b7536c357dc8d3-vf2cf3cbd463b7ad942381f1c6d077626485a1e9e", + "selectedInstanceId": "instance_nodebb__nodebb-be43cd25974681c9743d424238b7536c357dc8d3-vf2cf3cbd463b7ad942381f1c6d077626485a1e9e", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "NodeBB/NodeBB", + "baseCommit": "f24b630e1afcb8135144be66d67a09a61b21753e", + "language": "js", + "tags": [ + "js", + "core_feat", + "ui_ux_feat", + "localization_feat", + "customization_feat", + "back_end_knowledge", + "ui_ux_knowledge", + "database_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "nodebb.nodebb-NodeBB__NodeBB-be43cd25974681c9743d424238b7536c357dc8d3-vf2cf3cbd463b7ad942381f1c6d077626485a1e9e", + "beforeRepoSetCommand": "git reset --hard f24b630e1afcb8135144be66d67a09a61b21753e\ngit clean -fd \ngit checkout f24b630e1afcb8135144be66d67a09a61b21753e \ngit checkout be43cd25974681c9743d424238b7536c357dc8d3 -- test/posts.js", + "selectedTestFiles": [ + "test/posts.js" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-138", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_NodeBB__NodeBB-f2082d7de85eb62a70819f4f3396dd85626a0c0a-vd59a5728dfc977f44533186ace531248c2917516", + "selectedInstanceId": "instance_nodebb__nodebb-f2082d7de85eb62a70819f4f3396dd85626a0c0a-vd59a5728dfc977f44533186ace531248c2917516", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "NodeBB/NodeBB", + "baseCommit": "f0d989e4ba5b0dccff6e56022fc6d378d05ab404", + "language": "js", + "tags": [ + "js", + "api_feat", + "core_feat", + "integration_feat", + "back_end_knowledge", + "api_knowledge", + "web_knowledge", + "authentication_authorization_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "nodebb.nodebb-NodeBB__NodeBB-f2082d7de85eb62a70819f4f3396dd85626a0c0a-vd59a5728dfc977f44533186ace531248c2917516", + "beforeRepoSetCommand": "git reset --hard f0d989e4ba5b0dccff6e56022fc6d378d05ab404\ngit clean -fd \ngit checkout f0d989e4ba5b0dccff6e56022fc6d378d05ab404 \ngit checkout f2082d7de85eb62a70819f4f3396dd85626a0c0a -- test/posts.js", + "selectedTestFiles": [ + "test/posts.js" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-139", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_protonmail__webclients-2f2f6c311c6128fe86976950d3c0c2db07b03921", + "selectedInstanceId": "instance_protonmail__webclients-2f2f6c311c6128fe86976950d3c0c2db07b03921", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "protonmail/webclients", + "baseCommit": "4d0ef1ed136e80692723f148dc0390dcf28ba9dc", + "language": "js", + "tags": [ + "js", + "compatibility_bug", + "back_end_knowledge", + "api_knowledge", + "security_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "protonmail.webclients-protonmail__webclients-2f2f6c311c6128fe86976950d3c0c2db07b03921", + "beforeRepoSetCommand": "git reset --hard 4d0ef1ed136e80692723f148dc0390dcf28ba9dc\ngit clean -fd \ngit checkout 4d0ef1ed136e80692723f148dc0390dcf28ba9dc \ngit checkout 2f2f6c311c6128fe86976950d3c0c2db07b03921 -- applications/drive/src/app/store/_shares/useShareActions.test.ts", + "selectedTestFiles": [ + "applications/drive/src/app/store/_shares/useShareActions.test.ts", + "src/app/store/_shares/useShareActions.test.ts" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-140", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_protonmail__webclients-863d524b5717b9d33ce08a0f0535e3fd8e8d1ed8", + "selectedInstanceId": "instance_protonmail__webclients-863d524b5717b9d33ce08a0f0535e3fd8e8d1ed8", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "protonmail/webclients", + "baseCommit": "464a02f3da87d165d1bfc330e5310c7c6e5e9734", + "language": "js", + "tags": [ + "js", + "performance_enh", + "ui_ux_enh", + "front_end_knowledge", + "web_knowledge", + "ui_ux_knowledge", + "performance_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "protonmail.webclients-protonmail__webclients-863d524b5717b9d33ce08a0f0535e3fd8e8d1ed8", + "beforeRepoSetCommand": "git reset --hard 464a02f3da87d165d1bfc330e5310c7c6e5e9734\ngit clean -fd \ngit checkout 464a02f3da87d165d1bfc330e5310c7c6e5e9734 \ngit checkout 863d524b5717b9d33ce08a0f0535e3fd8e8d1ed8 -- packages/components/payments/client-extensions/usePollEvents.test.ts", + "selectedTestFiles": [ + "packages/components/payments/client-extensions/usePollEvents.test.ts", + "payments/client-extensions/usePollEvents.test.ts" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-141", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_protonmail__webclients-b530a3db50cb33e5064464addbcbef1465856ce6", + "selectedInstanceId": "instance_protonmail__webclients-b530a3db50cb33e5064464addbcbef1465856ce6", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "protonmail/webclients", + "baseCommit": "32394eda944448efcd48d2487c2878cd402e612c", + "language": "js", + "tags": [ + "js", + "code_quality_enh", + "refactoring_enh", + "front_end_knowledge", + "web_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "protonmail.webclients-protonmail__webclients-b530a3db50cb33e5064464addbcbef1465856ce6", + "beforeRepoSetCommand": "git reset --hard 32394eda944448efcd48d2487c2878cd402e612c\ngit clean -fd \ngit checkout 32394eda944448efcd48d2487c2878cd402e612c \ngit checkout b530a3db50cb33e5064464addbcbef1465856ce6 -- applications/mail/src/app/containers/onboardingChecklist/hooks/useCanCheckItem.test.ts", + "selectedTestFiles": [ + "applications/mail/src/app/containers/onboardingChecklist/hooks/useCanCheckItem.test.ts", + "src/app/containers/onboardingChecklist/hooks/useCanCheckItem.test.ts" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-142", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_qutebrowser__qutebrowser-36ade4bba504eb96f05d32ceab9972df7eb17bcc-v2ef375ac784985212b1805e1d0431dc8f1b3c171", + "selectedInstanceId": "instance_qutebrowser__qutebrowser-36ade4bba504eb96f05d32ceab9972df7eb17bcc-v2ef375ac784985212b1805e1d0431dc8f1b3c171", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "qutebrowser/qutebrowser", + "baseCommit": "73f93008f6c8104dcb317b10bae2c6d156674b33", + "language": "python", + "tags": [ + "python", + "customization_feat", + "code_quality_enh", + "back_end_knowledge", + "desktop_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "qutebrowser.qutebrowser-qutebrowser__qutebrowser-36ade4bba504eb96f05d32ceab9972df7eb17bcc-v2ef375ac784985212b1805e1d0431dc8f1b3c", + "beforeRepoSetCommand": "git reset --hard 73f93008f6c8104dcb317b10bae2c6d156674b33\ngit clean -fd \ngit checkout 73f93008f6c8104dcb317b10bae2c6d156674b33 \ngit checkout 36ade4bba504eb96f05d32ceab9972df7eb17bcc -- tests/unit/config/test_qtargs.py", + "selectedTestFiles": [ + "tests/unit/config/test_qtargs.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-143", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_qutebrowser__qutebrowser-394bfaed6544c952c6b3463751abab3176ad4997-vafb3e8e01b31319c66c4e666b8a3b1d8ba55db24", + "selectedInstanceId": "instance_qutebrowser__qutebrowser-394bfaed6544c952c6b3463751abab3176ad4997-vafb3e8e01b31319c66c4e666b8a3b1d8ba55db24", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "qutebrowser/qutebrowser", + "baseCommit": "d1164925c55f2417f1c3130b0196830bc2a3d25d", + "language": "python", + "tags": [ + "python", + "refactoring_enh", + "code_quality_enh", + "technical_debt_enh", + "desktop_knowledge", + "back_end_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "qutebrowser.qutebrowser-qutebrowser__qutebrowser-394bfaed6544c952c6b3463751abab3176ad4997-vafb3e8e01b31319c66c4e666b8a3b1d8ba55d", + "beforeRepoSetCommand": "git reset --hard d1164925c55f2417f1c3130b0196830bc2a3d25d\ngit clean -fd \ngit checkout d1164925c55f2417f1c3130b0196830bc2a3d25d \ngit checkout 394bfaed6544c952c6b3463751abab3176ad4997 -- tests/unit/misc/test_elf.py tests/unit/utils/test_version.py", + "selectedTestFiles": [ + "tests/unit/utils/test_version.py", + "tests/unit/misc/test_elf.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-144", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_qutebrowser__qutebrowser-3e21c8214a998cb1058defd15aabb24617a76402-v5fc38aaf22415ab0b70567368332beee7955b367", + "selectedInstanceId": "instance_qutebrowser__qutebrowser-3e21c8214a998cb1058defd15aabb24617a76402-v5fc38aaf22415ab0b70567368332beee7955b367", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "qutebrowser/qutebrowser", + "baseCommit": "fce306d5f184f185b660f35ec2808b6745d57520", + "language": "python", + "tags": [ + "python", + "code_quality_enh", + "refactoring_enh", + "desktop_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "qutebrowser.qutebrowser-qutebrowser__qutebrowser-3e21c8214a998cb1058defd15aabb24617a76402-v5fc38aaf22415ab0b70567368332beee7955b", + "beforeRepoSetCommand": "git reset --hard fce306d5f184f185b660f35ec2808b6745d57520\ngit clean -fd \ngit checkout fce306d5f184f185b660f35ec2808b6745d57520 \ngit checkout 3e21c8214a998cb1058defd15aabb24617a76402 -- tests/unit/keyinput/test_keyutils.py", + "selectedTestFiles": [ + "tests/unit/keyinput/test_bindingtrie.py", + "tests/unit/keyinput/test_keyutils.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-145", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_qutebrowser__qutebrowser-473a15f7908f2bb6d670b0e908ab34a28d8cf7e2-v363c8a7e5ccdf6968fc7ab84a2053ac78036691d", + "selectedInstanceId": "instance_qutebrowser__qutebrowser-473a15f7908f2bb6d670b0e908ab34a28d8cf7e2-v363c8a7e5ccdf6968fc7ab84a2053ac78036691d", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "qutebrowser/qutebrowser", + "baseCommit": "8e08f046aeffb38445cc659050f7a083b4a9649c", + "language": "python", + "tags": [ + "python", + "regression_bug", + "localization_feat", + "critical_bug", + "desktop_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "qutebrowser.qutebrowser-qutebrowser__qutebrowser-473a15f7908f2bb6d670b0e908ab34a28d8cf7e2-v363c8a7e5ccdf6968fc7ab84a2053ac780366", + "beforeRepoSetCommand": "git reset --hard 8e08f046aeffb38445cc659050f7a083b4a9649c\ngit clean -fd \ngit checkout 8e08f046aeffb38445cc659050f7a083b4a9649c \ngit checkout 473a15f7908f2bb6d670b0e908ab34a28d8cf7e2 -- tests/unit/config/test_qtargs_locale_workaround.py", + "selectedTestFiles": [ + "tests/unit/config/test_qtargs_locale_workaround.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-146", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_qutebrowser__qutebrowser-e34dfc68647d087ca3175d9ad3f023c30d8c9746-v363c8a7e5ccdf6968fc7ab84a2053ac78036691d", + "selectedInstanceId": "instance_qutebrowser__qutebrowser-e34dfc68647d087ca3175d9ad3f023c30d8c9746-v363c8a7e5ccdf6968fc7ab84a2053ac78036691d", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "qutebrowser/qutebrowser", + "baseCommit": "c984983bc4cf6f9148e16ea17369597f67774ff9", + "language": "python", + "tags": [ + "python", + "edge_case_bug", + "data_bug", + "web_knowledge", + "desktop_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "qutebrowser.qutebrowser-qutebrowser__qutebrowser-e34dfc68647d087ca3175d9ad3f023c30d8c9746-v363c8a7e5ccdf6968fc7ab84a2053ac780366", + "beforeRepoSetCommand": "git reset --hard c984983bc4cf6f9148e16ea17369597f67774ff9\ngit clean -fd \ngit checkout c984983bc4cf6f9148e16ea17369597f67774ff9 \ngit checkout e34dfc68647d087ca3175d9ad3f023c30d8c9746 -- tests/unit/utils/test_urlutils.py", + "selectedTestFiles": [ + "tests/unit/utils/test_urlutils.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-147", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_qutebrowser__qutebrowser-fcfa069a06ade76d91bac38127f3235c13d78eb1-v5fc38aaf22415ab0b70567368332beee7955b367", + "selectedInstanceId": "instance_qutebrowser__qutebrowser-fcfa069a06ade76d91bac38127f3235c13d78eb1-v5fc38aaf22415ab0b70567368332beee7955b367", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "qutebrowser/qutebrowser", + "baseCommit": "74671c167f6f18a5494ffe44809e6a0e1c6ea8e9", + "language": "python", + "tags": [ + "python", + "core_feat", + "technical_debt_enh", + "back_end_knowledge", + "database_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "qutebrowser.qutebrowser-qutebrowser__qutebrowser-fcfa069a06ade76d91bac38127f3235c13d78eb1-v5fc38aaf22415ab0b70567368332beee7955b", + "beforeRepoSetCommand": "git reset --hard 74671c167f6f18a5494ffe44809e6a0e1c6ea8e9\ngit clean -fd \ngit checkout 74671c167f6f18a5494ffe44809e6a0e1c6ea8e9 \ngit checkout fcfa069a06ade76d91bac38127f3235c13d78eb1 -- tests/unit/misc/test_sql.py", + "selectedTestFiles": [ + "tests/unit/misc/test_sql.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-148", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_qutebrowser__qutebrowser-ff1c025ad3210506fc76e1f604d8c8c27637d88e-v363c8a7e5ccdf6968fc7ab84a2053ac78036691d", + "selectedInstanceId": "instance_qutebrowser__qutebrowser-ff1c025ad3210506fc76e1f604d8c8c27637d88e-v363c8a7e5ccdf6968fc7ab84a2053ac78036691d", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "qutebrowser/qutebrowser", + "baseCommit": "e545faaf7b18d451c082f697675f0ab0e7599ed1", + "language": "python", + "tags": [ + "python", + "ui_ux_feat", + "customization_feat", + "code_quality_enh", + "full_stack_knowledge", + "desktop_knowledge", + "ui_ux_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "qutebrowser.qutebrowser-qutebrowser__qutebrowser-ff1c025ad3210506fc76e1f604d8c8c27637d88e-v363c8a7e5ccdf6968fc7ab84a2053ac780366", + "beforeRepoSetCommand": "git reset --hard e545faaf7b18d451c082f697675f0ab0e7599ed1\ngit clean -fd \ngit checkout e545faaf7b18d451c082f697675f0ab0e7599ed1 \ngit checkout ff1c025ad3210506fc76e1f604d8c8c27637d88e -- tests/helpers/fixtures.py tests/unit/config/test_configinit.py tests/unit/config/test_configtypes.py", + "selectedTestFiles": [ + "tests/unit/config/test_configfiles.py", + "tests/unit/config/test_configcache.py", + "tests/unit/config/test_configtypes.py", + "tests/unit/config/test_config.py", + "tests/helpers/fixtures.py", + "tests/unit/config/test_configinit.py", + "tests/unit/config/test_stylesheet.py", + "tests/unit/config/test_configcommands.py", + "tests/unit/config/test_configdata.py" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-149", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_tutao__tutanota-1ff82aa365763cee2d609c9d19360ad87fdf2ec7-vc4e41fd0029957297843cb9dec4a25c7c756f029", + "selectedInstanceId": "instance_tutao__tutanota-1ff82aa365763cee2d609c9d19360ad87fdf2ec7-vc4e41fd0029957297843cb9dec4a25c7c756f029", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "tutao/tutanota", + "baseCommit": "2e5d877afe8a1b8fc9ad3ac0a0d579bbad364224", + "language": "ts", + "tags": [ + "ts", + "data_bug", + "integration_bug", + "back_end_knowledge", + "security_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "tutao.tutanota-tutao__tutanota-1ff82aa365763cee2d609c9d19360ad87fdf2ec7-vc4e41fd0029957297843cb9dec4a25c7c756f029", + "beforeRepoSetCommand": "git reset --hard 2e5d877afe8a1b8fc9ad3ac0a0d579bbad364224\ngit clean -fd \ngit checkout 2e5d877afe8a1b8fc9ad3ac0a0d579bbad364224 \ngit checkout 1ff82aa365763cee2d609c9d19360ad87fdf2ec7 -- test/tests/api/worker/rest/EntityRestCacheTest.ts", + "selectedTestFiles": [ + "test/tests/api/worker/crypto/OwnerEncSessionKeysUpdateQueueTest.js", + "test/tests/api/worker/search/EventQueueTest.js", + "test/tests/misc/parsing/MailAddressParserTest.js", + "test/tests/api/worker/search/IndexUtilsTest.js", + "test/tests/gui/GuiUtilsTest.js", + "test/tests/api/worker/rest/EntityRestCacheTest.js", + "test/tests/api/worker/SuspensionHandlerTest.js", + "test/tests/misc/credentials/CredentialsProviderTest.js", + "test/tests/misc/credentials/CredentialsKeyProviderTest.js", + "test/tests/contacts/VCardImporterTest.js", + "test/tests/misc/SchedulerTest.js", + "test/tests/mail/export/ExporterTest.js", + "test/tests/misc/webauthn/WebauthnClientTest.js", + "test/tests/mail/TemplateSearchFilterTest.js", + "test/tests/calendar/eventeditor/CalendarEventModelTest.js", + "test/tests/api/worker/crypto/CompatibilityTest.js", + "test/tests/api/worker/rest/CustomCacheHandlerTest.js", + "test/tests/api/main/EntropyCollectorTest.js", + "test/tests/misc/ParserTest.js", + "test/tests/misc/UsageTestModelTest.js", + "test/tests/translations/TranslationKeysTest.js", + "test/tests/api/worker/facades/BlobAccessTokenFacadeTest.js", + "test/tests/gui/base/WizardDialogNTest.js", + "test/tests/api/common/utils/FileUtilsTest.js", + "test/tests/api/worker/search/TokenizerTest.js", + "test/tests/mail/KnowledgeBaseSearchFilterTest.js", + "test/tests/misc/HtmlSanitizerTest.js", + "test/tests/api/worker/rest/EntityRestCacheTest.ts", + "test/tests/gui/ColorTest.js", + "test/tests/misc/PasswordUtilsTest.js", + "test/tests/api/worker/facades/MailFacadeTest.js", + "test/tests/settings/login/secondfactor/SecondFactorEditModelTest.js", + "test/tests/api/common/error/RestErrorTest.js", + "test/tests/misc/FormatValidatorTest.js", + "test/tests/api/worker/CompressionTest.js", + "test/tests/api/worker/EventBusClientTest.js", + "test/tests/api/common/error/TutanotaErrorTest.js", + "test/tests/api/worker/utils/SleepDetectorTest.js", + "test/tests/misc/FormatterTest.js", + "test/tests/api/worker/search/SuggestionFacadeTest.js", + "test/tests/calendar/AlarmSchedulerTest.js", + "test/tests/api/worker/rest/ServiceExecutorTest.js", + "test/tests/contacts/ContactUtilsTest.js", + "test/tests/api/worker/rest/CborDateEncoderTest.js", + "test/tests/calendar/CalendarImporterTest.js", + "test/tests/login/LoginViewModelTest.js", + "test/tests/api/worker/search/GroupInfoIndexerTest.js", + "test/tests/api/worker/facades/BlobFacadeTest.js", + "test/tests/mail/SendMailModelTest.js", + "test/tests/api/worker/search/MailIndexerTest.js", + "test/tests/mail/export/BundlerTest.js", + "test/tests/contacts/ContactMergeUtilsTest.js", + "test/tests/mail/model/FolderSystemTest.js", + "test/tests/calendar/CalendarViewModelTest.js", + "test/tests/api/common/utils/LoggerTest.js", + "test/tests/misc/NewsModelTest.js", + "test/tests/misc/OutOfOfficeNotificationTest.js", + "test/tests/serviceworker/SwTest.js", + "test/tests/api/worker/UrlifierTest.js", + "test/tests/api/common/utils/CommonFormatterTest.js", + "test/tests/api/worker/facades/LoginFacadeTest.js", + "test/tests/calendar/CalendarGuiUtilsTest.js", + "test/tests/misc/LanguageViewModelTest.js", + "test/tests/subscription/CreditCardViewModelTest.js", + "test/tests/calendar/CalendarUtilsTest.js", + "test/tests/api/worker/rest/EntityRestClientTest.js", + "test/tests/calendar/CalendarModelTest.js", + "test/tests/support/FaqModelTest.js", + "test/tests/api/worker/rest/EphemeralCacheStorageTest.js", + "test/tests/calendar/EventDragHandlerTest.js", + "test/tests/gui/animation/AnimationsTest.js", + "test/tests/calendar/eventeditor/CalendarEventWhoModelTest.js", + "test/tests/mail/InboxRuleHandlerTest.js", + "test/tests/api/common/utils/PlainTextSearchTest.js", + "test/tests/file/FileControllerTest.js", + "test/tests/calendar/eventeditor/CalendarEventAlarmModelTest.js", + "test/tests/mail/MailModelTest.js", + "test/tests/api/common/utils/BirthdayUtilsTest.js", + "test/tests/api/worker/search/SearchFacadeTest.js", + "test/tests/calendar/CalendarParserTest.js", + "test/tests/misc/ListModelTest.js", + "test/tests/api/worker/crypto/CryptoFacadeTest.js", + "test/tests/api/worker/facades/CalendarFacadeTest.js", + "test/tests/api/worker/search/ContactIndexerTest.js", + "test/tests/misc/news/items/ReferralLinkNewsTest.js", + "test/tests/api/worker/search/SearchIndexEncodingTest.js", + "test/tests/api/worker/search/IndexerCoreTest.js", + "test/tests/api/worker/facades/UserFacadeTest.js", + "test/tests/api/worker/rest/CacheStorageProxyTest.js", + "test/tests/subscription/PriceUtilsTest.js", + "test/tests/misc/credentials/NativeCredentialsEncryptionTest.js", + "test/tests/misc/RecipientsModelTest.js", + "test/tests/contacts/VCardExporterTest.js", + "test/tests/settings/whitelabel/CustomColorEditorTest.js", + "test/tests/api/worker/facades/ConfigurationDbTest.js", + "test/tests/mail/MailUtilsSignatureTest.js", + "test/tests/api/worker/facades/MailAddressFacadeTest.js", + "test/tests/subscription/SubscriptionUtilsTest.js", + "test/tests/settings/TemplateEditorModelTest.js", + "test/tests/misc/DeviceConfigTest.js", + "test/tests/calendar/eventeditor/CalendarEventWhenModelTest.js", + "test/tests/settings/mailaddress/MailAddressTableModelTest.js", + "test/tests/misc/ClientDetectorTest.js", + "test/tests/api/common/utils/EntityUtilsTest.js", + "test/tests/api/worker/rest/RestClientTest.js", + "test/tests/api/worker/search/IndexerTest.js", + "test/tests/settings/UserDataExportTest.js", + "test/tests/gui/ThemeControllerTest.js" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-150", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_tutao__tutanota-51818218c6ae33de00cbea3a4d30daac8c34142e-vc4e41fd0029957297843cb9dec4a25c7c756f029", + "selectedInstanceId": "instance_tutao__tutanota-51818218c6ae33de00cbea3a4d30daac8c34142e-vc4e41fd0029957297843cb9dec4a25c7c756f029", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "tutao/tutanota", + "baseCommit": "dac77208814de95c4018bcf13137324153cc9a3a", + "language": "ts", + "tags": [ + "ts", + "regression_bug", + "ui_ux_bug", + "compatibility_bug", + "desktop_knowledge", + "ui_ux_knowledge", + "full_stack_knowledge" + ], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "tutao.tutanota-tutao__tutanota-51818218c6ae33de00cbea3a4d30daac8c34142e-vc4e41fd0029957297843cb9dec4a25c7c756f029", + "beforeRepoSetCommand": "git reset --hard dac77208814de95c4018bcf13137324153cc9a3a\ngit clean -fd \ngit checkout dac77208814de95c4018bcf13137324153cc9a3a \ngit checkout 51818218c6ae33de00cbea3a4d30daac8c34142e -- test/client/desktop/DesktopDownloadManagerTest.ts test/client/nodemocker.ts", + "selectedTestFiles": [ + "test/client/desktop/DesktopDownloadManagerTest.ts", + "test/api/Suite.ts", + "test/client/nodemocker.ts" + ], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + } + ] +} diff --git a/benchmarks/agentrail-swe-lifecycle/fixtures/fake-evaluator.mjs b/benchmarks/agentrail-swe-lifecycle/fixtures/fake-evaluator.mjs new file mode 100644 index 0000000..eb43c0b --- /dev/null +++ b/benchmarks/agentrail-swe-lifecycle/fixtures/fake-evaluator.mjs @@ -0,0 +1,19 @@ +#!/usr/bin/env node +import { readFile, writeFile } from "node:fs/promises"; + +const args = parseArgs(process.argv.slice(2)); +const patch = await readFile(args.patch, "utf8"); +const passed = patch.includes("fake runner completed"); +await writeFile(args.output, `${JSON.stringify({ passed, reason: passed ? "expected change found" : "expected change missing" }, null, 2)}\n`, "utf8"); +process.exitCode = passed ? 0 : 1; + +function parseArgs(argv) { + const parsed = {}; + for (let index = 0; index < argv.length; index += 1) { + if (argv[index] === "--patch") parsed.patch = argv[++index]; + else if (argv[index] === "--output") parsed.output = argv[++index]; + else throw new Error(`Unknown evaluator argument: ${argv[index]}`); + } + if (!parsed.patch || !parsed.output) throw new Error("--patch and --output are required."); + return parsed; +} diff --git a/benchmarks/agentrail-swe-lifecycle/fixtures/fake-lifecycle-runner.mjs b/benchmarks/agentrail-swe-lifecycle/fixtures/fake-lifecycle-runner.mjs new file mode 100644 index 0000000..a2d0754 --- /dev/null +++ b/benchmarks/agentrail-swe-lifecycle/fixtures/fake-lifecycle-runner.mjs @@ -0,0 +1,67 @@ +#!/usr/bin/env node +import { execFileSync } from "node:child_process"; +import { appendFile, readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; + +const repoPath = process.cwd(); +const promptPath = process.env.AGENTRAIL_BENCH_PROMPT_PATH; +const prompt = promptPath ? await readFile(promptPath, "utf8").catch(() => "") : ""; +const hasLifecycleProvider = Boolean(process.env.AGENTRAIL_BENCH_LIFECYCLE_URL); +const lifecycleCli = process.env.AGENTRAIL_BENCH_LIFECYCLE_CLI + ?? path.resolve("benchmarks/agentrail-swe-lifecycle/fixtures/mock-lifecycle-cli.mjs"); + +const issue = hasLifecycleProvider ? runLifecycle(["issue", "get"]) : {}; +if (hasLifecycleProvider) { + runLifecycle(["branch", "create", "--name", "benchmark-fix"]); +} + +const readmePath = path.join(repoPath, "README.md"); +const previous = await readFile(readmePath, "utf8").catch(() => ""); +await writeFile(readmePath, `${previous.trimEnd()}\nfake lifecycle runner completed ${issue.instanceId ?? ""}\n`, "utf8"); + +execFileSync("git", ["-C", repoPath, "add", "README.md"]); +execFileSync("git", ["-C", repoPath, "config", "user.email", "bench@example.com"]); +execFileSync("git", ["-C", repoPath, "config", "user.name", "Benchmark Runner"]); +execFileSync("git", ["-C", repoPath, "commit", "-m", "fake lifecycle runner change"]); +const commitSha = execFileSync("git", ["-C", repoPath, "rev-parse", "HEAD"], { encoding: "utf8" }).trim(); + +if (hasLifecycleProvider) { + const pr = runLifecycle(["pr", "create", "--head", "benchmark-fix", "--base", "main", "--title", "Benchmark fix"]); + runLifecycle(["ci", "get", "--pr", String(pr.number)]); + runLifecycle(["review", "get", "--pr", String(pr.number)]); + runLifecycle(["complete", "--pr", String(pr.number), "--summary", "Fake lifecycle runner completed benchmark task."]); +} + +if (process.env.AGENTRAIL_HANDOFF_PATH) { + await writeFile(process.env.AGENTRAIL_HANDOFF_PATH, `${JSON.stringify({ + version: 1, + status: "completed", + summary: "Fake lifecycle runner completed benchmark task.", + checks: [], + artifacts: [], + blocker: null, + commitSha, + }, null, 2)}\n`, "utf8"); +} + +const usage = { + input_tokens: Math.max(1, Math.ceil(prompt.length / 4)), + output_tokens: 35, + reasoning_tokens: 7, +}; +usage.total_tokens = usage.input_tokens + usage.output_tokens + usage.reasoning_tokens; + +if (process.env.AGENTRAIL_BENCH_EXTRA_LOG_PATH) { + await appendFile(process.env.AGENTRAIL_BENCH_EXTRA_LOG_PATH, `${JSON.stringify({ usage })}\n`); +} + +process.stdout.write(`${JSON.stringify({ usage })}\n`); + +function runLifecycle(args) { + const stdout = execFileSync("node", [lifecycleCli, ...args], { + cwd: repoPath, + env: process.env, + encoding: "utf8", + }); + return JSON.parse(stdout); +} diff --git a/benchmarks/agentrail-swe-lifecycle/fixtures/fake-phase-runner.mjs b/benchmarks/agentrail-swe-lifecycle/fixtures/fake-phase-runner.mjs new file mode 100644 index 0000000..1c450e7 --- /dev/null +++ b/benchmarks/agentrail-swe-lifecycle/fixtures/fake-phase-runner.mjs @@ -0,0 +1,66 @@ +#!/usr/bin/env node +import { execFileSync } from "node:child_process"; +import { appendFile, mkdir, readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; + +const phaseId = process.env.AGENTRAIL_BENCH_PHASE_ID ?? "unknown"; +const promptPath = process.env.AGENTRAIL_BENCH_PROMPT_PATH; +const prompt = promptPath ? await readFile(promptPath, "utf8").catch(() => "") : ""; + +runLifecycleStep(phaseId); + +if (phaseId.includes("code") || phaseId.startsWith("fix_")) { + const readmePath = path.join(process.cwd(), "README.md"); + const previous = await readFile(readmePath, "utf8").catch(() => ""); + await writeFile(readmePath, `${previous.trimEnd()}\nphase runner coding change ${phaseId}\n`, "utf8"); +} + +if (phaseId === "agentrail_handoff") { + const resultPath = process.env.AGENTRAIL_RESULT_PATH; + if (resultPath) { + await mkdir(path.dirname(resultPath), { recursive: true }); + await writeFile(resultPath, `${JSON.stringify({ + version: 1, + status: "completed", + summary: "phase attribution benchmark complete", + checks: [], + artifacts: [], + })}\n`, "utf8"); + } +} + +const base = phaseId.startsWith("provider_") || phaseId.endsWith("_triage") ? 100 : 25; +const usage = { + input_tokens: Math.max(base, Math.ceil(prompt.length / 4)), + output_tokens: phaseId === "agentrail_handoff" ? 5 : 10, + reasoning_tokens: phaseId.includes("fix") ? 4 : 2, +}; +usage.total_tokens = usage.input_tokens + usage.output_tokens + usage.reasoning_tokens; + +if (process.env.AGENTRAIL_BENCH_EXTRA_LOG_PATH) { + await appendFile(process.env.AGENTRAIL_BENCH_EXTRA_LOG_PATH, `${JSON.stringify({ phaseId, usage })}\n`, "utf8"); +} + +process.stdout.write(`${JSON.stringify({ phaseId, usage })}\n`); + +function runLifecycleStep(id) { + const lifecycleCli = process.env.AGENTRAIL_BENCH_LIFECYCLE_CLI; + if (!lifecycleCli) return; + const argsByPhase = { + provider_issue_intake: ["issue", "get"], + provider_create_pr: ["pr", "create", "--head", "benchmark-fix", "--base", "main", "--title", "Benchmark fix"], + provider_wait_for_ci: ["ci", "get", "--pr", "1"], + provider_wait_for_review: ["review", "get", "--pr", "1"], + provider_update_pr: ["pr", "update", "--pr", "1", "--summary", "phase attribution update"], + provider_final_ci_check: ["ci", "get", "--pr", "1"], + provider_final_review_check: ["review", "get", "--pr", "1"], + provider_complete: ["complete", "--pr", "1", "--summary", "phase attribution benchmark complete"], + }; + const args = argsByPhase[id]; + if (!args) return; + execFileSync("node", [lifecycleCli, ...args], { + cwd: process.cwd(), + env: process.env, + stdio: "pipe", + }); +} diff --git a/benchmarks/agentrail-swe-lifecycle/fixtures/fake-runner.mjs b/benchmarks/agentrail-swe-lifecycle/fixtures/fake-runner.mjs new file mode 100644 index 0000000..bd89399 --- /dev/null +++ b/benchmarks/agentrail-swe-lifecycle/fixtures/fake-runner.mjs @@ -0,0 +1,45 @@ +#!/usr/bin/env node +import { execFileSync } from "node:child_process"; +import { appendFile, readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; + +const repoPath = process.cwd(); +const promptPath = process.env.AGENTRAIL_BENCH_PROMPT_PATH; +const prompt = promptPath ? await readFile(promptPath, "utf8").catch(() => "") : ""; +const usage = { + input_tokens: Math.max(1, Math.ceil(prompt.length / 4)), + output_tokens: 20, + reasoning_tokens: 5, +}; +usage.total_tokens = usage.input_tokens + usage.output_tokens + usage.reasoning_tokens; + +const readmePath = path.join(repoPath, "README.md"); +const previous = await readFile(readmePath, "utf8").catch(() => ""); +await writeFile(readmePath, `${previous.trimEnd()}\nfake runner completed\n`, "utf8"); + +execFileSync("git", ["-C", repoPath, "add", "README.md"]); +execFileSync("git", ["-C", repoPath, "config", "user.email", "bench@example.com"]); +execFileSync("git", ["-C", repoPath, "config", "user.name", "Benchmark Runner"]); +execFileSync("git", ["-C", repoPath, "commit", "-m", "fake runner change"]); +const commitSha = execFileSync("git", ["-C", repoPath, "rev-parse", "HEAD"], { encoding: "utf8" }).trim(); + +if (process.env.AGENTRAIL_HANDOFF_PATH) { + await writeFile(process.env.AGENTRAIL_HANDOFF_PATH, `${JSON.stringify({ + version: 1, + status: "completed", + summary: "Fake runner completed benchmark task.", + checks: [], + artifacts: [], + blocker: null, + commitSha, + }, null, 2)}\n`, "utf8"); +} + +if (process.env.AGENTRAIL_BENCH_EXTRA_LOG_PATH) { + await appendFile( + process.env.AGENTRAIL_BENCH_EXTRA_LOG_PATH, + `${JSON.stringify({ usage })}\n`, + ); +} + +process.stdout.write(`${JSON.stringify({ usage })}\n`); diff --git a/benchmarks/agentrail-swe-lifecycle/fixtures/mock-lifecycle-cli.mjs b/benchmarks/agentrail-swe-lifecycle/fixtures/mock-lifecycle-cli.mjs new file mode 100644 index 0000000..87dae1c --- /dev/null +++ b/benchmarks/agentrail-swe-lifecycle/fixtures/mock-lifecycle-cli.mjs @@ -0,0 +1,168 @@ +#!/usr/bin/env node +import { readFile, writeFile } from "node:fs/promises"; + +const baseUrl = process.env.AGENTRAIL_BENCH_LIFECYCLE_URL; +const statePath = process.env.AGENTRAIL_BENCH_LIFECYCLE_STATE_PATH; +if (!baseUrl && !statePath) { + process.stderr.write("AGENTRAIL_BENCH_LIFECYCLE_URL or AGENTRAIL_BENCH_LIFECYCLE_STATE_PATH is required.\n"); + process.exit(2); +} + +const [resource, action, ...args] = process.argv.slice(2); +const flags = parseFlags(args); + +try { + if (resource === "issue" && action === "get") { + await printJson(await lifecycleOperation("issue.get", {}, () => request("GET", "/issue"))); + } else if (resource === "branch" && action === "create") { + await printJson(await lifecycleOperation("branch.create", { name: flags.name ?? "benchmark-fix" }, () => ( + request("POST", "/branches", { name: flags.name ?? "benchmark-fix" }) + ))); + } else if (resource === "pr" && action === "create") { + const body = { + head: flags.head ?? "benchmark-fix", + base: flags.base ?? "main", + title: flags.title ?? "Benchmark fix", + }; + await printJson(await lifecycleOperation("pr.create", body, () => request("POST", "/pulls", body))); + } else if (resource === "pr" && action === "update") { + const body = { + pr: flags.pr ?? "1", + summary: flags.summary ?? "", + }; + await printJson(await lifecycleOperation("pr.update", body, () => request("POST", "/pulls/update", body))); + } else if (resource === "ci" && action === "get") { + await printJson(await lifecycleOperation("ci.get", { pr: flags.pr ?? "" }, () => ( + request("GET", `/ci?pr=${encodeURIComponent(flags.pr ?? "")}`) + ))); + } else if (resource === "review" && action === "get") { + await printJson(await lifecycleOperation("review.get", { pr: flags.pr ?? "" }, () => ( + request("GET", `/review?pr=${encodeURIComponent(flags.pr ?? "")}`) + ))); + } else if (resource === "complete") { + const body = { + pr: flags.pr ?? "", + summary: flags.summary ?? "", + }; + await printJson(await lifecycleOperation("complete", body, () => request("POST", "/complete", body))); + } else { + process.stderr.write("Usage: issue get | branch create | pr create | pr update | ci get | review get | complete\n"); + process.exit(2); + } +} catch (error) { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + process.exit(1); +} + +async function lifecycleOperation(type, body, httpRequest) { + if (statePath) { + return await fileLifecycleOperation(type, body); + } + return await httpRequest(); +} + +async function fileLifecycleOperation(type, body) { + const state = JSON.parse(await readFile(statePath, "utf8")); + state.events = Array.isArray(state.events) ? state.events : []; + state.pullRequests = Array.isArray(state.pullRequests) ? state.pullRequests : []; + + if (type === "issue.get") { + state.events.push({ type }); + await writeState(state); + return { + provider: state.task?.lifecycle?.issueProvider ?? "github", + repo: state.task?.repo, + instanceId: state.upstream?.instanceId, + baseCommit: state.upstream?.baseCommit, + problemStatement: state.upstream?.problemStatement, + requirements: state.upstream?.requirements ?? null, + interface: state.upstream?.interface ?? null, + }; + } + + if (type === "branch.create") { + const name = stringValue(body.name) || "benchmark-fix"; + state.events.push({ type, name }); + await writeState(state); + return { status: "created", name }; + } + + if (type === "pr.create") { + const number = state.pullRequests.length + 1; + const pullRequest = { + number, + url: `file://${statePath}#pull-${number}`, + head: stringValue(body.head) || "benchmark-fix", + base: stringValue(body.base) || "main", + title: stringValue(body.title) || `Benchmark fix ${number}`, + }; + state.pullRequests.push(pullRequest); + state.events.push({ type, pullRequest }); + await writeState(state); + return pullRequest; + } + + if (type === "pr.update") { + const pr = stringValue(body.pr) || "1"; + state.events.push({ type, pr, summary: stringValue(body.summary) }); + await writeState(state); + return { status: "updated", pr }; + } + + if (type === "ci.get") { + const pr = stringValue(body.pr); + state.events.push({ type, pr }); + await writeState(state); + return { provider: "mock-ci", pr, status: state.ciStatus ?? "passed" }; + } + + if (type === "review.get") { + const pr = stringValue(body.pr); + state.events.push({ type, pr }); + await writeState(state); + return { provider: "mock-review", pr, state: state.reviewState ?? "approved", comments: [] }; + } + + if (type === "complete") { + state.completed = true; + state.events.push({ type, pr: stringValue(body.pr), summary: stringValue(body.summary) }); + await writeState(state); + return { status: "completed" }; + } + + throw new Error(`Unsupported lifecycle operation: ${type}`); +} + +async function writeState(state) { + await writeFile(statePath, `${JSON.stringify(state, null, 2)}\n`, "utf8"); +} + +async function request(method, pathname, body) { + const response = await fetch(new URL(pathname, baseUrl), { + method, + headers: body ? { "content-type": "application/json" } : {}, + body: body ? JSON.stringify(body) : undefined, + }); + const text = await response.text(); + if (!response.ok) throw new Error(text); + return JSON.parse(text); +} + +function parseFlags(args) { + const result = {}; + for (let index = 0; index < args.length; index += 1) { + const key = args[index]; + if (!key.startsWith("--")) continue; + result[key.slice(2)] = args[index + 1] ?? ""; + index += 1; + } + return result; +} + +async function printJson(value) { + process.stdout.write(`${JSON.stringify(value)}\n`); +} + +function stringValue(value) { + return typeof value === "string" ? value : ""; +} diff --git a/benchmarks/agentrail-swe-lifecycle/fixtures/sample-dataset.json b/benchmarks/agentrail-swe-lifecycle/fixtures/sample-dataset.json new file mode 100644 index 0000000..942715b --- /dev/null +++ b/benchmarks/agentrail-swe-lifecycle/fixtures/sample-dataset.json @@ -0,0 +1,75 @@ +{ + "version": 1, + "id": "agentrail-swe-lifecycle-sample", + "defaultScenario": "local-mock-lifecycle", + "tasks": [ + { + "id": "agentrail-swe-pro-hard-aa-001", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "instance_owner__repo-aaaa", + "selectedInstanceId": "instance_owner__repo-aaaa", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "owner/repo", + "baseCommit": "1111111111111111111111111111111111111111", + "language": "python", + "tags": ["python", "bugfix"], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "owner.repo-test", + "beforeRepoSetCommand": "git reset --hard 1111111111111111111111111111111111111111", + "selectedTestFiles": ["tests/test_bug.py"], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + }, + { + "id": "agentrail-swe-pro-hard-aa-002", + "upstream": { + "dataset": "ScaleAI/SWE-bench_Pro", + "config": "default", + "split": "test", + "instanceId": "INSTANCE_OWNER__REPO-BBBB", + "selectedInstanceId": "instance_owner__repo-selected-bbbb", + "sourceUrl": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" + }, + "repo": "owner/ui-repo", + "baseCommit": "2222222222222222222222222222222222222222", + "language": "ts", + "tags": ["ts", "ui"], + "lifecycle": { + "scenario": "local-mock-lifecycle", + "issueProvider": "github", + "issueBodyFrom": "upstream.problem_statement", + "intake": "agentrail_task_ingestion", + "assignment": "agentrail_managed_agent", + "submission": "synthetic_pull_request_or_patch_record", + "ci": "swebench_evaluator", + "review": "deterministic_review", + "ship": "merge_ready_record" + }, + "evaluation": { + "oracle": "swebench_fail_to_pass", + "dockerhubTag": "owner.repo-test", + "beforeRepoSetCommand": "git reset --hard 2222222222222222222222222222222222222222", + "selectedTestFiles": ["tests/test_ui.ts"], + "maxCiRetries": 2, + "maxReviewRetries": 1 + } + } + ] +} diff --git a/benchmarks/agentrail-swe-lifecycle/lib/agentrail-harness.mjs b/benchmarks/agentrail-swe-lifecycle/lib/agentrail-harness.mjs new file mode 100644 index 0000000..8833638 --- /dev/null +++ b/benchmarks/agentrail-swe-lifecycle/lib/agentrail-harness.mjs @@ -0,0 +1,196 @@ +import { once } from "node:events"; +import { mkdir, writeFile } from "node:fs/promises"; +import path from "node:path"; + +import { AgentAuthStore } from "../../../src/agent-auth-store.ts"; +import { AgentRunStore } from "../../../src/agent-run-store.ts"; +import { AgentTaskQueue } from "../../../src/agent-task-queue.ts"; +import { createServer } from "../../../src/app.ts"; +import { TaskEventStore } from "../../../src/task-event-store.ts"; +import { buildAgentRailTaskFields } from "./prompt.mjs"; +import { summarizeLifecycleEvents, AGENTRAIL_LIFECYCLE_EVENTS } from "./lifecycle-provider-harness.mjs"; + +export async function withAgentRailBenchmarkHarness({ runDir, task, upstream, agentId = "agt_bench", flags = {} }, handler) { + const storeDir = path.join(runDir, "agentrail-store"); + await mkdir(storeDir, { recursive: true }); + + const now = () => new Date("2026-05-25T00:00:00.000Z"); + const eventStore = new TaskEventStore({ now, storagePath: path.join(storeDir, "events.json") }); + const lifecycleEvents = []; + const lifecycleEventsPath = path.join(runDir, "lifecycle-events.json"); + const recordLifecycleEvent = (type, data = {}) => { + lifecycleEvents.push({ type, ...data }); + }; + const submissions = []; + let baseUrl = "http://127.0.0.1:0"; + const taskQueue = new AgentTaskQueue({ + now, + eventStore, + storagePath: path.join(storeDir, "tasks.json"), + delegate: { + submitTask: async (taskId, payload) => { + const payloadObject = isRecord(payload) ? payload : {}; + const pullRequest = isRecord(payloadObject.pullRequest) ? payloadObject.pullRequest : {}; + const submissionId = `sub_bench_${submissions.length + 1}`; + const response = { + data: { + taskId, + submissionId, + prUrl: `${baseUrl}/benchmark/pr/${submissions.length + 1}`, + prNumber: submissions.length + 1, + head: typeof pullRequest.head === "string" ? pullRequest.head : null, + base: typeof pullRequest.base === "string" ? pullRequest.base : null, + headSha: typeof pullRequest.headSha === "string" ? pullRequest.headSha : null, + }, + availableActions: ["view_ci_status", "view_review_feedback"], + }; + submissions.push(response.data); + if (flags.benchmarkSurface === "full-lifecycle") { + recordLifecycleEvent("pr.created", response.data); + recordLifecycleEvent("ci.checked", { provider: "mock-ci", status: "passed", prNumber: response.data.prNumber }); + recordLifecycleEvent("review.checked", { provider: "mock-review", state: "approved", prNumber: response.data.prNumber }); + recordLifecycleEvent("task.completed", { taskId }); + } + return response; + }, + }, + }); + const authStore = new AgentAuthStore({ now, storagePath: path.join(storeDir, "auth.json") }); + const { data } = authStore.createKey({ + agent: { + id: agentId, + displayName: "Benchmark Agent", + role: "benchmark_agent", + externalIdentities: [], + }, + scopes: ["tasks:read", "tasks:write", "events:read", "ci:read", "reviews:read", "ship:write"], + }, `benchmark-key:${agentId}`); + const agentRailTask = taskQueue.createTask(buildTaskRecord({ task, upstream, agentId })); + if (flags.benchmarkSurface === "full-lifecycle") { + recordLifecycleEvent("issue.ingested", { taskId: agentRailTask.id, repo: task.repo, instanceId: upstream.instanceId }); + } + + const server = createServer({ + store: eventStore, + authStore, + agentRunStore: new AgentRunStore({ now, storagePath: path.join(storeDir, "agent-runs.json") }), + taskLifecycleStore: taskQueue, + ciStatusAdapter: flags.benchmarkSurface === "full-lifecycle" ? mockCiStatusAdapter() : null, + reviewFeedbackAdapter: flags.benchmarkSurface === "full-lifecycle" ? mockReviewFeedbackAdapter() : null, + now, + }); + + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("Missing AgentRail benchmark server address."); + baseUrl = `http://${address.address}:${address.port}`; + + const envFilePath = path.join(runDir, "agentrail.env"); + await writeAgentEnv({ + envFilePath, + baseUrl, + apiKey: data.apiKey, + agentId, + task, + flags, + }); + + try { + return await handler({ + baseUrl, + apiKey: data.apiKey, + agentId, + agentRailTaskId: agentRailTask.id, + envFilePath, + submissions, + lifecycleEvents, + lifecycleEventsPath, + recordLifecycleEvent, + lifecycleSummary: () => summarizeLifecycleEvents({ + eventsPath: lifecycleEventsPath, + events: lifecycleEvents, + requiredEvents: AGENTRAIL_LIFECYCLE_EVENTS, + }), + }); + } finally { + await writeFile(lifecycleEventsPath, `${JSON.stringify(lifecycleEvents, null, 2)}\n`, "utf8"); + await new Promise((resolve, reject) => { + server.close((error) => { + if (error) reject(error); + else resolve(); + }); + }); + } +} + +function mockCiStatusAdapter() { + return { + async getTaskCiStatus() { + return { + data: { + provider: "mock-ci", + status: "passed", + overallStatus: "passed", + }, + }; + }, + }; +} + +function mockReviewFeedbackAdapter() { + return { + async getTaskReviewFeedback() { + return { + data: { + provider: "mock-review", + state: "approved", + comments: [], + }, + }; + }, + }; +} + +function buildTaskRecord({ task, upstream, agentId }) { + const fields = buildAgentRailTaskFields({ task, upstream }); + const [owner, repo] = task.repo.split("/"); + return { + identifier: fields.identifier, + title: fields.title, + description: fields.description, + status: "todo", + priority: "medium", + assignee: { id: agentId, name: "Benchmark Agent" }, + assigneeAgentId: agentId, + acceptanceCriteria: fields.acceptanceCriteria, + context: fields.context, + availableActions: ["start"], + source: { + provider: "github", + owner, + repo, + baseBranch: "main", + }, + }; +} + +async function writeAgentEnv({ envFilePath, baseUrl, apiKey, agentId, task, flags }) { + await mkdir(path.dirname(envFilePath), { recursive: true }); + await writeFile(envFilePath, [ + `AGENTRAIL_BASE_URL=${baseUrl}`, + `AGENTRAIL_API_KEY=${apiKey}`, + `AGENTRAIL_AGENT_ID=${agentId}`, + `AGENTRAIL_AGENT_RUNNER=${flags.runner ?? "custom"}`, + `AGENTRAIL_AGENT_MODEL=${flags.model ?? "fake-model"}`, + "AGENTRAIL_MAX_CONCURRENT_TASKS=1", + `AGENTRAIL_REPO_ALLOWLIST=${task.repo}`, + `AGENTRAIL_RUNNER_TIMEOUT_SECONDS=${flags.timeoutSeconds ?? 30}`, + flags.runnerCommand ? `AGENTRAIL_BENCH_RUNNER_COMMAND=${flags.runnerCommand}` : null, + "", + ].filter((line) => line !== null).join("\n"), { encoding: "utf8", mode: 0o600 }); +} + +function isRecord(value) { + return value !== null && typeof value === "object" && !Array.isArray(value); +} diff --git a/benchmarks/agentrail-swe-lifecycle/lib/agentrail-lane.mjs b/benchmarks/agentrail-swe-lifecycle/lib/agentrail-lane.mjs new file mode 100644 index 0000000..3025e53 --- /dev/null +++ b/benchmarks/agentrail-swe-lifecycle/lib/agentrail-lane.mjs @@ -0,0 +1,385 @@ +import { execFile } from "node:child_process"; +import { appendFile, mkdir, readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { promisify } from "node:util"; + +import { defaultPrepareWorktree } from "../../../src/cli/agent-runner.ts"; +import { runCli } from "../../../src/cli/index.ts"; +import { createSetupConfig } from "../../../src/cli/setup-config.ts"; +import { writeSetupFiles } from "../../../src/cli/setup-files.ts"; +import { capturePatch } from "./patch.mjs"; +import { runEvaluator } from "./evaluator.mjs"; +import { runRunnerCommand } from "./runner-command.mjs"; +import { parseTokenUsageFromFiles } from "./token-usage.mjs"; +import { prepareLaneWorkspace } from "./workspace.mjs"; +import { withAgentRailBenchmarkHarness } from "./agentrail-harness.mjs"; + +const execFileAsync = promisify(execFile); +const BASE_REF = "refs/agentrail-bench/base"; + +export async function runAgentRailLane({ task, upstream, runDir, repoCache, sourceRepoPath, flags }) { + const workspace = await prepareLaneWorkspace({ + task, + upstream, + lane: "agentrail", + runDir, + repoCache, + sourceRepoPath, + }); + const agentRailHome = path.join(workspace.laneDir, "agentrail-home"); + const stdout = createMemoryWriter(); + const stderr = createMemoryWriter(); + let runnerResult = null; + let managedWorktreePath = null; + + const result = await withAgentRailBenchmarkHarness({ + runDir: workspace.laneDir, + task, + upstream, + flags, + }, async (harness) => { + await writeBenchmarkSetup({ + homePath: agentRailHome, + repoPath: workspace.worktreePath, + task, + baseUrl: harness.baseUrl, + flags, + }); + + const previousHome = process.env.AGENTRAIL_HOME; + const previousTelemetryDisabled = process.env.AGENTRAIL_TELEMETRY_DISABLED; + process.env.AGENTRAIL_HOME = agentRailHome; + process.env.AGENTRAIL_TELEMETRY_DISABLED = "1"; + try { + if (flags.benchmarkSurface === "full-lifecycle") { + harness.recordLifecycleEvent("task.started", { taskId: harness.agentRailTaskId }); + } + const exitCode = await runCli([ + "agent", + "run", + "--once", + "--env-file", + harness.envFilePath, + "--worktree-root", + path.join(workspace.laneDir, "managed-worktrees"), + "--json", + ], { + cwd: workspace.worktreePath, + stdout, + stderr, + agentRunner: { + prepareWorktree: async (params) => { + await defaultPrepareWorktree(params); + await git(["-C", params.worktreePath, "update-ref", BASE_REF, upstream.baseCommit]); + }, + launchRunner: async ({ worktreePath, prompt, logPath, handoffPath, env }) => { + managedWorktreePath = worktreePath; + const preparedPrompt = await prepareBenchmarkPromptVariant({ + worktreePath, + prompt, + handoffPath, + task, + upstream, + flags, + }); + runnerResult = await runRunnerCommand({ + runner: flags.runner, + runnerCommand: flags.runnerCommand, + model: flags.model ?? null, + repoPath: worktreePath, + prompt: preparedPrompt, + logDir: path.dirname(logPath), + timeoutSeconds: flags.timeoutSeconds ?? 1800, + env: { + ...env, + ...(handoffPath ? { + AGENTRAIL_HANDOFF_PATH: handoffPath, + AGENTRAIL_RESULT_PATH: handoffPath, + } : {}), + }, + }); + return { + status: runnerResult.status === "completed" ? "succeeded" : "failed", + exitCode: runnerResult.exitCode, + summary: runnerResult.status === "completed" + ? "Benchmark runner completed through AgentRail." + : "Benchmark runner failed through AgentRail.", + }; + }, + publishBranch: async () => {}, + }, + }); + return { exitCode, submissions: harness.submissions, lifecycle: harness.lifecycleSummary() }; + } finally { + restoreEnv("AGENTRAIL_HOME", previousHome); + restoreEnv("AGENTRAIL_TELEMETRY_DISABLED", previousTelemetryDisabled); + } + }); + + const patch = managedWorktreePath + ? await capturePatch({ + worktreePath: managedWorktreePath, + outputPath: path.join(workspace.laneDir, "patch.diff"), + }) + : { status: "unavailable", path: null, changedFiles: [] }; + const tokenUsage = runnerResult + ? await parseTokenUsageFromFiles([runnerResult.stdoutPath, runnerResult.stderrPath]) + : { status: "unavailable", totalTokens: null, inputTokens: null, outputTokens: null, reasoningTokens: null, sourceFiles: [] }; + const evaluator = await runEvaluator({ + evaluator: flags.evaluator, + evalCommandTemplate: flags.evalCommandTemplate, + patchPath: patch.path, + outputPath: path.join(workspace.laneDir, "evaluator.json"), + timeoutSeconds: flags.timeoutSeconds ?? 1800, + }); + const status = result.exitCode === 0 && runnerResult?.status === "completed" ? "completed" : "failed"; + const success = status === "completed" + && patch.status === "available" + && (flags.benchmarkSurface !== "full-lifecycle" || result.lifecycle?.complete === true) + && (evaluator.status === "passed" || evaluator.status === "unavailable"); + + return { + status, + success, + exitCode: result.exitCode, + runner: runnerResult, + tokenUsage, + patch, + evaluator, + lifecycle: result.lifecycle ?? null, + stdout: stdout.toString(), + stderr: stderr.toString(), + submissions: result.submissions, + workspace: { + laneDir: workspace.laneDir, + worktreePath: managedWorktreePath, + sourceWorktreePath: workspace.worktreePath, + }, + }; +} + +async function writeBenchmarkSetup({ homePath, repoPath, task, baseUrl, flags }) { + const config = createSetupConfig({ + cwd: repoPath, + detectedRepo: { + repoPath, + remoteSlug: task.repo, + defaultBranch: "main", + gitIgnoreHasAgentrail: true, + }, + interactionMode: "non_interactive", + acceptedDefaults: true, + baseUrl, + providerMode: "disabled", + repoAllowlist: [task.repo], + runnerPolicy: benchmarkRunnerPolicy(), + }); + await writeSetupFiles({ homePath, config }); +} + +export function benchmarkRunnerPolicy() { + return { preset: "advisory", enforcementMode: "advisory" }; +} + +async function prepareBenchmarkPromptVariant({ worktreePath, prompt, handoffPath, task, upstream, flags }) { + const variant = flags.agentrailPromptVariant ?? "current"; + if (variant === "current") return prompt; + if (variant !== "contract" && variant !== "skill") { + throw new Error(`Unsupported AgentRail prompt variant: ${variant}`); + } + + const paths = await writeAgentRailRunContractFiles({ worktreePath, handoffPath, task, upstream }); + if (variant === "skill") { + await writeAgentRailSkillFiles({ worktreePath, contractPath: paths.agentrailPath }); + } + return buildContractPrompt({ + task, + paths, + includeSkill: variant === "skill", + }); +} + +async function writeAgentRailRunContractFiles({ worktreePath, handoffPath, task, upstream }) { + const runDir = path.join(worktreePath, ".agentrail-run"); + await mkdir(runDir, { recursive: true }); + const taskPath = path.join(runDir, "task.md"); + const schemaPath = path.join(runDir, "result.schema.json"); + const agentrailPath = path.join(runDir, "agentrail.md"); + await writeFile(taskPath, renderTaskFile({ task, upstream }), "utf8"); + await writeFile(schemaPath, `${JSON.stringify(resultSchema(), null, 2)}\n`, "utf8"); + await writeFile(agentrailPath, renderAgentRailContract({ resultPath: relativePath(worktreePath, handoffPath) }), "utf8"); + return { + taskPath: relativePath(worktreePath, taskPath), + schemaPath: relativePath(worktreePath, schemaPath), + agentrailPath: relativePath(worktreePath, agentrailPath), + resultPath: relativePath(worktreePath, handoffPath), + }; +} + +async function writeAgentRailSkillFiles({ worktreePath, contractPath }) { + const skillDir = path.join(worktreePath, ".codex", "skills", "agentrail-run"); + const referencesDir = path.join(skillDir, "references"); + await mkdir(referencesDir, { recursive: true }); + await addGitExcludeEntry(worktreePath, ".codex/skills/agentrail-run/"); + await writeFile(path.join(skillDir, "SKILL.md"), [ + "---", + "name: agentrail-run", + "description: Use inside AgentRail-managed benchmark runs only when lifecycle, reporting, or result-file details are needed.", + "---", + "", + "# AgentRail Run", + "", + "You are in an AgentRail-managed run. AgentRail owns task assignment, pull requests, CI/review polling, shipping, and rollback.", + "", + "Read `.agentrail-run/task.md` for the assigned task. Write the result JSON to `$AGENTRAIL_RESULT_PATH` using `.agentrail-run/result.schema.json`.", + "", + "Do repository work only. Do not push, create pull requests, ship, roll back, commit manually, or call broad AgentRail APIs.", + "", + "For details, read `references/contract.md`.", + "", + ].join("\n"), "utf8"); + await writeFile(path.join(referencesDir, "contract.md"), [ + "# AgentRail Contract Reference", + "", + `The canonical run contract for this benchmark task is \`${contractPath}\`.`, + "", + "Completion means the repository patch is left in the worktree and the result file says `completed`.", + "Blocked means no safe repository patch can be produced without user action; the result file must include a blocker object.", + "", + ].join("\n"), "utf8"); +} + +function buildContractPrompt({ task, paths, includeSkill }) { + return [ + `You are working on AgentRail task ${task.upstream?.instanceId ? `swe:${task.upstream.instanceId}` : task.id}.`, + "", + `Read ${paths.taskPath}.`, + `Write ${paths.resultPath} matching ${paths.schemaPath}.`, + includeSkill + ? "If you need AgentRail lifecycle/reporting details, use the agentrail-run skill or read .codex/skills/agentrail-run/SKILL.md." + : `If you need AgentRail lifecycle/reporting details, read ${paths.agentrailPath}.`, + "Modify only repository files needed to solve the task.", + "Do not push, create pull requests, ship, roll back, run git add, or run git commit.", + "Leave the final patch in the working tree for AgentRail to collect.", + "Do not expose secrets in output.", + "", + ].join("\n"); +} + +function renderTaskFile({ task, upstream }) { + return [ + "# AgentRail Benchmark Task", + "", + `Repository: ${task.repo}`, + `Instance: ${upstream.instanceId}`, + `Base commit: ${upstream.baseCommit}`, + "", + "## Problem Statement", + "", + upstream.problemStatement, + upstream.requirements ? ["", "## Requirements", "", upstream.requirements].join("\n") : "", + upstream.interface ? ["", "## Interface Notes", "", upstream.interface].join("\n") : "", + "", + "## Acceptance Criteria", + "", + "- The repository is patched to solve the problem statement.", + "- The final patch is left in the git working tree.", + "- No gold patch or hidden test metadata is used.", + "", + ].filter(Boolean).join("\n"); +} + +function renderAgentRailContract({ resultPath }) { + return [ + "# AgentRail Run Contract", + "", + "AgentRail owns lifecycle state, pull request creation, CI reads, review reads, shipping, and rollback.", + "The runner only edits repository files, validates the smallest relevant surface, and writes a structured result.", + "", + "## Rules", + "", + "- Work only inside this git worktree.", + "- Do not hand-roll AgentRail API calls.", + "- Do not run `git add`, `git commit`, `git push`, create pull requests, ship, or roll back.", + "- Leave completed changes in place for AgentRail to collect.", + `- Write the result JSON to \`${resultPath}\` or \`$AGENTRAIL_RESULT_PATH\`.`, + "", + "## Result Examples", + "", + "Completed:", + "```json", + "{\"version\":1,\"status\":\"completed\",\"summary\":\"what changed\",\"checks\":[],\"artifacts\":[]}", + "```", + "", + "Blocked:", + "```json", + "{\"version\":1,\"status\":\"blocked\",\"summary\":\"what is blocked\",\"blocker\":{\"reason\":\"short reason\",\"actionRequired\":\"what the user must do\",\"resumeInstructions\":\"how to continue after the user acts\"}}", + "```", + "", + ].join("\n"); +} + +function resultSchema() { + return { + $schema: "https://json-schema.org/draft/2020-12/schema", + type: "object", + required: ["version", "status", "summary"], + properties: { + version: { const: 1 }, + status: { enum: ["completed", "blocked"] }, + summary: { type: "string" }, + checks: { type: "array", items: { type: "string" }, default: [] }, + artifacts: { type: "array", items: { type: "string" }, default: [] }, + blocker: { + type: "object", + required: ["reason", "actionRequired", "resumeInstructions"], + properties: { + reason: { type: "string" }, + actionRequired: { type: "string" }, + resumeInstructions: { type: "string" }, + }, + additionalProperties: true, + }, + }, + additionalProperties: true, + }; +} + +async function addGitExcludeEntry(worktreePath, entry) { + const excludePath = (await execFileAsync("git", ["-C", worktreePath, "rev-parse", "--git-path", "info/exclude"])).stdout.trim(); + const resolvedExcludePath = path.isAbsolute(excludePath) ? excludePath : path.join(worktreePath, excludePath); + await mkdir(path.dirname(resolvedExcludePath), { recursive: true }); + const existing = await readFile(resolvedExcludePath, "utf8").catch((error) => { + if (error?.code === "ENOENT") return ""; + throw error; + }); + if (existing.split(/\r?\n/u).map((line) => line.trim()).includes(entry)) return; + const prefix = existing.length === 0 || existing.endsWith("\n") ? "" : "\n"; + await appendFile(resolvedExcludePath, `${prefix}${entry}\n`, "utf8"); +} + +function relativePath(from, to) { + return path.relative(from, to).replaceAll(path.sep, "/") || "."; +} + +function createMemoryWriter() { + const chunks = []; + return { + write(chunk) { + chunks.push(Buffer.isBuffer(chunk) ? chunk.toString("utf8") : String(chunk)); + return true; + }, + toString() { + return chunks.join(""); + }, + }; +} + +function restoreEnv(key, value) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; +} + +async function git(args) { + await execFileAsync("git", args, { maxBuffer: 1024 * 1024 * 20 }); +} diff --git a/benchmarks/agentrail-swe-lifecycle/lib/args.mjs b/benchmarks/agentrail-swe-lifecycle/lib/args.mjs new file mode 100644 index 0000000..3d6bc01 --- /dev/null +++ b/benchmarks/agentrail-swe-lifecycle/lib/args.mjs @@ -0,0 +1,142 @@ +const VALID_MODES = new Set(["dry-run", "control", "agentrail", "paired", "phase-attribution"]); +const REAL_MODES = new Set(["control", "agentrail", "paired", "phase-attribution"]); +const VALID_RUNNERS = new Set(["codex", "cursor", "claude-code", "custom"]); +const VALID_EVALUATORS = new Set(["none", "command"]); +const VALID_AGENTRAIL_PROMPT_VARIANTS = new Set(["current", "contract", "skill"]); +const VALID_BENCHMARK_SURFACES = new Set(["patch-only", "full-lifecycle"]); +const VALID_PHASE_SCENARIOS = new Set(["ci-and-review-feedback"]); + +export function parseBenchmarkArgs(argv) { + const flags = { + mode: "dry-run", + evaluator: "none", + cleanup: true, + benchmarkSurface: "patch-only", + phaseScenario: "ci-and-review-feedback", + phaseRepeat: 1 + }; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === "--dry-run") { + flags.mode = "dry-run"; + } else if (arg === "--mode") { + flags.mode = readFlagValue({ argv, index: ++index, flag: arg }); + } else if (arg === "--runner") { + flags.runner = readFlagValue({ argv, index: ++index, flag: arg }); + } else if (arg === "--runner-command") { + flags.runnerCommand = readFlagValue({ argv, index: ++index, flag: arg }); + } else if (arg === "--model") { + flags.model = readFlagValue({ argv, index: ++index, flag: arg }); + } else if (arg === "--timeout-seconds") { + flags.timeoutSeconds = parsePositiveSafeInteger(readFlagValue({ argv, index: ++index, flag: arg }), arg); + } else if (arg === "--repo-cache") { + flags.repoCache = readFlagValue({ argv, index: ++index, flag: arg }); + } else if (arg === "--source-repo-path") { + flags.sourceRepoPath = readFlagValue({ argv, index: ++index, flag: arg }); + } else if (arg === "--evaluator") { + flags.evaluator = readFlagValue({ argv, index: ++index, flag: arg }); + } else if (arg === "--eval-command-template") { + flags.evalCommandTemplate = readFlagValue({ argv, index: ++index, flag: arg }); + } else if (arg === "--require-token-usage") { + flags.requireTokenUsage = true; + } else if (arg === "--require-evaluator") { + flags.requireEvaluator = true; + } else if (arg === "--continue-on-error") { + flags.continueOnError = true; + } else if (arg === "--agentrail-prompt-variant") { + flags.agentrailPromptVariant = readFlagValue({ argv, index: ++index, flag: arg }); + } else if (arg === "--benchmark-surface") { + flags.benchmarkSurface = readFlagValue({ argv, index: ++index, flag: arg }); + } else if (arg === "--phase-scenario") { + flags.phaseScenario = readFlagValue({ argv, index: ++index, flag: arg }); + } else if (arg === "--phase-repeat") { + flags.phaseRepeat = parsePositiveSafeInteger(readFlagValue({ argv, index: ++index, flag: arg }), arg); + } else if (arg === "--require-phase-token-usage") { + flags.requirePhaseTokenUsage = true; + } else if (arg === "--no-cleanup") { + flags.cleanup = false; + } else if (arg === "--dataset") { + flags.dataset = readFlagValue({ argv, index: ++index, flag: arg }); + } else if (arg === "--count") { + flags.count = readFlagValue({ argv, index: ++index, flag: arg }); + } else if (arg === "--repo") { + flags.repo = readFlagValue({ argv, index: ++index, flag: arg }); + } else if (arg === "--language") { + flags.language = readFlagValue({ argv, index: ++index, flag: arg }); + } else if (arg === "--tag") { + flags.tag = readFlagValue({ argv, index: ++index, flag: arg }); + } else if (arg === "--task") { + flags.taskId = readFlagValue({ argv, index: ++index, flag: arg }); + } else if (arg === "--seed") { + flags.seed = readFlagValue({ argv, index: ++index, flag: arg }); + } else if (arg === "--output-dir") { + flags.outputDir = readFlagValue({ argv, index: ++index, flag: arg }); + } else if (arg === "--run-id") { + flags.runId = readFlagValue({ argv, index: ++index, flag: arg }); + } else { + throw new Error(`Unknown argument: ${arg}`); + } + } + + validateBenchmarkArgs(flags); + return flags; +} + +function validateBenchmarkArgs(flags) { + if (!VALID_MODES.has(flags.mode)) { + throw new Error(`--mode must be one of: ${[...VALID_MODES].join(", ")}.`); + } + if (flags.runner !== undefined && !VALID_RUNNERS.has(flags.runner)) { + throw new Error(`--runner must be one of: ${[...VALID_RUNNERS].join(", ")}.`); + } + if (!VALID_EVALUATORS.has(flags.evaluator)) { + throw new Error(`--evaluator must be one of: ${[...VALID_EVALUATORS].join(", ")}.`); + } + if (REAL_MODES.has(flags.mode) && flags.runner === undefined) { + throw new Error("--runner is required for control, agentrail, and paired modes."); + } + if (flags.runner === "custom" && flags.runnerCommand === undefined) { + throw new Error("--runner-command is required when --runner custom is used."); + } + if (flags.evaluator === "command" && flags.evalCommandTemplate === undefined) { + throw new Error("--eval-command-template is required when --evaluator command is used."); + } + if (flags.runner === "cursor" && (flags.requireTokenUsage || flags.requirePhaseTokenUsage)) { + throw new Error( + "Cursor Agent CLI does not expose token usage in its JSON output. " + + "Use --runner codex for token-required benchmarks, or omit token usage requirements for Cursor smoke runs.", + ); + } + if ( + flags.agentrailPromptVariant !== undefined + && !VALID_AGENTRAIL_PROMPT_VARIANTS.has(flags.agentrailPromptVariant) + ) { + throw new Error(`--agentrail-prompt-variant must be one of: ${[...VALID_AGENTRAIL_PROMPT_VARIANTS].join(", ")}.`); + } + if (!VALID_BENCHMARK_SURFACES.has(flags.benchmarkSurface)) { + throw new Error(`--benchmark-surface must be one of: ${[...VALID_BENCHMARK_SURFACES].join(", ")}.`); + } + if (!VALID_PHASE_SCENARIOS.has(flags.phaseScenario)) { + throw new Error(`--phase-scenario must be one of: ${[...VALID_PHASE_SCENARIOS].join(", ")}.`); + } +} + +function readFlagValue({ argv, index, flag }) { + const value = argv[index]; + if (value === undefined || value.startsWith("--")) { + throw new Error(`${flag} requires a value.`); + } + return value; +} + +function parsePositiveSafeInteger(value, flag) { + if (!/^[1-9]\d*$/u.test(value)) { + throw new Error(`${flag} must be a positive safe integer.`); + } + const parsed = Number(value); + if (!Number.isSafeInteger(parsed)) { + throw new Error(`${flag} must be a positive safe integer.`); + } + return parsed; +} diff --git a/benchmarks/agentrail-swe-lifecycle/lib/control-lane.mjs b/benchmarks/agentrail-swe-lifecycle/lib/control-lane.mjs new file mode 100644 index 0000000..d221e7b --- /dev/null +++ b/benchmarks/agentrail-swe-lifecycle/lib/control-lane.mjs @@ -0,0 +1,149 @@ +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { runEvaluator } from "./evaluator.mjs"; +import { capturePatch } from "./patch.mjs"; +import { buildControlPrompt } from "./prompt.mjs"; +import { runRunnerCommand } from "./runner-command.mjs"; +import { parseTokenUsageFromFiles } from "./token-usage.mjs"; +import { prepareLaneWorkspace } from "./workspace.mjs"; +import { buildFullLifecycleControlPrompt } from "./lifecycle-control-prompt.mjs"; +import { + CONTROL_LIFECYCLE_EVENTS, + summarizeLifecycleEvents, + withLifecycleProviderHarness, +} from "./lifecycle-provider-harness.mjs"; + +export async function runControlLane({ task, upstream, runDir, repoCache, sourceRepoPath, flags }) { + if (flags.benchmarkSurface === "full-lifecycle") { + return await runFullLifecycleControlLane({ task, upstream, runDir, repoCache, sourceRepoPath, flags }); + } + return await runPatchOnlyControlLane({ task, upstream, runDir, repoCache, sourceRepoPath, flags }); +} + +async function runPatchOnlyControlLane({ task, upstream, runDir, repoCache, sourceRepoPath, flags }) { + const workspace = await prepareLaneWorkspace({ + task, + upstream, + lane: "control", + runDir, + repoCache, + sourceRepoPath, + }); + const prompt = buildControlPrompt({ task, upstream }); + const runner = await runRunnerCommand({ + runner: flags.runner, + runnerCommand: flags.runnerCommand, + model: flags.model ?? null, + repoPath: workspace.worktreePath, + prompt, + logDir: workspace.laneDir, + timeoutSeconds: flags.timeoutSeconds ?? 1800, + env: {}, + }); + const patch = await capturePatch({ + worktreePath: workspace.worktreePath, + outputPath: path.join(workspace.laneDir, "patch.diff"), + }); + const tokenUsage = await parseTokenUsageFromFiles([runner.stdoutPath, runner.stderrPath]); + const evaluator = await runEvaluator({ + evaluator: flags.evaluator, + evalCommandTemplate: flags.evalCommandTemplate, + patchPath: patch.path, + outputPath: path.join(workspace.laneDir, "evaluator.json"), + timeoutSeconds: flags.timeoutSeconds ?? 1800, + }); + const success = runner.status === "completed" + && patch.status === "available" + && (evaluator.status === "passed" || evaluator.status === "unavailable"); + + return { + status: runner.status, + success, + runner, + tokenUsage, + patch, + evaluator, + workspace: { + laneDir: workspace.laneDir, + worktreePath: workspace.worktreePath, + }, + }; +} + +async function runFullLifecycleControlLane({ task, upstream, runDir, repoCache, sourceRepoPath, flags }) { + const workspace = await prepareLaneWorkspace({ + task, + upstream, + lane: "control", + runDir, + repoCache, + sourceRepoPath, + }); + + return await withLifecycleProviderHarness({ + runDir: workspace.laneDir, + stateDir: workspace.worktreePath, + task, + upstream, + }, async (lifecycle) => { + const lifecycleCli = path.join(repoRoot(), "benchmarks", "agentrail-swe-lifecycle", "fixtures", "mock-lifecycle-cli.mjs"); + const prompt = buildFullLifecycleControlPrompt({ + repo: task.repo, + lifecycleCli: `node ${lifecycleCli}`, + }); + const runner = await runRunnerCommand({ + runner: flags.runner, + runnerCommand: flags.runnerCommand, + model: flags.model ?? null, + repoPath: workspace.worktreePath, + prompt, + logDir: workspace.laneDir, + timeoutSeconds: flags.timeoutSeconds ?? 1800, + env: { + AGENTRAIL_BENCH_LIFECYCLE_URL: lifecycle.baseUrl, + AGENTRAIL_BENCH_LIFECYCLE_STATE_PATH: lifecycle.statePath, + AGENTRAIL_BENCH_LIFECYCLE_CLI: lifecycleCli, + }, + }); + const patch = await capturePatch({ + worktreePath: workspace.worktreePath, + outputPath: path.join(workspace.laneDir, "patch.diff"), + }); + const tokenUsage = await parseTokenUsageFromFiles([runner.stdoutPath, runner.stderrPath]); + const evaluator = await runEvaluator({ + evaluator: flags.evaluator, + evalCommandTemplate: flags.evalCommandTemplate, + patchPath: patch.path, + outputPath: path.join(workspace.laneDir, "evaluator.json"), + timeoutSeconds: flags.timeoutSeconds ?? 1800, + }); + const lifecycleSummary = summarizeLifecycleEvents({ + eventsPath: lifecycle.eventsPath, + events: await lifecycle.readEvents(), + requiredEvents: CONTROL_LIFECYCLE_EVENTS, + }); + const success = runner.status === "completed" + && patch.status === "available" + && lifecycleSummary.complete + && (evaluator.status === "passed" || evaluator.status === "unavailable"); + + return { + status: runner.status, + success, + runner, + tokenUsage, + patch, + evaluator, + lifecycle: lifecycleSummary, + workspace: { + laneDir: workspace.laneDir, + worktreePath: workspace.worktreePath, + }, + }; + }); +} + +function repoRoot() { + return path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", "..", ".."); +} diff --git a/benchmarks/agentrail-swe-lifecycle/lib/dataset.mjs b/benchmarks/agentrail-swe-lifecycle/lib/dataset.mjs new file mode 100644 index 0000000..e165c62 --- /dev/null +++ b/benchmarks/agentrail-swe-lifecycle/lib/dataset.mjs @@ -0,0 +1,75 @@ +import { readFile } from "node:fs/promises"; + +export async function loadLifecycleDataset(filePath) { + const dataset = JSON.parse(await readFile(filePath, "utf8")); + if (dataset.version !== 1 || !Array.isArray(dataset.tasks) || !dataset.tasks.every(isValidTask)) { + throw new Error(`Invalid AgentRail SWE lifecycle dataset at ${filePath}.`); + } + return dataset; +} + +function isValidTask(task) { + return isRecord(task) + && isNonEmptyString(task.id) + && isNonEmptyString(task.repo) + && (task.language === null || typeof task.language === "string") + && (task.tags === undefined || Array.isArray(task.tags)) + && isRecord(task.upstream) + && isNonEmptyString(task.upstream.instanceId) + && ( + task.upstream.selectedInstanceId === undefined + || task.upstream.selectedInstanceId === null + || typeof task.upstream.selectedInstanceId === "string" + ); +} + +function isRecord(value) { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +function isNonEmptyString(value) { + return typeof value === "string" && value.length > 0; +} + +export function selectLifecycleTasks(dataset, options = {}) { + let tasks = [...dataset.tasks].sort((left, right) => left.id.localeCompare(right.id)); + if (options.taskId) tasks = tasks.filter((task) => matchesTaskId(task, options.taskId)); + if (options.repo) tasks = tasks.filter((task) => task.repo === options.repo); + if (options.language) tasks = tasks.filter((task) => task.language === options.language); + if (options.tag) tasks = tasks.filter((task) => Array.isArray(task.tags) && task.tags.includes(options.tag)); + if (options.seed !== undefined) tasks = seededShuffle(tasks, String(options.seed)); + const count = normalizeCount(options.count, tasks.length); + return tasks.slice(0, count); +} + +function matchesTaskId(task, taskId) { + if (task.id === taskId) return true; + const normalizedTaskId = String(taskId).toLowerCase(); + return [task.upstream?.instanceId, task.upstream?.selectedInstanceId] + .filter((upstreamId) => typeof upstreamId === "string") + .some((upstreamId) => upstreamId.toLowerCase() === normalizedTaskId); +} + +function normalizeCount(value, fallback) { + if (value === undefined || value === null) return fallback; + if (!/^[1-9]\d*$/u.test(String(value))) { + throw new Error(`--count must be a positive integer, received ${value}.`); + } + const parsed = Number(value); + if (!Number.isSafeInteger(parsed)) { + throw new Error(`--count must be a positive integer, received ${value}.`); + } + return parsed; +} + +function seededShuffle(items, seed) { + const shuffled = [...items]; + let state = 0; + for (const char of seed) state = (state * 31 + char.charCodeAt(0)) >>> 0; + for (let index = shuffled.length - 1; index > 0; index -= 1) { + state = (1664525 * state + 1013904223) >>> 0; + const swapIndex = state % (index + 1); + [shuffled[index], shuffled[swapIndex]] = [shuffled[swapIndex], shuffled[index]]; + } + return shuffled; +} diff --git a/benchmarks/agentrail-swe-lifecycle/lib/evaluator.mjs b/benchmarks/agentrail-swe-lifecycle/lib/evaluator.mjs new file mode 100644 index 0000000..fd1f090 --- /dev/null +++ b/benchmarks/agentrail-swe-lifecycle/lib/evaluator.mjs @@ -0,0 +1,51 @@ +import { spawn } from "node:child_process"; + +export async function runEvaluator({ evaluator, evalCommandTemplate, patchPath, outputPath, cwd = process.cwd(), timeoutSeconds = 300 }) { + if (evaluator === "none" || !evaluator) { + return { status: "unavailable", exitCode: null, outputPath: null }; + } + if (evaluator !== "command") throw new Error(`Unsupported evaluator: ${evaluator}`); + + const command = materializeTemplate(evalCommandTemplate, { patchPath, outputPath }); + const [executable, ...args] = splitCommand(command); + const result = await spawnWithTimeout({ executable, args, cwd, timeoutMs: timeoutSeconds * 1000 }); + return { + status: result.exitCode === 0 ? "passed" : "failed", + exitCode: result.exitCode, + outputPath, + stdout: result.stdout, + stderr: result.stderr, + }; +} + +function materializeTemplate(template, values) { + return String(template) + .replaceAll("{patchPath}", values.patchPath) + .replaceAll("{outputPath}", values.outputPath); +} + +function splitCommand(command) { + const parts = String(command).match(/(?:[^\s"]+|"[^"]*")+/gu)?.map((part) => part.replace(/^"|"$/gu, "")) ?? []; + if (parts.length === 0) throw new Error("Evaluator command must not be empty."); + return parts; +} + +async function spawnWithTimeout({ executable, args, cwd, timeoutMs }) { + return await new Promise((resolve, reject) => { + const child = spawn(executable, args, { cwd, stdio: ["ignore", "pipe", "pipe"] }); + const stdout = []; + const stderr = []; + const timer = setTimeout(() => child.kill("SIGTERM"), timeoutMs); + child.stdout.on("data", (chunk) => stdout.push(Buffer.from(chunk))); + child.stderr.on("data", (chunk) => stderr.push(Buffer.from(chunk))); + child.on("error", reject); + child.on("close", (exitCode) => { + clearTimeout(timer); + resolve({ + exitCode, + stdout: Buffer.concat(stdout).toString("utf8"), + stderr: Buffer.concat(stderr).toString("utf8"), + }); + }); + }); +} diff --git a/benchmarks/agentrail-swe-lifecycle/lib/lifecycle-control-prompt.mjs b/benchmarks/agentrail-swe-lifecycle/lib/lifecycle-control-prompt.mjs new file mode 100644 index 0000000..259df02 --- /dev/null +++ b/benchmarks/agentrail-swe-lifecycle/lib/lifecycle-control-prompt.mjs @@ -0,0 +1,30 @@ +export function buildFullLifecycleControlPrompt({ repo, lifecycleCli }) { + return [ + "You are solving a software engineering task through the normal provider lifecycle.", + "", + `Repository: ${repo}`, + "", + "You must use the lifecycle CLI to fetch the issue and complete the provider workflow.", + "", + "Lifecycle CLI:", + `- Fetch issue: ${lifecycleCli} issue get`, + `- Create branch record: ${lifecycleCli} branch create --name benchmark-fix`, + `- Create pull request: ${lifecycleCli} pr create --head benchmark-fix --base main --title "Benchmark fix"`, + `- Check CI: ${lifecycleCli} ci get --pr `, + `- Check code review: ${lifecycleCli} review get --pr `, + `- Complete lifecycle: ${lifecycleCli} complete --pr --summary "short summary"`, + "", + "Required flow:", + "1. Fetch the issue through the lifecycle CLI before inspecting the repository.", + "2. Modify the repository to solve the fetched issue.", + "3. Leave the final patch in the git working tree.", + "4. Create a pull request through the lifecycle CLI.", + "5. Check CI through the lifecycle CLI.", + "6. Check code review through the lifecycle CLI.", + "7. Complete the lifecycle through the lifecycle CLI.", + "", + "Do not use gold patches or hidden test metadata.", + "Do not invent credentials or external services.", + "", + ].join("\n"); +} diff --git a/benchmarks/agentrail-swe-lifecycle/lib/lifecycle-provider-harness.mjs b/benchmarks/agentrail-swe-lifecycle/lib/lifecycle-provider-harness.mjs new file mode 100644 index 0000000..48c1e97 --- /dev/null +++ b/benchmarks/agentrail-swe-lifecycle/lib/lifecycle-provider-harness.mjs @@ -0,0 +1,183 @@ +import { once } from "node:events"; +import { createServer } from "node:http"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; + +export const CONTROL_LIFECYCLE_EVENTS = ["issue.get", "branch.create", "pr.create", "ci.get", "review.get", "complete"]; +export const PHASE_ATTRIBUTION_LIFECYCLE_EVENTS = [ + "issue.get", + "pr.create", + "ci.get", + "review.get", + "pr.update", + "complete", +]; +export const AGENTRAIL_LIFECYCLE_EVENTS = ["issue.ingested", "task.started", "pr.created", "ci.checked", "review.checked", "task.completed"]; + +export async function withLifecycleProviderHarness({ runDir, stateDir, task, upstream }, handler) { + const events = []; + const resolvedRunDir = path.resolve(runDir); + const resolvedStateDir = path.resolve(stateDir ?? runDir); + const eventsPath = path.join(resolvedRunDir, "lifecycle-events.json"); + const statePath = path.join(resolvedStateDir, "lifecycle-state.json"); + await mkdir(resolvedRunDir, { recursive: true }); + await mkdir(resolvedStateDir, { recursive: true }); + + let baseUrl = "http://127.0.0.1:0"; + const state = { + task, + upstream, + pullRequests: [], + ciStatus: "passed", + reviewState: "approved", + completed: false, + }; + const recordEvent = (type, data = {}) => { + events.push({ type, ...data }); + }; + await writeLifecycleState({ statePath, state, events }); + + const server = createServer(async (request, response) => { + try { + const url = new URL(request.url ?? "/", "http://127.0.0.1"); + const body = request.method === "POST" ? await readJsonBody(request) : {}; + + if (request.method === "GET" && url.pathname === "/issue") { + recordEvent("issue.get"); + return sendJson(response, { + provider: task.lifecycle?.issueProvider ?? "github", + repo: task.repo, + instanceId: upstream.instanceId, + baseCommit: upstream.baseCommit, + problemStatement: upstream.problemStatement, + requirements: upstream.requirements ?? null, + interface: upstream.interface ?? null, + }); + } + + if (request.method === "POST" && url.pathname === "/branches") { + const name = stringValue(body.name) || "benchmark-fix"; + recordEvent("branch.create", { name }); + return sendJson(response, { status: "created", name }); + } + + if (request.method === "POST" && url.pathname === "/pulls") { + const number = state.pullRequests.length + 1; + const pullRequest = { + number, + url: `${baseUrl}/pull/${number}`, + head: stringValue(body.head) || "benchmark-fix", + base: stringValue(body.base) || "main", + title: stringValue(body.title) || `Benchmark fix ${number}`, + }; + state.pullRequests.push(pullRequest); + recordEvent("pr.create", { pullRequest }); + return sendJson(response, pullRequest); + } + + if (request.method === "POST" && url.pathname === "/pulls/update") { + const pr = stringValue(body.pr) || "1"; + recordEvent("pr.update", { pr, summary: stringValue(body.summary) }); + return sendJson(response, { status: "updated", pr }); + } + + if (request.method === "GET" && url.pathname === "/ci") { + const pr = url.searchParams.get("pr") ?? ""; + recordEvent("ci.get", { pr }); + return sendJson(response, { provider: "mock-ci", pr, status: state.ciStatus }); + } + + if (request.method === "GET" && url.pathname === "/review") { + const pr = url.searchParams.get("pr") ?? ""; + recordEvent("review.get", { pr }); + return sendJson(response, { provider: "mock-review", pr, state: state.reviewState, comments: [] }); + } + + if (request.method === "POST" && url.pathname === "/complete") { + state.completed = true; + recordEvent("complete", { pr: stringValue(body.pr), summary: stringValue(body.summary) }); + return sendJson(response, { status: "completed" }); + } + + response.writeHead(404, { "content-type": "application/json" }); + response.end(`${JSON.stringify({ error: "not_found" })}\n`); + } catch (error) { + response.writeHead(500, { "content-type": "application/json" }); + response.end(`${JSON.stringify({ error: error instanceof Error ? error.message : String(error) })}\n`); + } + }); + + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("Missing lifecycle provider address."); + baseUrl = `http://${address.address}:${address.port}`; + + try { + return await handler({ + baseUrl, + statePath, + eventsPath, + events, + recordEvent, + readEvents: async () => await readLifecycleStateEvents({ statePath, fallbackEvents: events }), + state, + }); + } finally { + const recordedEvents = await readLifecycleStateEvents({ statePath, fallbackEvents: events }); + await writeFile(eventsPath, `${JSON.stringify(recordedEvents, null, 2)}\n`, "utf8"); + await new Promise((resolve, reject) => { + server.close((error) => error ? reject(error) : resolve()); + }); + } +} + +async function writeLifecycleState({ statePath, state, events }) { + await writeFile(statePath, `${JSON.stringify({ + task: state.task, + upstream: state.upstream, + pullRequests: state.pullRequests, + ciStatus: state.ciStatus, + reviewState: state.reviewState, + completed: state.completed, + events, + }, null, 2)}\n`, "utf8"); +} + +async function readLifecycleStateEvents({ statePath, fallbackEvents }) { + try { + const state = JSON.parse(await readFile(statePath, "utf8")); + if (Array.isArray(state.events) && state.events.length > 0) return state.events; + return fallbackEvents; + } catch { + return fallbackEvents; + } +} + +export function summarizeLifecycleEvents({ eventsPath, events, requiredEvents }) { + const eventTypes = Array.isArray(events) ? events.map((event) => event.type).filter(Boolean) : []; + const missingEvents = requiredEvents.filter((eventType) => !eventTypes.includes(eventType)); + return { + status: "available", + eventsPath, + events: eventTypes, + missingEvents, + complete: missingEvents.length === 0, + }; +} + +async function readJsonBody(request) { + const chunks = []; + for await (const chunk of request) chunks.push(Buffer.from(chunk)); + const text = Buffer.concat(chunks).toString("utf8").trim(); + return text ? JSON.parse(text) : {}; +} + +function sendJson(response, data) { + response.writeHead(200, { "content-type": "application/json" }); + response.end(`${JSON.stringify(data)}\n`); +} + +function stringValue(value) { + return typeof value === "string" ? value : ""; +} diff --git a/benchmarks/agentrail-swe-lifecycle/lib/local-lifecycle.mjs b/benchmarks/agentrail-swe-lifecycle/lib/local-lifecycle.mjs new file mode 100644 index 0000000..45602f2 --- /dev/null +++ b/benchmarks/agentrail-swe-lifecycle/lib/local-lifecycle.mjs @@ -0,0 +1,121 @@ +import { createHash } from "node:crypto"; + +const DETERMINISTIC_CREATED_AT = "1970-01-01T00:00:00.000Z"; +const RUN_ID_PATTERN = /^run_[A-Za-z0-9_-]+$/u; +const TASK_ID_PATTERN = /^[a-z0-9]+(?:-[a-z0-9]+)*$/u; +const REPO_PATTERN = /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u; +const BASE_COMMIT_PATTERN = /^[a-f0-9]{40}$/u; +const SUPPORTED_ISSUE_PROVIDERS = new Set(["github", "linear"]); + +export function buildLocalLifecycleArtifacts({ task, upstream, runId }) { + const issueProvider = validateArtifactInput({ task, upstream, runId }); + return { + issue: { + provider: issueProvider, + mode: "local_mock", + externalId: `${runId}:${task.id}:issue`, + title: issueTitleForTask(task, upstream), + body: upstream.problemStatement, + repo: task.repo, + baseCommit: task.baseCommit, + createdAt: DETERMINISTIC_CREATED_AT + }, + agentrailTask: { + id: syntheticTaskId({ runId, taskId: task.id }), + identifier: `${issueProvider}:${task.repo}:issues/${runId}-${task.id}`, + status: "todo", + sourceInstanceId: upstream.instanceId, + availableActions: ["start"], + createdAt: DETERMINISTIC_CREATED_AT + }, + pullRequest: { + provider: issueProvider, + mode: "synthetic_pr_record", + status: "not_created", + number: null, + headSha: null, + branch: null + }, + ci: { + provider: "swebench_evaluator", + overallStatus: "not_run", + failureSummaries: [] + }, + review: { + provider: "deterministic_review", + outcome: "not_reviewed", + blockingFindings: [] + }, + final: { + status: "pending", + success: false, + reason: null + } + }; +} + +function issueTitleForTask(task, upstream) { + return `[${task.upstream?.instanceId ?? upstream.instanceId}] ${task.repo}`; +} + +function syntheticTaskId({ runId, taskId }) { + const digest = createHash("sha256") + .update(runId) + .update("\0") + .update(taskId) + .digest("hex") + .slice(0, 24); + return `tsk_${digest}`; +} + +function validateArtifactInput({ task, upstream, runId }) { + if (!isRecord(task)) { + throw new Error("task must be a non-null object."); + } + if (!isRecord(upstream)) { + throw new Error("upstream must be a non-null object."); + } + if (task.lifecycle !== undefined && !isRecord(task.lifecycle)) { + throw new Error("task.lifecycle must be a non-null object when present."); + } + if (typeof runId !== "string" || !RUN_ID_PATTERN.test(runId)) { + throw new Error("runId must match /^run_[A-Za-z0-9_-]+$/."); + } + if (typeof task.id !== "string" || !TASK_ID_PATTERN.test(task.id)) { + throw new Error("task.id must match /^[a-z0-9]+(?:-[a-z0-9]+)*$/."); + } + if (typeof task.repo !== "string" || !REPO_PATTERN.test(task.repo)) { + throw new Error("task.repo must be an owner/name repository."); + } + if (typeof task.baseCommit !== "string" || !BASE_COMMIT_PATTERN.test(task.baseCommit)) { + throw new Error("task.baseCommit must be a 40-character lowercase hex SHA."); + } + if (typeof upstream.instanceId !== "string" || upstream.instanceId.length === 0) { + throw new Error("upstream.instanceId must be a non-empty string."); + } + if (typeof upstream.problemStatement !== "string" || upstream.problemStatement.length === 0) { + throw new Error("upstream.problemStatement must be a non-empty string."); + } + if ( + task.upstream !== undefined + && !isRecord(task.upstream) + ) { + throw new Error("task.upstream must be a non-null object when present."); + } + if ( + isRecord(task.upstream) + && task.upstream.instanceId !== undefined + && (typeof task.upstream.instanceId !== "string" || task.upstream.instanceId.length === 0) + ) { + throw new Error("task.upstream.instanceId must be a non-empty string."); + } + const issueProvider = task.lifecycle?.issueProvider ?? "github"; + if (!SUPPORTED_ISSUE_PROVIDERS.has(issueProvider)) { + throw new Error("issueProvider must be github or linear."); + } + return issueProvider; +} + +function isRecord(value) { + return typeof value === "object" && value !== null && !Array.isArray(value); +} diff --git a/benchmarks/agentrail-swe-lifecycle/lib/patch.mjs b/benchmarks/agentrail-swe-lifecycle/lib/patch.mjs new file mode 100644 index 0000000..12111ed --- /dev/null +++ b/benchmarks/agentrail-swe-lifecycle/lib/patch.mjs @@ -0,0 +1,148 @@ +import { execFile } from "node:child_process"; +import { lstat, mkdir, readlink, realpath, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { promisify } from "node:util"; + +const execFileAsync = promisify(execFile); +const GIT_MAX_BUFFER = 1024 * 1024 * 20; +const BASE_REF = "refs/agentrail-bench/base"; +const OUTPUT_PATH_ERROR = "capturePatch outputPath must be outside the benchmark worktree."; +const BENCHMARK_PATCH_EXCLUDES = new Set([ + "lifecycle-state.json", + ".agentrail-run/result.json", +]); + +export async function capturePatch({ worktreePath, outputPath }) { + await assertOutputPathOutsideWorktree({ worktreePath, outputPath }); + + if (!(await baseRefExists(worktreePath))) { + throw new Error("capturePatch requires refs/agentrail-bench/base in benchmark worktrees."); + } + + await includeUntrackedFiles(worktreePath); + + const diff = await git(["-C", worktreePath, "diff", "--binary", BASE_REF, ...patchPathspecArgs()]); + const changedFilesOutput = await git(["-C", worktreePath, "diff", "--name-only", "-z", BASE_REF, ...patchPathspecArgs()]); + const commitSha = (await git(["-C", worktreePath, "rev-parse", "HEAD"])).trim(); + const changedFiles = changedFilesOutput + .split("\0") + .filter((line) => line.length > 0 && !BENCHMARK_PATCH_EXCLUDES.has(line)); + + await mkdir(path.dirname(outputPath), { recursive: true }); + await writeFile(outputPath, diff, "utf8"); + + return { + status: diff.trim().length > 0 ? "available" : "empty", + path: outputPath, + changedFiles, + commitSha + }; +} + +async function assertOutputPathOutsideWorktree({ worktreePath, outputPath }) { + const realWorktreePath = await realpath(worktreePath); + const realOutputLocation = await resolveOutputLocation(outputPath); + if (isInsideOrEqual(realWorktreePath, realOutputLocation)) { + throw new Error(OUTPUT_PATH_ERROR); + } +} + +async function resolveOutputLocation(outputPath) { + const outputAbsolutePath = path.resolve(outputPath); + + const symlinkTarget = await outputSymlinkTarget(outputAbsolutePath); + if (symlinkTarget) return resolveSymlinkTargetLocation({ symlinkPath: outputAbsolutePath, symlinkTarget }); + + try { + return await realpath(outputAbsolutePath); + } catch (error) { + if (error?.code !== "ENOENT") throw error; + } + + const outputParentPath = path.dirname(outputAbsolutePath); + const { existingPath, missingSegments } = await nearestExistingPath(outputParentPath); + const realExistingPath = await realpath(existingPath); + return path.resolve(realExistingPath, ...missingSegments, path.basename(outputAbsolutePath)); +} + +async function outputSymlinkTarget(outputAbsolutePath) { + try { + const entry = await lstat(outputAbsolutePath); + if (!entry.isSymbolicLink()) return null; + return await readlink(outputAbsolutePath); + } catch (error) { + if (error?.code === "ENOENT") return null; + throw error; + } +} + +async function resolveSymlinkTargetLocation({ symlinkPath, symlinkTarget }) { + const targetPath = path.resolve(path.dirname(symlinkPath), symlinkTarget); + + try { + return await realpath(targetPath); + } catch (error) { + if (error?.code !== "ENOENT") throw error; + } + + const targetParentPath = path.dirname(targetPath); + const { existingPath, missingSegments } = await nearestExistingPath(targetParentPath); + const realExistingPath = await realpath(existingPath); + return path.resolve(realExistingPath, ...missingSegments, path.basename(targetPath)); +} + +async function nearestExistingPath(targetPath) { + const missingSegments = []; + let currentPath = path.resolve(targetPath); + + while (true) { + try { + await realpath(currentPath); + return { existingPath: currentPath, missingSegments: missingSegments.reverse() }; + } catch (error) { + if (error?.code !== "ENOENT") throw error; + } + + const parentPath = path.dirname(currentPath); + if (parentPath === currentPath) throw new Error(OUTPUT_PATH_ERROR); + missingSegments.push(path.basename(currentPath)); + currentPath = parentPath; + } +} + +function isInsideOrEqual(parentPath, candidatePath) { + const relative = path.relative(parentPath, candidatePath); + return relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative)); +} + +async function includeUntrackedFiles(worktreePath) { + const output = await git(["-C", worktreePath, "ls-files", "--others", "--exclude-standard", "-z"]); + const files = output + .split("\0") + .filter((file) => file.length > 0 && !BENCHMARK_PATCH_EXCLUDES.has(file)); + if (files.length === 0) return; + await git(["-C", worktreePath, "add", "--intent-to-add", "--", ...files]); +} + +function patchPathspecArgs() { + return [ + "--", + ".", + ...[...BENCHMARK_PATCH_EXCLUDES].map((file) => `:(exclude)${file}`), + ]; +} + +async function baseRefExists(worktreePath) { + try { + await git(["-C", worktreePath, "rev-parse", "--verify", `${BASE_REF}^{commit}`]); + return true; + } catch (error) { + if (typeof error?.code === "number") return false; + throw error; + } +} + +async function git(args) { + const { stdout } = await execFileAsync("git", args, { encoding: "utf8", maxBuffer: GIT_MAX_BUFFER }); + return stdout; +} diff --git a/benchmarks/agentrail-swe-lifecycle/lib/phase-accounting.mjs b/benchmarks/agentrail-swe-lifecycle/lib/phase-accounting.mjs new file mode 100644 index 0000000..8054df0 --- /dev/null +++ b/benchmarks/agentrail-swe-lifecycle/lib/phase-accounting.mjs @@ -0,0 +1,53 @@ +const TOKEN_FIELDS = [ + "inputTokens", + "cachedInputTokens", + "nonCachedInputTokens", + "outputTokens", + "reasoningTokens", + "totalTokens", +]; + +export function computePhaseAccounting({ phases }) { + const controlPhases = phases.filter((phase) => phase.controlIncluded); + const agentRailPhases = phases.filter((phase) => phase.agentRailIncluded); + const control = sumTokenUsage(controlPhases.map((phase) => phase.tokenUsage)); + const agentrail = sumTokenUsage(agentRailPhases.map((phase) => phase.tokenUsage)); + const tokenDelta = control.status === "available" && agentrail.status === "available" + ? agentrail.totalTokens - control.totalTokens + : null; + + return { + control, + agentrail, + tokenDelta, + tokenReductionRate: tokenDelta !== null && control.totalTokens > 0 + ? (control.totalTokens - agentrail.totalTokens) / control.totalTokens + : null, + controlPhaseIds: controlPhases.map((phase) => phase.id), + agentRailPhaseIds: agentRailPhases.map((phase) => phase.id), + }; +} + +export function sumTokenUsage(tokenUsages) { + const totals = { + status: "available", + inputTokens: 0, + cachedInputTokens: 0, + nonCachedInputTokens: 0, + outputTokens: 0, + reasoningTokens: 0, + totalTokens: 0, + }; + + for (const usage of tokenUsages) { + if (usage?.status !== "available") { + totals.status = "unavailable"; + continue; + } + for (const field of TOKEN_FIELDS) { + totals[field] += typeof usage[field] === "number" ? usage[field] : 0; + } + } + + return totals; +} diff --git a/benchmarks/agentrail-swe-lifecycle/lib/phase-definitions.mjs b/benchmarks/agentrail-swe-lifecycle/lib/phase-definitions.mjs new file mode 100644 index 0000000..d79e5a1 --- /dev/null +++ b/benchmarks/agentrail-swe-lifecycle/lib/phase-definitions.mjs @@ -0,0 +1,141 @@ +const CI_AND_REVIEW_FEEDBACK_PHASES = [ + { + id: "provider_issue_intake", + title: "Read provider issue", + measured: true, + actor: "llm_provider_lifecycle", + controlIncluded: true, + agentRailIncluded: false, + }, + { + id: "initial_code_change", + title: "Implement initial code change", + measured: true, + actor: "llm_coding", + controlIncluded: true, + agentRailIncluded: true, + }, + { + id: "provider_create_pr", + title: "Create provider pull request", + measured: true, + actor: "llm_provider_lifecycle", + controlIncluded: true, + agentRailIncluded: false, + }, + { + id: "provider_wait_for_ci", + title: "Wait for CI and read status", + measured: true, + actor: "llm_provider_lifecycle", + controlIncluded: true, + agentRailIncluded: false, + }, + { + id: "ci_failure_triage", + title: "Read CI failure and plan fix", + measured: true, + actor: "llm_provider_lifecycle", + controlIncluded: true, + agentRailIncluded: false, + }, + { + id: "fix_ci_failure", + title: "Fix CI failure", + measured: true, + actor: "llm_coding", + controlIncluded: true, + agentRailIncluded: true, + }, + { + id: "provider_wait_for_review", + title: "Wait for review and read comments", + measured: true, + actor: "llm_provider_lifecycle", + controlIncluded: true, + agentRailIncluded: false, + }, + { + id: "review_feedback_triage", + title: "Read review feedback and plan fix", + measured: true, + actor: "llm_provider_lifecycle", + controlIncluded: true, + agentRailIncluded: false, + }, + { + id: "fix_review_feedback", + title: "Fix review feedback", + measured: true, + actor: "llm_coding", + controlIncluded: true, + agentRailIncluded: true, + }, + { + id: "provider_update_pr", + title: "Update pull request after fixes", + measured: true, + actor: "llm_provider_lifecycle", + controlIncluded: true, + agentRailIncluded: false, + }, + { + id: "provider_final_ci_check", + title: "Check final CI", + measured: true, + actor: "llm_provider_lifecycle", + controlIncluded: true, + agentRailIncluded: false, + }, + { + id: "provider_final_review_check", + title: "Check final review approval", + measured: true, + actor: "llm_provider_lifecycle", + controlIncluded: true, + agentRailIncluded: false, + }, + { + id: "agentrail_handoff", + title: "Write AgentRail result handoff", + measured: true, + actor: "llm_agentrail_handoff", + controlIncluded: false, + agentRailIncluded: true, + }, + { + id: "provider_complete", + title: "Complete provider lifecycle", + measured: true, + actor: "llm_provider_lifecycle", + controlIncluded: true, + agentRailIncluded: false, + }, +]; + +export const PHASE_SCENARIOS = new Map([ + ["ci-and-review-feedback", { + id: "ci-and-review-feedback", + phases: CI_AND_REVIEW_FEEDBACK_PHASES, + }], +]); + +export const CONTROL_INCLUDED_PHASES = new Set( + CI_AND_REVIEW_FEEDBACK_PHASES.filter((phase) => phase.controlIncluded).map((phase) => phase.id), +); + +export const AGENTRAIL_INCLUDED_PHASES = new Set( + CI_AND_REVIEW_FEEDBACK_PHASES.filter((phase) => phase.agentRailIncluded).map((phase) => phase.id), +); + +export function getPhaseScenario(id) { + const scenario = PHASE_SCENARIOS.get(id); + if (!scenario) throw new Error(`Unknown phase scenario: ${id}`); + return scenario; +} + +export function validatePhaseResult({ phase, result, requireTokenUsage }) { + if (requireTokenUsage && phase.measured && result.tokenUsage?.status !== "available") { + throw new Error(`${phase.id} did not produce parseable token usage.`); + } +} diff --git a/benchmarks/agentrail-swe-lifecycle/lib/phase-prompts.mjs b/benchmarks/agentrail-swe-lifecycle/lib/phase-prompts.mjs new file mode 100644 index 0000000..2224809 --- /dev/null +++ b/benchmarks/agentrail-swe-lifecycle/lib/phase-prompts.mjs @@ -0,0 +1,77 @@ +export function buildPhasePrompt({ phase, task, upstream, artifacts = {} }) { + if (phase.actor === "llm_coding") { + return buildCodingPrompt({ phase, task, upstream, artifacts }); + } + if (phase.actor === "llm_provider_lifecycle") { + return buildProviderLifecyclePrompt({ phase, task, artifacts }); + } + if (phase.actor === "llm_agentrail_handoff") { + return buildAgentRailHandoffPrompt({ artifacts }); + } + throw new Error(`Unsupported phase actor: ${phase.actor}`); +} + +function buildCodingPrompt({ phase, task, upstream, artifacts }) { + return [ + `You are working on benchmark task ${task.id}.`, + `Repository: ${task.repo}`, + `Phase: ${phase.id}`, + "", + "Modify the repository only for this coding phase.", + "Do not operate provider lifecycle tooling or lifecycle state.", + "Leave changes in the working tree.", + "", + `Instance: ${upstream.instanceId}`, + `Base commit: ${upstream.baseCommit}`, + "", + `Problem statement:\n${upstream.problemStatement}`, + upstream.requirements ? `\nRequirements:\n${upstream.requirements}` : "", + upstream.interface ? `\nInterface notes:\n${upstream.interface}` : "", + artifacts.ciFailure ? `\nCI failure to address:\n${artifacts.ciFailure}` : "", + artifacts.reviewFeedback ? `\nReview feedback to address:\n${artifacts.reviewFeedback}` : "", + "", + "When this phase is complete, stop. Do not continue into later phases.", + ].filter(Boolean).join("\n"); +} + +function buildProviderLifecyclePrompt({ phase, task, artifacts }) { + const lifecycleCli = artifacts.lifecycleCli ?? "node benchmarks/agentrail-swe-lifecycle/fixtures/mock-lifecycle-cli.mjs"; + const instructions = { + provider_issue_intake: `Fetch the issue with: ${lifecycleCli} issue get`, + provider_create_pr: `Use ${lifecycleCli} to create the pull request.`, + provider_wait_for_ci: `Check CI with: ${lifecycleCli} ci get --pr 1`, + ci_failure_triage: `Read the provided CI failure and summarize the fix needed. CI failure:\n${artifacts.ciFailure ?? "Synthetic CI failure."}`, + provider_wait_for_review: `Check review with: ${lifecycleCli} review get --pr 1`, + review_feedback_triage: `Read the provided review feedback and summarize the fix needed. Review feedback:\n${artifacts.reviewFeedback ?? "Synthetic review feedback."}`, + provider_update_pr: `Update the pull request status with: ${lifecycleCli} pr update --pr 1`, + provider_final_ci_check: `Check final CI with: ${lifecycleCli} ci get --pr 1`, + provider_final_review_check: `Check final review with: ${lifecycleCli} review get --pr 1`, + provider_complete: `Complete the lifecycle with: ${lifecycleCli} complete --pr 1`, + }; + + const instruction = instructions[phase.id]; + if (!instruction) throw new Error(`Unsupported provider lifecycle phase: ${phase.id}`); + + return [ + `You are performing only a provider lifecycle phase for ${task.repo}.`, + `Phase: ${phase.id}`, + "", + instruction, + "", + "Do not edit source files in this phase.", + "Do not solve coding tasks in this phase.", + "When this phase is complete, stop.", + ].join("\n"); +} + +function buildAgentRailHandoffPrompt({ artifacts }) { + if (typeof artifacts.resultPath !== "string" || artifacts.resultPath.length === 0) { + throw new Error("AgentRail handoff phase requires artifacts.resultPath."); + } + + return [ + "Write the completed AgentRail result JSON.", + `Result path: ${artifacts.resultPath}`, + `JSON: {"version":1,"status":"completed","summary":"phase attribution benchmark complete","checks":[],"artifacts":[]}`, + ].join("\n"); +} diff --git a/benchmarks/agentrail-swe-lifecycle/lib/phase-runner.mjs b/benchmarks/agentrail-swe-lifecycle/lib/phase-runner.mjs new file mode 100644 index 0000000..b46b721 --- /dev/null +++ b/benchmarks/agentrail-swe-lifecycle/lib/phase-runner.mjs @@ -0,0 +1,165 @@ +import { mkdir, readFile } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { capturePatch } from "./patch.mjs"; +import { + PHASE_ATTRIBUTION_LIFECYCLE_EVENTS, + summarizeLifecycleEvents, + withLifecycleProviderHarness, +} from "./lifecycle-provider-harness.mjs"; +import { computePhaseAccounting } from "./phase-accounting.mjs"; +import { getPhaseScenario, validatePhaseResult } from "./phase-definitions.mjs"; +import { buildPhasePrompt } from "./phase-prompts.mjs"; +import { runRunnerCommand } from "./runner-command.mjs"; +import { parseTokenUsageFromFiles } from "./token-usage.mjs"; +import { prepareLaneWorkspace } from "./workspace.mjs"; + +export async function runPhaseAttributionTask({ + task, + upstream, + runId, + runDir, + repoCache, + sourceRepoPath, + flags, +}) { + const workspace = await prepareLaneWorkspace({ + task, + upstream, + lane: "phase-attribution", + runDir, + repoCache, + sourceRepoPath, + }); + + return await withLifecycleProviderHarness({ + runDir: workspace.laneDir, + stateDir: workspace.worktreePath, + task, + upstream, + }, async (lifecycle) => await runPhaseAttributionInWorkspace({ + task, + upstream, + runId, + workspace, + flags, + lifecycle, + })); +} + +async function runPhaseAttributionInWorkspace({ + task, + upstream, + runId, + workspace, + flags, + lifecycle, +}) { + const scenario = getPhaseScenario(flags.phaseScenario ?? "ci-and-review-feedback"); + const resultPath = path.join(workspace.worktreePath, ".agentrail-run", "result.json"); + const lifecycleCli = path.join(repoRoot(), "benchmarks", "agentrail-swe-lifecycle", "fixtures", "mock-lifecycle-cli.mjs"); + const phases = []; + const artifacts = { + lifecycleCli: `node ${lifecycleCli}`, + ciFailure: "Synthetic CI failure: targeted test failed after initial patch.", + reviewFeedback: "Synthetic review feedback: simplify the fix and add focused regression coverage.", + resultPath, + }; + + const repeatCount = flags.phaseRepeat ?? 1; + for (let repeatIndex = 0; repeatIndex < repeatCount; repeatIndex += 1) { + for (const phase of scenario.phases) { + const phaseDir = repeatCount === 1 + ? path.join(workspace.laneDir, "phases", phase.id) + : path.join(workspace.laneDir, "phases", `repeat-${repeatIndex + 1}`, phase.id); + await mkdir(phaseDir, { recursive: true }); + const prompt = buildPhasePrompt({ phase, task, upstream, artifacts }); + const runner = await runRunnerCommand({ + runner: flags.runner, + runnerCommand: flags.runnerCommand, + model: flags.model ?? null, + repoPath: workspace.worktreePath, + prompt, + logDir: phaseDir, + timeoutSeconds: flags.timeoutSeconds ?? 1800, + env: { + AGENTRAIL_BENCH_PHASE_ID: phase.id, + AGENTRAIL_BENCH_PHASE_REPEAT: String(repeatIndex + 1), + AGENTRAIL_RESULT_PATH: resultPath, + AGENTRAIL_BENCH_LIFECYCLE_URL: lifecycle.baseUrl, + AGENTRAIL_BENCH_LIFECYCLE_STATE_PATH: lifecycle.statePath, + AGENTRAIL_BENCH_LIFECYCLE_CLI: lifecycleCli, + }, + }); + await validateCompletedRunner({ phase, runner }); + const tokenUsage = await parseTokenUsageFromFiles([runner.stdoutPath, runner.stderrPath]); + const phaseResult = { + ...phase, + repeat: repeatIndex + 1, + runner, + tokenUsage, + status: runner.status, + success: runner.status === "completed", + }; + validatePhaseResult({ + phase, + result: phaseResult, + requireTokenUsage: flags.requirePhaseTokenUsage, + }); + phases.push(phaseResult); + } + } + + const patch = await capturePatch({ + worktreePath: workspace.worktreePath, + outputPath: path.join(workspace.laneDir, "patch.diff"), + }); + const accounting = computePhaseAccounting({ phases }); + const lifecycleSummary = summarizeLifecycleEvents({ + eventsPath: lifecycle.eventsPath, + events: await lifecycle.readEvents(), + requiredEvents: PHASE_ATTRIBUTION_LIFECYCLE_EVENTS, + }); + const success = phases.every((phase) => phase.success) + && patch.status === "available" + && lifecycleSummary.complete; + + return { + taskId: task.id, + upstreamInstanceId: upstream.instanceId, + repo: task.repo, + runId, + mode: "phase-attribution", + scenario: scenario.id, + phases, + accounting, + patch, + lifecycle: lifecycleSummary, + final: { + success, + status: success ? "completed" : "failed", + }, + tokenUsage: accounting.agentrail, + }; +} + +async function validateCompletedRunner({ phase, runner }) { + if (runner.status === "completed") return; + const stderr = await readTextFile(runner.stderrPath); + const suffix = stderr.trim().length > 0 ? ` Stderr: ${stderr.trim().slice(-1000)}` : ""; + throw new Error(`${phase.id} runner ${runner.status}.${suffix}`); +} + +async function readTextFile(filePath) { + try { + return await readFile(filePath, "utf8"); + } catch (error) { + if (error?.code === "ENOENT") return ""; + throw error; + } +} + +function repoRoot() { + return path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", "..", ".."); +} diff --git a/benchmarks/agentrail-swe-lifecycle/lib/prompt.mjs b/benchmarks/agentrail-swe-lifecycle/lib/prompt.mjs new file mode 100644 index 0000000..9ca185f --- /dev/null +++ b/benchmarks/agentrail-swe-lifecycle/lib/prompt.mjs @@ -0,0 +1,43 @@ +export function buildControlPrompt({ task, upstream }) { + const lines = [ + "You are solving a software engineering benchmark task.", + "", + `Repository: ${task.repo}`, + `Instance: ${upstream.instanceId}`, + `Base commit: ${upstream.baseCommit}`, + "", + "Problem statement:", + upstream.problemStatement, + ]; + + if (upstream.requirements) lines.push("", "Requirements:", upstream.requirements); + if (upstream.interface) lines.push("", "Interface notes:", upstream.interface); + + lines.push( + "", + "Instructions:", + "- Modify the repository to solve the problem.", + "- Do not invent external services or credentials.", + "- Keep changes focused on the requested fix.", + "- When finished, leave the final patch in the git working tree.", + ); + + return lines.join("\n"); +} + +export function buildAgentRailTaskFields({ task, upstream }) { + return { + identifier: `swe:${upstream.instanceId}`, + title: `SWE benchmark task ${upstream.instanceId}`, + description: buildControlPrompt({ task, upstream }), + acceptanceCriteria: [ + "The repository is patched to solve the problem statement.", + "The final patch is left in the git working tree.", + "No gold patch or hidden test metadata is used.", + ], + context: { + project: task.repo, + goal: "Solve a SWE-style benchmark issue through AgentRail.", + }, + }; +} diff --git a/benchmarks/agentrail-swe-lifecycle/lib/results.mjs b/benchmarks/agentrail-swe-lifecycle/lib/results.mjs new file mode 100644 index 0000000..4c22cfb --- /dev/null +++ b/benchmarks/agentrail-swe-lifecycle/lib/results.mjs @@ -0,0 +1,157 @@ +import { mkdir, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; + +export function aggregateLifecycleResults(results) { + const totalTasks = results.length; + const successfulTasks = results.filter((result) => result.final?.success === true).length; + const normalizedTokenTotals = results.map((result) => normalizeTokenTotal(result.tokenUsage)); + const totalTokens = normalizedTokenTotals.reduce((sum, total) => sum + (total ?? 0), 0); + const tokenUsageUnavailableTasks = normalizedTokenTotals.filter((total) => total === null).length; + const comparisons = results + .map((result) => result.comparison) + .filter((comparison) => comparison && typeof comparison === "object"); + const controlTokenTotals = comparisons.map((comparison) => normalizeTokenTotal(comparison.control?.tokenUsage)); + const agentRailTokenTotals = comparisons.map((comparison) => normalizeTokenTotal(comparison.agentrail?.tokenUsage)); + const comparableTokenTasks = comparisons.filter((_, index) => ( + controlTokenTotals[index] !== null && agentRailTokenTotals[index] !== null + )).length; + const controlTotalTokens = controlTokenTotals.reduce((sum, total) => sum + (total ?? 0), 0); + const agentRailTotalTokens = agentRailTokenTotals.reduce((sum, total) => sum + (total ?? 0), 0); + const tokenDelta = comparableTokenTasks > 0 ? agentRailTotalTokens - controlTotalTokens : null; + const tokenFieldTotals = (lane, field) => comparisons.reduce((sum, comparison, index) => { + if (controlTokenTotals[index] === null || agentRailTokenTotals[index] === null) return sum; + return sum + normalizeTokenComponent(comparison[lane]?.tokenUsage, field); + }, 0); + const controlCachedInputTokens = tokenFieldTotals("control", "cachedInputTokens"); + const agentRailCachedInputTokens = tokenFieldTotals("agentrail", "cachedInputTokens"); + const controlNonCachedInputTokens = tokenFieldTotals("control", "nonCachedInputTokens"); + const agentRailNonCachedInputTokens = tokenFieldTotals("agentrail", "nonCachedInputTokens"); + const controlOutputTokens = tokenFieldTotals("control", "outputTokens"); + const agentRailOutputTokens = tokenFieldTotals("agentrail", "outputTokens"); + const controlReasoningTokens = tokenFieldTotals("control", "reasoningTokens"); + const agentRailReasoningTokens = tokenFieldTotals("agentrail", "reasoningTokens"); + + return { + totalTasks, + successfulTasks, + successRate: totalTasks > 0 ? successfulTasks / totalTasks : 0, + totalTokens, + tokenUsageUnavailableTasks, + tokensPerSuccessfulTask: successfulTasks > 0 ? totalTokens / successfulTasks : null, + phaseAttribution: aggregatePhaseAttributionResults(results), + paired: comparisons.length > 0 + ? { + comparableTokenTasks, + controlTotalTokens, + agentRailTotalTokens, + tokenDelta, + tokenReductionRate: tokenDelta !== null && controlTotalTokens > 0 + ? (controlTotalTokens - agentRailTotalTokens) / controlTotalTokens + : null, + controlCachedInputTokens, + agentRailCachedInputTokens, + cachedInputDelta: agentRailCachedInputTokens - controlCachedInputTokens, + controlNonCachedInputTokens, + agentRailNonCachedInputTokens, + nonCachedInputDelta: agentRailNonCachedInputTokens - controlNonCachedInputTokens, + controlOutputTokens, + agentRailOutputTokens, + outputDelta: agentRailOutputTokens - controlOutputTokens, + controlReasoningTokens, + agentRailReasoningTokens, + reasoningDelta: agentRailReasoningTokens - controlReasoningTokens, + } + : null + }; +} + +function aggregatePhaseAttributionResults(results) { + const phaseResults = results.filter((result) => result.accounting); + if (phaseResults.length === 0) return null; + + let comparableTasks = 0; + let controlTotalTokens = 0; + let agentRailTotalTokens = 0; + + for (const result of phaseResults) { + const controlTokens = normalizeTokenTotal(result.accounting?.control); + const agentRailTokens = normalizeTokenTotal(result.accounting?.agentrail); + if (controlTokens === null || agentRailTokens === null) continue; + comparableTasks += 1; + controlTotalTokens += controlTokens; + agentRailTotalTokens += agentRailTokens; + } + + const tokenDelta = comparableTasks > 0 ? agentRailTotalTokens - controlTotalTokens : null; + return { + comparableTasks, + controlTotalTokens, + agentRailTotalTokens, + tokenDelta, + tokenReductionRate: tokenDelta !== null && controlTotalTokens > 0 + ? (controlTotalTokens - agentRailTotalTokens) / controlTotalTokens + : null, + }; +} + +export async function writeLifecycleResultFiles({ outputDir, run }) { + await mkdir(outputDir, { recursive: true }); + const completeMarkerPath = path.join(outputDir, "complete.marker"); + await rm(completeMarkerPath, { force: true }); + await writeFile(path.join(outputDir, "results.json"), `${JSON.stringify(run, null, 2)}\n`, "utf8"); + await writeFile(path.join(outputDir, "summary.md"), renderSummary(run), "utf8"); + await writeFile(completeMarkerPath, "complete\n", "utf8"); +} + +function normalizeTokenTotal(tokenUsage) { + if (!tokenUsage || tokenUsage.totalTokens === undefined) return null; + if (tokenUsage.status === "unavailable") return null; + if (typeof tokenUsage.totalTokens === "number") { + return Number.isSafeInteger(tokenUsage.totalTokens) && tokenUsage.totalTokens >= 0 ? tokenUsage.totalTokens : null; + } + if (typeof tokenUsage.totalTokens === "string" && /^(0|[1-9]\d*)$/u.test(tokenUsage.totalTokens)) { + const totalTokens = Number(tokenUsage.totalTokens); + return Number.isSafeInteger(totalTokens) ? totalTokens : null; + } + return null; +} + +function normalizeTokenComponent(tokenUsage, field) { + const value = tokenUsage?.[field]; + return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : 0; +} + +function renderSummary(run) { + const aggregate = run.aggregate; + return [ + "# AgentRail SWE Lifecycle Benchmark", + "", + `- Run: ${run.runId}`, + `- Benchmark surface: ${run.benchmarkSurface ?? "patch-only"}`, + `- Tasks: ${aggregate.totalTasks}`, + `- Successful tasks: ${aggregate.successfulTasks}`, + `- Success rate: ${(aggregate.successRate * 100).toFixed(2)}%`, + `- Total tokens: ${aggregate.totalTokens ?? "unavailable"}`, + `- Token usage unavailable tasks: ${aggregate.tokenUsageUnavailableTasks}`, + `- Tokens per successful task: ${aggregate.tokensPerSuccessfulTask ?? "unavailable"}`, + ...(aggregate.paired ? [ + `- Paired comparable token tasks: ${aggregate.paired.comparableTokenTasks}`, + `- Control total tokens: ${aggregate.paired.controlTotalTokens}`, + `- AgentRail total tokens: ${aggregate.paired.agentRailTotalTokens}`, + `- AgentRail token delta: ${aggregate.paired.tokenDelta ?? "unavailable"}`, + `- AgentRail token reduction: ${aggregate.paired.tokenReductionRate === null ? "unavailable" : `${(aggregate.paired.tokenReductionRate * 100).toFixed(2)}%`}`, + `- Cached input token delta: ${aggregate.paired.cachedInputDelta}`, + `- Non-cached input token delta: ${aggregate.paired.nonCachedInputDelta}`, + `- Output token delta: ${aggregate.paired.outputDelta}`, + `- Reasoning token delta: ${aggregate.paired.reasoningDelta}` + ] : []), + ...(aggregate.phaseAttribution ? [ + `- Phase attribution comparable tasks: ${aggregate.phaseAttribution.comparableTasks}`, + `- Phase attribution control total tokens: ${aggregate.phaseAttribution.controlTotalTokens}`, + `- Phase attribution AgentRail total tokens: ${aggregate.phaseAttribution.agentRailTotalTokens}`, + `- Phase attribution AgentRail token delta: ${aggregate.phaseAttribution.tokenDelta ?? "unavailable"}`, + `- Phase attribution AgentRail token reduction: ${aggregate.phaseAttribution.tokenReductionRate === null ? "unavailable" : `${(aggregate.phaseAttribution.tokenReductionRate * 100).toFixed(2)}%`}`, + ] : []), + "" + ].join("\n"); +} diff --git a/benchmarks/agentrail-swe-lifecycle/lib/runner-command.mjs b/benchmarks/agentrail-swe-lifecycle/lib/runner-command.mjs new file mode 100644 index 0000000..a1f4eab --- /dev/null +++ b/benchmarks/agentrail-swe-lifecycle/lib/runner-command.mjs @@ -0,0 +1,151 @@ +import { spawn } from "node:child_process"; +import { access, mkdir, readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; + +export async function runRunnerCommand({ runner, runnerCommand, model, repoPath, prompt, logDir, timeoutSeconds, env = {} }) { + await mkdir(logDir, { recursive: true }); + const promptPath = path.join(logDir, "prompt.md"); + const stdoutPath = path.join(logDir, "stdout.log"); + const stderrPath = path.join(logDir, "stderr.log"); + await writeFile(promptPath, prompt, "utf8"); + + const command = await resolveRunnerCommand({ runner, runnerCommand, model, promptPath }); + const startedAt = Date.now(); + const result = await spawnWithTimeout({ + command: command.executable, + args: command.args, + stdin: command.stdin, + cwd: repoPath, + env: { ...process.env, ...env, AGENTRAIL_BENCH_PROMPT_PATH: promptPath }, + timeoutMs: timeoutSeconds * 1000, + }); + + await writeFile(stdoutPath, result.stdout, "utf8"); + await writeFile(stderrPath, result.stderr, "utf8"); + + return { + status: result.timedOut ? "timed_out" : result.exitCode === 0 ? "completed" : "failed", + exitCode: result.exitCode, + durationMs: Date.now() - startedAt, + promptPath, + stdoutPath, + stderrPath, + command: command.redacted, + }; +} + +export async function resolveRunnerCommand({ runner, runnerCommand, model, promptPath }) { + if (runner === "custom") { + const [executable, ...args] = splitCommand(runnerCommand); + return { + executable, + args: await resolveRepoRelativeArgs(args), + redacted: [executable, ...args].join(" "), + }; + } + if (runner === "codex") { + const prompt = await readPromptFile(promptPath); + const args = [ + "-a", + "never", + "-c", + "features.hooks=false", + "exec", + "--sandbox", + "workspace-write", + "--ephemeral", + "--ignore-user-config", + "--ignore-rules", + ...(model ? ["--model", model] : []), + "--json", + "-", + ]; + return { executable: "codex", args, stdin: prompt, redacted: ["codex", ...args].join(" ") }; + } + if (runner === "cursor") { + const prompt = await readPromptFile(promptPath); + const args = [ + "agent", + "--print", + "--output-format", + "stream-json", + "--trust", + "--force", + ...(model ? ["--model", model] : []), + ]; + return { executable: "cursor", args, stdin: prompt, redacted: ["cursor", ...args].join(" ") }; + } + if (runner === "claude-code") { + const prompt = await readPromptFile(promptPath); + const args = [ + "--print", + "--output-format", + "stream-json", + "--verbose", + "--permission-mode", + "bypassPermissions", + "--dangerously-skip-permissions", + ...(model ? ["--model", model] : []), + ]; + return { executable: "claude", args, stdin: prompt, redacted: ["claude", ...args].join(" ") }; + } + throw new Error(`Unsupported runner: ${runner}`); +} + +function splitCommand(command) { + const parts = String(command).match(/(?:[^\s"]+|"[^"]*")+/gu)?.map((part) => part.replace(/^"|"$/gu, "")) ?? []; + if (parts.length === 0) throw new Error("--runner-command must not be empty."); + return parts; +} + +async function resolveRepoRelativeArgs(args) { + const resolved = []; + for (const arg of args) { + resolved.push(await resolveExistingRelativePathArg(arg)); + } + return resolved; +} + +async function resolveExistingRelativePathArg(arg) { + if (arg.startsWith("-") || path.isAbsolute(arg)) return arg; + const absolute = path.resolve(process.cwd(), arg); + try { + await access(absolute); + return absolute; + } catch (error) { + if (error?.code === "ENOENT") return arg; + throw error; + } +} + +async function readPromptFile(promptPath) { + return await readFile(promptPath, "utf8"); +} + +async function spawnWithTimeout({ command, args, stdin, cwd, env, timeoutMs }) { + return await new Promise((resolve, reject) => { + const child = spawn(command, args, { cwd, env, stdio: [stdin === undefined ? "ignore" : "pipe", "pipe", "pipe"] }); + const stdout = []; + const stderr = []; + let timedOut = false; + const timer = setTimeout(() => { + timedOut = true; + child.kill("SIGTERM"); + }, timeoutMs); + child.stdout.on("data", (chunk) => stdout.push(Buffer.from(chunk))); + child.stderr.on("data", (chunk) => stderr.push(Buffer.from(chunk))); + child.on("error", reject); + if (stdin !== undefined) { + child.stdin.end(stdin); + } + child.on("close", (exitCode) => { + clearTimeout(timer); + resolve({ + exitCode, + timedOut, + stdout: Buffer.concat(stdout).toString("utf8"), + stderr: Buffer.concat(stderr).toString("utf8"), + }); + }); + }); +} diff --git a/benchmarks/agentrail-swe-lifecycle/lib/token-usage.mjs b/benchmarks/agentrail-swe-lifecycle/lib/token-usage.mjs new file mode 100644 index 0000000..153bef2 --- /dev/null +++ b/benchmarks/agentrail-swe-lifecycle/lib/token-usage.mjs @@ -0,0 +1,241 @@ +import { readFile } from "node:fs/promises"; + +const TOKEN_VALUE_PATTERN = /^(0|[1-9]\d*)$/u; +const TOKEN_AGGREGATE_OVERFLOW_ERROR = "Token usage aggregate exceeds Number.MAX_SAFE_INTEGER."; +const TOKEN_TOTAL_VALUE_ERROR = "Token usage total_tokens must be a non-negative safe integer."; + +export async function parseTokenUsageFromFiles(paths) { + const aggregate = createEmptyAggregate(); + const sources = []; + + for (const filePath of paths) { + const text = await readTokenUsageFile(filePath); + if (text === null) continue; + + let contributed = false; + for (const record of parseJsonRecords(text)) { + for (const usage of findUsageObjects(record)) { + aggregateUsage(aggregate, usage); + contributed = true; + } + } + if (contributed) sources.push(filePath); + } + + if (!aggregate.available) { + return { + status: "unavailable", + inputTokens: null, + outputTokens: null, + reasoningTokens: null, + cachedInputTokens: null, + nonCachedInputTokens: null, + totalTokens: null, + sources: [] + }; + } + + const inputTokens = aggregate.hasInput ? aggregate.inputTokens : null; + const cachedInputTokens = aggregate.hasCachedInput ? aggregate.cachedInputTokens : null; + return { + status: "available", + inputTokens, + outputTokens: aggregate.hasOutput ? aggregate.outputTokens : null, + reasoningTokens: aggregate.hasReasoning ? aggregate.reasoningTokens : null, + cachedInputTokens, + nonCachedInputTokens: inputTokens === null || cachedInputTokens === null + ? null + : Math.max(0, inputTokens - cachedInputTokens), + totalTokens: aggregate.hasTotal ? aggregate.totalTokens : null, + sources + }; +} + +async function readTokenUsageFile(filePath) { + try { + return await readFile(filePath, "utf8"); + } catch (error) { + if (error?.code === "ENOENT") return null; + throw error; + } +} + +function parseJsonRecords(text) { + const trimmedText = text.trimStart(); + if (trimmedText.startsWith("{") || trimmedText.startsWith("[")) { + try { + return [JSON.parse(text)]; + } catch { + // Fall through to JSONL scanning. + } + } + + const records = []; + for (const line of text.split(/\r?\n/u)) { + const trimmed = line.trimStart(); + if (!trimmed.startsWith("{") && !trimmed.startsWith("[")) continue; + try { + records.push(JSON.parse(trimmed)); + } catch { + // Runner logs can contain partial JSON-looking lines; ignore non-records. + } + } + + return records; +} + +function* findUsageObjects(value) { + if (Array.isArray(value)) { + for (const nested of value) { + yield* findUsageObjects(nested); + } + return; + } + + if (!isRecord(value)) return; + if (isUsageObject(value)) yield value; + + for (const [key, nested] of Object.entries(value)) { + if (isTokenDetailsKey(key)) continue; + yield* findUsageObjects(nested); + } +} + +function isTokenDetailsKey(key) { + return key === "input_tokens_details" + || key === "inputTokensDetails" + || key === "output_tokens_details" + || key === "outputTokensDetails"; +} + +function isUsageObject(value) { + return getTokenValue(value, "input_tokens", "inputTokens") !== null + || getTokenValue(value, "output_tokens", "outputTokens") !== null + || getTokenValue(value, "reasoning_tokens", "reasoningTokens") !== null + || getTokenValue(value, "reasoning_output_tokens", "reasoningOutputTokens") !== null + || getTokenValue(value, "cached_input_tokens", "cachedInputTokens") !== null + || hasTokenField(value, "total_tokens", "totalTokens") + || getCachedInputTokensFromDetails(value) !== null + || getReasoningTokensFromDetails(value) !== null; +} + +function aggregateUsage(aggregate, usage) { + const inputTokens = getTokenValue(usage, "input_tokens", "inputTokens"); + const outputTokens = getTokenValue(usage, "output_tokens", "outputTokens"); + const reasoningTokens = getTokenValue(usage, "reasoning_tokens", "reasoningTokens") + ?? getTokenValue(usage, "reasoning_output_tokens", "reasoningOutputTokens") + ?? getReasoningTokensFromDetails(usage); + const cachedInputTokens = getTokenValue(usage, "cached_input_tokens", "cachedInputTokens") + ?? getCachedInputTokensFromDetails(usage); + const totalTokens = getTotalTokenValue(usage); + + addComponent(aggregate, "inputTokens", "hasInput", inputTokens); + addComponent(aggregate, "outputTokens", "hasOutput", outputTokens); + addComponent(aggregate, "reasoningTokens", "hasReasoning", reasoningTokens); + addComponent(aggregate, "cachedInputTokens", "hasCachedInput", cachedInputTokens); + + if (totalTokens.present) { + addComponent(aggregate, "totalTokens", "hasTotal", totalTokens.value); + return; + } + + if (inputTokens !== null || outputTokens !== null || reasoningTokens !== null) { + addComponent( + aggregate, + "totalTokens", + "hasTotal", + sumTokenValues(inputTokens ?? 0, outputTokens ?? 0, reasoningTokens ?? 0) + ); + } +} + +function addComponent(aggregate, totalKey, flagKey, value) { + if (value === null) return; + aggregate[totalKey] = sumTokenValues(aggregate[totalKey], value); + aggregate[flagKey] = true; + aggregate.available = true; +} + +function sumTokenValues(...values) { + let sum = 0; + for (const value of values) { + sum += value; + if (!Number.isSafeInteger(sum) || sum < 0) { + throw new Error(TOKEN_AGGREGATE_OVERFLOW_ERROR); + } + } + return sum; +} + +function getTokenValue(record, snakeKey, camelKey) { + return normalizeTokenValue(record[snakeKey] ?? record[camelKey]); +} + +function getTotalTokenValue(record) { + if (!hasTokenField(record, "total_tokens", "totalTokens")) { + return { present: false, value: null }; + } + + let value = null; + for (const key of ["total_tokens", "totalTokens"]) { + if (!hasOwn(record, key)) continue; + const normalized = normalizeTokenValue(record[key]); + if (normalized === null) throw new Error(TOKEN_TOTAL_VALUE_ERROR); + value ??= normalized; + } + + return { present: true, value }; +} + +function hasTokenField(record, snakeKey, camelKey) { + return hasOwn(record, snakeKey) || hasOwn(record, camelKey); +} + +function getCachedInputTokensFromDetails(record) { + return getDetailTokenValue(record, "input_tokens_details", "inputTokensDetails", "cached_tokens", "cachedTokens"); +} + +function getReasoningTokensFromDetails(record) { + return getDetailTokenValue(record, "output_tokens_details", "outputTokensDetails", "reasoning_tokens", "reasoningTokens"); +} + +function getDetailTokenValue(record, snakeContainerKey, camelContainerKey, snakeTokenKey, camelTokenKey) { + const container = record[snakeContainerKey] ?? record[camelContainerKey]; + if (!isRecord(container)) return null; + return normalizeTokenValue(container[snakeTokenKey] ?? container[camelTokenKey]); +} + +function normalizeTokenValue(value) { + if (typeof value === "number") { + return Number.isSafeInteger(value) && value >= 0 ? value : null; + } + if (typeof value === "string" && TOKEN_VALUE_PATTERN.test(value)) { + const parsed = Number(value); + return Number.isSafeInteger(parsed) ? parsed : null; + } + return null; +} + +function createEmptyAggregate() { + return { + available: false, + inputTokens: 0, + outputTokens: 0, + reasoningTokens: 0, + cachedInputTokens: 0, + totalTokens: 0, + hasInput: false, + hasOutput: false, + hasReasoning: false, + hasCachedInput: false, + hasTotal: false + }; +} + +function isRecord(value) { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +function hasOwn(record, key) { + return Object.prototype.hasOwnProperty.call(record, key); +} diff --git a/benchmarks/agentrail-swe-lifecycle/lib/upstream.mjs b/benchmarks/agentrail-swe-lifecycle/lib/upstream.mjs new file mode 100644 index 0000000..c4c2c65 --- /dev/null +++ b/benchmarks/agentrail-swe-lifecycle/lib/upstream.mjs @@ -0,0 +1,87 @@ +const HF_ROWS_URL = "https://datasets-server.huggingface.co/rows"; +const HF_DATASET = "ScaleAI/SWE-bench_Pro"; +const HF_CONFIG = "default"; +const HF_SPLIT = "test"; +const PAGE_LENGTH = 100; + +export function buildHuggingFaceRowsUrl({ offset = 0, length = PAGE_LENGTH } = {}) { + const url = new URL(HF_ROWS_URL); + url.searchParams.set("dataset", HF_DATASET); + url.searchParams.set("config", HF_CONFIG); + url.searchParams.set("split", HF_SPLIT); + url.searchParams.set("offset", String(offset)); + url.searchParams.set("length", String(length)); + return url; +} + +export async function resolveUpstreamRowsForTasks(tasks, { fetchImpl = globalThis.fetch } = {}) { + if (typeof fetchImpl !== "function") { + throw new Error("A fetch implementation is required to resolve upstream rows."); + } + + const wanted = new Set(tasks.map((task) => upstreamInstanceIdForTask(task).toLowerCase())); + const rowsByInstanceId = new Map(); + + for (let offset = 0; rowsByInstanceId.size < wanted.size; offset += PAGE_LENGTH) { + const response = await fetchImpl(buildHuggingFaceRowsUrl({ offset, length: PAGE_LENGTH })); + const text = await response.text(); + if (!response.ok) { + throw new Error(`Failed to fetch SWE-bench Pro rows: HTTP ${response.status}: ${text.slice(0, 200)}`); + } + + const body = JSON.parse(text); + const pageRows = rowsFromPageBody(body); + if (pageRows.length === 0) break; + + for (const row of pageRows) { + const instanceId = String(row?.instance_id ?? "").toLowerCase(); + if (wanted.has(instanceId)) rowsByInstanceId.set(instanceId, row); + } + + if (isFinalPage({ body, offset, pageLength: PAGE_LENGTH, rowCount: pageRows.length })) break; + } + + const missing = [...wanted].filter((instanceId) => !rowsByInstanceId.has(instanceId)); + if (missing.length > 0) { + throw new Error(`Missing upstream rows: ${missing.join(", ")}`); + } + + return tasks.map((task) => redactUpstreamRowForLifecycle(rowsByInstanceId.get(upstreamInstanceIdForTask(task).toLowerCase()))); +} + +export function redactUpstreamRowForLifecycle(row) { + return { + repo: row.repo, + instanceId: row.instance_id, + baseCommit: row.base_commit, + problemStatement: row.problem_statement, + requirements: nullableString(row.requirements), + interface: nullableString(row.interface), + repoLanguage: nullableString(row.repo_language) + }; +} + +function rowsFromPageBody(body) { + if (!Array.isArray(body?.rows)) return []; + return body.rows.map((entry) => entry?.row).filter((row) => row && typeof row === "object"); +} + +function isFinalPage({ body, offset, pageLength, rowCount }) { + const totalRows = Number(body?.num_rows_total); + if (Number.isFinite(totalRows) && offset + rowCount >= totalRows) return true; + return rowCount < pageLength; +} + +function upstreamInstanceIdForTask(task) { + const instanceId = task?.upstream?.instanceId; + if (typeof instanceId !== "string" || instanceId.length === 0) { + throw new Error("Each task must include upstream.instanceId to resolve upstream rows."); + } + return instanceId; +} + +function nullableString(value) { + if (typeof value !== "string") return null; + const trimmed = value.trim(); + return trimmed.length > 0 && trimmed.toLowerCase() !== "nan" ? trimmed : null; +} diff --git a/benchmarks/agentrail-swe-lifecycle/lib/workspace.mjs b/benchmarks/agentrail-swe-lifecycle/lib/workspace.mjs new file mode 100644 index 0000000..ed5a4a8 --- /dev/null +++ b/benchmarks/agentrail-swe-lifecycle/lib/workspace.mjs @@ -0,0 +1,156 @@ +import { createHash } from "node:crypto"; +import { execFile } from "node:child_process"; +import { lstat, mkdir, rm, stat } from "node:fs/promises"; +import path from "node:path"; +import { promisify } from "node:util"; + +const execFileAsync = promisify(execFile); +const GIT_MAX_BUFFER = 1024 * 1024 * 20; +const BASE_REF = "refs/agentrail-bench/base"; +const ORIGIN_MAIN_REF = "refs/heads/main"; +const LANE_PATTERN = /^[A-Za-z0-9._-]+$/u; +const REPO_PATTERN = /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u; +const BASE_COMMIT_PATTERN = /^[a-f0-9]{40}$/u; +const TASK_ID_SEGMENT_ERROR = "prepareLaneWorkspace requires task.id to resolve to a safe path segment."; +const SYMLINK_WORKSPACE_PATH_ERROR = "prepareLaneWorkspace refuses to operate through symlinked workspace paths."; +const REPO_PATTERN_ERROR = "prepareLaneWorkspace requires task.repo to be an owner/name repository."; +const BASE_COMMIT_PATTERN_ERROR = "prepareLaneWorkspace requires upstream.baseCommit to be a 40-character lowercase hex SHA."; + +export async function prepareLaneWorkspace({ task, upstream, lane, runDir, repoCache, sourceRepoPath }) { + if (!task?.id || !task?.repo) { + throw new Error("prepareLaneWorkspace requires task.id and task.repo."); + } + if (typeof task.repo !== "string" || !isSafeRepo(task.repo)) { + throw new Error(REPO_PATTERN_ERROR); + } + if (typeof upstream?.baseCommit !== "string" || !BASE_COMMIT_PATTERN.test(upstream.baseCommit)) { + throw new Error(BASE_COMMIT_PATTERN_ERROR); + } + if (typeof lane !== "string" || !LANE_PATTERN.test(lane) || lane === "." || lane === "..") { + throw new Error("prepareLaneWorkspace requires lane to match /^[A-Za-z0-9._-]+$/."); + } + + const taskId = String(task.id); + const safeTaskId = taskId.replaceAll(/[^A-Za-z0-9._-]/gu, "_"); + if (safeTaskId.length === 0 || safeTaskId === "." || safeTaskId === "..") { + throw new Error(TASK_ID_SEGMENT_ERROR); + } + const taskPathSegment = `${safeTaskId}__${shortHash(taskId)}`; + const tasksDir = path.join(runDir, "tasks"); + const legacyTaskDir = path.join(tasksDir, safeTaskId); + const taskDir = path.join(tasksDir, taskPathSegment); + const laneDir = path.join(taskDir, lane); + assertPathInsideTaskDir({ laneDir, taskDir }); + const worktreePath = path.join(laneDir, "worktree"); + const remotePath = path.join(laneDir, "remote.git"); + + await assertNoSymlinkWorkspacePaths([tasksDir, legacyTaskDir, taskDir, laneDir]); + await rm(laneDir, { recursive: true, force: true }); + await mkdir(laneDir, { recursive: true }); + + const cloneSource = sourceRepoPath ?? (await prepareCachedRepo({ repo: task.repo, repoCache })); + await git(["clone", "--no-local", "--no-hardlinks", cloneSource, worktreePath]); + await git(["-C", worktreePath, "checkout", "--detach", upstream.baseCommit]); + await git(["-C", worktreePath, "update-ref", BASE_REF, upstream.baseCommit]); + await git(["-C", worktreePath, "update-ref", ORIGIN_MAIN_REF, upstream.baseCommit]); + await git(["init", "--bare", remotePath]); + await git(["-C", worktreePath, "remote", "set-url", "origin", remotePath]); + await git(["-C", worktreePath, "push", "origin", `${upstream.baseCommit}:${ORIGIN_MAIN_REF}`]); + await git(["--git-dir", remotePath, "symbolic-ref", "HEAD", ORIGIN_MAIN_REF]); + + return { laneDir, worktreePath, remotePath }; +} + +function shortHash(value) { + return createHash("sha256").update(value).digest("hex").slice(0, 12); +} + +function isSafeRepo(repo) { + if (!REPO_PATTERN.test(repo)) return false; + return repo.split("/").every((segment) => segment !== "." && segment !== ".."); +} + +function assertPathInsideTaskDir({ laneDir, taskDir }) { + const resolvedLaneDir = path.resolve(laneDir); + const resolvedTaskDir = path.resolve(taskDir); + const relative = path.relative(resolvedTaskDir, resolvedLaneDir); + if (relative === "" || relative.startsWith("..") || path.isAbsolute(relative)) { + throw new Error("prepareLaneWorkspace resolved lane outside task directory."); + } +} + +async function assertNoSymlinkWorkspacePaths(paths) { + for (const workspacePath of paths) { + let entry; + try { + entry = await lstat(workspacePath); + } catch (error) { + if (error?.code === "ENOENT") continue; + throw error; + } + + if (entry.isSymbolicLink()) { + throw new Error(SYMLINK_WORKSPACE_PATH_ERROR); + } + } +} + +async function prepareCachedRepo({ repo, repoCache }) { + if (!repoCache) { + throw new Error("prepareLaneWorkspace requires repoCache when sourceRepoPath is absent."); + } + + const cachePath = path.join(repoCache, String(repo).replaceAll("/", "__")); + const remoteUrl = `https://github.com/${repo}.git`; + if (await pathExists(cachePath)) { + if (await isBareGitRepo(cachePath)) { + await git(["-C", cachePath, "remote", "update", "--prune"]); + } else { + const existingRemoteUrl = (await originUrl(cachePath)) ?? remoteUrl; + await rm(cachePath, { recursive: true, force: true }); + await git(["clone", "--mirror", existingRemoteUrl, cachePath]); + } + } else { + await mkdir(repoCache, { recursive: true }); + await git(["clone", "--mirror", remoteUrl, cachePath]); + } + return cachePath; +} + +async function isBareGitRepo(repoPath) { + try { + return (await gitOutput(["-C", repoPath, "rev-parse", "--is-bare-repository"])).trim() === "true"; + } catch (error) { + if (typeof error?.code === "number") return false; + throw error; + } +} + +async function originUrl(repoPath) { + try { + const output = await gitOutput(["-C", repoPath, "remote", "get-url", "origin"]); + return output.trim() || null; + } catch (error) { + if (typeof error?.code === "number") return null; + throw error; + } +} + +async function pathExists(targetPath) { + try { + await stat(targetPath); + return true; + } catch (error) { + if (error?.code === "ENOENT") return false; + throw error; + } +} + +async function git(args) { + await execFileAsync("git", args, { maxBuffer: GIT_MAX_BUFFER }); +} + +async function gitOutput(args) { + const { stdout } = await execFileAsync("git", args, { encoding: "utf8", maxBuffer: GIT_MAX_BUFFER }); + return stdout; +} diff --git a/benchmarks/agentrail-swe-lifecycle/run.mjs b/benchmarks/agentrail-swe-lifecycle/run.mjs new file mode 100644 index 0000000..9272339 --- /dev/null +++ b/benchmarks/agentrail-swe-lifecycle/run.mjs @@ -0,0 +1,244 @@ +#!/usr/bin/env node +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { parseBenchmarkArgs } from "./lib/args.mjs"; +import { runAgentRailLane } from "./lib/agentrail-lane.mjs"; +import { runControlLane } from "./lib/control-lane.mjs"; +import { loadLifecycleDataset, selectLifecycleTasks } from "./lib/dataset.mjs"; +import { buildLocalLifecycleArtifacts } from "./lib/local-lifecycle.mjs"; +import { runPhaseAttributionTask } from "./lib/phase-runner.mjs"; +import { aggregateLifecycleResults, writeLifecycleResultFiles } from "./lib/results.mjs"; +import { resolveUpstreamRowsForTasks } from "./lib/upstream.mjs"; + +const RUN_ID_PATTERN = /^run_[A-Za-z0-9_-]+$/u; +const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", ".."); +const defaultDatasetPath = path.join(repoRoot, "benchmarks", "agentrail-swe-lifecycle", "dataset.json"); + +const flags = parseBenchmarkArgs(process.argv.slice(2)); +const dataset = await loadLifecycleDataset(flags.dataset ?? defaultDatasetPath); +const tasks = selectLifecycleTasks(dataset, flags); +if (tasks.length === 0) { + throw new Error("No lifecycle tasks matched the provided filters."); +} +const runId = flags.runId ?? `run_${new Date().toISOString().replaceAll(/[:.]/gu, "-")}`; +validateRunId(runId); +const outputDir = flags.outputDir + ? path.resolve(flags.outputDir) + : path.join(repoRoot, ".agentrail-bench", "agentrail-swe-lifecycle", runId); +const results = flags.mode === "dry-run" + ? tasks.map((task) => buildDryRunTaskResult({ task, runId })) + : flags.mode === "phase-attribution" + ? await runPhaseAttributionBenchmarkTasks({ tasks, runId, outputDir, flags }) + : await runRealBenchmarkTasks({ tasks, runId, outputDir, flags }); +const run = { + version: 1, + runId, + mode: flags.mode === "dry-run" ? "dry_run" : flags.mode, + benchmarkSurface: flags.benchmarkSurface, + dataset: dataset.id, + scenario: dataset.defaultScenario, + aggregate: aggregateLifecycleResults(results), + tasks: results +}; + +await writeLifecycleResultFiles({ outputDir, run }); + +console.log(JSON.stringify({ status: "ok", outputDir, tasks: tasks.length }, null, 2)); + +async function runPhaseAttributionBenchmarkTasks({ tasks, runId, outputDir, flags }) { + const upstreams = await resolveUpstreamsForExecution(tasks); + const results = []; + for (let index = 0; index < tasks.length; index += 1) { + const task = tasks[index]; + const upstream = upstreams[index]; + process.stderr.write(`[agentrail-swe] phase-attribution ${index + 1}/${tasks.length} ${task.id}\n`); + try { + results.push(await runPhaseAttributionTask(buildLaneOptions({ task, upstream, outputDir, flags, runId }))); + } catch (error) { + if (!flags.continueOnError) throw error; + const message = error instanceof Error ? error.message : String(error); + process.stderr.write(`[agentrail-swe] failed ${task.id}: ${message}\n`); + results.push(buildErrorTaskResult({ task, upstream, runId, error })); + } + } + return results; +} + +async function runRealBenchmarkTasks({ tasks, runId, outputDir, flags }) { + const upstreams = await resolveUpstreamsForExecution(tasks); + const results = []; + for (let index = 0; index < tasks.length; index += 1) { + const task = tasks[index]; + const upstream = upstreams[index]; + process.stderr.write(`[agentrail-swe] ${flags.mode} ${index + 1}/${tasks.length} ${task.id}\n`); + try { + if (flags.mode === "control") { + const control = await runControlLane(buildLaneOptions({ task, upstream, outputDir, flags })); + validateRequiredOutputs({ label: "control", lane: control, flags }); + results.push(buildSingleLaneTaskResult({ task, upstream, runId, laneName: "control", lane: control })); + } else if (flags.mode === "agentrail") { + const agentrail = await runAgentRailLane(buildLaneOptions({ task, upstream, outputDir, flags })); + validateRequiredOutputs({ label: "agentrail", lane: agentrail, flags }); + results.push(buildSingleLaneTaskResult({ task, upstream, runId, laneName: "agentrail", lane: agentrail })); + } else if (flags.mode === "paired") { + const control = await runControlLane(buildLaneOptions({ task, upstream, outputDir, flags })); + validateRequiredOutputs({ label: "control", lane: control, flags }); + const agentrail = await runAgentRailLane(buildLaneOptions({ task, upstream, outputDir, flags })); + validateRequiredOutputs({ label: "agentrail", lane: agentrail, flags }); + results.push(buildPairedTaskResult({ task, upstream, runId, control, agentrail })); + } else { + throw new Error(`Unsupported execution mode: ${flags.mode}`); + } + } catch (error) { + if (!flags.continueOnError) throw error; + const message = error instanceof Error ? error.message : String(error); + process.stderr.write(`[agentrail-swe] failed ${task.id}: ${message}\n`); + results.push(buildErrorTaskResult({ task, upstream, runId, error })); + } + } + return results; +} + +async function resolveUpstreamsForExecution(tasks) { + const manifestUpstreams = tasks.map(upstreamFromTaskManifest); + if (manifestUpstreams.every(Boolean)) return manifestUpstreams; + return await resolveUpstreamRowsForTasks(tasks); +} + +function upstreamFromTaskManifest(task) { + const upstream = task.upstream ?? {}; + const baseCommit = upstream.baseCommit ?? task.baseCommit; + const problemStatement = upstream.problemStatement ?? task.problemStatement; + if (typeof baseCommit !== "string" || typeof problemStatement !== "string") return null; + return { + repo: task.repo, + instanceId: upstream.instanceId, + baseCommit, + problemStatement, + requirements: upstream.requirements ?? null, + interface: upstream.interface ?? null, + repoLanguage: task.language ?? null + }; +} + +function buildLaneOptions({ task, upstream, outputDir, flags, runId }) { + return { + task, + upstream, + runId, + runDir: outputDir, + repoCache: flags.repoCache ?? path.join(repoRoot, ".agentrail-bench", "repo-cache"), + sourceRepoPath: flags.sourceRepoPath, + flags + }; +} + +function buildSingleLaneTaskResult({ task, upstream, runId, laneName, lane }) { + return { + taskId: task.id, + upstreamInstanceId: upstream.instanceId, + repo: task.repo, + lifecycle: buildLocalLifecycleArtifacts({ task, upstream, runId }), + lanes: { [laneName]: summarizeLane(lane) }, + final: { success: lane.success, status: lane.status }, + tokenUsage: lane.tokenUsage + }; +} + +function buildPairedTaskResult({ task, upstream, runId, control, agentrail }) { + return { + taskId: task.id, + upstreamInstanceId: upstream.instanceId, + repo: task.repo, + lifecycle: buildLocalLifecycleArtifacts({ task, upstream, runId }), + lanes: { + control: summarizeLane(control), + agentrail: summarizeLane(agentrail) + }, + comparison: { + control: { tokenUsage: control.tokenUsage }, + agentrail: { tokenUsage: agentrail.tokenUsage }, + tokenDelta: tokenDelta({ control, agentrail }), + tokenReductionRate: tokenReductionRate({ control, agentrail }) + }, + final: { + success: control.success && agentrail.success, + status: control.success && agentrail.success ? "completed" : "failed" + }, + tokenUsage: agentrail.tokenUsage + }; +} + +function buildErrorTaskResult({ task, upstream, runId, error }) { + const message = error instanceof Error ? error.message : String(error); + return { + taskId: task.id, + upstreamInstanceId: upstream.instanceId, + repo: task.repo, + lifecycle: buildLocalLifecycleArtifacts({ task, upstream, runId }), + lanes: {}, + final: { success: false, status: "error", error: message }, + tokenUsage: null + }; +} + +function summarizeLane(lane) { + return { + status: lane.status, + success: lane.success, + tokenUsage: lane.tokenUsage, + patch: lane.patch, + evaluator: lane.evaluator, + lifecycle: lane.lifecycle ?? null, + workspace: lane.workspace + }; +} + +function tokenDelta({ control, agentrail }) { + const controlTokens = tokenTotal(control.tokenUsage); + const agentRailTokens = tokenTotal(agentrail.tokenUsage); + return controlTokens === null || agentRailTokens === null ? null : agentRailTokens - controlTokens; +} + +function tokenReductionRate({ control, agentrail }) { + const controlTokens = tokenTotal(control.tokenUsage); + const agentRailTokens = tokenTotal(agentrail.tokenUsage); + if (controlTokens === null || agentRailTokens === null || controlTokens === 0) return null; + return (controlTokens - agentRailTokens) / controlTokens; +} + +function tokenTotal(tokenUsage) { + return typeof tokenUsage?.totalTokens === "number" ? tokenUsage.totalTokens : null; +} + +function validateRequiredOutputs({ label, lane, flags }) { + if (flags.requireTokenUsage && lane.tokenUsage?.status !== "available") { + throw new Error(`${label} lane did not produce parseable token usage.`); + } + if (flags.requireEvaluator && lane.evaluator?.status === "unavailable") { + throw new Error(`${label} lane did not produce an evaluator result.`); + } +} + +function buildDryRunTaskResult({ task, runId }) { + const upstream = { + instanceId: task.upstream.instanceId, + problemStatement: `Dry-run placeholder for ${task.upstream.instanceId}. Use upstream resolver in execution mode.` + }; + + return { + taskId: task.id, + upstreamInstanceId: task.upstream.instanceId, + repo: task.repo, + lifecycle: buildLocalLifecycleArtifacts({ task, upstream, runId }), + final: { success: false, status: "dry_run" }, + tokenUsage: null + }; +} + +function validateRunId(runId) { + if (!RUN_ID_PATTERN.test(runId)) { + throw new Error("runId must match /^run_[A-Za-z0-9_-]+$/."); + } +} diff --git a/benchmarks/agentrail-swe-lifecycle/scenarios/local-mock-lifecycle.json b/benchmarks/agentrail-swe-lifecycle/scenarios/local-mock-lifecycle.json new file mode 100644 index 0000000..a81b563 --- /dev/null +++ b/benchmarks/agentrail-swe-lifecycle/scenarios/local-mock-lifecycle.json @@ -0,0 +1,63 @@ +{ + "version": 1, + "id": "local-mock-lifecycle", + "description": "Run SWE-bench Pro tasks through an AgentRail lifecycle using local deterministic provider doubles instead of live GitHub, Linear, hosted CI, or human review.", + "providers": { + "issue": { + "default": "github", + "supported": ["github", "linear"], + "mode": "local_mock", + "bodyFrom": "upstream.problem_statement" + }, + "intake": { + "mode": "agentrail_local", + "required": true + }, + "pullRequest": { + "mode": "synthetic_pr_record", + "required": true + }, + "ci": { + "provider": "swebench_evaluator", + "mode": "local", + "required": true + }, + "review": { + "provider": "deterministic_review", + "mode": "local", + "required": true, + "blockingRules": [ + "evaluator_failed", + "forbidden_file_changed", + "gold_patch_or_test_patch_referenced", + "oversized_generated_or_vendor_churn" + ] + }, + "ship": { + "mode": "merge_ready_record", + "mergeRequired": false + } + }, + "retryPolicy": { + "maxCiRetries": 2, + "maxReviewRetries": 1 + }, + "successCriteria": [ + "upstream_evaluator_passed", + "no_blocking_review_findings", + "agentrail_task_reached_merge_ready_or_done" + ], + "tokenAccounting": { + "sourceOfTruth": "provider_reported_usage", + "include": [ + "input_tokens", + "output_tokens", + "reasoning_tokens_when_reported", + "cached_tokens_when_reported", + "tool_result_tokens_inserted_into_model_context", + "all_retry_ci_review_and_ship_turns" + ], + "doNotEstimateMissingProviderFields": true + } +} + diff --git a/landing-next/app/benchmarks/page.tsx b/landing-next/app/benchmarks/page.tsx new file mode 100644 index 0000000..1fbc9a7 --- /dev/null +++ b/landing-next/app/benchmarks/page.tsx @@ -0,0 +1,1400 @@ +import type { Metadata } from "next"; +import Link from "next/link"; +import Nav from "@/components/Nav"; +import Footer from "@/components/Footer"; + +export const metadata: Metadata = { + title: "Benchmarks — AgentRail", + description: + "AgentRail SWE-lifecycle benchmark: 60% fewer tokens on Cursor Composer 2.5, 21% lower cost on Claude Opus 4.7 and GPT-5.5, across 5 real software-engineering tasks.", + alternates: { canonical: "/benchmarks" }, +}; + +type RunnerRow = { + metric: string; + control: string; + agentrail: string; + deltaAbs: string; + deltaPct: string; + emphasised?: boolean; +}; + +type Runner = { + id: string; + name: string; + provider: string; + via: string; + headline: string; + headlineMetric: string; + rows: RunnerRow[]; + runId: string; + note: string; +}; + +const runners: Runner[] = [ + { + id: "cursor", + name: "Cursor Composer 2.5", + provider: "Cursor", + via: "native agent", + headline: "−60.0%", + headlineMetric: "non-cache tokens", + rows: [ + { + metric: "non-cache tokens", + control: "8,800,719", + agentrail: "3,520,245", + deltaAbs: "−5,280,474", + deltaPct: "−60.0%", + emphasised: true, + }, + { + metric: "cache-inclusive tokens", + control: "100,969,521", + agentrail: "77,663,576", + deltaAbs: "−23,305,945", + deltaPct: "−23.1%", + }, + { + metric: "input tokens", + control: "8,524,884", + agentrail: "3,312,805", + deltaAbs: "−5,212,079", + deltaPct: "−61.1%", + }, + { + metric: "output tokens", + control: "275,835", + agentrail: "207,440", + deltaAbs: "−68,395", + deltaPct: "−24.8%", + }, + ], + runId: "run_2026-05-25T18-15-35-314Z", + note: "Non-cache tokens are the headline figure for Cursor — they correspond to billable activity outside the prompt-cache hit path. Composer 2.5 is plan-billed; per-token dollar cost is not directly comparable to API providers.", + }, + { + id: "opus", + name: "Claude Opus 4.7", + provider: "Anthropic", + via: "via Claude Code", + headline: "−21.3%", + headlineMetric: "dollar cost", + rows: [ + { + metric: "dollar cost (USD)", + control: "$48.93", + agentrail: "$38.53", + deltaAbs: "−$10.40", + deltaPct: "−21.3%", + emphasised: true, + }, + { + metric: "cache-inclusive tokens", + control: "49,367,189", + agentrail: "45,202,488", + deltaAbs: "−4,164,701", + deltaPct: "−8.4%", + }, + { + metric: "cache creation", + control: "2,729,630", + agentrail: "1,388,016", + deltaAbs: "−1,341,614", + deltaPct: "−49.2%", + }, + { + metric: "cache read", + control: "46,374,169", + agentrail: "43,575,956", + deltaAbs: "−2,798,213", + deltaPct: "−6.0%", + }, + { + metric: "non-cache tokens", + control: "263,390", + agentrail: "238,516", + deltaAbs: "−24,874", + deltaPct: "−9.4%", + }, + ], + runId: "run_phase_opus47_5_20260525_213304", + note: "Cost reduction outpaces raw token reduction because AgentRail shifts work from cache-creation (priced higher) toward cache-read. Direct dollar figures come from Anthropic's billed rates at run time.", + }, + { + id: "gpt55", + name: "GPT-5.5", + provider: "OpenAI", + via: "via Codex", + headline: "−21.0%", + headlineMetric: "dollar cost", + rows: [ + { + metric: "dollar cost (USD)", + control: "~$8.02", + agentrail: "~$6.34", + deltaAbs: "−$1.68", + deltaPct: "−21.0%", + emphasised: true, + }, + { + metric: "total tokens", + control: "27,392,589", + agentrail: "24,228,912", + deltaAbs: "−3,163,677", + deltaPct: "−11.5%", + }, + { + metric: "input tokens", + control: "27,131,547", + agentrail: "24,003,102", + deltaAbs: "−3,128,445", + deltaPct: "−11.5%", + }, + { + metric: "cached input", + control: "24,862,976", + agentrail: "22,605,952", + deltaAbs: "−2,257,024", + deltaPct: "−9.1%", + }, + { + metric: "output tokens", + control: "207,314", + agentrail: "176,400", + deltaAbs: "−30,914", + deltaPct: "−14.9%", + }, + ], + runId: "run_phase_gpt55_5_20260525_212153", + note: "Output-token reductions matter most here — output is the priciest unit on GPT-5.5 and the one most sensitive to scaffolding overhead. Dollar cost computed at OpenAI's published GPT-5-class rates.", + }, +]; + +const tasks = [ + { + id: "T-01", + repo: "ansible/ansible", + summary: "Password lookup + encrypt: fix subjects_alt_name handling", + files: 5, + loc: "+143 / −89", + }, + { + id: "T-02", + repo: "ansible/ansible", + summary: "YAML dumper: propagate filter trust through templating", + files: 6, + loc: "+129 / −19", + }, + { + id: "T-03", + repo: "ansible/ansible", + summary: "Linux facts: correct nproc/cpuinfo on container hosts", + files: 3, + loc: "+73 / −1", + }, + { + id: "T-04", + repo: "ansible/ansible", + summary: "mount_facts module: introduce structured mount inventory", + files: 7, + loc: "+1622 / −0", + }, + { + id: "T-05", + repo: "ansible/ansible", + summary: "Module respawn + SELinux compat shim (libselinux removal)", + files: 13, + loc: "+649 / −304", + }, +]; + +const metricDefs = [ + { + term: "non-cache tokens", + desc: "Tokens billed at the model's full input rate — i.e. not served from the provider's prompt cache. The cleanest signal of net new work the model had to do. Where AgentRail's lifecycle routing has the largest effect.", + }, + { + term: "cache-inclusive tokens", + desc: "Every token the agent sent or received, including cache reads. Larger absolute numbers and smaller percentage deltas — context-window-bound, not cost-bound.", + }, + { + term: "cache creation", + desc: "First-time write of a prompt prefix into the provider's cache. Priced higher than cache-read for Anthropic models; AgentRail's stable role briefs cut repeat creations.", + }, + { + term: "cache read", + desc: "Reuse of a previously cached prefix. Discounted; counts toward cache-inclusive but not non-cache. Higher is generally better — it means the cache is doing its job.", + }, + { + term: "output tokens", + desc: "Tokens the model generated. Priced highest on most providers. Disproportionate driver of cost on GPT-5.5 and a tight proxy for agent verbosity.", + }, + { + term: "dollar cost", + desc: "Provider-reported USD spend, summed across all five tasks in the run-set. For Claude Opus, computed from Anthropic's billed rates at the time of the run. For GPT-5.5, computed from OpenAI's published GPT-5-class rates applied to the captured token mix.", + }, +]; + +const fontMono = "var(--font-mono)"; +const fontSerif = "var(--font-serif)"; + +// ── Layout primitives ────────────────────────────────────────────────────── + +function SectionLabel({ num, title }: { num: string; title: string }) { + return ( +
+ + + {num} / {title} + + +
+ ); +} + +// ── Page sections ────────────────────────────────────────────────────────── + +function PageHeader() { + return ( +
+
+
+ agentrail / benchmarks / run-set 5 + + 2026-05-26 + +
+

+ Run-set 5. +

+

+ Three coding agents — Cursor Composer 2.5, Claude Opus 4.7 (via Claude + Code), and GPT-5.5 (via Codex) — on five real software-engineering + tasks, full issue-to-ship lifecycle, two lanes per task. All totals + below are summed across the run-set. +

+ + {/* TL;DR strip */} +
+ + tl;dr + + {[ + { + runner: "cursor 2.5", + stat: "−60.0%", + metric: "non-cache tokens", + detail: "8.80M → 3.52M", + }, + { + runner: "opus 4.7", + stat: "−21.3%", + metric: "cost", + detail: "$48.93 → $38.53", + }, + { + runner: "gpt-5.5", + stat: "−21.0%", + metric: "cost", + detail: "$8.02 → $6.34", + }, + ].map((entry) => ( +
+
+ {entry.runner} +
+
+ {entry.stat}{" "} + + {entry.metric} + +
+
+ {entry.detail} +
+
+ ))} +
+
+
+ ); +} + +function TasksSection() { + return ( +
+
+ +

+ Five real Ansible tasks. +

+

+ Drawn from the SWE-bench-Pro hard set against the{" "} + + ansible/ansible + {" "} + repository. Each task runs the full 14-phase lifecycle: issue intake → + plan → implement → CI → review → ship. +

+ +
+
+
id
+
task
+
files
+
loc
+
+ {tasks.map((t, i) => ( +
+
+ {t.id} +
+
{t.summary}
+
+ {t.files} +
+
+ {t.loc} +
+
+ ))} +
+
+
+ ); +} + +function LanesSection() { + return ( +
+
+ +

+ Control vs AgentRail. +

+
+ {[ + { + label: "control", + body: "The agent owns every phase: intake, triage, plan, implement, CI watch, fix CI, review, revise, ship. Tokens are charged for every prompt the agent issues across the loop.", + }, + { + label: "agentrail", + body: "The agent is invoked only for the phases AgentRail's lifecycle leaves to it (predominantly implement and revise). Other phases are handled by AgentRail and incur no agent-side spend.", + }, + ].map((lane, i, arr) => ( +
+
+ {lane.label} +
+

+ {lane.body} +

+
+ ))} +
+

+ Both lanes share the same model, same repository state, and same + acceptance criteria. They differ only in which phases the agent runs + — mirroring the deployment decision in production, not a controlled + lab study. +

+
+
+ ); +} + +function ResultsTable() { + return ( +
+
+ +

+ Every metric, every cell. +

+

+ 3 runners × 2 lanes +

+ +
+ {/* Header */} +
+
runner
+
metric
+
control
+
agentrail
+
Δ
+
Δ%
+
+ + {runners.map((runner, ri) => ( +
+ {runner.rows.map((row, i) => ( +
+
+ {i === 0 ? runner.name : "·"} +
+
+ {row.metric} +
+
+ {row.control} +
+
+ {row.agentrail} +
+
+ {row.deltaAbs} +
+
+ {row.deltaPct} +
+
+ ))} +
+ ))} +
+
+
+ ); +} + +// ── Paired-bar chart (CSS-only) ──────────────────────────────────────────── + +function PairedBar({ + controlLabel, + controlValue, + agentrailLabel, + agentrailValue, + controlWidthPct, + agentrailWidthPct, + deltaLabel, +}: { + controlLabel: string; + controlValue: string; + agentrailLabel: string; + agentrailValue: string; + controlWidthPct: number; + agentrailWidthPct: number; + deltaLabel: string; +}) { + return ( +
+
+ + {controlLabel} + +
+
+
+ + {controlValue} + +
+
+ + {agentrailLabel} + +
+
+ + {deltaLabel} + +
+ + {agentrailValue} + +
+
+ ); +} + +function chartData(runner: Runner) { + // For each row, compute normalised widths against the row's control value. + return runner.rows.map((row) => { + const parse = (s: string) => + Number(s.replace(/[$,~Mk]/g, "")) || 0; + const c = parse(row.control); + const a = parse(row.agentrail); + const max = Math.max(c, a) || 1; + return { + ...row, + controlWidthPct: (c / max) * 100, + agentrailWidthPct: (a / max) * 100, + }; + }); +} + +function PerRunnerSection({ runner, idx }: { runner: Runner; idx: number }) { + const data = chartData(runner); + return ( +
+
+ + 0{idx + 1} + +

+ {runner.name} +

+ + {runner.provider} · {runner.via} + + + {runner.headline}{" "} + + {runner.headlineMetric} + + +
+ +
+ {data.map((row) => ( +
+
+ {row.metric} +
+ +
+ ))} +
+ +
+ {runner.runId} +
+
+ ); +} + +function ChartsSection() { + return ( +
+
+ +

+ Control vs AgentRail, per metric. +

+

+ ink = control · green = agentrail · normalised per row +

+ {runners.map((runner, i) => ( + + ))} +
+
+ ); +} + +function NotesSection() { + return ( +
+
+ +

+ Why the deltas differ. +

+
+ {runners.map((runner, i, arr) => ( +
+
+ {runner.provider} +
+
+ {runner.name} +
+

+ {runner.note} +

+
+ {runner.runId} +
+
+ ))} +
+
+
+ ); +} + +function DefinitionsSection() { + return ( +
+
+ +

+ What each number means. +

+
+ {metricDefs.map((def) => ( +
+
+ {def.term} +
+
+ {def.desc} +
+
+ ))} +
+
+
+ ); +} + +function ReproduceSection() { + return ( +
+
+ +

+ Run it yourself. +

+

+ Every prompt, every task harness, and every run id above is in the + benchmark repo. Numbers should reproduce within ±2% on a fresh + sample. +

+ 
+          $ git clone
+          https://github.com/oxnw/agentrail
+          {"\n"}
+          $ cd
+          agentrail/benchmarks/agentrail-swe-lifecycle
+          {"\n"}
+          $ npm run
+          benchmark:phase-attribution -- --runner claude-code --count 5
+
+
+ byok · BUSL-1.1 +
+
+
+ ); +} + +// ── Page ────────────────────────────────────────────────────────────────── + +export default function BenchmarksPage() { + return ( + <> +