[CI][Feature] Add nightly auto-bisect tool for E2E test failures (vllm-project#10564)

### What this PR does / why we need it?

This PR introduces an automated bisect tool (`auto_bisect`) for nightly
E2E test failures in `vllm-ascend`. It binary-searches the commit
history between the last known-good commit (tracked in a CSV table) and
the failing commit to identify the first bad commit and its associated
PR. It supports both single-node and multi-node environments, reuses
existing nightly launch entries, and optimizes the build process by only
rebuilding when native or build-definition files are modified.

### Does this PR introduce _any_ user-facing change?

No. This is a developer/CI tool and does not introduce any user-facing
changes.

### How was this patch tested?

Tested locally by running the auto-bisect tool on single-node and
multi-node scenarios.

- vLLM version: v0.23.0
- vLLM main:
vllm-project/vllm@1f486d9

Signed-off-by: DreamerLeader <88812830+DreamerLeader@users.noreply.github.com>

Author: DreamerLeader

.github/workflows/scripts/select_tests.py

@@ -43,6 +43,12 @@
   the broad regression triggered by ``optional: false`` modules when the
   intent of the PR is purely to add or adjust tests.
 
+Bisect-tool optimization:
+  If a PR is scoped to ``tools/bisect`` and its paired UT/config/format files,
+  the always-on modules are skipped and only modules whose dependencies match
+  the changed files are selected. This keeps maintenance-only tool changes
+  from triggering the full CPU and NPU regression suite.
+
 Routing is driven by ``test_config.yaml`` ``runner_mapping:`` (regex patterns).
 Partition sizing by ``partition:`` config block.
 See ``test_config.yaml`` for details.
@@ -90,6 +96,15 @@ class RunnerInfo:
 # is selected for UT runs (along with the changed test files).
 DEFAULT_CPU_UT_MODULE = "default_cpu_ut"
 
+_BISECT_TOOL_ROOTS = ("tools/bisect", "tests/ut/tools/bisect")
+_BISECT_TOOL_SUPPORT_FILES = {
+    ".github/workflows/scripts/select_tests.py",
+    ".github/workflows/scripts/test_config.yaml",
+    ".github/workflows/scripts/test_select_tests.py",
+    "csrc/build.sh",
+    "tests/ut/tools/__init__.py",
+}
+
 # Populated by _load_runner_mapping(). Ordered list of (regex, {key: RunnerKey}).
 _RUNNER_MAPPING: list[tuple[re.Pattern, dict[str, RunnerKey]]] = []
 
@@ -249,12 +264,14 @@ def resolve(name: str) -> dict:
 def _match_modules(
     changed_files: list[str],
     config: list[dict],
+    *,
+    include_always: bool = True,
 ) -> list[str]:
     if not changed_files:
         return []
     matched: list[str] = []
     for module in config:
-        if not module.get("optional", True):
+        if include_always and not module.get("optional", True):
             matched.append(module["name"])
             continue
         deps = module.get("source_file_dependencies", [])
@@ -348,6 +365,20 @@ def _is_test_only_change(changed_files: list[str]) -> bool:
     return bool(changed_files) and all(_is_test_path(f) for f in changed_files)
 
 
+def _is_bisect_tool_scoped_path(file_path: str) -> bool:
+    return file_path in _BISECT_TOOL_SUPPORT_FILES or any(
+        _matches_path_dependency(file_path, root) for root in _BISECT_TOOL_ROOTS
+    )
+
+
+def _is_bisect_tool_scoped_change(changed_files: list[str]) -> bool:
+    return (
+        bool(changed_files)
+        and any(_matches_path_dependency(f, root) for f in changed_files for root in _BISECT_TOOL_ROOTS)
+        and all(_is_bisect_tool_scoped_path(f) for f in changed_files)
+    )
+
+
 def _scan_ut_test_dir(
     dir_path: str,
     groups: dict[RunnerKey, list[str]],
@@ -731,15 +762,23 @@ def main():
             _scan_e2e_test_dir(path, all_groups)
     else:
         changed_files = _get_changed_files(args.diff_base) if args.diff_base else args.changed_files
+        bisect_tool_scoped_change = _is_bisect_tool_scoped_change(changed_files)
         test_only_change = _is_test_only_change(changed_files)
-        if test_only_change:
+        if bisect_tool_scoped_change:
+            print(
+                "Detected bisect tool-scoped change: running only matching tool modules (skipping always-on modules).",
+                file=sys.stderr,
+            )
+        elif test_only_change:
             print(
                 "Detected test-only change: running only default_cpu_ut"
                 " and the changed test files (skipping source-driven modules).",
                 file=sys.stderr,
             )
         if args.run_all_modules:
             matched_modules = [module["name"] for module in config]
+        elif bisect_tool_scoped_change:
+            matched_modules = _match_modules(changed_files, config, include_always=False)
         elif test_only_change:
             matched_modules = [m["name"] for m in config if m["name"] == DEFAULT_CPU_UT_MODULE]
         else:

.github/workflows/scripts/test_config.yaml

@@ -647,6 +647,15 @@
     - tests/e2e/pull_request/two_card/test_qwen3_5_35b_a3b_w8a8.py
 
 # === Others ===
+- name: nightly_bisect
+  optional: true
+  cpu_only: true
+  source_file_dependencies:
+    - tools/bisect
+    - tests/ut/tools/bisect
+  tests:
+    - tests/ut/tools/bisect
+
 - name: _tools
   optional: false
   source_file_dependencies:
@@ -824,4 +833,3 @@ partition:
 
 
 
-

.github/workflows/scripts/test_select_tests.py

@@ -468,6 +468,99 @@ def test_main_end_to_end_changed_files_options_and_skip(tmp_path, monkeypatch, c
     assert selected_tests == {"tests/e2e/pull_request/two_card/test_two_card.py::test_specific_case"}
 
 
+def test_bisect_tool_scoped_change_skips_global_modules(tmp_path, monkeypatch, capsys):
+    test_root = tmp_path / "tests"
+    default_cpu_dir = test_root / "ut" / "default"
+    bisect_dir = test_root / "ut" / "tools" / "bisect"
+    tools_dir = test_root / "ut" / "_tools"
+    e2e_dir = test_root / "e2e" / "pull_request" / "one_card"
+    for path in (default_cpu_dir, bisect_dir, tools_dir, e2e_dir):
+        path.mkdir(parents=True)
+
+    default_cpu_test = default_cpu_dir / "test_default.py"
+    bisect_test = bisect_dir / "test_auto_bisect.py"
+    tools_test = tools_dir / "test_base_tool.py"
+    e2e_test = e2e_dir / "test_model.py"
+    for path in (default_cpu_test, bisect_test, tools_test, e2e_test):
+        path.write_text("")
+
+    config = [
+        {
+            "name": "default_cpu_ut",
+            "optional": False,
+            "cpu_only": True,
+            "tests": ["tests/ut"],
+        },
+        {
+            "name": "always_e2e",
+            "optional": False,
+            "tests": ["tests/e2e/pull_request/one_card"],
+        },
+        {
+            "name": "nightly_bisect",
+            "optional": True,
+            "cpu_only": True,
+            "source_file_dependencies": ["tools/bisect", "tests/ut/tools/bisect"],
+            "tests": ["tests/ut/tools/bisect"],
+        },
+        {
+            "name": "_tools",
+            "optional": False,
+            "source_file_dependencies": ["tools/"],
+            "tests": ["tests/ut/_tools"],
+        },
+    ]
+    config_path = tmp_path / "config.yaml"
+    runner_mapping = {"tests/e2e/pull_request/one_card": {"default": "a2_x1"}}
+    _write_two_doc_config(config_path, config, {"runner_mapping": runner_mapping})
+    runner_file = tmp_path / "runner_label.json"
+    runner_file.write_text(
+        json.dumps(
+            {
+                "cpu-runner": {"chip": "cpu", "npu_num": 0},
+                "a2-runner": {"chip": "a2", "npu_num": 1},
+            }
+        )
+    )
+    monkeypatch.setattr(select_tests, "_RUNNER_LABEL_PATH", runner_file)
+    monkeypatch.chdir(tmp_path)
+
+    changed_files = [
+        "tools/bisect/auto_bisect.py",
+        "tests/ut/tools/bisect/test_auto_bisect.py",
+        ".github/workflows/scripts/select_tests.py",
+        ".github/workflows/scripts/test_config.yaml",
+        "csrc/build.sh",
+    ]
+    assert select_tests._is_bisect_tool_scoped_change(changed_files)
+    assert not select_tests._is_bisect_tool_scoped_change([*changed_files, "vllm_ascend/envs.py"])
+    monkeypatch.setattr(
+        sys,
+        "argv",
+        [
+            "select_tests.py",
+            "--config",
+            str(config_path),
+            "--changed-files",
+            *changed_files,
+        ],
+    )
+
+    select_tests.main()
+    captured = capsys.readouterr()
+    assert "Detected bisect tool-scoped change" in captured.err
+    assert "matched_modules=nightly_bisect,_tools" in captured.out
+    groups_line = next(line for line in captured.out.splitlines() if line.startswith("test_groups="))
+    test_groups = json.loads(groups_line.removeprefix("test_groups="))
+    selected_tests = {test for group in test_groups for test in group["tests"].split()}
+    assert selected_tests == {
+        "tests/ut/_tools/test_base_tool.py",
+        "tests/ut/tools/bisect/test_auto_bisect.py",
+    }
+    assert "tests/ut/default/test_default.py" not in selected_tests
+    assert "tests/e2e/pull_request/one_card/test_model.py" not in selected_tests
+
+
 def test_default_cpu_ut_always_runs(tmp_path, monkeypatch, capsys):
     test_root = tmp_path / "tests"
     cpu_dir = test_root / "ut" / "cpu"

csrc/build.sh

@@ -521,7 +521,7 @@ function gen_bisheng(){
     fi
 
     pushd ${gen_bisheng_dir}
-    $(> bisheng)
+    : > bisheng
     echo "#!/bin/bash" >> bisheng
     echo "ccache_args=""\"""${ccache_program} ${BISHENG_REAL_PATH}""\"" >> bisheng
     echo "args=""$""@" >> bisheng

tests/ut/tools/__init__.py

@@ -0,0 +1 @@
+

tests/ut/tools/bisect/__init__.py

@@ -0,0 +1 @@
+

tests/ut/tools/bisect/conftest.py

@@ -0,0 +1,12 @@
+import importlib.util
+import sys
+import types
+from unittest.mock import MagicMock
+
+if importlib.util.find_spec("psutil") is None:
+    psutil = types.ModuleType("psutil")
+    psutil.__spec__ = importlib.util.spec_from_loader("psutil", loader=None)
+    psutil.Error = RuntimeError  # type: ignore[attr-defined]
+    psutil.Process = MagicMock()  # type: ignore[attr-defined]
+    psutil.process_iter = MagicMock(return_value=[])  # type: ignore[attr-defined]
+    sys.modules["psutil"] = psutil

tests/ut/tools/bisect/test_auto_bisect.py

@@ -0,0 +1,76 @@
+import argparse
+from pathlib import Path
+
+import pytest
+
+from tools.bisect.auto_bisect import Bisector, _parse_args, _resolve_num_nodes
+from tools.bisect.config import SCENE_MULTI
+
+
+def test_pick_mid_prefers_midpoint_then_nearest_unskipped_index():
+    assert Bisector._pick_mid(0, 8, skipped=set()) == 4
+    assert Bisector._pick_mid(0, 8, skipped={4}) == 5
+    assert Bisector._pick_mid(0, 8, skipped={4, 5}) == 3
+    assert Bisector._pick_mid(0, 3, skipped={0, 1, 2}) is None
+
+
+def test_parse_args_maps_no_assume_built_head_flag():
+    args = _parse_args(
+        [
+            "--scene",
+            "single_node",
+            "--config-yaml",
+            "case.yaml",
+            "--good-commit",
+            "good",
+            "--no-assume-built-head",
+            "--native-check",
+            "since-build",
+        ]
+    )
+
+    assert args.scene == "single_node"
+    assert args.config_yaml == "case.yaml"
+    assert args.good_commit == "good"
+    assert args.no_assume_built_head is True
+    assert args.native_check == "since-build"
+
+
+def test_resolve_num_nodes_prefers_explicit_value(tmp_path: Path):
+    args = argparse.Namespace(
+        num_nodes=4,
+        scene=SCENE_MULTI,
+        config_base_path=None,
+        config_yaml="missing.yaml",
+    )
+
+    assert _resolve_num_nodes(args, tmp_path) == 4
+
+
+def test_resolve_num_nodes_reads_multi_node_yaml(tmp_path: Path):
+    config = tmp_path / "configs" / "case.yaml"
+    config.parent.mkdir()
+    config.write_text("num_nodes: 2\n", encoding="utf-8")
+    args = argparse.Namespace(
+        num_nodes=None,
+        scene=SCENE_MULTI,
+        config_base_path="configs",
+        config_yaml="case.yaml",
+    )
+
+    assert _resolve_num_nodes(args, tmp_path) == 2
+
+
+def test_resolve_num_nodes_fails_when_multi_node_yaml_has_no_node_count(tmp_path: Path):
+    config = tmp_path / "configs" / "case.yaml"
+    config.parent.mkdir()
+    config.write_text("test_cases: []\n", encoding="utf-8")
+    args = argparse.Namespace(
+        num_nodes=None,
+        scene=SCENE_MULTI,
+        config_base_path="configs",
+        config_yaml="case.yaml",
+    )
+
+    with pytest.raises(SystemExit, match="Could not determine --num-nodes"):
+        _resolve_num_nodes(args, tmp_path)