Skip to content

Commit 251f21f

Browse files
authored
[CI][Fix]: summarize CPU unit test failures (vllm-project#12945)
Preserve downloaded artifact directories so CPU logs can be identified independently. Collapse CPU pytest failures into a single cpu-ut result while retaining existing matching for other devices. ### What this PR does / why we need it? Add logic to match cpu-ut errors separately, exclude from global matching Fix Chinese character display issue in logs ### Does this PR introduce _any_ user-facing change? ### How was this patch tested? - vLLM version: v0.25.1 - vLLM main: vllm-project/vllm@fe784ff --------- Signed-off-by: Xuyzhen <958522639@qq.com>
1 parent 81c3b37 commit 251f21f

2 files changed

Lines changed: 76 additions & 42 deletions

File tree

.github/workflows/_analyze_failure.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@ jobs:
7979
with:
8080
pattern: ${{ inputs.log_artifact_pattern }}
8181
path: ./test-logs
82-
merge-multiple: true
82+
merge-multiple: false
8383
continue-on-error: true
8484

8585
- name: Show downloaded logs structure

.github/workflows/scripts/analyze_failure_report.py

Lines changed: 75 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -48,62 +48,94 @@ def clean_line(line):
4848
# ============================================================
4949

5050

51+
FAILED_PATTERN = re.compile(r"^FAILED\s+(tests/\S+?\.py::\S+?)\s")
52+
SUMMARY_SEPARATOR_PATTERN = re.compile(r"^=+\s")
53+
CPU_LOG_PATH_PATTERN = re.compile(r"(?:^|-)cpu-\d+card(?:-|$)", re.IGNORECASE)
54+
CPU_FAILURE_LABEL = "cpu-ut"
55+
56+
57+
def extract_failed_from_log(log_path):
58+
"""Extract pytest node IDs from one log's short test summary."""
59+
try:
60+
lines = log_path.read_text(encoding="utf-8", errors="replace").splitlines()
61+
except Exception as exc:
62+
print(f"::warning:: Cannot read {log_path}: {exc}")
63+
return []
64+
65+
failed = []
66+
in_summary = False
67+
for line in lines:
68+
text = clean_line(line)
69+
70+
if "short test summary info" in text:
71+
in_summary = True
72+
continue
73+
74+
if not in_summary:
75+
continue
76+
77+
if SUMMARY_SEPARATOR_PATTERN.match(text):
78+
in_summary = False
79+
continue
80+
81+
match = FAILED_PATTERN.match(text)
82+
if match:
83+
failed.append(match.group(1))
84+
85+
return failed
86+
87+
88+
def is_cpu_log(log_path):
89+
"""Return whether a log belongs to a CPU selected-test artifact."""
90+
if log_path.stem.lower().endswith("-cpu-ut"):
91+
return True
92+
93+
return any(CPU_LOG_PATH_PATTERN.search(part) for part in log_path.parent.parts)
94+
95+
5196
def extract_failed_from_logs(log_dir):
5297
"""
53-
Recursively scan .log and .txt log files:
54-
- Locate "short test summary info" marker
55-
- Read subsequent lines until the next "=====" separator
56-
- Match "FAILED tests/...::..." lines
57-
- Deduplicate across all files
98+
Scan CPU logs first and represent all CPU failures as one ``cpu-ut`` item.
99+
Scan all remaining logs with the existing pytest node-ID behavior.
58100
"""
59101
base = Path(log_dir)
60102
if not base.is_dir():
61103
print(f"::warning:: Log directory not found: {log_dir}")
62104
return []
63105

64-
FAILED_PAT = re.compile(r"^FAILED\s+(tests/\S+?\.py::\S+?)\s")
65-
SEP_PAT = re.compile(r"^=+\s")
66-
67-
all_failed = []
68-
seen = set()
69-
70106
# Scan both real .log files (from run_selected_tests.sh) and mock .txt files
71107
candidates = []
72108
candidates.extend(base.rglob("*.log"))
73109
candidates.extend(base.rglob("*.txt"))
74-
for candidate in sorted(candidates):
75-
if candidate.suffix == ".txt" and "run-selected-tests" not in candidate.name:
76-
continue
77-
try:
78-
lines = candidate.read_text(encoding="utf-8", errors="replace").splitlines()
79-
except Exception as exc:
80-
print(f"::warning:: Cannot read {candidate.name}: {exc}")
81-
continue
110+
candidates = [
111+
candidate
112+
for candidate in sorted(candidates)
113+
if candidate.suffix != ".txt" or "run-selected-tests" in candidate.name
114+
]
115+
116+
cpu_candidates = []
117+
regular_candidates = []
118+
for candidate in candidates:
119+
target = cpu_candidates if is_cpu_log(candidate) else regular_candidates
120+
target.append(candidate)
82121

83-
in_summary = False
84-
for line in lines:
85-
text = clean_line(line)
86-
87-
# Enter: found the bookmark
88-
if "short test summary info" in text:
89-
in_summary = True
90-
continue
122+
all_failed = []
123+
seen = set()
91124

92-
if not in_summary:
93-
continue
125+
cpu_failed = False
126+
for candidate in cpu_candidates:
127+
if extract_failed_from_log(candidate):
128+
cpu_failed = True
94129

95-
# Exit: hit the separator line ("======= 2 failed, 100 passed =======")
96-
if SEP_PAT.match(text):
97-
in_summary = False
98-
continue
130+
if cpu_failed:
131+
seen.add(CPU_FAILURE_LABEL)
132+
all_failed.append(CPU_FAILURE_LABEL)
99133

100-
# Collect: FAILED line inside the block
101-
m = FAILED_PAT.match(text)
102-
if m:
103-
tp = m.group(1)
104-
if tp not in seen:
105-
seen.add(tp)
106-
all_failed.append(tp)
134+
for candidate in regular_candidates:
135+
for test_path in extract_failed_from_log(candidate):
136+
if test_path not in seen:
137+
seen.add(test_path)
138+
all_failed.append(test_path)
107139

108140
return all_failed
109141

@@ -137,6 +169,8 @@ def normalize_test_path(test_path):
137169
normalized = test_path.strip().replace("\\", "/").removeprefix("./")
138170
file_path, separator, test_name = normalized.partition("::")
139171
file_path = file_path.removesuffix(".py")
172+
if separator:
173+
test_name = test_name.partition("[")[0]
140174
return f"{file_path}{separator}{test_name}" if separator else file_path
141175

142176

@@ -210,7 +244,7 @@ def generate_report(failed, recommended, matched, log_dir, recommendations_sourc
210244
# ================================================================
211245
out.append("---")
212246
out.append("")
213-
out.append(f"## Failed Test Cases( {len(failed)} )")
247+
out.append(f"## Failed Test Cases( {len(failed)} total)")
214248
out.append("")
215249
if failed:
216250
for i, t in enumerate(failed, 1):

0 commit comments

Comments
 (0)