Skip to content

Commit e475908

Browse files
shiqiangAshiqiangA
authored
[CI]accurcy Test (vllm-project#10530)
### What this PR does / why we need it? Accurate testing requires coverage data as a foundation. This PR is used to run all test cases, collect coverage data, and upload it to OBS. ### Does this PR introduce _any_ user-facing change? No user plane changes are involved. This is used to run full test case coverage data at a scheduled time every early morning. ### How was this patch tested? The format of the generated data is correct. The precise test tool can download and parse the data from OBS. - vLLM version: v0.23.0 - vLLM main: vllm-project/vllm@1f486d9 --------- Signed-off-by: shiqiangA <shiqiang15@huawei.com>
1 parent a8cfc19 commit e475908

6 files changed

Lines changed: 330 additions & 5 deletions

File tree

‎.github/workflows/_selected_tests.yaml‎

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,17 @@ on:
4747
required: false
4848
default: false
4949
description: 'Continue running the job even if tests fail'
50+
enable-coverage:
51+
type: boolean
52+
required: false
53+
default: false
54+
description: 'Whether to run tests with coverage enabled.'
55+
secrets:
56+
OBS_ACCESS_KEY_PRECISION:
57+
required: false
58+
OBS_SECRET_ACCESS_KEY_PRECISION:
59+
required: false
60+
5061

5162
# Bash shells do not use ~/.profile or ~/.bashrc so these shells need to be explicitly
5263
# declared as "shell: bash -el {0}" on steps that need to be properly activated.
@@ -244,8 +255,12 @@ jobs:
244255
continue-on-error: ${{ inputs.continue_on_error }}
245256
env:
246257
VLLM_WORKER_MULTIPROC_METHOD: spawn
258+
ENABLE_COVERAGE: ${{ inputs.enable-coverage}}
247259
run: |
248260
. /usr/local/Ascend/ascend-toolkit/set_env.sh
261+
if [ "${{ inputs.enable-coverage }}" = "true" ]; then
262+
export ENABLE_COVERAGE=true
263+
fi
249264
TIMING_FLAG=""
250265
if [ "${{ inputs.upload_timing }}" = "true" ]; then
251266
TIMING_FLAG="--timing"
@@ -263,7 +278,11 @@ jobs:
263278
env:
264279
VLLM_WORKER_MULTIPROC_METHOD: spawn
265280
TORCH_DEVICE_BACKEND_AUTOLOAD: 0
281+
ENABLE_COVERAGE: ${{ inputs.enable-coverage}}
266282
run: |
283+
if [ "${{ inputs.enable-coverage }}" = "true" ]; then
284+
export ENABLE_COVERAGE=true
285+
fi
267286
.github/workflows/scripts/run_selected_tests.sh \
268287
"${{ matrix.group.npu_type }}" \
269288
"${{ matrix.group.num_npus }}" \
@@ -280,6 +299,17 @@ jobs:
280299
if-no-files-found: ignore
281300
retention-days: 7
282301

302+
- name: Upload coverage data
303+
if: ${{ always() && inputs.enable-coverage }}
304+
continue-on-error: true
305+
uses: actions/upload-artifact@v7
306+
with:
307+
name: selected-test-coverage-vllm-${{ inputs.vllm }}-${{ matrix.group.npu_type }}-${{ matrix.group.num_npus }}card-${{ matrix.group.partition }}
308+
path: tests/outputs/**/covdata/**
309+
if-no-files-found: ignore
310+
retention-days: 14
311+
compression-level: 0
312+
283313
- name: Upload selected test logs
284314
if: always()
285315
continue-on-error: true
@@ -290,3 +320,112 @@ jobs:
290320
if-no-files-found: ignore
291321
retention-days: 14
292322
compression-level: 0
323+
324+
upload-coverage-to-obs:
325+
if: ${{ always() && inputs.enable-coverage }}
326+
needs: selected-tests
327+
runs-on: ubuntu-latest
328+
continue-on-error: true
329+
env:
330+
OBS_ACCESS_KEY: ${{ secrets.OBS_ACCESS_KEY_PRECISION }}
331+
OBS_SECRET_KEY: ${{ secrets.OBS_SECRET_ACCESS_KEY_PRECISION }}
332+
steps:
333+
- name: Checkout vllm-ascend source code
334+
uses: actions/checkout@v6
335+
with:
336+
ref: ${{ inputs.ref || github.ref }}
337+
338+
- name: Download all coverage artifacts
339+
uses: actions/download-artifact@v5
340+
with:
341+
path: all-coverage
342+
pattern: selected-test-coverage-*
343+
merge-multiple: true
344+
345+
- name: Debug - List downloaded coverage files
346+
run: |
347+
echo "=== all-coverage top level ==="
348+
find all-coverage -maxdepth 1 2>/dev/null | head -20 || echo "all-coverage directory not exist"
349+
echo ""
350+
echo "=== all-coverage total file count ==="
351+
find all-coverage -type f 2>/dev/null | wc -l
352+
echo "=== sample covdata dirs ==="
353+
find all-coverage -type d -name covdata 2>/dev/null | head -10
354+
echo "=== sample files ==="
355+
find all-coverage -path '*/covdata/*' -type f 2>/dev/null | head -10
356+
357+
- name: Assemble and compress coverage files
358+
run: |
359+
set -euo pipefail
360+
TASK_DATE=$(date +%Y%m%d%H)
361+
TASK_NAME="VLLM-ASCEND@task_${TASK_DATE}"
362+
COVERAGE_PKG_DIR="coverage/vllm-ascend"
363+
TASK_DIR="${COVERAGE_PKG_DIR}/${TASK_NAME}"
364+
mkdir -p "${TASK_DIR}" "${COVERAGE_PKG_DIR}/covstub/vllm_ascend"
365+
366+
COVERAGE_FILE_COUNT=0
367+
while IFS= read -r f; do
368+
rel="${f#all-coverage/}"
369+
case "${rel}" in
370+
selected-test-coverage-*/*) rel="${rel#*/}" ;;
371+
esac
372+
dest="${TASK_DIR}/${rel}"
373+
mkdir -p "$(dirname "${dest}")"
374+
cp "${f}" "${dest}"
375+
COVERAGE_FILE_COUNT=$((COVERAGE_FILE_COUNT + 1))
376+
done < <(find all-coverage -path '*/covdata/*' -type f 2>/dev/null)
377+
378+
echo "Copied ${COVERAGE_FILE_COUNT} coverage files to ${TASK_DIR}"
379+
if [ "${COVERAGE_FILE_COUNT}" -eq 0 ]; then
380+
echo "::error::No coverage files found under all-coverage/*/covdata/"
381+
exit 1
382+
fi
383+
384+
cp -r vllm_ascend/. "${COVERAGE_PKG_DIR}/covstub/vllm_ascend/"
385+
cp tests/coverage_settingInfo.xml "${COVERAGE_PKG_DIR}/settingInfo.xml"
386+
echo "${TASK_NAME}" > coverage_version.txt
387+
388+
echo "=== package top level ==="
389+
find "${COVERAGE_PKG_DIR}" -maxdepth 1 2>/dev/null | head -20 || true
390+
echo "=== task dir file count ==="
391+
find "${TASK_DIR}" -type f | wc -l
392+
393+
tar cf coverage.tar -C coverage vllm-ascend
394+
echo "Packed coverage/vllm-ascend into coverage.tar"
395+
du -h coverage.tar
396+
echo "=== tar contents (sample) ==="
397+
tar tf coverage.tar | head -20 || true
398+
echo "=== tar task file count ==="
399+
tar tf coverage.tar | grep -c "vllm-ascend/${TASK_NAME}/" || true
400+
401+
{
402+
echo "TASK_DATE=${TASK_DATE}"
403+
echo "TASK_NAME=${TASK_NAME}"
404+
} >> "${GITHUB_ENV}"
405+
406+
- name: Upload coverage to OBS
407+
run: |
408+
pip install esdk-obs-python --quiet
409+
python3 - <<'EOF'
410+
import os
411+
from obs import ObsClient
412+
413+
OBS_BUCKET = 'vllm-ascend'
414+
OBS_PREFIX = 'ci/precision-test'
415+
client = ObsClient(
416+
access_key_id=os.environ['OBS_ACCESS_KEY'],
417+
secret_access_key=os.environ['OBS_SECRET_KEY'],
418+
server='https://obs.cn-north-4.myhuaweicloud.com'
419+
)
420+
421+
uploads = [
422+
('coverage.tar', f'{OBS_PREFIX}/coverage.tar'),
423+
('coverage_version.txt', f'{OBS_PREFIX}/coverage_version.txt'),
424+
]
425+
for local_path, obs_path in uploads:
426+
resp = client.putFile(OBS_BUCKET, obs_path, local_path)
427+
if resp.status < 300:
428+
print(f'Uploaded: {local_path} -> {obs_path}')
429+
else:
430+
raise Exception(f'Failed to upload {local_path}: {resp.errorMessage}')
431+
EOF

‎.github/workflows/schedule_test_coverage.yaml‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
#
2+
# Copyright (c) 2026 Huawei Technologies Co., Ltd. All Rights Reserved.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
# This file is a part of the vllm-ascend project.
16+
#
17+
18+
name: Schedule Test Coverage
19+
20+
on:
21+
schedule:
22+
- cron: "0 18 * * *"
23+
workflow_dispatch:
24+
inputs:
25+
vllm_ascend_ref:
26+
description: "vllm-ascend ref (branch, tag, or SHA)"
27+
required: false
28+
default: "main"
29+
type: string
30+
31+
# Bash shells do not use ~/.profile or ~/.bashrc so these shells need to be explicitly
32+
# declared as "shell: bash -el {0}" on steps that need to be properly activated.
33+
# It's used to activate ascend-toolkit environment variables.
34+
defaults:
35+
run:
36+
shell: bash -el {0}
37+
38+
concurrency:
39+
group: ${{ github.workflow }}-${{ github.ref }}
40+
cancel-in-progress: true
41+
42+
jobs:
43+
select-full-tests:
44+
if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }}
45+
runs-on: linux-amd64-cpu-8-hk
46+
container:
47+
image: quay.io/ascend-ci/vllm-ascend:lint
48+
outputs:
49+
has_tests: ${{ steps.full-scope.outputs.has_tests }}
50+
test_groups: ${{ steps.full-scope.outputs.test_groups }}
51+
matched_modules: ${{ steps.full-scope.outputs.matched_modules }}
52+
vllm_ascend_ref: ${{ steps.resolve-refs.outputs.vllm_ascend_ref }}
53+
main_commit: ${{ steps.resolve-refs.outputs.main_commit }}
54+
release_tag: ${{ steps.resolve-refs.outputs.release_tag }}
55+
steps:
56+
- name: Checkout vllm-project/vllm-ascend repo
57+
uses: actions/checkout@v6
58+
with:
59+
ref: ${{ inputs.vllm_ascend_ref || 'main' }}
60+
fetch-depth: 0
61+
62+
- name: Resolve refs
63+
id: resolve-refs
64+
run: |
65+
if [ "${{ github.event_name }}" = "pull_request" ]; then
66+
echo "vllm_ascend_ref=${{ github.event.pull_request.head.sha }}" >> "$GITHUB_OUTPUT"
67+
else
68+
echo "vllm_ascend_ref=${{ inputs.vllm_ascend_ref || 'main' }}" >> "$GITHUB_OUTPUT"
69+
fi
70+
main_commit="$(tr -d '[:space:]' < .github/vllm-main-verified.commit)"
71+
release_tag="$(tr -d '[:space:]' < .github/vllm-release-tag.commit)"
72+
[[ "${main_commit}" =~ ^[0-9a-f]{7,40}$ ]] || {
73+
echo "::error file=.github/vllm-main-verified.commit::invalid vLLM main commit: ${main_commit}"
74+
exit 1
75+
}
76+
[[ "${release_tag}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+([.-].*)?$ ]] || {
77+
echo "::error file=.github/vllm-release-tag.commit::invalid vLLM release tag: ${release_tag}"
78+
exit 1
79+
}
80+
{
81+
echo "main_commit=${main_commit}"
82+
echo "release_tag=${release_tag}"
83+
} >> "$GITHUB_OUTPUT"
84+
85+
- name: Select all tests
86+
id: full-scope
87+
run: |
88+
pip install regex pyyaml
89+
git config --global --add safe.directory /__w/vllm-ascend/vllm-ascend
90+
python3 .github/workflows/scripts/select_tests.py \
91+
--changed-files vllm_ascend/dummy.py \
92+
--run-all-modules
93+
94+
run-full-tests:
95+
needs: select-full-tests
96+
if: ${{ needs.select-full-tests.outputs.has_tests == 'true' }}
97+
strategy:
98+
fail-fast: false
99+
matrix:
100+
vllm_version:
101+
- ${{ needs.select-full-tests.outputs.release_tag }}
102+
uses: ./.github/workflows/_selected_tests.yaml
103+
with:
104+
vllm: ${{ matrix.vllm_version }}
105+
ref: ${{ needs.select-full-tests.outputs.vllm_ascend_ref }}
106+
test_groups: ${{ needs.select-full-tests.outputs.test_groups }}
107+
enable-coverage: true
108+
secrets:
109+
OBS_ACCESS_KEY_PRECISION: ${{ secrets.OBS_ACCESS_KEY_PRECISION }}
110+
OBS_SECRET_ACCESS_KEY_PRECISION: ${{ secrets.OBS_SECRET_ACCESS_KEY_PRECISION }}

‎.github/workflows/scripts/run_selected_tests.sh‎

Lines changed: 50 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,25 @@
11
#!/usr/bin/env bash
22
set -euo pipefail
33

4+
enable_coverage=false
5+
if [ "${ENABLE_COVERAGE:-}" = "true" ]; then
6+
enable_coverage=true
7+
fi
8+
9+
while [ "$#" -gt 0 ]; do
10+
case "$1" in
11+
--enable-coverage)
12+
enable_coverage=true
13+
shift
14+
;;
15+
*)
16+
break
17+
;;
18+
esac
19+
done
20+
421
if [ "$#" -lt 4 ]; then
5-
echo "Usage: $0 <npu_type> <num_npus> <with-device|without-device> [--timing] <test> [test ...]"
22+
echo "Usage: $0 [--enable-coverage] <npu_type> <num_npus> <with-device|without-device> [--timing] <test> [test ...]"
623
exit 1
724
fi
825

@@ -28,10 +45,23 @@ test_results=()
2845
failed_logs=()
2946
timing_entries=()
3047
test_index=0
48+
overall_status=0
3149
pytest_log_dir="${RUNNER_TEMP:-/tmp}/selected-tests-${npu_type}-${num_npus}card"
50+
project_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
3251

3352
mkdir -p "${pytest_log_dir}"
3453

54+
setup_coverage() {
55+
local target="$1"
56+
local test_basename="${target%.py}"
57+
test_basename="${test_basename//\//__}"
58+
test_basename="${test_basename//::/--}"
59+
local covdata_dir="${project_root}/tests/outputs/${test_basename}/covdata"
60+
mkdir -p "${covdata_dir}"
61+
export COVERAGE_FILE="${covdata_dir}/coverage"
62+
echo -e " \033[33mCOVERAGE_FILE:\033[0m ${COVERAGE_FILE}"
63+
}
64+
3565
setup_vllm_cache_root() {
3666
if [ "${CI:-}" != "true" ]; then
3767
return
@@ -47,6 +77,7 @@ print_test_info() {
4777
if [ "${npu_type}" != "cpu" ]; then
4878
echo -e " \033[33mNPU count:\033[0m ${num_npus}"
4979
fi
80+
echo -e " \033[33mCoverage:\033[0m ${enable_coverage}"
5081
echo -e " \033[33mTargets:\033[0m"
5182
for target in "${targets[@]}"; do
5283
echo -e " \033[32m-\033[0m ${target}"
@@ -86,8 +117,14 @@ run_pytest_target() {
86117
if [ "${record_timing}" = true ]; then
87118
start_time=$(date +%s%N)
88119
fi
89-
set +e
90-
pytest -sv --color=yes "${target}" 2>&1 | tee "${log_file}"
120+
if [ "${enable_coverage}" = "true" ]; then
121+
setup_coverage "${target}"
122+
set +e
123+
python -m coverage run --rcfile="${project_root}/tests/coveragerc" -m pytest -sv --color=yes "${target}" 2>&1 | tee "${log_file}"
124+
else
125+
set +e
126+
pytest -sv --color=yes "${target}" 2>&1 | tee "${log_file}"
127+
fi
91128
local status=${PIPESTATUS[0]}
92129
set -e
93130
if [ "${record_timing}" = true ]; then
@@ -121,8 +158,15 @@ run_pytest_batch() {
121158
if [ "${record_timing}" = true ]; then
122159
start_time=$(date +%s%N)
123160
fi
124-
set +e
125-
pytest -sv --color=yes "${batch_targets[@]}" 2>&1 | tee "${log_file}"
161+
if [ "${enable_coverage}" = "true" ]; then
162+
echo "DEBUG: Go to the [Coverage Branch] page."
163+
setup_coverage "cpu-ut"
164+
set +e
165+
python -m coverage run --rcfile="${project_root}/tests/coveragerc" -m pytest -sv --color=yes "${batch_targets[@]}" 2>&1 | tee "${log_file}"
166+
else
167+
set +e
168+
pytest -sv --color=yes "${batch_targets[@]}" 2>&1 | tee "${log_file}"
169+
fi
126170
local status=${PIPESTATUS[0]}
127171
set -e
128172
if [ "${record_timing}" = true ]; then
@@ -188,3 +232,4 @@ fi
188232

189233
print_timing_json
190234
print_summary
235+
exit "${overall_status}"

‎requirements-dev.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,3 +24,4 @@ mindstudio-probe>=8.3.0
2424
xlite==0.1.0rc11.dev210
2525
uc-manager
2626
ninja
27+
coverage

‎tests/coverage_settingInfo.xml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
2+
<setting>
3+
<type>3</type>
4+
<stage>1</stage>
5+
<executor>admin</executor>
6+
</setting>

0 commit comments

Comments
 (0)