Skip to content

Commit db36970

Browse files
JavaPythonAIForBATJavaPythonAIForBAT
committed
[Test] Verify cpu-32-hk Vault/buildkitd fix by building a real CI image
Replaces the previous static diagnostic checks with an end-to-end verification modeled on vllm-project#11000: a reusable _build_test_image.yaml workflow is wired into pr_test.yaml and runs buildctl against the shared buildkitd daemon on linux-amd64-cpu-32-hk, using the Vault-injected TLS certs under DOCKER_CONFIG. This confirms the ascend-ci-deployment fix (serviceAccount: runner-service-account added to the cpu-32 job pod template, see opensourceways/ascend-ci-deployment#928) by actually exercising the full path: job pod starts -> vault-agent-init succeeds -> certs land in /home/user/.docker -> buildctl authenticates to buildkitd -> image builds. Differs from vllm-project#11000 in one respect: the image is built with push=false and no registry login step, since this PR only needs to prove buildctl connectivity works, not publish an image. No registry credentials are required or referenced.
1 parent b748870 commit db36970

4 files changed

Lines changed: 258 additions & 108 deletions

File tree

‎.github/workflows/_build_test_image.yaml‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
#
2+
# Copyright (c) 2025 Huawei Technologies Co., Ltd. All Rights Reserved.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
# This file is a part of the vllm-ascend project.
16+
#
17+
18+
# Reusable workflow that exercises the cpu-32-hk buildkitd pipeline end to
19+
# end: it runs on linux-amd64-cpu-32-hk, relies on the Vault-injected
20+
# buildkitd TLS certs (ca.pem/cert.pem/key.pem) under DOCKER_CONFIG, and
21+
# drives `buildctl` against the shared buildkitd daemon to build the CI test
22+
# image. This is a dedicated verification of the ascend-ci-deployment fix
23+
# that adds `serviceAccount: runner-service-account` to the cpu-32 job pod
24+
# template (see opensourceways/ascend-ci-deployment#928) — the image is
25+
# built but intentionally not pushed anywhere, so no registry credentials
26+
# are required.
27+
28+
name: Build Test Image
29+
30+
on:
31+
workflow_call:
32+
inputs:
33+
vllm_main_commit:
34+
type: string
35+
required: true
36+
description: 'vLLM main commit hash (from .github/vllm-main-verified.commit).'
37+
vllm_release_tag:
38+
type: string
39+
required: true
40+
description: 'vLLM release tag (from .github/vllm-release-tag.commit).'
41+
pr_number:
42+
type: string
43+
required: true
44+
description: 'PR number for image tag construction.'
45+
ascend_ref:
46+
type: string
47+
required: true
48+
description: 'The vllm-ascend commit hash (PR head sha) to build from.'
49+
50+
defaults:
51+
run:
52+
shell: bash -el {0}
53+
54+
jobs:
55+
build-push:
56+
name: build-${{ matrix.npu_type }}-${{ matrix.vllm_label }}
57+
runs-on: ${{ matrix.runner }}
58+
strategy:
59+
fail-fast: false
60+
matrix:
61+
include:
62+
# A3 (ARM64, device) — built on the cpu-32-hk runner so this job
63+
# doubles as an end-to-end check of the Vault/buildkitd fix.
64+
- npu_type: a3
65+
cann_tag: "9.0.0-a3-ubuntu22.04-py3.12"
66+
soc: "ascend910_9391"
67+
arch: arm64
68+
mode: device
69+
runner: linux-amd64-cpu-32-hk
70+
vllm_label: main
71+
vllm_version: ${{ inputs.vllm_main_commit }}
72+
container:
73+
image: swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:9.0.0-a3-ubuntu22.04-py3.11
74+
steps:
75+
- name: Checkout vllm-project/vllm-ascend repo
76+
uses: actions/checkout@v7
77+
with:
78+
ref: ${{ inputs.ascend_ref }}
79+
fetch-depth: 0
80+
submodules: recursive
81+
82+
- name: Verify Vault-injected buildkitd certs
83+
run: |
84+
set -e
85+
DOCKER_CONFIG_DIR="${DOCKER_CONFIG:-/home/user/.docker}"
86+
for f in ca.pem cert.pem key.pem config.json; do
87+
path="${DOCKER_CONFIG_DIR}/${f}"
88+
if [ ! -s "$path" ]; then
89+
echo "::error::vault-agent-init did not inject ${path}"
90+
exit 1
91+
fi
92+
echo "OK: ${path} ($(wc -c < "$path") bytes)"
93+
done
94+
95+
- name: Build image with buildctl (no push)
96+
run: |
97+
set -ex
98+
IMAGE_TAG="vllm-ascend-pr${{ inputs.pr_number }}-${{ matrix.vllm_label }}-${{ matrix.npu_type }}-${{ matrix.arch }}"
99+
100+
buildctl \
101+
--addr="${BUILDKITD_ADDR}" \
102+
--tlscacert="/home/user/.docker/ca.pem" \
103+
--tlscert="/home/user/.docker/cert.pem" \
104+
--tlskey="/home/user/.docker/key.pem" \
105+
build \
106+
--frontend dockerfile.v0 \
107+
--local context=. \
108+
--local dockerfile=.github/workflows/dockerfiles \
109+
--opt filename=Dockerfile.ci_test \
110+
--opt platform=linux/${{ matrix.arch }} \
111+
--opt build-arg:CANN_TAG=${{ matrix.cann_tag }} \
112+
--opt build-arg:VLLM_COMMIT=${{ matrix.vllm_version }} \
113+
--opt build-arg:SOC_VERSION=${{ matrix.soc }} \
114+
--opt build-arg:INSTALL_MODE=${{ matrix.mode }} \
115+
--output type=image,name="${IMAGE_TAG}",push=false \
116+
--progress=plain
117+
118+
echo "Image built (not pushed): ${IMAGE_TAG}"

‎.github/workflows/dockerfiles/Dockerfile.ci_test‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
#
2+
# Copyright (c) 2025 Huawei Technologies Co., Ltd. All Rights Reserved.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
# This file is a part of the vllm-ascend project.
16+
#
17+
18+
# CI-specific test image that pre-installs vllm + vllm-ascend on top of the
19+
# CANN base image. The source code is baked in at /vllm-workspace/vllm-ascend
20+
# so that the test job can skip checkout / install entirely.
21+
#
22+
# Build args:
23+
# CANN_TAG – CANN image tag (e.g. 9.0.0-910b-ubuntu22.04-py3.12)
24+
# VLLM_COMMIT – vLLM commit hash to checkout
25+
# SOC_VERSION – Ascend SOC version (e.g. ascend910b1)
26+
# INSTALL_MODE – "device" (compile kernels, install mooncake + triton-ascend)
27+
# or "nodevice" (COMPILE_CUSTOM_KERNELS=0, no mooncake/triton-ascend)
28+
29+
ARG CANN_TAG=9.0.0-910b-ubuntu22.04-py3.12
30+
FROM swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:${CANN_TAG}
31+
32+
ARG VLLM_COMMIT
33+
ARG SOC_VERSION=ascend910b1
34+
ARG INSTALL_MODE=device
35+
ARG VLLM_REPO=https://github.com/vllm-project/vllm.git
36+
37+
ENV DEBIAN_FRONTEND=noninteractive
38+
ENV SOC_VERSION=${SOC_VERSION} \
39+
UV_INDEX_URL=http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \
40+
UV_EXTRA_INDEX_URL="https://repo.huaweicloud.com/ascend/repos/pypi http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu/" \
41+
UV_INDEX_STRATEGY=unsafe-best-match \
42+
UV_INSECURE_HOST=cache-service.nginx-pypi-cache.svc.cluster.local \
43+
UV_HTTP_TIMEOUT=120 \
44+
UV_NO_CACHE=1 \
45+
UV_SYSTEM_PYTHON=1
46+
47+
# ---------------------------------------------------------------------------
48+
# Install packages (mirrors _selected_tests.yaml "Install packages" step)
49+
# ---------------------------------------------------------------------------
50+
RUN sed -Ei 's@(ports|archive).ubuntu.com@cache-service.nginx-pypi-cache.svc.cluster.local:8081@g' /etc/apt/sources.list && \
51+
pip config set global.index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple && \
52+
pip config set global.trusted-host cache-service.nginx-pypi-cache.svc.cluster.local && \
53+
apt-get update -y && \
54+
apt-get install -y python3-pip git vim wget net-tools gcc g++ cmake libnuma-dev curl gnupg2 zstd && \
55+
pip install uv
56+
57+
# ---------------------------------------------------------------------------
58+
# Checkout vllm and install (empty target, uninstall triton)
59+
# ---------------------------------------------------------------------------
60+
RUN git init /vllm-workspace/vllm && \
61+
git -C /vllm-workspace/vllm fetch --depth 1 "$VLLM_REPO" "$VLLM_COMMIT" && \
62+
git -C /vllm-workspace/vllm checkout FETCH_HEAD && \
63+
cd /vllm-workspace/vllm && \
64+
VLLM_TARGET_DEVICE=empty uv pip install . && \
65+
pip uninstall -y triton
66+
67+
# ---------------------------------------------------------------------------
68+
# Copy vllm-ascend source from build context (already rebased on latest main)
69+
# ---------------------------------------------------------------------------
70+
COPY . /vllm-workspace/vllm-ascend/
71+
WORKDIR /vllm-workspace/vllm-ascend
72+
73+
# ---------------------------------------------------------------------------
74+
# Install Mooncake wheel (device mode only — aarch64 wheel)
75+
# ---------------------------------------------------------------------------
76+
RUN if [ "$INSTALL_MODE" = "device" ]; then \
77+
apt-get install -y --no-install-recommends \
78+
libibverbs1 ibverbs-providers librdmacm1 libnuma1 libcurl4 && \
79+
ldconfig && \
80+
MOONCAKE_WHEEL="mooncake_transfer_engine_ascend-0.3.9-cp312-cp312-manylinux_2_35_aarch64.whl" && \
81+
pip install --no-cache-dir --no-deps \
82+
"https://vllm-ascend.obs.cn-north-4.myhuaweicloud.com/vllm-ascend/${MOONCAKE_WHEEL}" && \
83+
pip show mooncake-transfer-engine-ascend || true; \
84+
fi
85+
86+
# ---------------------------------------------------------------------------
87+
# Install vllm-ascend (with csrc cache logic matching _selected_tests.yaml)
88+
# ---------------------------------------------------------------------------
89+
RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
90+
source /usr/local/Ascend/nnal/atb/set_env.sh && \
91+
arch=$(uname -i) && \
92+
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/${arch}-linux/devlib && \
93+
if [ "$(uname -m)" = "x86_64" ]; then \
94+
export UV_EXTRA_INDEX_URL="${UV_EXTRA_INDEX_URL} https://download.pytorch.org/whl/cpu/"; \
95+
fi && \
96+
export MAX_JOBS=16 && \
97+
pip install uc-manager && \
98+
uv pip install -r requirements-dev.txt && \
99+
if [ "$INSTALL_MODE" = "device" ]; then \
100+
uv pip install --force-reinstall --no-deps triton-ascend==3.2.1; \
101+
fi && \
102+
if [ "$INSTALL_MODE" = "nodevice" ] || find vllm_ascend -maxdepth 1 -name '*.so' -type f 2>/dev/null | grep -q .; then \
103+
echo "CSRC cache hit or nodevice: skip kernel compilation" && \
104+
COMPILE_CUSTOM_KERNELS=0 uv pip install -e .; \
105+
else \
106+
echo "CSRC cache miss: compile kernels" && \
107+
uv pip install -e . --no-build-isolation; \
108+
fi
109+
110+
CMD ["/bin/bash"]

‎.github/workflows/pr_test.yaml‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -200,6 +200,36 @@ jobs:
200200
echo "matched_modules="
201201
} >> "$GITHUB_OUTPUT"
202202
203+
prepare-build:
204+
name: Prepare build inputs
205+
runs-on: linux-amd64-cpu-8-hk
206+
outputs:
207+
vllm_main_commit: ${{ steps.vllm.outputs.main_commit }}
208+
vllm_release_tag: ${{ steps.vllm.outputs.release_tag }}
209+
steps:
210+
- uses: actions/checkout@v7
211+
with:
212+
fetch-depth: 1
213+
214+
- name: Read verified vLLM refs
215+
id: vllm
216+
run: |
217+
main_commit="$(tr -d '[:space:]' < .github/vllm-main-verified.commit)"
218+
release_tag="$(tr -d '[:space:]' < .github/vllm-release-tag.commit)"
219+
{
220+
echo "main_commit=${main_commit}"
221+
echo "release_tag=${release_tag}"
222+
} >> "$GITHUB_OUTPUT"
223+
224+
build-test-image:
225+
needs: prepare-build
226+
uses: ./.github/workflows/_build_test_image.yaml
227+
with:
228+
vllm_main_commit: ${{ needs.prepare-build.outputs.vllm_main_commit }}
229+
vllm_release_tag: ${{ needs.prepare-build.outputs.vllm_release_tag }}
230+
pr_number: ${{ github.event.pull_request.number }}
231+
ascend_ref: ${{ github.event.pull_request.head.sha }}
232+
203233
run-selected-tests:
204234
needs: lint-and-select-tests
205235
if: ${{ needs.lint-and-select-tests.outputs.has_tests == 'true' && contains(github.event.pull_request.labels.*.name, 'ready') }}

‎.github/workflows/test-cpu-32-hk-vault-fix.yaml‎

Lines changed: 0 additions & 108 deletions
This file was deleted.

0 commit comments

Comments
 (0)