Skip to content

Commit a705f21

Browse files
JavaPythonAIForBATJavaPythonAIForBAT
committed
feat(ci): add buildkit fuse-overlayfs test for 910b and 310p Dockerfiles
Add workflow buildkit-dockerfile-test-group1 and sync Dockerfiles from nv-action/vllm-benchmarks#279 Signed-off-by: JavaPythonAIForBAT <wuhejun@h-partners.com>
1 parent dd13a7f commit a705f21

4 files changed

Lines changed: 117 additions & 81 deletions

File tree

‎.github/workflows/buildkit-dockerfile-test-group1.yaml‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
name: buildkit-dockerfile-test-group1
2+
3+
on:
4+
pull_request:
5+
paths:
6+
- '.github/workflows/buildkit-dockerfile-test-group1.yaml'
7+
- 'Dockerfile'
8+
- 'Dockerfile.310p'
9+
- 'Dockerfile.310p.openEuler'
10+
workflow_dispatch:
11+
12+
jobs:
13+
build-test:
14+
name: "${{ matrix.dockerfile }} (${{ matrix.runner_info.arch }})"
15+
runs-on: ${{ matrix.runner_info.runner }}
16+
container:
17+
image: swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:9.0.0-a3-ubuntu22.04-py3.12
18+
19+
strategy:
20+
fail-fast: false
21+
matrix:
22+
dockerfile:
23+
- Dockerfile
24+
- Dockerfile.310p
25+
- Dockerfile.310p.openEuler
26+
runner_info:
27+
- {runner: linux-aarch64-cpu-4-buildkit, arch: arm64}
28+
- {runner: linux-amd64-cpu-4-buildkit, arch: amd64}
29+
30+
steps:
31+
- name: Checkout
32+
uses: actions/checkout@v7
33+
with:
34+
fetch-depth: 0
35+
persist-credentials: false
36+
37+
- name: Determine SOC_VERSION
38+
id: vars
39+
run: |
40+
case "${{ matrix.dockerfile }}" in
41+
Dockerfile|Dockerfile.openEuler)
42+
echo "soc_version=ascend910b1" >> $GITHUB_OUTPUT
43+
;;
44+
Dockerfile.310p|Dockerfile.310p.openEuler)
45+
echo "soc_version=ascend310p1" >> $GITHUB_OUTPUT
46+
;;
47+
esac
48+
49+
- name: Build and push
50+
uses: docker/build-push-action@v7
51+
with:
52+
context: .
53+
file: ${{ matrix.dockerfile }}
54+
push: true
55+
tags: |
56+
swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/test-buildkit:${{ matrix.dockerfile }}-${{ matrix.runner_info.arch }}-${{ github.sha }}
57+
swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/buildkit-cache:${{ matrix.dockerfile }}-${{ matrix.runner_info.arch }}
58+
build-args: |
59+
APTMIRROR=http://cache-service.nginx-pypi-cache.svc.cluster.local:8081
60+
PIP_INDEX_URL=http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple
61+
PIP_TRUSTED_HOST=cache-service.nginx-pypi-cache.svc.cluster.local
62+
PYTORCH_INDEX_URL=http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu
63+
ASCEND_INDEX_URL=http://cache-service.nginx-pypi-cache.svc.cluster.local/ascend/repos/pypi
64+
GIT_PROXY=https://gh-proxy.test.osinfra.cn/
65+
VLLM_ASCEND_IMAGE=swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/vllm-ascend/vllm-ascend-ci
66+
SOC_VERSION=${{ steps.vars.outputs.soc_version }}
67+
COMPILE_CUSTOM_KERNELS=0
68+
cache-from: type=registry,ref=swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/buildkit-cache:${{ matrix.dockerfile }}-${{ matrix.runner_info.arch }}
69+
cache-to: type=inline
70+
provenance: false

‎Dockerfile‎

Lines changed: 20 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -14,23 +14,30 @@
1414
# limitations under the License.
1515
# This file is a part of the vllm-ascend project.
1616
#
17-
ARG CANN_QUAY_URL="quay.io/ascend/cann"
18-
ARG CANN_VERSION="9.0.1"
19-
ARG BASE_OS="ubuntu22.04"
20-
FROM ${CANN_QUAY_URL}:${CANN_VERSION}-910b-${BASE_OS}-py3.12
2117

22-
ARG PIP_INDEX_URL="https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple"
18+
FROM swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:9.0.1-910b-ubuntu22.04-py3.12
19+
20+
ARG PIP_INDEX_URL="https://pypi.tuna.tsinghua.edu.cn/simple"
21+
ARG MOONCAKE_INDEX_URL="https://mirrors.aliyun.com/pypi/web/simple"
22+
ARG ASCEND_INDEX_URL="https://mirrors.huaweicloud.com/ascend/repos/pypi"
23+
ARG PYTORCH_INDEX_URL="https://download.pytorch.org/whl/cpu"
24+
ARG PIP_TRUSTED_HOST=""
25+
ARG GIT_PROXY=""
26+
ARG APTMIRROR
2327

2428
WORKDIR /workspace
2529

2630
# Install clang-15 (for triton-ascend) and Mooncake
2731
ARG MOONCAKE_TAG=0.3.11.post1
28-
RUN apt-get update -y && \
32+
RUN if [ -n "$APTMIRROR" ]; then \
33+
sed -Ei "s@(ports|archive).ubuntu.com@${APTMIRROR#http://}@g" /etc/apt/sources.list; \
34+
fi && \
35+
apt-get update -y && \
2936
apt-get install -y git vim wget net-tools gcc g++ cmake numactl libnuma-dev libibverbs-dev libjemalloc2 libhiredis-dev clang-15 && \
3037
update-alternatives --install /usr/bin/clang clang /usr/bin/clang-15 20 && \
3138
update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-15 20 && \
3239
source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
33-
python3 -m pip install mooncake-transfer-engine-npu==${MOONCAKE_TAG} --extra-index-url https://mirrors.aliyun.com/pypi/web/simple && \
40+
python3 -m pip install mooncake-transfer-engine-npu==${MOONCAKE_TAG} --extra-index-url ${MOONCAKE_INDEX_URL} && \
3441
rm -rf /var/cache/apt/* && \
3542
rm -rf /var/lib/apt/lists/*
3643

@@ -41,17 +48,18 @@ RUN pip config set global.index-url ${PIP_INDEX_URL} && \
4148

4249
# Install vLLM
4350
ARG VLLM_REPO=https://github.com/vllm-project/vllm.git
44-
ARG VLLM_TAG=v0.26.0
51+
ARG VLLM_TAG=v0.25.1
4552
ARG VLLM_COMMIT=""
4653
RUN if [ -n "$VLLM_COMMIT" ]; then \
4754
git init /vllm-workspace/vllm && \
4855
git -C /vllm-workspace/vllm fetch --depth 1 $VLLM_REPO "$VLLM_COMMIT" && \
4956
git -C /vllm-workspace/vllm checkout FETCH_HEAD; \
5057
else \
58+
if [ -n "$GIT_PROXY" ]; then git config --global url."${GIT_PROXY}https://github.com/".insteadOf https://github.com/; fi && \
5159
git clone --depth 1 -b $VLLM_TAG $VLLM_REPO /vllm-workspace/vllm; \
5260
fi
5361
# In x86, triton will be installed by vllm. But in Ascend, triton doesn't work correctly. we need to uninstall it.
54-
RUN VLLM_TARGET_DEVICE="empty" python3 -m pip install -e /vllm-workspace/vllm/[audio] --extra-index-url https://download.pytorch.org/whl/cpu/ && \
62+
RUN VLLM_TARGET_DEVICE="empty" python3 -m pip install -e /vllm-workspace/vllm/[audio] && \
5563
python3 -m pip uninstall -y triton && \
5664
python3 -m pip cache purge
5765

@@ -64,38 +72,17 @@ ENV SOC_VERSION=$SOC_VERSION \
6472
OMP_NUM_THREADS=1
6573
COPY . /vllm-workspace/vllm-ascend/
6674

67-
RUN export PIP_EXTRA_INDEX_URL="https://mirrors.huaweicloud.com/ascend/repos/pypi" && \
75+
RUN export PIP_EXTRA_INDEX_URL="${ASCEND_INDEX_URL}" && \
6876
export VLLM_BATCH_INVARIANT=1 && \
6977
source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
7078
source /usr/local/Ascend/nnal/atb/set_env.sh && \
71-
python3 -m pip install -e /vllm-workspace/vllm-ascend/ --extra-index-url https://download.pytorch.org/whl/cpu/ && \
79+
python3 -m pip install -e /vllm-workspace/vllm-ascend/ --find-links ${PYTORCH_INDEX_URL}/torch/ --find-links ${PYTORCH_INDEX_URL}/torchvision/ && \
7280
python3 -m pip uninstall -y triton triton-ascend && \
73-
python3 -m pip install triton-ascend==3.2.1 --extra-index-url https://mirrors.huaweicloud.com/ascend/repos/pypi && \
81+
python3 -m pip install triton-ascend==3.2.1 --extra-index-url ${ASCEND_INDEX_URL} && \
7482
python3 -m pip cache purge
7583

7684
# Append `libascend_hal.so` path (devlib) to LD_LIBRARY_PATH
7785
RUN echo "export LD_PRELOAD=/usr/lib/$(uname -m)-linux-gnu/libjemalloc.so.2:$LD_PRELOAD" >> ~/.bashrc
7886
RUN echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib" >> ~/.bashrc
7987

80-
# ===== Conditional installation based on BUILD_TYPE =====
81-
# All ARG definitions are in the same stage for better maintainability
82-
ARG BUILD_TYPE="release"
83-
ARG MEMCACHE_VERSION
84-
ARG MEMCACHE_DATE
85-
ARG MEMFABRIC_VERSION
86-
ARG MEMFABRIC_DATE
87-
ARG TORCH_NPU_VERSION
88-
ARG TORCH_NPU_DATE
89-
ARG TRITON_ASCEND_VERSION
90-
ARG TRITON_ASCEND_PACKAGE_VERSION
91-
ARG DAILY_DEPS_MODE="full"
92-
93-
# Install daily packages via shared script
94-
COPY .github/workflows/scripts/install_daily_deps.sh /tmp/
95-
RUN if [ "$BUILD_TYPE" = "daily" ]; then \
96-
bash /tmp/install_daily_deps.sh; \
97-
else \
98-
echo "Building release version without daily packages"; \
99-
fi && rm -f /tmp/install_daily_deps.sh
100-
10188
CMD ["/bin/bash"]

‎Dockerfile.310p‎

Lines changed: 16 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -15,16 +15,21 @@
1515
# This file is a part of the vllm-ascend project.
1616
#
1717

18-
ARG CANN_QUAY_URL="quay.io/ascend/cann"
19-
ARG CANN_VERSION="9.1.0-beta.1"
20-
ARG BASE_OS="ubuntu22.04"
21-
FROM ${CANN_QUAY_URL}:${CANN_VERSION}-310p-${BASE_OS}-py3.12
18+
FROM swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:9.1.0-beta.1-310p-ubuntu22.04-py3.12
2219

23-
ARG PIP_INDEX_URL="https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple"
20+
ARG PIP_INDEX_URL="https://pypi.tuna.tsinghua.edu.cn/simple"
21+
ARG ASCEND_INDEX_URL="https://mirrors.huaweicloud.com/ascend/repos/pypi"
22+
ARG PYTORCH_INDEX_URL="https://download.pytorch.org/whl/cpu"
23+
ARG PIP_TRUSTED_HOST=""
24+
ARG GIT_PROXY=""
25+
ARG APTMIRROR
2426

2527
WORKDIR /workspace
2628

27-
RUN apt-get update -y && \
29+
RUN if [ -n "$APTMIRROR" ]; then \
30+
sed -Ei "s@(ports|archive).ubuntu.com@${APTMIRROR#http://}@g" /etc/apt/sources.list; \
31+
fi && \
32+
apt-get update -y && \
2833
apt-get install -y python3-pip git vim wget net-tools gcc g++ cmake numactl libnuma-dev libjemalloc2 pciutils && \
2934
rm -rf /var/cache/apt/* && \
3035
rm -rf /var/lib/apt/lists/*
@@ -36,17 +41,18 @@ RUN pip config set global.index-url ${PIP_INDEX_URL} && \
3641

3742
# Install vLLM
3843
ARG VLLM_REPO=https://github.com/vllm-project/vllm.git
39-
ARG VLLM_TAG=v0.26.0
44+
ARG VLLM_TAG=v0.25.1
4045
ARG VLLM_COMMIT=""
4146
RUN if [ -n "$VLLM_COMMIT" ]; then \
4247
git init /vllm-workspace/vllm && \
4348
git -C /vllm-workspace/vllm fetch --depth 1 $VLLM_REPO "$VLLM_COMMIT" && \
4449
git -C /vllm-workspace/vllm checkout FETCH_HEAD; \
4550
else \
51+
if [ -n "$GIT_PROXY" ]; then git config --global url."${GIT_PROXY}https://github.com/".insteadOf https://github.com/; fi && \
4652
git clone --depth 1 -b $VLLM_TAG $VLLM_REPO /vllm-workspace/vllm; \
4753
fi
4854
# In x86, triton will be installed by vllm. But in Ascend, triton doesn't work correctly. we need to uninstall it.
49-
RUN VLLM_TARGET_DEVICE="empty" python3 -m pip install -e /vllm-workspace/vllm/[audio] --extra-index-url https://download.pytorch.org/whl/cpu/ && \
55+
RUN VLLM_TARGET_DEVICE="empty" python3 -m pip install -e /vllm-workspace/vllm/[audio] && \
5056
python3 -m pip uninstall -y triton && \
5157
python3 -m pip cache purge
5258

@@ -59,10 +65,10 @@ ENV SOC_VERSION=$SOC_VERSION \
5965
OMP_NUM_THREADS=1
6066
COPY . /vllm-workspace/vllm-ascend/
6167

62-
RUN export PIP_EXTRA_INDEX_URL="https://mirrors.huaweicloud.com/ascend/repos/pypi" && \
68+
RUN export PIP_EXTRA_INDEX_URL="${ASCEND_INDEX_URL}" && \
6369
source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
6470
source /usr/local/Ascend/nnal/atb/set_env.sh && \
65-
python3 -m pip install -e /vllm-workspace/vllm-ascend/ --extra-index-url https://download.pytorch.org/whl/cpu/ && \
71+
python3 -m pip install -e /vllm-workspace/vllm-ascend/ --find-links ${PYTORCH_INDEX_URL}/torch/ --find-links ${PYTORCH_INDEX_URL}/torchvision/ && \
6672
python3 -m pip uninstall -y triton-ascend triton && \
6773
python3 -m pip cache purge
6874

@@ -108,21 +114,6 @@ RUN echo '#!/bin/bash' > /usr/local/bin/entrypoint.sh && \
108114
echo 'exec "$@"' >> /usr/local/bin/entrypoint.sh && \
109115
chmod +x /usr/local/bin/entrypoint.sh
110116

111-
# ===== Conditional installation based on BUILD_TYPE =====
112-
# All ARG definitions are in the same stage for better maintainability
113-
ARG BUILD_TYPE="release"
114-
ARG TORCH_NPU_VERSION
115-
ARG TORCH_NPU_DATE
116-
ARG DAILY_DEPS_MODE="torch_npu_only"
117-
118-
# Install daily packages via shared script
119-
COPY .github/workflows/scripts/install_daily_deps.sh /tmp/
120-
RUN if [ "$BUILD_TYPE" = "daily" ]; then \
121-
bash /tmp/install_daily_deps.sh; \
122-
else \
123-
echo "Building release version without daily packages"; \
124-
fi && rm -f /tmp/install_daily_deps.sh
125-
126117
# Set entrypoint and default command
127118
ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
128119

‎Dockerfile.310p.openEuler‎

Lines changed: 11 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,13 @@
1515
# This file is a part of the vllm-ascend project.
1616
#
1717

18-
ARG CANN_QUAY_URL="quay.io/ascend/cann"
19-
ARG CANN_VERSION="9.1.0-beta.1"
20-
ARG BASE_OS="openeuler24.03"
21-
FROM ${CANN_QUAY_URL}:${CANN_VERSION}-310p-${BASE_OS}-py3.12
18+
FROM swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:9.1.0-beta.1-310p-openeuler24.03-py3.12
2219

23-
ARG PIP_INDEX_URL="https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple"
20+
ARG PIP_INDEX_URL="https://pypi.tuna.tsinghua.edu.cn/simple"
21+
ARG ASCEND_INDEX_URL="https://mirrors.huaweicloud.com/ascend/repos/pypi"
22+
ARG PYTORCH_INDEX_URL="https://download.pytorch.org/whl/cpu"
23+
ARG PIP_TRUSTED_HOST=""
24+
ARG GIT_PROXY=""
2425

2526
WORKDIR /workspace
2627

@@ -35,17 +36,18 @@ RUN pip config set global.index-url ${PIP_INDEX_URL} && \
3536

3637
# Install vLLM
3738
ARG VLLM_REPO=https://github.com/vllm-project/vllm.git
38-
ARG VLLM_TAG=v0.26.0
39+
ARG VLLM_TAG=v0.25.1
3940
ARG VLLM_COMMIT=""
4041
RUN if [ -n "$VLLM_COMMIT" ]; then \
4142
git init /vllm-workspace/vllm && \
4243
git -C /vllm-workspace/vllm fetch --depth 1 $VLLM_REPO "$VLLM_COMMIT" && \
4344
git -C /vllm-workspace/vllm checkout FETCH_HEAD; \
4445
else \
46+
if [ -n "$GIT_PROXY" ]; then git config --global url."${GIT_PROXY}https://github.com/".insteadOf https://github.com/; fi && \
4547
git clone --depth 1 -b $VLLM_TAG $VLLM_REPO /vllm-workspace/vllm; \
4648
fi
4749
# In x86, triton will be installed by vllm. But in Ascend, triton doesn't work correctly. we need to uninstall it.
48-
RUN VLLM_TARGET_DEVICE="empty" python3 -m pip install -e /vllm-workspace/vllm/[audio] --extra-index-url https://download.pytorch.org/whl/cpu/ && \
50+
RUN VLLM_TARGET_DEVICE="empty" python3 -m pip install -e /vllm-workspace/vllm/[audio] && \
4951
python3 -m pip uninstall -y triton && \
5052
python3 -m pip cache purge
5153

@@ -57,10 +59,10 @@ ENV SOC_VERSION=$SOC_VERSION \
5759
OMP_NUM_THREADS=1
5860
COPY . /vllm-workspace/vllm-ascend/
5961

60-
RUN export PIP_EXTRA_INDEX_URL="https://mirrors.huaweicloud.com/ascend/repos/pypi" && \
62+
RUN export PIP_EXTRA_INDEX_URL="${ASCEND_INDEX_URL}" && \
6163
source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
6264
source /usr/local/Ascend/nnal/atb/set_env.sh && \
63-
python3 -m pip install -e /vllm-workspace/vllm-ascend/ --extra-index-url https://download.pytorch.org/whl/cpu/ && \
65+
python3 -m pip install -e /vllm-workspace/vllm-ascend/ --find-links ${PYTORCH_INDEX_URL}/torch/ --find-links ${PYTORCH_INDEX_URL}/torchvision/ && \
6466
python3 -m pip uninstall -y triton-ascend triton && \
6567
python3 -m pip cache purge
6668

@@ -104,20 +106,6 @@ RUN echo '#!/bin/bash' > /usr/local/bin/entrypoint.sh && \
104106
echo 'exec "$@"' >> /usr/local/bin/entrypoint.sh && \
105107
chmod +x /usr/local/bin/entrypoint.sh
106108

107-
# ===== Conditional installation based on BUILD_TYPE =====
108-
# All ARG definitions are in the same stage for better maintainability
109-
ARG BUILD_TYPE="release"
110-
ARG TORCH_NPU_VERSION
111-
ARG TORCH_NPU_DATE
112-
ARG DAILY_DEPS_MODE="torch_npu_only"
113-
114-
# Install daily packages via shared script
115-
COPY .github/workflows/scripts/install_daily_deps.sh /tmp/
116-
RUN if [ "$BUILD_TYPE" = "daily" ]; then \
117-
bash /tmp/install_daily_deps.sh; \
118-
else \
119-
echo "Building release version without daily packages"; \
120-
fi && rm -f /tmp/install_daily_deps.sh
121109
# Set entrypoint and default command
122110
ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
123111

0 commit comments

Comments
 (0)