feat(ci): add buildkit fuse-overlayfs test for a3 and openEuler Dockerfiles

Add workflow buildkit-dockerfile-test and sync Dockerfiles from nv-action/vllm-benchmarks#280

Signed-off-by: JavaPythonAIForBAT <wuhejun@h-partners.com>

Author: JavaPythonAIForBAT

.github/workflows/buildkit-dockerfile-test.yaml

@@ -0,0 +1,70 @@
+name: buildkit-dockerfile-test
+# Test Dockerfile builds via docker/build-push-action@v7 on self-hosted runners
+
+on:
+  pull_request:
+    paths:
+      - '.github/workflows/buildkit-dockerfile-test.yaml'
+      - 'Dockerfile.a3'
+      - 'Dockerfile.a3.openEuler'
+      - 'Dockerfile.openEuler'
+  workflow_dispatch:
+
+jobs:
+  build-test:
+    name: "${{ matrix.dockerfile }} (${{ matrix.runner_info.arch }})"
+    runs-on: ${{ matrix.runner_info.runner }}
+    container:
+      image: swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:9.0.0-a3-ubuntu22.04-py3.12
+
+    strategy:
+      fail-fast: false
+      matrix:
+        dockerfile:
+          - Dockerfile.a3
+          - Dockerfile.a3.openEuler
+          - Dockerfile.openEuler
+        runner_info:
+          - {runner: linux-aarch64-cpu-4-buildkit, arch: arm64}
+          - {runner: linux-amd64-cpu-4-buildkit, arch: amd64}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v7
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+
+      - name: Determine SOC_VERSION
+        id: vars
+        run: |
+          case "${{ matrix.dockerfile }}" in
+            Dockerfile|Dockerfile.openEuler)
+              echo "soc_version=ascend910b1" >> $GITHUB_OUTPUT
+              ;;
+            Dockerfile.a3|Dockerfile.a3.openEuler)
+              echo "soc_version=ascend910_9391" >> $GITHUB_OUTPUT
+              ;;
+          esac
+
+      - name: Build and push
+        uses: docker/build-push-action@v7
+        with:
+          context: .
+          file: ${{ matrix.dockerfile }}
+          push: true
+          tags: |
+            swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/test-buildkit:${{ matrix.dockerfile }}-${{ matrix.runner_info.arch }}-${{ github.sha }}
+            swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/buildkit-cache:${{ matrix.dockerfile }}-${{ matrix.runner_info.arch }}
+          build-args: |
+            APTMIRROR=http://cache-service.nginx-pypi-cache.svc.cluster.local:8081
+            PIP_INDEX_URL=http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple
+            PIP_TRUSTED_HOST=cache-service.nginx-pypi-cache.svc.cluster.local
+            PYTORCH_INDEX_URL=http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu
+            ASCEND_INDEX_URL=http://cache-service.nginx-pypi-cache.svc.cluster.local/ascend/repos/pypi
+            GIT_PROXY=https://gh-proxy.test.osinfra.cn/
+            SOC_VERSION=${{ steps.vars.outputs.soc_version }}
+            COMPILE_CUSTOM_KERNELS=0
+          cache-from: type=registry,ref=swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/buildkit-cache:${{ matrix.dockerfile }}-${{ matrix.runner_info.arch }}
+          cache-to: type=inline
+          provenance: false

Dockerfile.a3

@@ -15,25 +15,31 @@
 # This file is a part of the vllm-ascend project.
 #
 
-ARG CANN_QUAY_URL="quay.io/ascend/cann"
-ARG CANN_VERSION="9.0.1"
-ARG BASE_OS="ubuntu22.04"
-FROM ${CANN_QUAY_URL}:${CANN_VERSION}-a3-${BASE_OS}-py3.12
+FROM swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:9.0.1-a3-ubuntu22.04-py3.12
 
-ARG PIP_INDEX_URL="https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple"
+ARG PIP_INDEX_URL="https://pypi.tuna.tsinghua.edu.cn/simple"
+ARG MOONCAKE_INDEX_URL="https://mirrors.aliyun.com/pypi/web/simple"
+ARG ASCEND_INDEX_URL="https://mirrors.huaweicloud.com/ascend/repos/pypi"
+ARG PYTORCH_INDEX_URL="https://download.pytorch.org/whl/cpu"
+ARG PIP_TRUSTED_HOST=""
+ARG GIT_PROXY=""
+ARG APTMIRROR
 
 ENV DEBIAN_FRONTEND=noninteractive
 
 WORKDIR /workspace
 
 # Install clang-15 (for triton-ascend) and Mooncake
 ARG MOONCAKE_TAG=0.3.11.post1
-RUN apt-get update -y && \
+RUN if [ -n "$APTMIRROR" ]; then \
+        sed -Ei "s@(ports|archive).ubuntu.com@${APTMIRROR#http://}@g" /etc/apt/sources.list; \
+    fi && \
+    apt-get update -y && \
     apt-get install -y git vim wget net-tools gcc g++ cmake numactl libnuma-dev libibverbs-dev libjemalloc2 libhiredis-dev clang-15 && \
     update-alternatives --install /usr/bin/clang clang /usr/bin/clang-15 20 && \
     update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-15 20 && \
     source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
-    python3 -m pip install mooncake-transfer-engine-npu==${MOONCAKE_TAG} --extra-index-url https://mirrors.aliyun.com/pypi/web/simple && \
+    python3 -m pip install mooncake-transfer-engine-npu==${MOONCAKE_TAG} --extra-index-url ${MOONCAKE_INDEX_URL} && \
     rm -rf /var/cache/apt/* && \
     rm -rf /var/lib/apt/lists/*
 
@@ -44,17 +50,18 @@ RUN pip config set global.index-url ${PIP_INDEX_URL} && \
 
 # Install vLLM
 ARG VLLM_REPO=https://github.com/vllm-project/vllm.git
-ARG VLLM_TAG=v0.26.0
+ARG VLLM_TAG=v0.25.1
 ARG VLLM_COMMIT=""
 RUN if [ -n "$VLLM_COMMIT" ]; then \
       git init /vllm-workspace/vllm && \
       git -C /vllm-workspace/vllm fetch --depth 1 $VLLM_REPO "$VLLM_COMMIT" && \
       git -C /vllm-workspace/vllm checkout FETCH_HEAD; \
     else \
+      if [ -n "$GIT_PROXY" ]; then git config --global url."${GIT_PROXY}https://github.com/".insteadOf https://github.com/; fi && \
       git clone --depth 1 -b $VLLM_TAG $VLLM_REPO /vllm-workspace/vllm; \
     fi
 # In x86, triton will be installed by vllm. But in Ascend, triton doesn't work correctly. we need to uninstall it.
-RUN VLLM_TARGET_DEVICE="empty" python3 -m pip install -e /vllm-workspace/vllm/[audio] --extra-index-url https://download.pytorch.org/whl/cpu/ && \
+RUN VLLM_TARGET_DEVICE="empty" python3 -m pip install -e /vllm-workspace/vllm/[audio] && \
     python3 -m pip uninstall -y triton && \
     python3 -m pip cache purge
 
@@ -67,38 +74,17 @@ ENV SOC_VERSION=$SOC_VERSION \
     OMP_NUM_THREADS=1
 COPY . /vllm-workspace/vllm-ascend/
 
-RUN export PIP_EXTRA_INDEX_URL="https://mirrors.huaweicloud.com/ascend/repos/pypi" && \
+RUN export PIP_EXTRA_INDEX_URL="${ASCEND_INDEX_URL}" && \
     export VLLM_BATCH_INVARIANT=1 && \
     source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
     source /usr/local/Ascend/nnal/atb/set_env.sh && \
-    python3 -m pip install -e /vllm-workspace/vllm-ascend/ --extra-index-url https://download.pytorch.org/whl/cpu/ && \
+    python3 -m pip install -e /vllm-workspace/vllm-ascend/ --find-links ${PYTORCH_INDEX_URL}/torch/ --find-links ${PYTORCH_INDEX_URL}/torchvision/ && \
     python3 -m pip uninstall -y triton triton-ascend && \
-    python3 -m pip install triton-ascend==3.2.1 --extra-index-url https://mirrors.huaweicloud.com/ascend/repos/pypi && \
+    python3 -m pip install triton-ascend==3.2.1 --extra-index-url ${ASCEND_INDEX_URL} && \
     python3 -m pip cache purge
 
 # Append `libascend_hal.so` path (devlib) to LD_LIBRARY_PATH
 RUN echo "export LD_PRELOAD=/usr/lib/$(uname -m)-linux-gnu/libjemalloc.so.2:$LD_PRELOAD" >> ~/.bashrc
 RUN echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib" >> ~/.bashrc
 
-# ===== Conditional installation based on BUILD_TYPE =====
-# All ARG definitions are in the same stage for better maintainability
-ARG BUILD_TYPE="release"
-ARG MEMCACHE_VERSION
-ARG MEMCACHE_DATE
-ARG MEMFABRIC_VERSION
-ARG MEMFABRIC_DATE
-ARG TORCH_NPU_VERSION
-ARG TORCH_NPU_DATE
-ARG TRITON_ASCEND_VERSION
-ARG TRITON_ASCEND_PACKAGE_VERSION
-ARG DAILY_DEPS_MODE="full"
-
-# Install daily packages via shared script
-COPY .github/workflows/scripts/install_daily_deps.sh /tmp/
-RUN if [ "$BUILD_TYPE" = "daily" ]; then \
-        bash /tmp/install_daily_deps.sh; \
-    else \
-        echo "Building release version without daily packages"; \
-    fi && rm -f /tmp/install_daily_deps.sh
-
 CMD ["/bin/bash"]

Dockerfile.a3.openEuler

@@ -15,12 +15,14 @@
 # This file is a part of the vllm-ascend project.
 #
 
-ARG CANN_QUAY_URL="quay.io/ascend/cann"
-ARG CANN_VERSION="9.0.1"
-ARG BASE_OS="openeuler24.03"
-FROM ${CANN_QUAY_URL}:${CANN_VERSION}-a3-${BASE_OS}-py3.12
+FROM swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:9.0.1-a3-openeuler24.03-py3.12
 
-ARG PIP_INDEX_URL="https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple"
+ARG PIP_INDEX_URL="https://pypi.tuna.tsinghua.edu.cn/simple"
+ARG MOONCAKE_INDEX_URL="https://mirrors.aliyun.com/pypi/web/simple"
+ARG ASCEND_INDEX_URL="https://mirrors.huaweicloud.com/ascend/repos/pypi"
+ARG PYTORCH_INDEX_URL="https://download.pytorch.org/whl/cpu"
+ARG PIP_TRUSTED_HOST=""
+ARG GIT_PROXY=""
 
 WORKDIR /workspace
 
@@ -31,7 +33,7 @@ ARG MOONCAKE_TAG=0.3.11.post1
 RUN yum update -y && \
     yum install -y git vim wget net-tools gcc gcc-c++ make cmake numactl numactl-devel libibverbs-devel jemalloc hiredis-devel clang patch && \
     source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
-    python3 -m pip install mooncake-transfer-engine-npu==${MOONCAKE_TAG} --extra-index-url https://mirrors.aliyun.com/pypi/web/simple && \
+    python3 -m pip install mooncake-transfer-engine-npu==${MOONCAKE_TAG} --extra-index-url ${MOONCAKE_INDEX_URL} && \
     rm -rf /var/cache/yum/*
 
 # Install modelscope (for fast download) and ray (for multinode)
@@ -41,17 +43,18 @@ RUN pip config set global.index-url ${PIP_INDEX_URL} && \
 
 # Install vLLM
 ARG VLLM_REPO=https://github.com/vllm-project/vllm.git
-ARG VLLM_TAG=v0.26.0
+ARG VLLM_TAG=v0.25.1
 ARG VLLM_COMMIT=""
 RUN if [ -n "$VLLM_COMMIT" ]; then \
       git init /vllm-workspace/vllm && \
       git -C /vllm-workspace/vllm fetch --depth 1 $VLLM_REPO "$VLLM_COMMIT" && \
       git -C /vllm-workspace/vllm checkout FETCH_HEAD; \
     else \
+      if [ -n "$GIT_PROXY" ]; then git config --global url."${GIT_PROXY}https://github.com/".insteadOf https://github.com/; fi && \
       git clone --depth 1 -b $VLLM_TAG $VLLM_REPO /vllm-workspace/vllm; \
     fi
 # In x86, triton will be installed by vllm. But in Ascend, triton doesn't work correctly. we need to uninstall it.
-RUN VLLM_TARGET_DEVICE="empty" python3 -m pip install -e /vllm-workspace/vllm/[audio] --extra-index-url https://download.pytorch.org/whl/cpu/ && \
+RUN VLLM_TARGET_DEVICE="empty" python3 -m pip install -e /vllm-workspace/vllm/[audio] && \
     python3 -m pip uninstall -y triton && \
     python3 -m pip cache purge
 
@@ -63,37 +66,16 @@ ENV SOC_VERSION=$SOC_VERSION \
     OMP_NUM_THREADS=1
 COPY . /vllm-workspace/vllm-ascend/
 
-RUN export PIP_EXTRA_INDEX_URL="https://mirrors.huaweicloud.com/ascend/repos/pypi" && \
+RUN export PIP_EXTRA_INDEX_URL="${ASCEND_INDEX_URL}" && \
     export VLLM_BATCH_INVARIANT=1 && \
     source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
     source /usr/local/Ascend/nnal/atb/set_env.sh && \
-    python3 -m pip install -e /vllm-workspace/vllm-ascend/ --extra-index-url https://download.pytorch.org/whl/cpu/ && \
+    python3 -m pip install -e /vllm-workspace/vllm-ascend/ --find-links ${PYTORCH_INDEX_URL}/torch/ --find-links ${PYTORCH_INDEX_URL}/torchvision/ && \
     python3 -m pip uninstall -y triton triton-ascend && \
-    python3 -m pip install triton-ascend==3.2.1 --extra-index-url https://mirrors.huaweicloud.com/ascend/repos/pypi && \
+    python3 -m pip install triton-ascend==3.2.1 --extra-index-url ${ASCEND_INDEX_URL} && \
     python3 -m pip cache purge
 
 RUN echo "export LD_PRELOAD=/usr/lib64/libjemalloc.so.2:$LD_PRELOAD" >> ~/.bashrc
 RUN echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib" >> ~/.bashrc
 
-# ===== Conditional installation based on BUILD_TYPE =====
-# All ARG definitions are in the same stage for better maintainability
-ARG BUILD_TYPE="release"
-ARG MEMCACHE_VERSION
-ARG MEMCACHE_DATE
-ARG MEMFABRIC_VERSION
-ARG MEMFABRIC_DATE
-ARG TORCH_NPU_VERSION
-ARG TORCH_NPU_DATE
-ARG TRITON_ASCEND_VERSION
-ARG TRITON_ASCEND_PACKAGE_VERSION
-ARG DAILY_DEPS_MODE="full"
-
-# Install daily packages via shared script
-COPY .github/workflows/scripts/install_daily_deps.sh /tmp/
-RUN if [ "$BUILD_TYPE" = "daily" ]; then \
-        bash /tmp/install_daily_deps.sh; \
-    else \
-        echo "Building release version without daily packages"; \
-    fi && rm -f /tmp/install_daily_deps.sh
-
 CMD ["/bin/bash"]

Dockerfile.openEuler

@@ -15,12 +15,14 @@
 # This file is a part of the vllm-ascend project.
 #
 
-ARG CANN_QUAY_URL="quay.io/ascend/cann"
-ARG CANN_VERSION="9.0.1"
-ARG BASE_OS="openeuler24.03"
-FROM ${CANN_QUAY_URL}:${CANN_VERSION}-910b-${BASE_OS}-py3.12
+FROM swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:9.0.1-910b-openeuler24.03-py3.12
 
-ARG PIP_INDEX_URL="https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple"
+ARG PIP_INDEX_URL="https://pypi.tuna.tsinghua.edu.cn/simple"
+ARG MOONCAKE_INDEX_URL="https://mirrors.aliyun.com/pypi/web/simple"
+ARG ASCEND_INDEX_URL="https://mirrors.huaweicloud.com/ascend/repos/pypi"
+ARG PYTORCH_INDEX_URL="https://download.pytorch.org/whl/cpu"
+ARG PIP_TRUSTED_HOST=""
+ARG GIT_PROXY=""
 
 WORKDIR /workspace
 
@@ -31,7 +33,7 @@ ARG MOONCAKE_TAG=0.3.11.post1
 RUN yum update -y && \
     yum install -y git vim wget net-tools gcc gcc-c++ make cmake numactl numactl-devel libibverbs-devel jemalloc hiredis-devel clang patch && \
     source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
-    python3 -m pip install mooncake-transfer-engine-npu==${MOONCAKE_TAG} --extra-index-url https://mirrors.aliyun.com/pypi/web/simple && \
+    python3 -m pip install mooncake-transfer-engine-npu==${MOONCAKE_TAG} --extra-index-url ${MOONCAKE_INDEX_URL} && \
     rm -rf /var/cache/yum/*
 
 # Install modelscope (for fast download) and ray (for multinode)
@@ -41,17 +43,18 @@ RUN pip config set global.index-url ${PIP_INDEX_URL} && \
 
 # Install vLLM
 ARG VLLM_REPO=https://github.com/vllm-project/vllm.git
-ARG VLLM_TAG=v0.26.0
+ARG VLLM_TAG=v0.25.1
 ARG VLLM_COMMIT=""
 RUN if [ -n "$VLLM_COMMIT" ]; then \
       git init /vllm-workspace/vllm && \
       git -C /vllm-workspace/vllm fetch --depth 1 $VLLM_REPO "$VLLM_COMMIT" && \
       git -C /vllm-workspace/vllm checkout FETCH_HEAD; \
     else \
+      if [ -n "$GIT_PROXY" ]; then git config --global url."${GIT_PROXY}https://github.com/".insteadOf https://github.com/; fi && \
       git clone --depth 1 -b $VLLM_TAG $VLLM_REPO /vllm-workspace/vllm; \
     fi
 # In x86, triton will be installed by vllm. But in Ascend, triton doesn't work correctly. we need to uninstall it.
-RUN VLLM_TARGET_DEVICE="empty" python3 -m pip install -e /vllm-workspace/vllm/[audio] --extra-index-url https://download.pytorch.org/whl/cpu/ && \
+RUN VLLM_TARGET_DEVICE="empty" python3 -m pip install -e /vllm-workspace/vllm/[audio] && \
     python3 -m pip uninstall -y triton && \
     python3 -m pip cache purge
 
@@ -63,37 +66,16 @@ ENV SOC_VERSION=$SOC_VERSION \
     OMP_NUM_THREADS=1
 COPY . /vllm-workspace/vllm-ascend/
 
-RUN export PIP_EXTRA_INDEX_URL="https://mirrors.huaweicloud.com/ascend/repos/pypi" && \
+RUN export PIP_EXTRA_INDEX_URL="${ASCEND_INDEX_URL}" && \
     export VLLM_BATCH_INVARIANT=1 && \
     source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
     source /usr/local/Ascend/nnal/atb/set_env.sh && \
-    python3 -m pip install -e /vllm-workspace/vllm-ascend/ --extra-index-url https://download.pytorch.org/whl/cpu/ && \
+    python3 -m pip install -e /vllm-workspace/vllm-ascend/ --find-links ${PYTORCH_INDEX_URL}/torch/ --find-links ${PYTORCH_INDEX_URL}/torchvision/ && \
     python3 -m pip uninstall -y triton triton-ascend && \
-    python3 -m pip install triton-ascend==3.2.1 --extra-index-url https://mirrors.huaweicloud.com/ascend/repos/pypi && \
+    python3 -m pip install triton-ascend==3.2.1 --extra-index-url ${ASCEND_INDEX_URL} && \
     python3 -m pip cache purge
 
 RUN echo "export LD_PRELOAD=/usr/lib64/libjemalloc.so.2:$LD_PRELOAD" >> ~/.bashrc
 RUN echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib" >> ~/.bashrc
 
-# ===== Conditional installation based on BUILD_TYPE =====
-# All ARG definitions are in the same stage for better maintainability
-ARG BUILD_TYPE="release"
-ARG MEMCACHE_VERSION
-ARG MEMCACHE_DATE
-ARG MEMFABRIC_VERSION
-ARG MEMFABRIC_DATE
-ARG TORCH_NPU_VERSION
-ARG TORCH_NPU_DATE
-ARG TRITON_ASCEND_VERSION
-ARG TRITON_ASCEND_PACKAGE_VERSION
-ARG DAILY_DEPS_MODE="full"
-
-# Install daily packages via shared script
-COPY .github/workflows/scripts/install_daily_deps.sh /tmp/
-RUN if [ "$BUILD_TYPE" = "daily" ]; then \
-        bash /tmp/install_daily_deps.sh; \
-    else \
-        echo "Building release version without daily packages"; \
-    fi && rm -f /tmp/install_daily_deps.sh
-
 CMD ["/bin/bash"]