Merge branch 'main' into feat/test-cpu-32-hk-workflow

Author: JavaPythonAIForBAT

.github/vllm-main-verified.commit

@@ -1 +1 @@
-967c5c3bc38891f4465d3f4e99917ed837bb3833
+a30addc7548a9a8b9b3323a7bc3eb7d7c4895d1c

.github/workflows/_e2e_nightly_multi_node.yaml

@@ -73,6 +73,11 @@ on:
         default: 120
         type: string
         description: test case execution timeout period,default:120 minutes
+      aop_multi_enabled:
+        required: false
+        type: boolean
+        default: false
+        description: enable Pod AOP pipeline on failure
     secrets:
       KUBECONFIG_B64:
         required: true
@@ -225,6 +230,8 @@ jobs:
             -D pvc_name="$pvc_name" \
             -D benchmark_job_name="$benchmark_job_name" \
             -D runner="$runner" \
+            -D aop_multi_enabled="${{ inputs.aop_multi_enabled }}" \
+            -D good_table="/root/.cache/vllm-ascend/${{ inputs.vllm_ascend_branch }}/nightly/good_table.csv" \
             --outfile lws.yaml
 
           kubectl apply -f ./lws.yaml
@@ -301,13 +308,46 @@ jobs:
             pids+=($!)
           done
 
-          kubectl logs -f "$LEADER_POD" -n "$NAMESPACE" | while IFS= read -r line; do
+          LEADER_TMP=$(mktemp)
+          kubectl logs -f "$LEADER_POD" -n "$NAMESPACE" | tee "$LEADER_TMP" | while IFS= read -r line; do
             echo "$line"
             if echo "$line" | grep -q "$FAIL_TAG"; then
               exit 1
             fi
           done
 
+          eval "$(awk '
+            /^vLLM Git information$/        { sec="vllm"; next }
+            /^vLLM-Ascend Git information$/ { sec="vllm_ascend"; next }
+            sec && /^Commit hash: / {
+              print "HASH_" sec "=" $NF
+              sec=""
+            }
+          ' "$LEADER_TMP")"
+          echo "vllm_hash=${HASH_vllm:-N/A}" >> "$GITHUB_OUTPUT"
+          echo "vllm_ascend_hash=${HASH_vllm_ascend:-N/A}" >> "$GITHUB_OUTPUT"
+          rm -f "$LEADER_TMP"
+
+      - name: Update good_table.csv on success
+        # if: ${{ steps.stream_logs.outcome == 'success' && inputs.request_id == '' }}
+        if: ${{ steps.stream_logs.outcome == 'success'}}
+        env:
+          TEST_NAME: ${{ inputs.name || inputs.config_file_path }}
+          TEST_PATH: ${{ inputs.config_file_path }}
+          CONFIG_BASE_PATH: ${{ inputs.config_base_path }}
+          RUN_LINK: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        run: |
+          python3 tests/e2e/nightly/scripts/update_good_table.py \
+            --cache-csv "/root/.cache/vllm-ascend/${{ inputs.vllm_ascend_branch || 'main' }}/nightly/good_table.csv" \
+            --test-name "$TEST_NAME" \
+            --test-path "$TEST_PATH" \
+            --config-base-path "$CONFIG_BASE_PATH" \
+            --scene "multi_node" \
+            --run-link  "$RUN_LINK" \
+            --vllm-ascend-dir "." \
+            --vllm-ascend-version "${{ steps.stream_logs.outputs.vllm_ascend_hash }}" \
+            --vllm-version "${{ steps.stream_logs.outputs.vllm_hash }}"
+
       - name: Fetch benchmark results from PVC
         if: always()
         run: |

.github/workflows/_e2e_nightly_single_node.yaml

@@ -64,6 +64,11 @@ on:
         default: 120
         type: string
         description: test case execution timeout period,default:120 minutes
+      aop_single_enabled:
+        required: false
+        type: boolean
+        default: false
+        description: enable AOP hooks (capture / classify / skip-or-process)
     secrets:
       OBS_ACCESS_KEY_ID:
         required: false
@@ -220,6 +225,7 @@ jobs:
           fi
 
       - name: Run Pytest (py-driven)
+        id: pytest-driven
         if: ${{ inputs.tests != '' }}
         env:
           VLLM_WORKER_MULTIPROC_METHOD: spawn
@@ -229,10 +235,14 @@ jobs:
         run: |
           export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH
           echo "Running pytest with tests path: ${{ inputs.tests }}"
+          mkdir -p /tmp/test-logs
+          set -o pipefail
           pytest -sv "${{ inputs.tests }}" \
-          --ignore=tests/e2e/nightly/single_node/ops/singlecard_ops/test_fused_moe.py
+            --ignore=tests/e2e/nightly/single_node/ops/singlecard_ops/test_fused_moe.py \
+            2>&1 | tee /tmp/test-logs/pytest-driven.log
 
       - name: Run Pytest (YAML-driven)
+        id: yaml-test
         if: ${{ always() && inputs.config_file_path != '' }}
         env:
           VLLM_WORKER_MULTIPROC_METHOD: spawn
@@ -246,7 +256,125 @@ jobs:
           export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH
           echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib" >> ~/.bashrc
           echo "Running YAML-driven test with config: ${{ inputs.config_file_path }}"
-          pytest -sv tests/e2e/nightly/single_node/models/scripts/test_single_node.py
+          mkdir -p /tmp/test-logs
+          set -o pipefail
+          pytest -sv tests/e2e/nightly/single_node/models/scripts/test_single_node.py \
+            2>&1 | tee /tmp/test-logs/yaml-test.log
+
+      - name: Update good_table.csv on success
+        if: >-
+          ${{ always() &&
+          (steps.pytest-driven.outcome == 'success' || steps.pytest-driven.outcome == 'skipped') &&
+          (steps.yaml-test.outcome == 'success' || steps.yaml-test.outcome == 'skipped') &&
+          (steps.pytest-driven.outcome == 'success' || steps.yaml-test.outcome == 'success') }}
+        working-directory: /vllm-workspace/vllm-ascend
+        env:
+          TEST_NAME: ${{ inputs.name || inputs.config_file_path || inputs.tests }}
+          TEST_PATH: ${{ inputs.config_file_path || inputs.tests }}
+          CONFIG_BASE_PATH: ${{ inputs.config_base_path }}
+          RUN_LINK: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          VLLM_ASCEND_VERSION: ${{ env.VLLM_ASCEND_VERSION }}
+        run: |
+          python3 tests/e2e/nightly/scripts/update_good_table.py \
+            --cache-csv "/root/.cache/vllm-ascend/${{ inputs.vllm_ascend_branch }}/nightly/good_table.csv" \
+            --test-name "$TEST_NAME" \
+            --test-path "$TEST_PATH" \
+            --config-base-path "$CONFIG_BASE_PATH" \
+            --run-link  "$RUN_LINK" \
+            --vllm-dir  "/vllm-workspace/vllm" \
+            --vllm-ascend-dir "/vllm-workspace/vllm-ascend" \
+            --vllm-ascend-version "$VLLM_ASCEND_VERSION"
+
+      # ============================================
+      # AOP hooks: intercept test results uniformly
+      # ============================================
+
+      - name: Capture test result
+        id: test-result
+        if: ${{ always() && inputs.aop_single_enabled && (inputs.config_file_path != '' || inputs.tests != '') }}
+        working-directory: /vllm-workspace/vllm-ascend
+        run: |
+          bash tests/e2e/nightly/scripts/aop_capture.sh \
+            "${{ steps.yaml-test.outcome }}" \
+            "${{ steps.pytest-driven.outcome }}"
+
+      - name: Classify failure
+        id: classify
+        if: ${{ always() && inputs.aop_single_enabled && steps.test-result.outputs.result == 'failure' }}
+        working-directory: /vllm-workspace/vllm-ascend
+        run: |
+          bash tests/e2e/nightly/scripts/aop_classify.sh \
+            "${{ steps.test-result.outputs.failed_test }}"
+
+      # ============================================================
+      # Decision gate:
+      #   env_failure     → Skip directly (no need to check age)
+      #   not_env_failure → Check commit age → old? Skip : Process
+      # ============================================================
+
+      - name: Skip - env issue
+        if: >-
+          ${{ always() && inputs.aop_single_enabled &&
+          steps.classify.outputs.failure_type == 'env_failure' }}
+        working-directory: /vllm-workspace/vllm-ascend
+        run: |
+          bash tests/e2e/nightly/scripts/aop_skip.sh \
+            "env_failure" "" "" "" \
+            "${{ steps.test-result.outputs.pytest_summary }}" \
+            "${{ steps.test-result.outputs.yaml_summary }}"
+
+      - name: Check commit age
+        id: commit-age
+        if: ${{ always() && inputs.aop_single_enabled && steps.classify.outputs.failure_type == 'not_env_failure' }}
+        working-directory: /vllm-workspace/vllm-ascend
+        run: |
+          bash tests/e2e/nightly/scripts/aop_commit_age.sh \
+            "${{ inputs.name || inputs.config_file_path || inputs.tests }}" \
+            "/root/.cache/vllm-ascend/${{ inputs.vllm_ascend_branch }}/nightly/good_table.csv"
+
+      - name: Skip - old commit
+        if: >-
+          ${{ always() && inputs.aop_single_enabled &&
+          steps.classify.outputs.failure_type == 'not_env_failure' &&
+          steps.commit-age.outputs.is_old == 'true' }}
+        working-directory: /vllm-workspace/vllm-ascend
+        run: |
+          bash tests/e2e/nightly/scripts/aop_skip.sh \
+            "not_env_failure" \
+            "${{ steps.commit-age.outputs.last_status }}" \
+            "${{ steps.commit-age.outputs.last_date }}" \
+            "${{ steps.commit-age.outputs.commit_age_days }}" \
+            "${{ steps.test-result.outputs.pytest_summary }}" \
+            "${{ steps.test-result.outputs.yaml_summary }}"
+
+      - name: Process - recent real failure
+        if: >-
+          ${{ always() && inputs.aop_single_enabled &&
+          steps.classify.outputs.failure_type == 'not_env_failure' &&
+          steps.commit-age.outputs.is_old == 'false' }}
+        working-directory: /vllm-workspace/vllm-ascend
+        env:
+          GOOD_TABLE: /root/.cache/vllm-ascend/${{ inputs.vllm_ascend_branch }}/nightly/good_table.csv
+        run: |
+          bash tests/e2e/nightly/scripts/aop_process.sh \
+            "${{ steps.classify.outputs.failure_type }}" \
+            "${{ steps.commit-age.outputs.commit_age_days }}" \
+            "${{ inputs.runner }}" \
+            "${{ inputs.tests }}" \
+            "${{ inputs.config_file_path }}" \
+            "${{ steps.test-result.outputs.pytest_summary }}" \
+            "${{ steps.test-result.outputs.yaml_summary }}" \
+            "single_node" \
+            "HEAD" \
+            "" \
+            "${{ inputs.name }}"
+
+      - name: On success
+        if: ${{ always() && inputs.aop_single_enabled && steps.test-result.outputs.result == 'success' }}
+        run: |
+          echo ">>> All tests passed!"
+
+      # ============================================
 
       - name: Set benchmark artifact timestamp
         if: ${{ always() && inputs.config_file_path != '' }}

.github/workflows/_selected_tests.yaml

@@ -32,6 +32,11 @@ on:
         required: false
         default: ''
         description: 'The vllm-ascend ref to test.'
+      base_ref:
+        type: string
+        required: false
+        default: 'main'
+        description: 'The base branch to rebase onto (e.g. main, v0.8.x).'
       upload_timing:
         type: boolean
         required: false
@@ -112,12 +117,12 @@ jobs:
           fetch-depth: 0
           submodules: recursive
 
-      - name: Rebase on latest main
+      - name: Rebase on latest ${{ inputs.base_ref }}
         run: |
           git config user.name "vllm-ascend-ci"
           git config user.email "vllm-ascend-ci@users.noreply.github.com"
-          git fetch origin main
-          git rebase origin/main
+          git fetch origin ${{ inputs.base_ref }}
+          git rebase origin/${{ inputs.base_ref }}
 
       - name: Get csrc hash
         id: get_csrc_hash
@@ -172,11 +177,11 @@ jobs:
             libcurl4
           ldconfig
 
-          MOONCAKE_WHEEL="mooncake_transfer_engine_ascend-0.3.9-cp312-cp312-manylinux_2_35_aarch64.whl"
           pip install --no-cache-dir --no-deps \
-            "https://vllm-ascend.obs.cn-north-4.myhuaweicloud.com/vllm-ascend/${MOONCAKE_WHEEL}"
+            -i https://mirrors.aliyun.com/pypi/simple/ \
+            mooncake-transfer-engine-npu==0.3.11.post1
 
-          pip show mooncake-transfer-engine-ascend || true
+          pip show mooncake-transfer-engine-npu || true
 
       - name: Install vllm-project/vllm-ascend with device
         if: ${{ matrix.group.npu_type != 'cpu' }}