From 6690cc773bd6ff9f63540fd227166d8b918bfc5d Mon Sep 17 00:00:00 2001 From: EdwardXuy Date: Wed, 22 Jul 2026 10:38:32 +0800 Subject: [PATCH 1/4] fix(ci): isolate multi-node cache path and k8s job per run+version PR-triggered A2 multi-node pipeline frequently fails with: cp: cannot create regular file '/root/.cache/tests/sglang/CMakeLists.txt': File exists Root cause: all matrix jobs (8.5.0, 9.0.0) and concurrent PRs share the same fixed path /root/.cache/tests/sglang on a shared PVC, and the same fixed k8s Volcano Job name sglang-npu-multi. When the previous job's test pods are still terminating or the PVC has sync latency, the next job's cp -r hits residual files and fails. Fix in internode.yml: - Derive unique cache path /root/.cache/tests/sglang/{RUN_ID}_{CANN_VERSION} - Derive unique k8s job name sglang-npu-multi-{CANN_VERSION} - Replace rm -rf path/* (misses dotfiles, races on PVC sync) with rm -rf path - Copy hidden files (.gitmodules, .github) that cp .../* skips - Clean up the job PVC directory in Post process No change to workflow inputs/outputs, so daily-build-test.yml and a2-internode-test.yml callers are unaffected. The daily pipeline also benefits from this isolation (previously masked by the 5-7h needs delay). --- .github/workflows/internode.yml | 49 +++++++++++++++++++++++++++++---- 1 file changed, 44 insertions(+), 5 deletions(-) diff --git a/.github/workflows/internode.yml b/.github/workflows/internode.yml index f86b60cbe..00779a044 100644 --- a/.github/workflows/internode.yml +++ b/.github/workflows/internode.yml @@ -69,11 +69,43 @@ jobs: - name: Prepare scripts run: | - # prepare for test code - sglang_source_path=/root/.cache/tests/sglang - mkdir -p $sglang_source_path && chmod -R 777 $sglang_source_path - rm -rf $sglang_source_path/* - cp -r $GITHUB_WORKSPACE/* $sglang_source_path/ + # Extract CANN version from image tag for path and k8s job isolation. + # e.g. "swr.../cann:8.5.0-910b-ubuntu22.04-py3.11" -> "8.5.0" + CANN_VERSION=$(echo "${{ inputs.image }}" | sed -n 's/.*cann:\([0-9.]*\)-.*/\1/p') + if [ -z "$CANN_VERSION" ]; then CANN_VERSION="default"; fi + + # Unique path per run + CANN version to prevent PVC collisions across + # concurrent jobs. Different PRs and different matrix entries (8.5.0 + # vs 9.0.0) previously shared /root/.cache/tests/sglang and raced on + # cp -r ("File exists"). The run_id + cann_version suffix guarantees + # each job writes to its own directory on the shared PVC. + sglang_source_path="/root/.cache/tests/sglang/${GITHUB_RUN_ID}_${CANN_VERSION}" + echo "SGLANG_SOURCE_PATH=$sglang_source_path" >> $GITHUB_ENV + + # Unique k8s Volcano Job name per CANN version. Previously fixed to + # "sglang-npu-multi", so kubectl delete -f in one job could hit + # another job's resources when they ran concurrently or overlapped + # during pod termination. + KUBE_JOB_NAME="sglang-npu-multi-${CANN_VERSION}" + echo "KUBE_JOB_NAME=$KUBE_JOB_NAME" >> $GITHUB_ENV + + # Clean and prepare the unique directory. Path isolation guarantees + # no cross-job conflict, so rm -rf on the full dir is safe. + rm -rf "$sglang_source_path" + mkdir -p "$sglang_source_path" && chmod -R 777 "$sglang_source_path" + + # Copy source code. Directory is already unique and empty, so plain + # cp -r is safe (no "File exists" possible). + cp -r $GITHUB_WORKSPACE/* "$sglang_source_path/" + # cp .../* does not match dotfiles; copy hidden entries (.gitmodules, + # .github, etc.) so submodules and workflow files are available. + for f in $GITHUB_WORKSPACE/.*; do + case $(basename "$f") in + .|..) continue;; + esac + cp -r "$f" "$sglang_source_path/" 2>/dev/null || true + done + ls -l $sglang_source_path/tests/python/deepep/ echo "Code copied to $sglang_source_path, files in deepep: $(ls $sglang_source_path/tests/python/deepep/)" @@ -126,3 +158,10 @@ jobs: kubectl get pods -n $NAMESPACE cd $ASCEND_TEST_CASE_PATH kubectl delete -f ./k8s_multi.yaml --ignore-not-found=true || true + # Clean up this job's PVC directory to reclaim space. The path is + # unique per run + CANN version, so only this job's files are removed + # and other concurrent jobs' directories are untouched. + if [ -n "$SGLANG_SOURCE_PATH" ]; then + rm -rf "$SGLANG_SOURCE_PATH" || true + echo "Cleaned up PVC path: $SGLANG_SOURCE_PATH" + fi From ca5ee717821cbebf3e02a8fa5b1b467448f1f278 Mon Sep 17 00:00:00 2001 From: EdwardXuy Date: Wed, 22 Jul 2026 11:21:36 +0800 Subject: [PATCH 2/4] style(ci): trim verbose comments in internode.yml --- .github/workflows/internode.yml | 30 ++++++------------------------ 1 file changed, 6 insertions(+), 24 deletions(-) diff --git a/.github/workflows/internode.yml b/.github/workflows/internode.yml index 00779a044..83d8ba8d8 100644 --- a/.github/workflows/internode.yml +++ b/.github/workflows/internode.yml @@ -69,40 +69,24 @@ jobs: - name: Prepare scripts run: | - # Extract CANN version from image tag for path and k8s job isolation. - # e.g. "swr.../cann:8.5.0-910b-ubuntu22.04-py3.11" -> "8.5.0" + # Isolate per run + CANN version: shared PVC + fixed path caused + # "cp: File exists" races between concurrent matrix jobs/PRs. CANN_VERSION=$(echo "${{ inputs.image }}" | sed -n 's/.*cann:\([0-9.]*\)-.*/\1/p') if [ -z "$CANN_VERSION" ]; then CANN_VERSION="default"; fi - # Unique path per run + CANN version to prevent PVC collisions across - # concurrent jobs. Different PRs and different matrix entries (8.5.0 - # vs 9.0.0) previously shared /root/.cache/tests/sglang and raced on - # cp -r ("File exists"). The run_id + cann_version suffix guarantees - # each job writes to its own directory on the shared PVC. sglang_source_path="/root/.cache/tests/sglang/${GITHUB_RUN_ID}_${CANN_VERSION}" echo "SGLANG_SOURCE_PATH=$sglang_source_path" >> $GITHUB_ENV - # Unique k8s Volcano Job name per CANN version. Previously fixed to - # "sglang-npu-multi", so kubectl delete -f in one job could hit - # another job's resources when they ran concurrently or overlapped - # during pod termination. + # Unique k8s job name avoids Volcano Job name collisions on delete. KUBE_JOB_NAME="sglang-npu-multi-${CANN_VERSION}" echo "KUBE_JOB_NAME=$KUBE_JOB_NAME" >> $GITHUB_ENV - # Clean and prepare the unique directory. Path isolation guarantees - # no cross-job conflict, so rm -rf on the full dir is safe. rm -rf "$sglang_source_path" mkdir -p "$sglang_source_path" && chmod -R 777 "$sglang_source_path" - - # Copy source code. Directory is already unique and empty, so plain - # cp -r is safe (no "File exists" possible). cp -r $GITHUB_WORKSPACE/* "$sglang_source_path/" - # cp .../* does not match dotfiles; copy hidden entries (.gitmodules, - # .github, etc.) so submodules and workflow files are available. + # cp .../* skips dotfiles; copy them explicitly. for f in $GITHUB_WORKSPACE/.*; do - case $(basename "$f") in - .|..) continue;; - esac + case $(basename "$f") in .|..) continue;; esac cp -r "$f" "$sglang_source_path/" 2>/dev/null || true done @@ -158,9 +142,7 @@ jobs: kubectl get pods -n $NAMESPACE cd $ASCEND_TEST_CASE_PATH kubectl delete -f ./k8s_multi.yaml --ignore-not-found=true || true - # Clean up this job's PVC directory to reclaim space. The path is - # unique per run + CANN version, so only this job's files are removed - # and other concurrent jobs' directories are untouched. + # Reclaim PVC space; path is unique per job, safe to remove. if [ -n "$SGLANG_SOURCE_PATH" ]; then rm -rf "$SGLANG_SOURCE_PATH" || true echo "Cleaned up PVC path: $SGLANG_SOURCE_PATH" From f0204734fa391a1ae9204f6122bbba76149fd6cb Mon Sep 17 00:00:00 2001 From: EdwardXuy Date: Wed, 22 Jul 2026 14:36:53 +0800 Subject: [PATCH 3/4] fix(ci): serial global concurrency to fix PVC path race (revert path isolation) Previous commit (6690cc7) tried to isolate PVC path per run+version by creating subdirectory /root/.cache/tests/sglang/_. This broke: runner container's /root/.cache/tests/sglang is a dedicated mount point (subPath) on the shared PVC, while the test pod mounts the entire PVC at /root/.cache. Files written to the subdirectory land on the runner's local view but are NOT visible to the test pod, causing: ERROR: Script not found at .../29888246254_8.5.0/tests/python/deepep/run_ascend_testcase.sh Revert to the fixed path /root/.cache/tests/sglang (which both runner and test pod share correctly) and solve the original "cp: File exists" race differently: 1. concurrency: drop github.ref from group -> all multi-node runs for the same soc_version serialize globally (cross-PR). Set cancel-in-progress: false so an in-progress test is not interrupted (scarce NPU resources should not be wasted on partial runs). 2. rm -rf "$sglang_source_path" (whole dir) instead of rm -rf $path/* to also clear leftover dotfiles that caused stale-state issues. Result: no two multi-node jobs write the shared path simultaneously, and each job starts from a clean directory. Daily pipeline is unaffected (it calls internode.yml with the same soc_version and will simply queue behind a PR run if one is active, which is acceptable). --- .github/workflows/internode.yml | 31 +++++++------------------------ 1 file changed, 7 insertions(+), 24 deletions(-) diff --git a/.github/workflows/internode.yml b/.github/workflows/internode.yml index 83d8ba8d8..922bb1c9f 100644 --- a/.github/workflows/internode.yml +++ b/.github/workflows/internode.yml @@ -30,8 +30,11 @@ on: description: path of test case file concurrency: - group: ascend-nightly-multi-node-${{ github.ref }}-${{ inputs.soc_version }} - cancel-in-progress: true + # Global serial per soc_version: fixed PVC path /root/.cache/tests/sglang is + # shared, so concurrent runs (different PRs) would race on cp ("File exists"). + # do NOT cancel in-progress: multi-node test uses scarce NPU resources. + group: ascend-nightly-multi-node-${{ inputs.soc_version }} + cancel-in-progress: false jobs: multi-node: @@ -69,26 +72,11 @@ jobs: - name: Prepare scripts run: | - # Isolate per run + CANN version: shared PVC + fixed path caused - # "cp: File exists" races between concurrent matrix jobs/PRs. - CANN_VERSION=$(echo "${{ inputs.image }}" | sed -n 's/.*cann:\([0-9.]*\)-.*/\1/p') - if [ -z "$CANN_VERSION" ]; then CANN_VERSION="default"; fi - - sglang_source_path="/root/.cache/tests/sglang/${GITHUB_RUN_ID}_${CANN_VERSION}" - echo "SGLANG_SOURCE_PATH=$sglang_source_path" >> $GITHUB_ENV - - # Unique k8s job name avoids Volcano Job name collisions on delete. - KUBE_JOB_NAME="sglang-npu-multi-${CANN_VERSION}" - echo "KUBE_JOB_NAME=$KUBE_JOB_NAME" >> $GITHUB_ENV - + sglang_source_path=/root/.cache/tests/sglang + # rm -rf the whole dir (clears dotfiles too); safer than rm -rf $path/* rm -rf "$sglang_source_path" mkdir -p "$sglang_source_path" && chmod -R 777 "$sglang_source_path" cp -r $GITHUB_WORKSPACE/* "$sglang_source_path/" - # cp .../* skips dotfiles; copy them explicitly. - for f in $GITHUB_WORKSPACE/.*; do - case $(basename "$f") in .|..) continue;; esac - cp -r "$f" "$sglang_source_path/" 2>/dev/null || true - done ls -l $sglang_source_path/tests/python/deepep/ echo "Code copied to $sglang_source_path, files in deepep: $(ls $sglang_source_path/tests/python/deepep/)" @@ -142,8 +130,3 @@ jobs: kubectl get pods -n $NAMESPACE cd $ASCEND_TEST_CASE_PATH kubectl delete -f ./k8s_multi.yaml --ignore-not-found=true || true - # Reclaim PVC space; path is unique per job, safe to remove. - if [ -n "$SGLANG_SOURCE_PATH" ]; then - rm -rf "$SGLANG_SOURCE_PATH" || true - echo "Cleaned up PVC path: $SGLANG_SOURCE_PATH" - fi From d295204b5b896838c3c3d6c8dc255354b69537cf Mon Sep 17 00:00:00 2001 From: EdwardXuy Date: Wed, 22 Jul 2026 16:09:18 +0800 Subject: [PATCH 4/4] fix(ci): move concurrency to caller workflow (revert internode.yml) Previous commits tried two approaches that both failed: 1. Path isolation in internode.yml (6690cc7): Created subdirectory /root/.cache/tests/sglang/_. FAILED because runner container mounts /root/.cache/tests/sglang as a subPath, while test pod mounts the entire PVC at /root/.cache. Files in the subdirectory are invisible to the test pod: "ERROR: Script not found at .../run_ascend_testcase.sh" 2. Global concurrency in internode.yml (f020473): Changed concurrency group to ascend-nightly-multi-node-${{ inputs.soc_version }}. FAILED because concurrency in a reusable (called) workflow does NOT serialize across different caller runs. Evidence: run 29896822833 (different PR) and run 29897327319 (this PR) both executed multi-node jobs simultaneously, causing: "rm: cannot remove '/root/.cache/tests/sglang': Directory not empty" Root cause confirmed via GitHub docs and community: "You need to apply the concurrency setting to the CALLER workflow, not the reusable workflow itself." Fix: - Revert internode.yml to original (no changes from main). - Change a2-internode-test.yml concurrency: - group: a2-internode-test (was: a2-internode-test-${{ github.ref }}-${{ github.event_name }}) Removes github.ref so ALL PR runs share one global group and serialize. - cancel-in-progress: false (was: true) Don't interrupt in-progress multi-node tests (scarce NPU resources). Daily pipeline (daily-build-test.yml) is unaffected: it calls internode.yml directly, not a2-internode-test.yml, and runs once daily with a 5-7h delay from prerequisite jobs. --- .github/workflows/a2-internode-test.yml | 7 +++++-- .github/workflows/internode.yml | 16 ++++++---------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/.github/workflows/a2-internode-test.yml b/.github/workflows/a2-internode-test.yml index e612554e5..eca27fde5 100644 --- a/.github/workflows/a2-internode-test.yml +++ b/.github/workflows/a2-internode-test.yml @@ -16,8 +16,11 @@ on: workflow_call: concurrency: - group: a2-internode-test-${{ github.ref }}-${{ github.event_name }} - cancel-in-progress: true + # Global serial: shared PVC path /root/.cache/tests/sglang cannot handle + # concurrent writers. Concurrency in the reusable workflow (internode.yml) + # does not serialize across different caller runs, so we must serialize here. + group: a2-internode-test + cancel-in-progress: false jobs: multi-node-internode: diff --git a/.github/workflows/internode.yml b/.github/workflows/internode.yml index 922bb1c9f..f86b60cbe 100644 --- a/.github/workflows/internode.yml +++ b/.github/workflows/internode.yml @@ -30,11 +30,8 @@ on: description: path of test case file concurrency: - # Global serial per soc_version: fixed PVC path /root/.cache/tests/sglang is - # shared, so concurrent runs (different PRs) would race on cp ("File exists"). - # do NOT cancel in-progress: multi-node test uses scarce NPU resources. - group: ascend-nightly-multi-node-${{ inputs.soc_version }} - cancel-in-progress: false + group: ascend-nightly-multi-node-${{ github.ref }}-${{ inputs.soc_version }} + cancel-in-progress: true jobs: multi-node: @@ -72,12 +69,11 @@ jobs: - name: Prepare scripts run: | + # prepare for test code sglang_source_path=/root/.cache/tests/sglang - # rm -rf the whole dir (clears dotfiles too); safer than rm -rf $path/* - rm -rf "$sglang_source_path" - mkdir -p "$sglang_source_path" && chmod -R 777 "$sglang_source_path" - cp -r $GITHUB_WORKSPACE/* "$sglang_source_path/" - + mkdir -p $sglang_source_path && chmod -R 777 $sglang_source_path + rm -rf $sglang_source_path/* + cp -r $GITHUB_WORKSPACE/* $sglang_source_path/ ls -l $sglang_source_path/tests/python/deepep/ echo "Code copied to $sglang_source_path, files in deepep: $(ls $sglang_source_path/tests/python/deepep/)"