[CI]Result upload obs (vllm-project#12786)

### What this PR does / why we need it?
After E2E tests complete, the workflow:
Collects selected-test logs as GitHub Artifacts.
Generates failure_report.md by comparing failed tests with recommended
tests.
Uploads the report as a GitHub Artifact.
Archives each PR report into a date-based .tar.gz object in Huawei OBS.
Preserves the actual uploaded archive as a GitHub Artifact for
troubleshooting.
Adds a local OBS upload/download smoke test with SHA256 and content
verification.
Keeps the analysis and OBS archival path non-blocking for the main E2E
CI gate.
### Does this PR introduce _any_ user-facing change?
N/A
### How was this patch tested?

- vLLM version: v0.25.1
- vLLM main:
vllm-project/vllm@fe784ff

---------

Signed-off-by: Xuyzhen <958522639@qq.com>

Author: Xuyzhen

.github/workflows/_analyze_failure.yaml

@@ -32,6 +32,11 @@ on:
         required: false
         default: ''
         description: 'Content of recommendations from upstream job output (priority 1)'
+    secrets:
+      OBS_ACCESS_KEY_FAILURE_ANALYSIS:
+        required: false
+      OBS_SECRET_ACCESS_KEY_FAILURE_ANALYSIS:
+        required: false
 
 defaults:
   run:
@@ -116,3 +121,229 @@ jobs:
           path: ./test-logs/failure_report.md
           if-no-files-found: ignore
           retention-days: 14
+
+  archive-failure-report-to-obs:
+    name: Archive failure report to OBS
+    needs: analyze
+    if: ${{ always() }}
+    continue-on-error: true
+    runs-on: ubuntu-latest
+    concurrency:
+      group: failure-analysis-obs-daily-archive
+      cancel-in-progress: false
+    env:
+      OBS_ACCESS_KEY: ${{ secrets.OBS_ACCESS_KEY_FAILURE_ANALYSIS }}
+      OBS_SECRET_KEY: ${{ secrets.OBS_SECRET_ACCESS_KEY_FAILURE_ANALYSIS }}
+      PR_NUMBER: ${{ github.event.pull_request.number }}
+      PR_TITLE: ${{ github.event.pull_request.title }}
+      RUN_ID: ${{ github.run_id }}
+      RUN_ATTEMPT: ${{ github.run_attempt }}
+    steps:
+      - name: Download failure analysis report
+        continue-on-error: true
+        uses: actions/download-artifact@v7
+        with:
+          name: failure-analysis-report
+          path: ./failure-analysis-report
+
+      - name: Install OBS SDK
+        if: always()
+        run: |
+          if ! pip install esdk-obs-python --quiet --default-timeout=30 --retries 1; then
+            echo "::warning::Failed to install OBS SDK; skipping OBS archival."
+          fi
+
+      - name: Add report to daily archive and upload to OBS
+        if: always()
+        run: |
+          set +e
+          python3 - <<'PY'
+          import os
+          import shutil
+          import tarfile
+          import unicodedata
+          from datetime import datetime
+          from pathlib import Path
+          from zoneinfo import ZoneInfo
+
+          OBS_BUCKET = "vllm-ascend"
+          OBS_PREFIX = "ci/precision-test"
+          OBS_SERVER = "https://obs.cn-north-4.myhuaweicloud.com"
+          OBS_REQUEST_TIMEOUT_SECONDS = 30
+          OBS_MAX_RETRY_COUNT = 0
+          REPORT_PATH = Path("failure-analysis-report/failure_report.md")
+          WORK_DIR = Path("failure-analysis-archive")
+          CONTENT_DIR = WORK_DIR / "contents"
+          ARTIFACT_DIR = WORK_DIR / "uploaded"
+
+
+          def warn(message):
+              print(f"::warning::{message}")
+
+
+          def sanitize_filename(value):
+              normalized = unicodedata.normalize("NFKC", value)
+              sanitized = "".join(
+                  char if char.isalnum() or char in ".-_" else "_"
+                  for char in normalized
+              )
+              while "__" in sanitized:
+                  sanitized = sanitized.replace("__", "_")
+              return sanitized.strip("._-")[:80] or "untitled"
+
+
+          def extract_archive(archive_path, destination):
+              try:
+                  destination_root = destination.resolve()
+                  with tarfile.open(archive_path, "r:gz") as archive:
+                      for member in archive.getmembers():
+                          target = (destination / member.name).resolve()
+                          if target != destination_root and destination_root not in target.parents:
+                              warn(f"Unsafe archive path: {member.name}; skipping OBS archival.")
+                              return False
+                          if not (member.isfile() or member.isdir()):
+                              warn(f"Unsupported archive entry: {member.name}; skipping OBS archival.")
+                              return False
+                      archive.extractall(destination)
+                  return True
+              except Exception as exc:
+                  warn(f"Failed to extract existing OBS archive: {exc}")
+                  return False
+
+
+          def main():
+              if not REPORT_PATH.is_file() or REPORT_PATH.stat().st_size == 0:
+                  print("::notice::No failure analysis report was produced; skipping OBS upload.")
+                  return
+
+              access_key = os.environ.get("OBS_ACCESS_KEY", "")
+              secret_key = os.environ.get("OBS_SECRET_KEY", "")
+              if not access_key or not secret_key:
+                  warn("OBS failure-analysis credentials are not configured; skipping upload.")
+                  return
+
+              try:
+                  from obs import ObsClient
+              except Exception as exc:
+                  warn(f"OBS SDK is unavailable; skipping OBS archival: {exc}")
+                  return
+
+              now = datetime.now(ZoneInfo("Asia/Shanghai"))
+              archive_date = now.strftime("%Y-%m-%d")
+              timestamp = now.strftime("%Y%m%dT%H%M%S%z")
+              pr_number = os.environ.get("PR_NUMBER") or "unknown"
+              pr_title = sanitize_filename(os.environ.get("PR_TITLE", "untitled"))
+              run_id = os.environ.get("RUN_ID") or "unknown"
+              run_attempt = os.environ.get("RUN_ATTEMPT") or "1"
+              report_name = (
+                  f"PR-{pr_number}_{pr_title}_{timestamp}_"
+                  f"run-{run_id}-{run_attempt}.md"
+              )
+              archive_name = f"{archive_date}.tar.gz"
+              obs_key = f"{OBS_PREFIX}/{archive_name}"
+              archive_path = WORK_DIR / archive_name
+              new_archive_path = WORK_DIR / f"new-{archive_name}"
+
+              CONTENT_DIR.mkdir(parents=True, exist_ok=True)
+              client = ObsClient(
+                  access_key_id=access_key,
+                  secret_access_key=secret_key,
+                  server=OBS_SERVER,
+                  timeout=OBS_REQUEST_TIMEOUT_SECONDS,
+                  max_retry_count=OBS_MAX_RETRY_COUNT,
+              )
+              try:
+                  try:
+                      response = client.getObject(
+                          OBS_BUCKET,
+                          obs_key,
+                          downloadPath=str(archive_path),
+                      )
+                  except Exception as exc:
+                      warn(
+                          f"OBS download failed or timed out after "
+                          f"{OBS_REQUEST_TIMEOUT_SECONDS}s: {exc}"
+                      )
+                      return
+
+                  if response.status < 300:
+                      print(f"Downloaded existing archive: {obs_key}")
+                      if not extract_archive(archive_path, CONTENT_DIR):
+                          return
+                  elif response.status == 404:
+                      print(f"No archive exists for {archive_date}; creating a new one")
+                  else:
+                      warn(
+                          f"Failed to download {obs_key}: status={response.status}, "
+                          f"code={response.errorCode}, message={response.errorMessage}"
+                      )
+                      return
+
+                  shutil.copy2(REPORT_PATH, CONTENT_DIR / report_name)
+                  with tarfile.open(new_archive_path, "w:gz") as archive:
+                      for report in sorted(CONTENT_DIR.rglob("*")):
+                          if report.is_file():
+                              archive.add(report, arcname=report.relative_to(CONTENT_DIR))
+
+                  with tarfile.open(new_archive_path, "r:gz") as archive:
+                      if report_name not in archive.getnames():
+                          warn(f"New report is missing from {archive_name}; skipping upload.")
+                          return
+
+                  try:
+                      response = client.putFile(
+                          OBS_BUCKET,
+                          obs_key,
+                          str(new_archive_path),
+                      )
+                  except Exception as exc:
+                      warn(
+                          f"OBS upload failed or timed out after "
+                          f"{OBS_REQUEST_TIMEOUT_SECONDS}s: {exc}"
+                      )
+                      return
+
+                  if response.status >= 300:
+                      warn(
+                          f"Failed to upload {obs_key}: status={response.status}, "
+                          f"code={response.errorCode}, message={response.errorMessage}"
+                      )
+                      return
+                  print(f"Uploaded: {report_name} -> {obs_key}")
+                  ARTIFACT_DIR.mkdir(parents=True, exist_ok=True)
+                  shutil.copy2(new_archive_path, ARTIFACT_DIR / archive_name)
+              finally:
+                  try:
+                      client.close()
+                  except Exception as exc:
+                      warn(f"Failed to close OBS client: {exc}")
+
+              summary_path = os.environ.get("GITHUB_STEP_SUMMARY")
+              if summary_path:
+                  with open(summary_path, "a", encoding="utf-8") as summary:
+                      summary.write("## Failure analysis OBS archive\n\n")
+                      summary.write(f"- Report: {report_name}\n")
+                      summary.write(f"- Object: {obs_key}\n")
+
+
+          try:
+              main()
+          except Exception as exc:
+              warn(f"Unexpected failure while archiving the report: {exc}")
+          PY
+          status=$?
+          if [ "$status" -ne 0 ]; then
+            echo "::warning::Failure analysis archival exited with status $status; ignoring."
+          fi
+          exit 0
+
+      - name: Upload OBS archive copy
+        if: always()
+        continue-on-error: true
+        uses: actions/upload-artifact@v7
+        with:
+          name: failure-analysis-obs-archive-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ./failure-analysis-archive/uploaded/*.tar.gz
+          if-no-files-found: ignore
+          retention-days: 14
+          compression-level: 0

.github/workflows/pr_test.yaml

@@ -497,3 +497,6 @@ jobs:
       log_artifact_pattern: selected-test-logs-*
       recommendations_file: .github/workflows/scripts/recommended_pytest_paths.txt
       recommendations_content: ${{ needs.recommend-tests-from-coverage.outputs.coverage_paths || '' }}
+    secrets:
+      OBS_ACCESS_KEY_FAILURE_ANALYSIS: ${{ secrets.OBS_ACCESS_KEY_FAILURE_ANALYSIS }}
+      OBS_SECRET_ACCESS_KEY_FAILURE_ANALYSIS: ${{ secrets.OBS_SECRET_ACCESS_KEY_FAILURE_ANALYSIS }}