From 7d6b9467ca2e0310a031e483c7f3d6fcc118dc38 Mon Sep 17 00:00:00 2001
From: Ed Savage <ed.savage@elastic.co>
Date: Fri, 13 Mar 2026 12:02:59 +1300
Subject: [PATCH] [ML] Run allowlist validation in PyTorch edge pipeline

When the PyTorch edge build triggers a test run, also invoke the
Python-based allowlist validation (validate_allowlist.py) which
traces live HuggingFace models with the new PyTorch version and
checks every op against ALLOWED_OPERATIONS / FORBIDDEN_OPERATIONS.

This ensures that if a PyTorch upgrade introduces new TorchScript
operations for any supported model architecture, the pipeline fails
with a clear message before the change reaches a release.

Made-with: Cursor
---
 .buildkite/scripts/steps/run_tests.sh | 25 ++++++++++++++++++++++++-
 1 file changed, 24 insertions(+), 1 deletion(-)

diff --git a/.buildkite/scripts/steps/run_tests.sh b/.buildkite/scripts/steps/run_tests.sh
index 12b88c1bb..0add8ea37 100755
--- a/.buildkite/scripts/steps/run_tests.sh
+++ b/.buildkite/scripts/steps/run_tests.sh
@@ -105,6 +105,27 @@ else
         -P cmake/run-all-tests-parallel.cmake || TEST_OUTCOME=$?
 fi
 
+# --- PyTorch allowlist validation ---
+# When triggered from the PyTorch edge pipeline, run the Python-based
+# allowlist validation which traces live HuggingFace models with the
+# new PyTorch version and verifies every op is in ALLOWED_OPERATIONS.
+VALIDATION_OUTCOME=0
+if [[ "${GITHUB_PR_COMMENT_VAR_ACTION:-}" == "run_pytorch_tests" ]]; then
+    echo "--- Validating PyTorch allowlist against HuggingFace models"
+    cmake \
+        -DSOURCE_DIR="$(pwd)" \
+        -DVALIDATE_CONFIG="$(pwd)/dev-tools/extract_model_ops/validation_models.json" \
+        -DVALIDATE_PT_DIR="$(pwd)/dev-tools/extract_model_ops/es_it_models" \
+        -DVALIDATE_VERBOSE=TRUE \
+        -P cmake/run-validation.cmake || VALIDATION_OUTCOME=$?
+
+    if [[ $VALIDATION_OUTCOME -ne 0 ]]; then
+        echo "^^^ +++"
+        echo "Allowlist validation failed — the new PyTorch version may introduce ops not in ALLOWED_OPERATIONS."
+        echo "See dev-tools/extract_model_ops/README.md for how to update the allowlist."
+    fi
+fi
+
 # Upload test results
 echo "--- Uploading test results"
 TEST_RESULTS_ARCHIVE=${OS}-${HARDWARE_ARCH}-unit_test_results.tgz
@@ -117,4 +138,6 @@ else
     echo "No test results archive created"
 fi
 
-exit $TEST_OUTCOME
+if [[ $TEST_OUTCOME -ne 0 || $VALIDATION_OUTCOME -ne 0 ]]; then
+    exit 1
+fi