[ML] Run PyTorch allowlist validation as a dedicated Buildkite step

.buildkite/pipeline.json.py

@@ -84,6 +84,14 @@ def main():
                                                        ".buildkite/pipelines/check_build_regression.yml.sh",
                                                        soft_fail=True))
 
+    # Validate the PyTorch allowlist against HuggingFace models when
+    # triggered from the PyTorch edge pipeline.  Runs in a python:3
+    # container since the build/test images don't include Python.
+    if config.run_pytorch_tests:
+        pipeline_steps.append(pipeline_steps.generate_step("Upload PyTorch allowlist validation",
+                                                           ".buildkite/pipelines/validate_pytorch_allowlist.yml.sh",
+                                                           soft_fail=True))
+
     pipeline["env"] = env
     pipeline["steps"] = pipeline_steps
     print(json.dumps(pipeline, indent=2))

.buildkite/pipelines/validate_pytorch_allowlist.yml.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License
+# 2.0 and the following additional limitation. Functionality enabled by the
+# files subject to the Elastic License 2.0 may only be used in production when
+# invoked by an Elasticsearch process with a license key installed that permits
+# use of machine learning features. You may not use this file except in
+# compliance with the Elastic License 2.0 and the foregoing additional
+# limitation.
+
+cat <<'EOL'
+steps:
+  - label: "Validate PyTorch allowlist :torch:"
+    key: "validate_pytorch_allowlist"
+    command:
+        - "cmake -DSOURCE_DIR=$(pwd) -DVALIDATE_CONFIG=$(pwd)/dev-tools/extract_model_ops/validation_models.json -DVALIDATE_PT_DIR=$(pwd)/dev-tools/extract_model_ops/es_it_models -DVALIDATE_VERBOSE=TRUE -P cmake/run-validation.cmake"
+EOL
+
+# Depend on the build steps so validation doesn't start before the
+# pipeline is fully generated.
+if [ -n "${ML_BUILD_STEP_KEYS:-}" ]; then
+    echo '    depends_on:'
+    IFS=',' read -ra STEP_KEYS <<< "$ML_BUILD_STEP_KEYS"
+    for key in "${STEP_KEYS[@]}"; do
+        echo "        - \"${key}\""
+    done
+fi
+
+cat <<'EOL'
+    allow_dependency_failure: true
+    soft_fail: true
+    agents:
+      image: "python:3"
+EOL

.buildkite/scripts/steps/run_tests.sh

@@ -105,28 +105,6 @@ else
         -P cmake/run-all-tests-parallel.cmake || TEST_OUTCOME=$?
 fi
 
-# --- PyTorch allowlist validation ---
-# When triggered from the PyTorch edge pipeline, run the Python-based
-# allowlist validation which traces live HuggingFace models with the
-# new PyTorch version and verifies every op is in ALLOWED_OPERATIONS.
-VALIDATION_OUTCOME=0
-if [[ "${GITHUB_PR_COMMENT_VAR_ACTION:-}" == "run_pytorch_tests" ]] && [ -f cmake/run-validation.cmake ]; then
-    echo "--- Validating PyTorch allowlist against HuggingFace models"
-    cmake \
-        -DSOURCE_DIR="$(pwd)" \
-        -DVALIDATE_CONFIG="$(pwd)/dev-tools/extract_model_ops/validation_models.json" \
-        -DVALIDATE_PT_DIR="$(pwd)/dev-tools/extract_model_ops/es_it_models" \
-        -DVALIDATE_VERBOSE=TRUE \
-        -DOPTIONAL=TRUE \
-        -P cmake/run-validation.cmake || VALIDATION_OUTCOME=$?
-
-    if [[ $VALIDATION_OUTCOME -ne 0 ]]; then
-        echo "^^^ +++"
-        echo "Allowlist validation failed — the new PyTorch version may introduce ops not in ALLOWED_OPERATIONS."
-        echo "See dev-tools/extract_model_ops/README.md for how to update the allowlist."
-    fi
-fi
-
 # Upload test results
 echo "--- Uploading test results"
 TEST_RESULTS_ARCHIVE=${OS}-${HARDWARE_ARCH}-unit_test_results.tgz
@@ -139,6 +117,4 @@ else
     echo "No test results archive created"
 fi
 
-if [[ $TEST_OUTCOME -ne 0 || $VALIDATION_OUTCOME -ne 0 ]]; then
-    exit 1
-fi
+exit $TEST_OUTCOME