diff --git a/.buildkite/pipeline.json.py b/.buildkite/pipeline.json.py index 9b6d6616e..0ae577685 100755 --- a/.buildkite/pipeline.json.py +++ b/.buildkite/pipeline.json.py @@ -84,6 +84,14 @@ def main(): ".buildkite/pipelines/check_build_regression.yml.sh", soft_fail=True)) + # Validate the PyTorch allowlist against HuggingFace models when + # triggered from the PyTorch edge pipeline. Runs in a python:3 + # container since the build/test images don't include Python. + if config.run_pytorch_tests: + pipeline_steps.append(pipeline_steps.generate_step("Upload PyTorch allowlist validation", + ".buildkite/pipelines/validate_pytorch_allowlist.yml.sh", + soft_fail=True)) + pipeline["env"] = env pipeline["steps"] = pipeline_steps print(json.dumps(pipeline, indent=2)) diff --git a/.buildkite/pipelines/validate_pytorch_allowlist.yml.sh b/.buildkite/pipelines/validate_pytorch_allowlist.yml.sh new file mode 100755 index 000000000..48ea217a1 --- /dev/null +++ b/.buildkite/pipelines/validate_pytorch_allowlist.yml.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0 and the following additional limitation. Functionality enabled by the +# files subject to the Elastic License 2.0 may only be used in production when +# invoked by an Elasticsearch process with a license key installed that permits +# use of machine learning features. You may not use this file except in +# compliance with the Elastic License 2.0 and the foregoing additional +# limitation. + +cat <<'EOL' +steps: + - label: "Validate PyTorch allowlist :torch:" + key: "validate_pytorch_allowlist" + command: + - "cmake -DSOURCE_DIR=$(pwd) -DVALIDATE_CONFIG=$(pwd)/dev-tools/extract_model_ops/validation_models.json -DVALIDATE_PT_DIR=$(pwd)/dev-tools/extract_model_ops/es_it_models -DVALIDATE_VERBOSE=TRUE -P cmake/run-validation.cmake" +EOL + +# Depend on the build steps so validation doesn't start before the +# pipeline is fully generated. +if [ -n "${ML_BUILD_STEP_KEYS:-}" ]; then + echo ' depends_on:' + IFS=',' read -ra STEP_KEYS <<< "$ML_BUILD_STEP_KEYS" + for key in "${STEP_KEYS[@]}"; do + echo " - \"${key}\"" + done +fi + +cat <<'EOL' + allow_dependency_failure: true + soft_fail: true + agents: + image: "python:3" +EOL diff --git a/.buildkite/scripts/steps/run_tests.sh b/.buildkite/scripts/steps/run_tests.sh index 0c5c08125..12b88c1bb 100755 --- a/.buildkite/scripts/steps/run_tests.sh +++ b/.buildkite/scripts/steps/run_tests.sh @@ -105,28 +105,6 @@ else -P cmake/run-all-tests-parallel.cmake || TEST_OUTCOME=$? fi -# --- PyTorch allowlist validation --- -# When triggered from the PyTorch edge pipeline, run the Python-based -# allowlist validation which traces live HuggingFace models with the -# new PyTorch version and verifies every op is in ALLOWED_OPERATIONS. -VALIDATION_OUTCOME=0 -if [[ "${GITHUB_PR_COMMENT_VAR_ACTION:-}" == "run_pytorch_tests" ]] && [ -f cmake/run-validation.cmake ]; then - echo "--- Validating PyTorch allowlist against HuggingFace models" - cmake \ - -DSOURCE_DIR="$(pwd)" \ - -DVALIDATE_CONFIG="$(pwd)/dev-tools/extract_model_ops/validation_models.json" \ - -DVALIDATE_PT_DIR="$(pwd)/dev-tools/extract_model_ops/es_it_models" \ - -DVALIDATE_VERBOSE=TRUE \ - -DOPTIONAL=TRUE \ - -P cmake/run-validation.cmake || VALIDATION_OUTCOME=$? - - if [[ $VALIDATION_OUTCOME -ne 0 ]]; then - echo "^^^ +++" - echo "Allowlist validation failed — the new PyTorch version may introduce ops not in ALLOWED_OPERATIONS." - echo "See dev-tools/extract_model_ops/README.md for how to update the allowlist." - fi -fi - # Upload test results echo "--- Uploading test results" TEST_RESULTS_ARCHIVE=${OS}-${HARDWARE_ARCH}-unit_test_results.tgz @@ -139,6 +117,4 @@ else echo "No test results archive created" fi -if [[ $TEST_OUTCOME -ne 0 || $VALIDATION_OUTCOME -ne 0 ]]; then - exit 1 -fi +exit $TEST_OUTCOME