ci.yml

# Copyright 2022 The IREE Authors
#
# Licensed under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

name: CI

# A few notes:
#
# Variables:
# GitHub actions don't have variables or even support normal yaml anchors (they
# are specially disabled because...reasons?):
# See https://github.com/github-community/community/discussions/4501
# https://github.community/t/support-for-yaml-anchors/16128/92
# https://github.com/actions/runner/issues/1182
# Neither does it have any contexts that are available everywhere. The
# top-level `env` field is available in many places, but not all. We already
# have a "setup" job that every other job depends on, so we leverage that
# for variables that every other job can use, since that *is* available in all
# sub-fields of the job.
# See https://docs.github.com/en/actions/learn-github-actions/contexts#context-availability
# and https://github.com/community/community/discussions/27370
#
# Runner label ordering:
# - self-hosted always has to be listed first in a runs-on block:
# https://docs.github.com/en/actions/hosting-your-own-runners/using-self-hosted-runners-in-a-workflow

on:
  workflow_dispatch:
  pull_request:
  push:
    branches:
      - main

concurrency:
  # A PR number if a pull request and otherwise the commit hash. This cancels
  # queued and in-progress runs for the same PR (presubmit) or commit
  # (postsubmit). The workflow name is prepended to avoid conflicts between
  # different workflows.
  group: ${{ github.workflow }}-${{ github.event.number || github.sha }}
  cancel-in-progress: true

env:
  # This needs to be in env instead of the outputs of setup because it contains
  # the run attempt and we want that to be the current attempt, not whatever
  # attempt the setup step last ran in.
  GCS_DIR: gs://iree-github-actions-${{ github.event_name == 'pull_request' && 'presubmit' || 'postsubmit' }}-artifacts/${{ github.run_id }}/${{ github.run_attempt }}
  GCS_URL: https://storage.googleapis.com/iree-github-actions-${{ github.event_name == 'pull_request' && 'presubmit' || 'postsubmit' }}-artifacts/${{ github.run_id }}/${{ github.run_attempt }}

# Jobs are organized into groups and topologically sorted by dependencies
jobs:
  setup:
    uses: ./.github/workflows/setup.yml

  ################################### Basic ####################################
  # Jobs that build all of IREE "normally"
  ##############################################################################
  build_all:
    needs: setup
    if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'build_all')
    uses: ./.github/workflows/build_all.yml
    with:
      runner-group: ${{ needs.setup.outputs.runner-group }}
      runner-env: ${{ needs.setup.outputs.runner-env }}
      write-caches: ${{ needs.setup.outputs.write-caches }}

  build_test_all_arm64:
    needs: setup
    if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'build_test_all_arm64')
    runs-on:
      - self-hosted # must come first
      - runner-group=${{ needs.setup.outputs.runner-group }}
      - environment=${{ needs.setup.outputs.runner-env }}
      - arm64
      - os-family=Linux
    env:
      BUILD_DIR: build-arm64
    steps:
      - name: "Checking out repository"
        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
        with:
          submodules: true
      - name: "Building IREE"
        env:
          IREE_WRITE_REMOTE_CCACHE: ${{ needs.setup.outputs.write-caches }}
        run: |
          ./build_tools/github_actions/docker_run.sh \
            --env "IREE_CCACHE_GCP_TOKEN=$(gcloud auth application-default print-access-token)" \
            --env "IREE_WRITE_REMOTE_CCACHE=${IREE_WRITE_REMOTE_CCACHE}" \
            --env "CCACHE_NAMESPACE=gcr.io/iree-oss/base-arm64@sha256:9daa1cdbbf12da8527319ece76a64d06219e04ecb99a4cff6e6364235ddf6c59" \
            --env "IREE_BUILD_SETUP_PYTHON_VENV=${BUILD_DIR}/.venv" \
            gcr.io/iree-oss/base-arm64@sha256:9daa1cdbbf12da8527319ece76a64d06219e04ecb99a4cff6e6364235ddf6c59 \
            ./build_tools/cmake/build_all.sh \
            "${BUILD_DIR}"
      - name: "Testing IREE"
        run: |
          ./build_tools/github_actions/docker_run.sh \
            --env "IREE_ARM_SME_QEMU_AARCH64_BIN=/usr/bin/qemu-aarch64" \
            gcr.io/iree-oss/base-arm64@sha256:9daa1cdbbf12da8527319ece76a64d06219e04ecb99a4cff6e6364235ddf6c59 \
            ./build_tools/cmake/ctest_all.sh \
            "${BUILD_DIR}"

  # Disabled since
  #   * windows-2022 is too slow
  #   * windows-2022-64core is too expensive (and takes ~8 minutes just to checkout the repo)
  #   * we don't have self-hosted Windows runners with enough cores yet
  # build_test_all_windows:
  #   needs: setup
  #   if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'build_test_all_windows')
  #   # TODO(saienduri): switch to self-hosted
  #   runs-on: windows-2022
  #   defaults:
  #     run:
  #       shell: bash
  #   env:
  #     BUILD_DIR: build-windows
  #   steps:
  #     - name: "Checking out repository"
  #       uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
  #       with:
  #         submodules: true
  #     - id: "gcp-auth"
  #       name: "Authenticating to Google Cloud"
  #       if: needs.setup.outputs.write-caches == 1
  #       uses: "google-github-actions/auth@v1"
  #       with:
  #         token_format: "access_token"
  #         credentials_json: "${{ secrets.IREE_OSS_GITHUB_RUNNER_BASIC_TRUST_SERVICE_ACCOUNT_KEY }}"
  #         create_credentials_file: false
  #     - name: "Setting up Python"
  #       uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4.5.0
  #       with:
  #         python-version: "3.10" # Needs pybind >= 2.10.1 for Python >= 3.11
  #     - name: "Installing Python packages"
  #       run: |
  #         python3 -m venv .venv
  #         .venv/Scripts/activate.bat
  #         python3 -m pip install -r runtime/bindings/python/iree/runtime/build_requirements.txt
  #     - name: "Installing requirements"
  #       run: choco install ccache --yes
  #     - name: "Configuring MSVC"
  #       uses: ilammy/msvc-dev-cmd@7315a94840631165970262a99c72cfb48a65d25d # v1.12.0
  #     # Finally: build and run tests.
  #     - name: "Building IREE"
  #       env:
  #         IREE_WRITE_REMOTE_CCACHE: ${{ needs.setup.outputs.write-caches }}
  #         IREE_CCACHE_GCP_TOKEN: ${{ steps.gcp-auth.outputs.access_token }}
  #         CCACHE_NAMESPACE: github-windows-2022
  #       run: ./build_tools/cmake/build_all.sh "${BUILD_DIR}"
  #     - name: "Testing IREE"
  #       run: ./build_tools/cmake/ctest_all.sh "${BUILD_DIR}"

  build_test_all_macos_arm64:
    needs: setup
    if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'build_test_all_macos_arm64')
    runs-on:
      - ${{ github.repository == 'iree-org/iree' && 'self-hosted' || 'macos-12' }} # must come first
      - runner-group=postsubmit
      - os-family=macOS
    env:
      BUILD_DIR: build-macos
    steps:
      - name: "Checking out repository"
        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
      - name: "Updating git submodules"
        run: git submodule update --init --jobs 8 --depth 1
      - name: "Installing Python packages"
        run: |
          python3 -m venv .venv
          source .venv/bin/activate
          python3 -m pip install -r runtime/bindings/python/iree/runtime/build_requirements.txt
      - name: "Building IREE"
        env:
          IREE_READ_REMOTE_CCACHE: 0
          IREE_WRITE_REMOTE_CCACHE: 0
          IREE_READ_LOCAL_CCACHE: 1
          IREE_WRITE_LOCAL_CCACHE: ${{ needs.setup.outputs.write-caches }}
          # We'll remove the GITHUB_WORKSPACE directory after the job.
          # Persist the cache by storing in the parent directory.
          CCACHE_DIR: ${{ github.workspace }}/../.ccache
          CCACHE_MAXSIZE: 30G
        run: |
          source .venv/bin/activate
          bash ./build_tools/cmake/build_all.sh "${BUILD_DIR}"
      - name: "Testing IREE"
        run: |
          source .venv/bin/activate
          IREE_METAL_DISABLE=0 bash ./build_tools/cmake/ctest_all.sh "${BUILD_DIR}"

  build_test_all_macos_x86_64:
    needs: setup
    if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'build_test_all_macos_x86_64')
    runs-on: macos-13-xl
    env:
      BUILD_DIR: build-macos
    defaults:
      run:
        shell: bash
    steps:
      - name: "Checking out repository"
        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
      - id: "gcp-auth"
        name: "Authenticating to Google Cloud"
        if: needs.setup.outputs.write-caches == 1
        uses: "google-github-actions/auth@v1"
        with:
          token_format: "access_token"
          credentials_json: "${{ secrets.IREE_OSS_GITHUB_RUNNER_BASIC_TRUST_SERVICE_ACCOUNT_KEY }}"
          create_credentials_file: false
      - name: "Updating git submodules"
        run: git submodule update --init --jobs 8 --depth 1
      # There are multiple versions of Xcode and SDKs installed on the macOS runner.
      # Select the latest Xcode app to enable using Metal offline toolchain.
      - name: "Update Xcode command line tools path"
        run: |
          sudo xcode-select --switch /Applications/Xcode.app/Contents/Developer
          xcrun metal --version
          xcrun metallib --version
      - name: "Setting up Python"
        uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4.5.0
        with:
          python-version: "3.10"
          cache: "pip"
      - name: "Installing Python packages"
        run: pip install -r runtime/bindings/python/iree/runtime/build_requirements.txt
      - name: "Installing requirements"
        # We need coreutils for `realpath` used in scripts.
        # We need bash because the default one on macOS is fairly old and lacks
        # features we use in scripts.
        run: brew install ninja ccache coreutils bash
      # Finally: build and run tests.
      - name: "Building IREE"
        env:
          IREE_WRITE_REMOTE_CCACHE: ${{ needs.setup.outputs.write-caches }}
          IREE_CCACHE_GCP_TOKEN: ${{ steps.gcp-auth.outputs.access_token }}
          CCACHE_NAMESPACE: github-macos-12-xl
        run: bash ./build_tools/cmake/build_all.sh "${BUILD_DIR}"
      - name: "Testing IREE"
        run: bash ./build_tools/cmake/ctest_all.sh "${BUILD_DIR}"

  build_test_all_bazel:
    needs: setup
    if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'build_test_all_bazel')
    runs-on:
      - self-hosted # must come first
      - runner-group=${{ needs.setup.outputs.runner-group }}
      - environment=${{ needs.setup.outputs.runner-env }}
      - cpu
      - os-family=Linux
    steps:
      - name: "Checking out repository"
        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
        with:
          submodules: true
      - name: "Building with Bazel"
        env:
          IREE_WRITE_REMOTE_BAZEL_CACHE: ${{ needs.setup.outputs.write-caches }}
        run: |
          ./build_tools/github_actions/docker_run.sh \
            --env "IREE_WRITE_REMOTE_BAZEL_CACHE=${IREE_WRITE_REMOTE_BAZEL_CACHE}" \
            gcr.io/iree-oss/base-bleeding-edge@sha256:c5f28883e6c570c20128fb37d7af3a00a25df3ce4e2b3a24c3a8dcd183182a27 \
            ./build_tools/bazel/build_test_all.sh

  test_nvidia_gpu:
    needs: [setup, build_all]
    if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'test_nvidia_gpu')
    env:
      BUILD_DIR: build-tests
      INSTALL_DIR: ${{ needs.build_all.outputs.install-dir }}
      INSTALL_DIR_ARCHIVE: ${{ needs.build_all.outputs.install-dir-archive }}
      INSTALL_DIR_GCS_URL: ${{ needs.build_all.outputs.install-dir-gcs-url }}
      IREE_CPU_DISABLE: 1
      IREE_VULKAN_DISABLE: 0
      IREE_CUDA_DISABLE: 0
      IREE_HIP_DISABLE: 1
    runs-on:
      - self-hosted # must come first
      - runner-group=${{ needs.setup.outputs.runner-group }}
      - environment=${{ needs.setup.outputs.runner-env }}
      - gpu
      - os-family=Linux
    steps:
      - name: "Checking out repository"
        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
      - name: "Checking out runtime submodules"
        run: ./build_tools/scripts/git/update_runtime_submodules.sh
      - name: Querying GPU information
        run: |
          ./build_tools/scripts/check_cuda.sh
          ./build_tools/scripts/check_vulkan.sh
      - name: "Downloading install dir archive"
        run: wget "${INSTALL_DIR_GCS_URL}" -O "${INSTALL_DIR_ARCHIVE}"
      - name: "Extracting install directory"
        run: tar -xf "${INSTALL_DIR_ARCHIVE}"
      - name: "Building tests"
        run: |
          ./build_tools/github_actions/docker_run.sh \
              --env IREE_CPU_DISABLE \
              --env IREE_VULKAN_DISABLE \
              --env IREE_CUDA_DISABLE \
              --env IREE_HIP_DISABLE \
              gcr.io/iree-oss/nvidia@sha256:82fa00b5cdda1b35634796cd0f88cb5d6d22d80328b94bfb51e5f2820598ba23 \
              ./build_tools/pkgci/build_tests_using_package.sh ${INSTALL_DIR}
      - name: "Running GPU tests"
        env:
          IREE_CTEST_LABEL_REGEX: ^requires-gpu|^driver=vulkan$|^driver=cuda$
          IREE_NVIDIA_SM80_TESTS_DISABLE: 1
          IREE_MULTI_DEVICE_TESTS_DISABLE: 1
        run: |
          ./build_tools/github_actions/docker_run.sh \
              --env IREE_VULKAN_DISABLE \
              --env IREE_CUDA_DISABLE \
              --env IREE_HIP_DISABLE \
              --env IREE_CTEST_LABEL_REGEX \
              --env IREE_NVIDIA_SM80_TESTS_DISABLE \
              --env IREE_MULTI_DEVICE_TESTS_DISABLE \
              --env IREE_VULKAN_F16_DISABLE=0 \
              --env IREE_NVIDIA_GPU_TESTS_DISABLE=0 \
              --env CTEST_PARALLEL_LEVEL=2 \
              --env NVIDIA_DRIVER_CAPABILITIES=all \
              --gpus all \
              gcr.io/iree-oss/nvidia@sha256:82fa00b5cdda1b35634796cd0f88cb5d6d22d80328b94bfb51e5f2820598ba23 \
              bash -euo pipefail -c \
                "./build_tools/scripts/check_cuda.sh
                ./build_tools/scripts/check_vulkan.sh
                ./build_tools/cmake/ctest_all.sh ${BUILD_DIR}"

  test_nvidia_a100:
    needs: [setup, build_all]
    if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'test_nvidia_a100')
    env:
      BUILD_DIR: build-tests
      INSTALL_DIR: ${{ needs.build_all.outputs.install-dir }}
      INSTALL_DIR_ARCHIVE: ${{ needs.build_all.outputs.install-dir-archive }}
      INSTALL_DIR_GCS_URL: ${{ needs.build_all.outputs.install-dir-gcs-url }}
      IREE_CPU_DISABLE: 1
      IREE_VULKAN_DISABLE: 0
      IREE_CUDA_DISABLE: 0
      IREE_HIP_DISABLE: 1
    runs-on:
      - self-hosted # must come first
      - runner-group=${{ needs.setup.outputs.runner-group }}
      - environment=${{ needs.setup.outputs.runner-env }}
      - a100
      - os-family=Linux
    steps:
      - name: "Checking out repository"
        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
      - name: "Checking out runtime submodules"
        run: ./build_tools/scripts/git/update_runtime_submodules.sh
      - name: Querying GPU information
        run: |
          ./build_tools/scripts/check_cuda.sh
          ./build_tools/scripts/check_vulkan.sh
      - name: "Downloading install dir archive"
        run: wget "${INSTALL_DIR_GCS_URL}" -O "${INSTALL_DIR_ARCHIVE}"
      - name: "Extracting install directory"
        run: tar -xf "${INSTALL_DIR_ARCHIVE}"
      - name: "Building tests"
        run: |
          ./build_tools/github_actions/docker_run.sh \
              --env IREE_CPU_DISABLE \
              --env IREE_VULKAN_DISABLE \
              --env IREE_CUDA_DISABLE \
              --env IREE_HIP_DISABLE \
              gcr.io/iree-oss/nvidia@sha256:82fa00b5cdda1b35634796cd0f88cb5d6d22d80328b94bfb51e5f2820598ba23 \
              ./build_tools/pkgci/build_tests_using_package.sh ${INSTALL_DIR}
      - name: "Running GPU tests"
        env:
          IREE_CTEST_LABEL_REGEX: ^requires-gpu-sm80|^requires-gpu|^driver=vulkan$|^driver=cuda$
          IREE_NVIDIA_SM80_TESTS_DISABLE: 0
          IREE_MULTI_DEVICE_TESTS_DISABLE: 1
        run: |
          ./build_tools/github_actions/docker_run.sh \
              --env IREE_VULKAN_DISABLE \
              --env IREE_CUDA_DISABLE \
              --env IREE_HIP_DISABLE \
              --env IREE_CTEST_LABEL_REGEX \
              --env IREE_NVIDIA_SM80_TESTS_DISABLE \
              --env IREE_MULTI_DEVICE_TESTS_DISABLE \
              --env IREE_VULKAN_F16_DISABLE=0 \
              --env IREE_NVIDIA_GPU_TESTS_DISABLE=0 \
              --env CTEST_PARALLEL_LEVEL=4 \
              --env NVIDIA_DRIVER_CAPABILITIES=all \
              --gpus all \
              gcr.io/iree-oss/nvidia@sha256:82fa00b5cdda1b35634796cd0f88cb5d6d22d80328b94bfb51e5f2820598ba23 \
              bash -euo pipefail -c \
                "./build_tools/scripts/check_cuda.sh
                ./build_tools/scripts/check_vulkan.sh
                ./build_tools/cmake/ctest_all.sh ${BUILD_DIR}"

  ################################## Subsets ###################################
  # Jobs that build some subset of IREE
  ##############################################################################
  build_test_runtime:
    needs: setup
    if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'build_test_runtime')
    runs-on: ubuntu-20.04
    env:
      BUILD_DIR: build-runtime
    steps:
      - name: "Checking out repository"
        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
      - name: "Checking out runtime submodules"
        run: ./build_tools/scripts/git/update_runtime_submodules.sh
      - name: "Building runtime"
        run: |
          ./build_tools/github_actions/docker_run.sh \
            --env "BUILD_PRESET=test" \
            gcr.io/iree-oss/base@sha256:dc314b4fe30fc1315742512891357bffed4d1b62ffcb46258b1e0761c737b446 \
            ./build_tools/cmake/build_runtime.sh \
            "${BUILD_DIR}"
      - name: "Testing runtime"
        run: |
          ./build_tools/github_actions/docker_run.sh \
            gcr.io/iree-oss/base@sha256:dc314b4fe30fc1315742512891357bffed4d1b62ffcb46258b1e0761c737b446 \
            ./build_tools/cmake/ctest_all.sh \
            "${BUILD_DIR}"

  build_test_runtime_arm64:
    needs: setup
    if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'build_test_runtime_arm64')
    runs-on:
      - self-hosted # must come first
      - runner-group=${{ needs.setup.outputs.runner-group }}
      - environment=${{ needs.setup.outputs.runner-env }}
      - arm64
      - os-family=Linux
    env:
      BUILD_DIR: build-runtime-arm64
    steps:
      - name: "Checking out repository"
        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
      - name: "Checking out runtime submodules"
        run: ./build_tools/scripts/git/update_runtime_submodules.sh
      - name: "Building runtime"
        run: |
          ./build_tools/github_actions/docker_run.sh \
            --env "BUILD_PRESET=test" \
            gcr.io/iree-oss/base-arm64@sha256:9daa1cdbbf12da8527319ece76a64d06219e04ecb99a4cff6e6364235ddf6c59 \
            ./build_tools/cmake/build_runtime.sh \
            "${BUILD_DIR}"
      - name: "Testing runtime"
        run: |
          ./build_tools/github_actions/docker_run.sh \
            gcr.io/iree-oss/base-arm64@sha256:9daa1cdbbf12da8527319ece76a64d06219e04ecb99a4cff6e6364235ddf6c59 \
            ./build_tools/cmake/ctest_all.sh \
            "${BUILD_DIR}"

  build_test_runtime_windows:
    needs: setup
    if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'build_test_runtime_windows')
    runs-on: windows-2022
    defaults:
      run:
        shell: bash
    env:
      BUILD_DIR: build-runtime-windows
    steps:
      - name: "Checking out repository"
        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
      - name: "Checking out runtime submodules"
        run: ./build_tools/scripts/git/update_runtime_submodules.sh
      - name: "Configuring MSVC"
        uses: ilammy/msvc-dev-cmd@7315a94840631165970262a99c72cfb48a65d25d # v1.12.0
      - name: "Installing Python requirements"
        run: pip install -r ./runtime/bindings/python/iree/runtime/build_requirements.txt
      - name: "Building runtime"
        env:
          BUILD_PRESET: "test"
        run: ./build_tools/cmake/build_runtime.sh "${BUILD_DIR}"
      - name: "Testing runtime"
        run: ./build_tools/cmake/ctest_all.sh "${BUILD_DIR}"

  ############################### Configurations ###############################
  # Jobs that build IREE in some non-default configuration
  ##############################################################################
  python_release_packages:
    needs: setup
    if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'python_release_packages')
    runs-on:
      - self-hosted # must come first
      - runner-group=${{ needs.setup.outputs.runner-group }}
      - environment=${{ needs.setup.outputs.runner-env }}
      - cpu
      - os-family=Linux
    steps:
      - name: Prefetch Docker
        run: |
          docker pull ghcr.io/nod-ai/manylinux_x86_64:main &
      - name: "Checking out repository"
        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
        with:
          submodules: true
      - name: "Setting up Python"
        uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4.5.0
        with:
          python-version: "3.9"
      - name: Wait for docker pull
        run: |
          wait
      - name: Build runtime wheels (Linux)
        shell: bash
        env:
          packages: "iree-runtime"
          output_dir: "${{ github.workspace }}/bindist"
          # Note when upgrading: Build just one Python version synced to our
          # minimum.
          override_python_versions: cp39-cp39
        run: |
          output_dir="$PWD" ./build_tools/python_deploy/build_linux_packages.sh
      - name: Validate runtime wheel (Linux)
        shell: bash
        run: |
          pip install --upgrade pip
          pip install --no-index -f $PWD -v iree-runtime
          echo "Testing default runtime:"
          python -m iree.runtime._package_test
          echo "Testing tracy runtime:"
          # GH runners don't expose the TSC but we want to make sure the basic packaging
          # works, so override the check with TRACY_NO_INVARIANT_CHECK=1 (per instructions
          # if this is left off).
          TRACY_NO_INVARIANT_CHECK=1 IREE_PY_RUNTIME=tracy \
            python -m iree.runtime._package_test
      # Note that it is just a trade-off decision to have this serialized
      # as a separate step vs parallelized as a separate job. The runtime
      # build is fast, pays the clone/docker overhead and provides early
      # signal (since it runs in just a couple of minutes).
      - name: Build compiler wheels (Linux)
        shell: bash
        env:
          packages: "iree-compiler"
          # Note when upgrading: Build just one Python version synced to our
          # minimum.
          override_python_versions: cp39-cp39
        run: |
          output_dir="$PWD" ./build_tools/python_deploy/build_linux_packages.sh
      - name: Validate compiler wheel (Linux)
        shell: bash
        run: |
          pip install --upgrade pip
          # Pre-fetch optional deps that iree-compiler needs (but we constrain that
          # to not consult a package index).
          pip install onnx>=1.15.0
          pip install --no-index -f $PWD -v iree-compiler[onnx]
          echo "Testing default compiler:"
          python -m iree.compiler._package_test
  asan:
    needs: setup
    if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'asan')
    runs-on:
      - self-hosted # must come first
      - runner-group=${{ needs.setup.outputs.runner-group }}
      - environment=${{ needs.setup.outputs.runner-env }}
      - cpu
      - os-family=Linux
    steps:
      - name: "Checking out repository"
        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
        with:
          submodules: true
      - name: "Building and testing with AddressSanitizer"
        env:
          IREE_WRITE_REMOTE_CCACHE: ${{ needs.setup.outputs.write-caches }}
        run: |
          # Note that this uses the latest version of the clang compiler, etc.
          # This gives us access to the latest features and validates that IREE
          # builds using the latest versions.
          ./build_tools/github_actions/docker_run.sh \
            --env "IREE_CCACHE_GCP_TOKEN=$(gcloud auth application-default print-access-token)" \
            --env "IREE_WRITE_REMOTE_CCACHE=${IREE_WRITE_REMOTE_CCACHE}" \
            --env "CCACHE_NAMESPACE=base-bleeding-edge@sha256:14200dacca3a0f3a66f8aa87c6f64729b83a2eeb403b689c24204074ad157418" \
            gcr.io/iree-oss/base-bleeding-edge@sha256:c5f28883e6c570c20128fb37d7af3a00a25df3ce4e2b3a24c3a8dcd183182a27 \
            ./build_tools/cmake/build_and_test_asan.sh

  tsan:
    needs: setup
    if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'tsan')
    runs-on:
      - self-hosted # must come first
      - runner-group=${{ needs.setup.outputs.runner-group }}
      - environment=${{ needs.setup.outputs.runner-env }}
      - cpu
      - os-family=Linux
    steps:
      - name: "Checking out repository"
        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
        with:
          submodules: true
      - name: "Building and testing with ThreadSanitizer"
        env:
          IREE_WRITE_REMOTE_CCACHE: ${{ needs.setup.outputs.write-caches }}
        run: |
          ./build_tools/github_actions/docker_run.sh \
            --env "IREE_CCACHE_GCP_TOKEN=$(gcloud auth application-default print-access-token)" \
            --env "IREE_WRITE_REMOTE_CCACHE=${IREE_WRITE_REMOTE_CCACHE}" \
            --env "CCACHE_NAMESPACE=gcr.io/iree-oss/base@sha256:dc314b4fe30fc1315742512891357bffed4d1b62ffcb46258b1e0761c737b446" \
            gcr.io/iree-oss/base@sha256:dc314b4fe30fc1315742512891357bffed4d1b62ffcb46258b1e0761c737b446 \
            ./build_tools/cmake/build_and_test_tsan.sh

  small_runtime:
    needs: setup
    if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'small_runtime')
    runs-on: ubuntu-20.04
    env:
      BUILD_DIR: build-runtime
    steps:
      - name: "Checking out repository"
        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
      - name: "Checking out runtime submodules"
        run: ./build_tools/scripts/git/update_runtime_submodules.sh
      - name: "Building size-optimized runtime"
        run: |
          ./build_tools/github_actions/docker_run.sh \
            gcr.io/iree-oss/base@sha256:dc314b4fe30fc1315742512891357bffed4d1b62ffcb46258b1e0761c737b446 \
            ./build_tools/cmake/build_runtime_small.sh \
            "${BUILD_DIR}"
      - name: "Testing runtime"
        run: |
          ./build_tools/github_actions/docker_run.sh \
            gcr.io/iree-oss/base@sha256:dc314b4fe30fc1315742512891357bffed4d1b62ffcb46258b1e0761c737b446 \
            ./build_tools/cmake/ctest_all.sh \
            "${BUILD_DIR}"

  gcc:
    needs: setup
    if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'gcc')
    runs-on:
      - self-hosted # must come first
      - runner-group=${{ needs.setup.outputs.runner-group }}
      - environment=${{ needs.setup.outputs.runner-env }}
      - cpu
      - os-family=Linux
    env:
      BUILD_DIR: build-gcc
    steps:
      - name: "Checking out repository"
        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
        with:
          submodules: true
      - name: "Building IREE with gcc"
        env:
          IREE_WRITE_REMOTE_CCACHE: ${{ needs.setup.outputs.write-caches }}
        run: |
          ./build_tools/github_actions/docker_run.sh \
            --env CC=/usr/bin/gcc-9 \
            --env CXX=/usr/bin/g++-9 \
            --env CMAKE_BUILD_TYPE=Release \
            --env "IREE_TARGET_BACKEND_WEBGPU_SPIRV=OFF" \
            --env "IREE_CCACHE_GCP_TOKEN=$(gcloud auth application-default print-access-token)" \
            --env "IREE_WRITE_REMOTE_CCACHE=${IREE_WRITE_REMOTE_CCACHE}" \
            --env "CCACHE_NAMESPACE=gcr.io/iree-oss/base@sha256:dc314b4fe30fc1315742512891357bffed4d1b62ffcb46258b1e0761c737b446" \
            --env "IREE_BUILD_SETUP_PYTHON_VENV=${BUILD_DIR}/.venv" \
            gcr.io/iree-oss/base@sha256:dc314b4fe30fc1315742512891357bffed4d1b62ffcb46258b1e0761c737b446 \
            ./build_tools/cmake/build_all.sh \
            "${BUILD_DIR}"

  tracing:
    needs: setup
    if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'tracing')
    runs-on:
      - self-hosted # must come first
      - runner-group=${{ needs.setup.outputs.runner-group }}
      - environment=${{ needs.setup.outputs.runner-env }}
      - cpu
      - os-family=Linux
    env:
      BUILD_DIR: build-tracing
    steps:
      - name: "Checking out repository"
        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
        with:
          submodules: true
      - name: "Building IREE with tracing enabled"
        env:
          IREE_WRITE_REMOTE_CCACHE: ${{ needs.setup.outputs.write-caches }}
        run: |
          # TODO(#11394): Enable Web GPU
          ./build_tools/github_actions/docker_run.sh \
            --env "IREE_CCACHE_GCP_TOKEN=$(gcloud auth application-default print-access-token)" \
            --env "IREE_WRITE_REMOTE_CCACHE=${IREE_WRITE_REMOTE_CCACHE}" \
            --env "CCACHE_NAMESPACE=gcr.io/iree-oss/base@sha256:dc314b4fe30fc1315742512891357bffed4d1b62ffcb46258b1e0761c737b446" \
            gcr.io/iree-oss/base@sha256:dc314b4fe30fc1315742512891357bffed4d1b62ffcb46258b1e0761c737b446 \
            ./build_tools/cmake/build_tracing.sh \
            "${BUILD_DIR}"

  debug:
    needs: setup
    if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'debug')
    runs-on:
      - self-hosted # must come first
      - runner-group=${{ needs.setup.outputs.runner-group }}
      - environment=${{ needs.setup.outputs.runner-env }}
      - cpu
      - os-family=Linux
    env:
      BUILD_DIR: build-debug
    steps:
      - name: "Checking out repository"
        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
        with:
          submodules: true
      - name: "Building IREE in Debug configuration"
        env:
          IREE_WRITE_REMOTE_CCACHE: ${{ needs.setup.outputs.write-caches }}
        run: |
          ./build_tools/github_actions/docker_run.sh \
            --env "IREE_CCACHE_GCP_TOKEN=$(gcloud auth application-default print-access-token)" \
            --env "IREE_WRITE_REMOTE_CCACHE=${IREE_WRITE_REMOTE_CCACHE}" \
            --env "CMAKE_BUILD_TYPE=Debug" \
            --env "CCACHE_NAMESPACE=gcr.io/iree-oss/base@sha256:dc314b4fe30fc1315742512891357bffed4d1b62ffcb46258b1e0761c737b446" \
            --env "IREE_BUILD_SETUP_PYTHON_VENV=${BUILD_DIR}/.venv" \
            gcr.io/iree-oss/base@sha256:dc314b4fe30fc1315742512891357bffed4d1b62ffcb46258b1e0761c737b446 \
            ./build_tools/cmake/build_all.sh \
            "${BUILD_DIR}"

  byo_llvm:
    needs: setup
    if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'byo_llvm')
    runs-on:
      - self-hosted # must come first
      - runner-group=${{ needs.setup.outputs.runner-group }}
      - environment=${{ needs.setup.outputs.runner-env }}
      - cpu
      - os-family=Linux
    steps:
      - name: "Checking out repository"
        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
        with:
          submodules: true
      - name: "Building and testing with bring-your-own-LLVM"
        env:
          IREE_WRITE_REMOTE_CCACHE: ${{ needs.setup.outputs.write-caches }}
        run: |
          ./build_tools/github_actions/docker_run.sh \
            --env "IREE_CCACHE_GCP_TOKEN=$(gcloud auth application-default print-access-token)" \
            --env "IREE_WRITE_REMOTE_CCACHE=${IREE_WRITE_REMOTE_CCACHE}" \
            --env "CCACHE_NAMESPACE=gcr.io/iree-oss/base@sha256:dc314b4fe30fc1315742512891357bffed4d1b62ffcb46258b1e0761c737b446" \
            gcr.io/iree-oss/base@sha256:dc314b4fe30fc1315742512891357bffed4d1b62ffcb46258b1e0761c737b446 \
            ./build_tools/cmake/build_and_test_byo_llvm.sh

  ############################## Crosscompilation ##############################
  # Jobs that cross-compile IREE for other platforms
  ##############################################################################

  cross_compile_and_test:
    needs: [setup, build_all]
    if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'cross_compile_and_test')
    runs-on:
      - self-hosted # must come first
      - runner-group=${{ needs.setup.outputs.runner-group }}
      - environment=${{ needs.setup.outputs.runner-env }}
      - cpu
      - os-family=Linux
    strategy:
      matrix:
        target:
          - platform: linux
            arch: riscv_64
            abi: lp64d
            docker_image: "gcr.io/iree-oss/riscv@sha256:62e87bad3405d691ddba6f9be0ef44eeb60461a467c8d86f0842c81a1f97da79"
            build_script: "./build_tools/cmake/build_riscv.sh"
            test_script: "./build_tools/cmake/test_riscv.sh"
          - platform: linux
            arch: riscv_32
            abi: ilp32d
            docker_image: "gcr.io/iree-oss/riscv@sha256:62e87bad3405d691ddba6f9be0ef44eeb60461a467c8d86f0842c81a1f97da79"
            build_script: "./build_tools/cmake/build_riscv.sh"
            test_script: "./build_tools/cmake/test_riscv.sh"
          - platform: generic
            arch: riscv_32
            abi: ilp32
            docker_image: "gcr.io/iree-oss/riscv@sha256:62e87bad3405d691ddba6f9be0ef44eeb60461a467c8d86f0842c81a1f97da79"
            build_script: "./build_tools/cmake/build_riscv.sh"
            test_script: "./tests/riscv32/smoke.sh"
          - platform: emscripten
            arch: wasm32
            abi: wasm32
            docker_image: "gcr.io/iree-oss/emscripten@sha256:2dd4c52f1bb499ab365aad0111fe5538b685d88af38636b409b0cf6a576ab214"
            build_script: "./build_tools/cmake/build_runtime_emscripten.sh"
            # No test script

    env:
      PLATFORM: ${{ matrix.target.platform }}
      ARCH: ${{ matrix.target.arch }}
      ABI: ${{ matrix.target.abi }}
      DOCKER_IMAGE: ${{ matrix.target.docker_image }}
      BUILD_SCRIPT: ${{ matrix.target.build_script }}
      TEST_SCRIPT: ${{ matrix.target.test_script }}
      INSTALL_DIR: ${{ needs.build_all.outputs.install-dir }}
      INSTALL_DIR_ARCHIVE: ${{ needs.build_all.outputs.install-dir-archive }}
      INSTALL_DIR_GCS_ARTIFACT: ${{ needs.build_all.outputs.install-dir-gcs-artifact }}
      TARGET_BUILD_DIR: build-${{ matrix.target.platform }}-${{ matrix.target.arch }}
      IREE_WRITE_REMOTE_CCACHE: ${{ needs.setup.outputs.write-caches }}
    steps:
      - name: "Checking out repository"
        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
      - name: "Checking out runtime submodules"
        run: ./build_tools/scripts/git/update_runtime_submodules.sh
      - name: "Downloading install dir archive"
        run: gcloud storage cp "${INSTALL_DIR_GCS_ARTIFACT}" "${INSTALL_DIR_ARCHIVE}"
      - name: "Extracting install directory"
        run: tar -xf "${INSTALL_DIR_ARCHIVE}"
      - name: "Build cross-compiling target"
        run: |
          ./build_tools/github_actions/docker_run.sh \
            --env "IREE_CCACHE_GCP_TOKEN=$(gcloud auth application-default print-access-token)" \
            --env "IREE_WRITE_REMOTE_CCACHE=${IREE_WRITE_REMOTE_CCACHE}" \
            --env "CCACHE_NAMESPACE=${DOCKER_IMAGE}" \
            --env "IREE_TARGET_PLATFORM=${PLATFORM}" \
            --env "IREE_TARGET_ARCH=${ARCH}" \
            --env "IREE_TARGET_ABI=${ABI}" \
            --env "IREE_TARGET_BUILD_DIR=${TARGET_BUILD_DIR}" \
            --env "BUILD_PRESET=test" \
            --env "IREE_HOST_BIN_DIR=${INSTALL_DIR}/bin" \
            "${DOCKER_IMAGE}" \
            "${BUILD_SCRIPT}"
      - name: "Test cross-compiling target"
        if: matrix.target.test_script
        run: |
          ./build_tools/github_actions/docker_run.sh \
            --env "IREE_TARGET_PLATFORM=${PLATFORM}" \
            --env "IREE_TARGET_ARCH=${ARCH}" \
            --env "IREE_TARGET_BUILD_DIR=${TARGET_BUILD_DIR}" \
            --env "BUILD_PRESET=test" \
            "${DOCKER_IMAGE}" \
            "${TEST_SCRIPT}"

  # Android cross-compilation and test is separated from cross_compile_and_test
  # because some tests need to run on physical devices instead of emulators. And
  # we need a fine-control on which tests only run on postsubmit as some devices
  # are not scalable (while we want to test with Android emulator on ARM VMs on
  # presubmit), which requires dynamic matrix generation (because "if" condition
  # can't access matrix variable to skip certain matrix jobs for presubmit).
  build_and_test_android:
    needs: [setup, build_all]
    if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'build_and_test_android')
    uses: ./.github/workflows/build_and_test_android.yml
    with:
      runner-group: ${{ needs.setup.outputs.runner-group }}
      runner-env: ${{ needs.setup.outputs.runner-env }}
      write-caches: ${{ needs.setup.outputs.write-caches }}
      is-pr: ${{ fromJson(needs.setup.outputs.is-pr) }}
      install-dir: ${{ needs.build_all.outputs.install-dir }}
      install-dir-archive: ${{ needs.build_all.outputs.install-dir-archive }}
      install-dir-gcs-artifact: ${{ needs.build_all.outputs.install-dir-gcs-artifact }}

  ############################### Configurations ###############################
  # Jobs that build and run IREE e2e tests/benchmarks                          #
  ##############################################################################

  build_benchmark_tools:
    needs: [setup, build_all]
    if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'build_benchmark_tools') && needs.setup.outputs.benchmark-presets != ''
    uses: ./.github/workflows/build_benchmark_tools.yml
    with:
      runner-group: ${{ needs.setup.outputs.runner-group }}
      runner-env: ${{ needs.setup.outputs.runner-env }}
      install-dir: ${{ needs.build_all.outputs.install-dir }}
      install-dir-archive: ${{ needs.build_all.outputs.install-dir-archive }}
      install-dir-gcs-artifact: ${{ needs.build_all.outputs.install-dir-gcs-artifact }}

  build_e2e_test_artifacts:
    needs: [setup, build_all]
    if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'build_e2e_test_artifacts') && needs.setup.outputs.benchmark-presets != ''
    uses: ./.github/workflows/build_e2e_test_artifacts.yml
    with:
      runner-group: ${{ needs.setup.outputs.runner-group }}
      runner-env: ${{ needs.setup.outputs.runner-env }}
      install-dir: ${{ needs.build_all.outputs.install-dir }}
      install-dir-archive: ${{ needs.build_all.outputs.install-dir-archive }}
      install-dir-gcs-artifact: ${{ needs.build_all.outputs.install-dir-gcs-artifact }}
      benchmark-presets: ${{ needs.setup.outputs.benchmark-presets }}
      shard-count: "c2-standard-60=2,default=1"

  test_benchmark_suites:
    needs: [setup, build_all, build_e2e_test_artifacts]
    if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'test_benchmark_suites') && needs.setup.outputs.benchmark-presets != ''
    strategy:
      matrix:
        target:
          - platform: linux
            arch: riscv_64
            docker_image: "gcr.io/iree-oss/riscv@sha256:62e87bad3405d691ddba6f9be0ef44eeb60461a467c8d86f0842c81a1f97da79"
            run_scripts: "./build_tools/cmake/build_riscv.sh && ./build_tools/cmake/test_riscv.sh"
          - platform: linux
            arch: riscv_32
            docker_image: "gcr.io/iree-oss/riscv@sha256:62e87bad3405d691ddba6f9be0ef44eeb60461a467c8d86f0842c81a1f97da79"
            run_scripts: "./build_tools/cmake/build_riscv.sh && ./build_tools/cmake/test_riscv.sh"
          - platform: linux
            arch: x86_64
            docker_image: "gcr.io/iree-oss/base@sha256:dc314b4fe30fc1315742512891357bffed4d1b62ffcb46258b1e0761c737b446"
            run_scripts: "./build_tools/cmake/test_benchmark_suites_on_linux.sh"
            # Requires Intel CascadeLake CPU.
            host_machine: c2s601t
    runs-on:
      - self-hosted # must come first
      - runner-group=${{ needs.setup.outputs.runner-group }}
      - environment=${{ needs.setup.outputs.runner-env }}
      - ${{ matrix.target.host_machine || 'cpu' }} # Default to generic x86_64 VM.
    env:
      PLATFORM: ${{ matrix.target.platform }}
      ARCH: ${{ matrix.target.arch }}
      DOCKER_IMAGE: ${{ matrix.target.docker_image }}
      RUN_SCRIPTS: ${{ matrix.target.run_scripts }}
      INSTALL_DIR: ${{ needs.build_all.outputs.install-dir }}
      INSTALL_DIR_ARCHIVE: ${{ needs.build_all.outputs.install-dir-archive }}
      INSTALL_DIR_GCS_ARTIFACT: ${{ needs.build_all.outputs.install-dir-gcs-artifact }}
      TARGET_BUILD_DIR: build-${{ matrix.target.platform }}-${{ matrix.target.arch }}
      E2E_TEST_ARTIFACTS_DIR: ${{ needs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-dir }}
      E2E_TEST_ARTIFACTS_GCS_ARTIFACT_DIR: ${{ needs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-gcs-artifact-dir }}
    steps:
      - name: "Checking out repository"
        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
      - name: "Checking out runtime submodules"
        run: ./build_tools/scripts/git/update_runtime_submodules.sh
      - name: "Downloading install dir archive"
        run: gcloud storage cp "${INSTALL_DIR_GCS_ARTIFACT}" "${INSTALL_DIR_ARCHIVE}"
      - name: "Extracting install directory"
        run: tar -xf "${INSTALL_DIR_ARCHIVE}"
      # TODO(#11136): Only download the needed artifacts instead of everything.
      - name: "Downloading e2e test artifacts"
        run: |
          mkdir -p ${E2E_TEST_ARTIFACTS_DIR}
          gcloud storage cp -r "${E2E_TEST_ARTIFACTS_GCS_ARTIFACT_DIR}/*" "${E2E_TEST_ARTIFACTS_DIR}"
      - name: "Build iree-run-module and test benchmark suite modules"
        run: |
          ./build_tools/github_actions/docker_run.sh \
            --env "IREE_TARGET_PLATFORM=${PLATFORM}" \
            --env "IREE_TARGET_ARCH=${ARCH}" \
            --env "IREE_TARGET_BUILD_DIR=${TARGET_BUILD_DIR}" \
            --env "BUILD_PRESET=benchmark-suite-test" \
            --env "IREE_HOST_BIN_DIR=${INSTALL_DIR}/bin" \
            --env "E2E_TEST_ARTIFACTS_DIR=${E2E_TEST_ARTIFACTS_DIR}" \
            "${DOCKER_IMAGE}" \
            bash -euo pipefail -c \
              "${RUN_SCRIPTS}"

  compilation_benchmarks:
    needs: [setup, build_e2e_test_artifacts]
    if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'compilation_benchmarks') && needs.setup.outputs.benchmark-presets != ''
    uses: ./.github/workflows/benchmark_compilation.yml
    with:
      runner-group: ${{ needs.setup.outputs.runner-group }}
      runner-env: ${{ needs.setup.outputs.runner-env }}
      e2e-test-artifacts-dir: ${{ needs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-dir }}
      e2e-test-artifacts-gcs-artifact-dir: ${{ needs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-gcs-artifact-dir }}
      e2e-test-artifacts-build-log: ${{ needs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-build-log }}
      e2e-test-artifacts-build-log-gcs-artifact: ${{ needs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-build-log-gcs-artifact }}

  execution_benchmarks:
    needs: [setup, build_benchmark_tools, build_e2e_test_artifacts]
    if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'execution_benchmarks') && needs.setup.outputs.benchmark-presets != ''
    uses: ./.github/workflows/benchmark_execution.yml
    with:
      # env.GCS_DIR is also duplicated in this workflow. See the note there on
      # why this is.
      runner-group: ${{ needs.setup.outputs.runner-group }}
      runner-env: ${{ needs.setup.outputs.runner-env }}
      e2e-test-artifacts-dir: ${{ needs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-dir }}
      e2e-test-artifacts-gcs-artifact-dir: ${{ needs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-gcs-artifact-dir }}
      benchmark-tools-gcs-artifact-dir: ${{ needs.build_benchmark_tools.outputs.benchmark-tools-gcs-artifact-dir }}

  process_benchmark_results:
    needs: [setup, compilation_benchmarks, execution_benchmarks]
    if: contains(fromJson(needs.setup.outputs.enabled-jobs), 'process_benchmark_results') && needs.setup.outputs.benchmark-presets != ''
    runs-on:
      - self-hosted # must come first
      - runner-group=${{ needs.setup.outputs.runner-group }}
      - environment=${{ needs.setup.outputs.runner-env }}
      - cpu
      - os-family=Linux
    env:
      COMPILE_STATS_RESULTS: ${{ needs.compilation_benchmarks.outputs.compile-stats-results }}
      COMPILE_STATS_RESULTS_GCS_ARTIFACT: ${{ needs.compilation_benchmarks.outputs.compile-stats-results-gcs-artifact }}
      # Empty if no execution benchmark runs.
      EXECUTION_BENCHMARK_RESULTS_DIR: ${{ needs.execution_benchmarks.outputs.benchmark-results-dir }}
      # Empty if no execution benchmark runs.
      EXECUTION_BENCHMARK_RESULTS_GCS_ARTIFACT_DIR: ${{ needs.execution_benchmarks.outputs.benchmark-results-gcs-artifact-dir }}
    steps:
      - name: "Checking out repository"
        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
        with:
          # We need the full history (and main branch) to generate the report.
          fetch-depth: 0
      - name: Downloading compilation benchmark results
        run: |
          gcloud storage cp \
            "${COMPILE_STATS_RESULTS_GCS_ARTIFACT}" \
            "${COMPILE_STATS_RESULTS}"
      - name: Downloading execution benchmark results
        id: download-execution-results
        # Skip the download if there is no execution benchmark results (e.g. no
        # benchmark matches the preset/filter). In such case, no benchmark job
        # is run in benchmark_execution.yml and the output variables are empty.
        if: env.EXECUTION_BENCHMARK_RESULTS_GCS_ARTIFACT_DIR != ''
        run: |
          gcloud storage cp -r \
            "${EXECUTION_BENCHMARK_RESULTS_GCS_ARTIFACT_DIR}/benchmark-results-*.json" \
            "${EXECUTION_BENCHMARK_RESULTS_DIR}"
          echo "execution-benchmark-results-pattern=${EXECUTION_BENCHMARK_RESULTS_DIR}/benchmark-results-*.json" >> "${GITHUB_OUTPUT}"
      - name: Generating comment
        if: fromJson(needs.setup.outputs.is-pr)
        id: generate-comment
        env:
          # Wildcard pattern to match all execution benchmark results. Empty if
          # execution_benchmarks is skipped, which results in no match.
          EXECUTION_BENCHMARK_RESULTS_PATTERN: ${{ steps.download-execution-results.outputs.execution-benchmark-results-pattern }}
          IREE_BUILD_URL: https://github.com/iree-org/iree/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}
          PR_NUMBER: ${{ github.event.pull_request.number }}
          BENCHMARK_COMMENT_ARTIFACT: benchmark-comment.json
        run: |
          build_tools/github_actions/docker_run.sh \
            gcr.io/iree-oss/benchmark-report@sha256:7498c6f32f63f13faf085463cc38656d4297519c824e63e1c99c8c258147f6ff \
            ./build_tools/benchmarks/generate_benchmark_comment.py \
              --verbose \
              --pr_number="${PR_NUMBER}" \
              --pr_committish="${GITHUB_SHA}" \
              --pr_base_branch="origin/${GITHUB_BASE_REF}" \
              --comment_type="benchmark-summary" \
              --build_url="${IREE_BUILD_URL}" \
              --benchmark_files="${EXECUTION_BENCHMARK_RESULTS_PATTERN}" \
              --compile_stats_files="${COMPILE_STATS_RESULTS}" \
              --output="${BENCHMARK_COMMENT_ARTIFACT}"
          echo "benchmark-comment-artifact=${BENCHMARK_COMMENT_ARTIFACT}" >> "${GITHUB_OUTPUT}"
      - name: Uploading comment artifact
        # Due to security reasons, instead of posting the comment to PR, we only
        # upload the comment data in presubmit workflow and trigger the posting
        # workflow on the main branch. See post_benchmark_comment.yaml
        if: fromJson(needs.setup.outputs.is-pr)
        env:
          BENCHMARK_COMMENT_ARTIFACT: ${{ steps.generate-comment.outputs.benchmark-comment-artifact }}
          BENCHMARK_COMMENT_GCS_ARTIFACT: ${{ env.GCS_DIR }}/${{ steps.generate-comment.outputs.benchmark-comment-artifact }}
        run: |
          gcloud storage cp \
            "${BENCHMARK_COMMENT_ARTIFACT}" \
            "${BENCHMARK_COMMENT_GCS_ARTIFACT}"
      - name: Uploading results to dashboard
        if: github.ref_name == 'main'
        env:
          EXECUTION_BENCHMARK_RESULTS_PATTERN: ${{ steps.download-execution-results.outputs.execution-benchmark-results-pattern }}
          IREE_DASHBOARD_API_TOKEN: ${{ secrets.IREE_DASHBOARD_API_TOKEN }}
        run: |
          build_tools/github_actions/docker_run.sh \
            --env "IREE_DASHBOARD_API_TOKEN=${IREE_DASHBOARD_API_TOKEN}" \
            gcr.io/iree-oss/benchmark-report@sha256:7498c6f32f63f13faf085463cc38656d4297519c824e63e1c99c8c258147f6ff \
            ./build_tools/benchmarks/upload_benchmarks_to_dashboard.py \
              --verbose \
              --benchmark_files="${EXECUTION_BENCHMARK_RESULTS_PATTERN}" \
              --compile_stats_files="${COMPILE_STATS_RESULTS}"

  ##############################################################################

  # Depends on all the other jobs to provide a single anchor that indicates the
  # final status. Status reporting will become more sophisticated in the future
  # and we can hopefully avoid the need to explicitly list every single job...
  summary:
    # Even if you have an explicit if condition, you still need to override
    # GitHub's default behavior of not running if any dependencies failed.
    if: always()
    runs-on: ubuntu-20.04
    needs:
      - setup

      # Basic
      - build_all
      - build_test_all_bazel

      # Platforms
      - build_test_all_arm64
      # - build_test_all_windows
      - build_test_all_macos_arm64
      - build_test_all_macos_x86_64

      # Accelerators
      - test_nvidia_gpu
      - test_nvidia_a100

      # Subsets
      - build_test_runtime
      - build_test_runtime_arm64
      - build_test_runtime_windows

      # Configurations
      - python_release_packages
      - asan
      - tsan
      - small_runtime
      - gcc
      - tracing
      - debug
      - byo_llvm

      # Crosscompilation
      - cross_compile_and_test
      - build_and_test_android

      # Benchmark pipeline
      - build_benchmark_tools
      - build_e2e_test_artifacts
      - test_benchmark_suites
      - compilation_benchmarks
      - execution_benchmarks
      - process_benchmark_results
    steps:
      - name: "Checking out repository"
        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
      - name: Getting failed jobs
        id: failed_jobs
        run: |
          echo '${{ toJson(needs) }}'
          FAILED_JOBS="$(echo '${{ toJson(needs) }}' \
            | jq --raw-output \
            'map_values(select(.result!="success" and .result!="skipped")) | keys | join(",")' \
          )"
          echo "failed-jobs=${FAILED_JOBS}" >> $GITHUB_OUTPUT
          if [[ "${FAILED_JOBS}" != "" ]]; then
            echo "The following jobs failed: ${FAILED_JOBS}"
            exit 1
          fi
      - name: Show useful artifact links
        if: always()
        env:
          # If the job of an artifact is skipped or failed, we show "NOT_PRESENT".
          INSTALL_DIR_GCS_ARTIFACT: ${{ needs.build_all.outputs.install-dir-gcs-artifact || 'NOT_PRESENT' }}
          E2E_TEST_ARTIFACTS_GCS_ARTIFACT_DIR: ${{ needs.build_e2e_test_artifacts.outputs.e2e-test-artifacts-gcs-artifact-dir || 'NOT_PRESENT' }}
          BENCHMARK_TOOLS_GCS_ARTIFACT_DIR: ${{ needs.build_benchmark_tools.outputs.benchmark-tools-gcs-artifact-dir || 'NOT_PRESENT' }}
          EXECUTION_BENCHMARK_RESULTS_GCS_ARTIFACT_DIR: ${{ needs.execution_benchmarks.outputs.benchmark-results-gcs-artifact-dir || 'NOT_PRESENT' }}
          COMPILATION_BENCHMARK_RESULTS_GCS_ARTIFACT: ${{ needs.compilation_benchmarks.outputs.compile-stats-results-gcs-artifact || 'NOT_PRESENT' }}
        run: |
          envsubst < ./.github/workflows/ARTIFACT_SUMMARY_TEMPLATE.md >> "${GITHUB_STEP_SUMMARY}"
      - name: Posting to Discord
        uses: sarisia/actions-status-discord@61114b793b460ee85fe38ad3fccc78c7ead38d55 # v1.11.1
        if: failure() && github.ref_name == 'main'
        with:
          webhook: ${{ secrets.DISCORD_WEBHOOK }}
          description: "The following jobs failed: ${{ steps.failed_jobs.outputs.failed-jobs }}"
          url: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}"