Merge branch 'main' into cmsis-pack-infrastructure

Author: MatthiasHertelArm

.ci/docker/build.sh

@@ -81,9 +81,19 @@ case "${IMAGE_NAME}" in
     LINTRUNNER=""
     GCC_VERSION=11
     CUDA_WINDOWS_CROSS_COMPILE=yes
-    CUDA_VERSION=12.8
+    CUDA_VERSION=13.0
     SKIP_PYTORCH=yes
     ;;
+  executorch-ubuntu-24.04-gcc14)
+    LINTRUNNER=""
+    OS_VERSION=24.04
+    GCC_VERSION=14
+    ;;
+  executorch-ubuntu-26.04-gcc15)
+    LINTRUNNER=""
+    OS_VERSION=26.04
+    GCC_VERSION=15
+    ;;
   *)
     echo "Invalid image name ${IMAGE_NAME}"
     exit 1
@@ -92,17 +102,9 @@ esac
 TORCH_VERSION=$(cat ci_commit_pins/pytorch.txt)
 BUILD_DOCS=1
 
-# Pull channel + spec/url helpers out of torch_pin.py so install_pytorch.sh
-# (which runs inside the docker build, where torch_pin.py isn't available)
-# can decide between wheel install (test/release) and source build (nightly).
-# Self-hosted runners often have python3 but not the unversioned python alias.
-PYTHON_BIN=$(command -v python3 || command -v python)
-TORCH_PIN_HELPERS=$(cd ../.. && "$PYTHON_BIN" -c "from torch_pin import CHANNEL, torch_spec, torchaudio_spec, torchvision_spec, torch_index_url_base; print(CHANNEL); print(torch_spec()); print(torchaudio_spec()); print(torchvision_spec()); print(torch_index_url_base())")
-TORCH_CHANNEL=$(echo "${TORCH_PIN_HELPERS}" | sed -n '1p')
-TORCH_SPEC=$(echo "${TORCH_PIN_HELPERS}" | sed -n '2p')
-TORCHAUDIO_SPEC=$(echo "${TORCH_PIN_HELPERS}" | sed -n '3p')
-TORCHVISION_SPEC=$(echo "${TORCH_PIN_HELPERS}" | sed -n '4p')
-TORCH_INDEX_URL=$(echo "${TORCH_PIN_HELPERS}" | sed -n '5p')
+if [[ -n "${GCC_VERSION:-}" && -z "${SKIP_PYTORCH:-}" ]]; then
+  PYTORCH_BUILD_MAX_JOBS=6
+fi
 
 # Copy requirements-lintrunner.txt from root to here
 cp ../../requirements-lintrunner.txt ./
@@ -116,11 +118,7 @@ docker build \
   --build-arg "PYTHON_VERSION=${PYTHON_VERSION}" \
   --build-arg "MINICONDA_VERSION=${MINICONDA_VERSION}" \
   --build-arg "TORCH_VERSION=${TORCH_VERSION}" \
-  --build-arg "TORCH_CHANNEL=${TORCH_CHANNEL}" \
-  --build-arg "TORCH_SPEC=${TORCH_SPEC}" \
-  --build-arg "TORCHAUDIO_SPEC=${TORCHAUDIO_SPEC}" \
-  --build-arg "TORCHVISION_SPEC=${TORCHVISION_SPEC}" \
-  --build-arg "TORCH_INDEX_URL=${TORCH_INDEX_URL}" \
+  --build-arg "PYTORCH_BUILD_MAX_JOBS=${PYTORCH_BUILD_MAX_JOBS:-}" \
   --build-arg "BUCK2_VERSION=${BUCK2_VERSION}" \
   --build-arg "LINTRUNNER=${LINTRUNNER:-}" \
   --build-arg "BUILD_DOCS=${BUILD_DOCS}" \

.ci/docker/ci_commit_pins/pytorch.txt

@@ -1 +1 @@
-release/2.11
+release/2.12

.ci/docker/common/install_android.sh

@@ -43,10 +43,10 @@ install_ndk() {
   ARCH=$(uname -m)
   if [ "${ARCH}" = "aarch64" ]; then
     # aarch64 NDK is not cached on S3, download from Google directly
-    curl -Os --retry 3 "https://dl.google.com/android/repository/android-ndk-${ANDROID_NDK_VERSION}-linux.zip"
+    curl -Os --retry 3 --retry-all-errors "https://dl.google.com/android/repository/android-ndk-${ANDROID_NDK_VERSION}-linux.zip"
   else
     # The NDK installation is cached on ossci-android S3 bucket
-    curl -Os --retry 3 "https://ossci-android.s3.amazonaws.com/android-ndk-${ANDROID_NDK_VERSION}-linux.zip"
+    curl -Os --retry 3 --retry-all-errors "https://ossci-android.s3.amazonaws.com/android-ndk-${ANDROID_NDK_VERSION}-linux.zip"
   fi
   unzip -qo "android-ndk-${ANDROID_NDK_VERSION}-linux.zip"
 
@@ -62,7 +62,7 @@ install_cmdtools() {
 
   pushd /tmp
   # The file is cached on ossci-android S3 bucket
-  curl -Os --retry 3 "https://ossci-android.s3.us-west-1.amazonaws.com/${CMDTOOLS_FILENAME}"
+  curl -Os --retry 3 --retry-all-errors "https://ossci-android.s3.us-west-1.amazonaws.com/${CMDTOOLS_FILENAME}"
   unzip -qo "${CMDTOOLS_FILENAME}" -d /opt
 
   ls -lah /opt/cmdline-tools/bin

.ci/docker/common/install_cache.sh

@@ -34,7 +34,7 @@ install_ubuntu() {
 
 install_binary() {
   echo "Downloading sccache binary from S3 repo"
-  curl --retry 3 https://s3.amazonaws.com/ossci-linux/sccache -o /opt/cache/bin/sccache
+  curl --retry 3 --retry-all-errors https://s3.amazonaws.com/ossci-linux/sccache -o /opt/cache/bin/sccache
   chmod +x /opt/cache/bin/sccache
 }
 
@@ -76,6 +76,9 @@ init_sccache() {
   # This is the remote cache bucket
   export SCCACHE_BUCKET=ossci-compiler-cache-circleci-v2
   export SCCACHE_S3_KEY_PREFIX=executorch
+  export SCCACHE_REGION=us-east-1
+  export AWS_REGION=us-east-1
+  export AWS_DEFAULT_REGION=us-east-1
   export SCCACHE_IDLE_TIMEOUT=0
   export SCCACHE_ERROR_LOG=/tmp/sccache_error.log
   export RUST_LOG=sccache::server=error

.ci/docker/common/install_cuda.sh

@@ -10,10 +10,10 @@
 
 set -ex
 
-# CUDA version must be specified (e.g., 12.8)
+# CUDA version must be specified (e.g., 13.0)
 CUDA_VERSION="${CUDA_VERSION:?CUDA_VERSION must be set}"
 
-# Convert version format (e.g., 12.8 -> 12-8 for package names)
+# Convert version format (e.g., 13.0 -> 13-0 for package names)
 CUDA_VERSION_DASH=$(echo "${CUDA_VERSION}" | tr '.' '-')
 
 # Add NVIDIA package repository

.ci/docker/common/install_cuda_windows_cross_compile.sh

@@ -17,6 +17,7 @@ declare -A CUDA_DRIVER_MAP=(
     ["12.6"]="12.6.3:561.17"
     ["12.8"]="12.8.1:572.61"
     ["12.9"]="12.9.1:576.57"
+    ["13.0"]="13.0.2:"
 )
 
 install_mingw() {
@@ -76,19 +77,26 @@ install_windows_cuda() {
     CUDA_VERSION=$(echo "${CUDA_INFO}" | cut -d: -f1)
     CUDA_DRIVER_VERSION=$(echo "${CUDA_INFO}" | cut -d: -f2)
 
-    echo "Using CUDA ${CUDA_VERSION} with driver ${CUDA_DRIVER_VERSION}"
+    if [ -n "${CUDA_DRIVER_VERSION}" ]; then
+        echo "Using CUDA ${CUDA_VERSION} with driver ${CUDA_DRIVER_VERSION}"
+        CUDA_INSTALLER="cuda_${CUDA_VERSION}_${CUDA_DRIVER_VERSION}_windows.exe"
+    else
+        echo "Using CUDA ${CUDA_VERSION}"
+        CUDA_INSTALLER="cuda_${CUDA_VERSION}_windows.exe"
+    fi
 
     echo "Installing Windows CUDA toolkit ${CUDA_VERSION}..."
 
     mkdir -p "${INSTALL_DIR}"
     cd "${INSTALL_DIR}"
 
-    CUDA_INSTALLER="cuda_${CUDA_VERSION}_${CUDA_DRIVER_VERSION}_windows.exe"
     CUDA_URL="https://developer.download.nvidia.com/compute/cuda/${CUDA_VERSION}/local_installers/${CUDA_INSTALLER}"
 
     # Check if already downloaded and extracted
     if [ -d "${INSTALL_DIR}/extracted/cuda_cudart" ]; then
         echo "Windows CUDA toolkit already installed, skipping download..."
+        chmod -R a+rX "${INSTALL_DIR}"
+        chmod -R a+rwX "${INSTALL_DIR}/extracted/cuda_cudart/cudart/lib"
         return 0
     fi
 
@@ -98,8 +106,11 @@ install_windows_cuda() {
     echo "Extracting CUDA toolkit..."
     7z x "${CUDA_INSTALLER}" -o"extracted" -y
 
-    # Fix permissions so ci-user can access the files
+    # Fix permissions so ci-user can access the files. PyTorch Inductor also
+    # needs to write a MinGW import library beside cudart.lib during Windows
+    # cross-compilation.
     chmod -R a+rX "${INSTALL_DIR}"
+    chmod -R a+rwX "${INSTALL_DIR}/extracted/cuda_cudart/cudart/lib"
 
     # Clean up installer to save space
     rm -f "${CUDA_INSTALLER}"

.ci/docker/common/install_docs_reqs.sh

@@ -12,11 +12,11 @@ if [ -n "$BUILD_DOCS" ]; then
   # Ignore error if gpg-agent doesn't exist (for Ubuntu 16.04)
   apt-get install -y gpg-agent || :
 
-  curl --retry 3 -sL https://deb.nodesource.com/setup_16.x | sudo -E bash -
+  curl --retry 3 --retry-all-errors -sL https://deb.nodesource.com/setup_16.x | sudo -E bash -
   sudo apt-get install -y nodejs
 
-  curl --retry 3 -sS https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add -
-  echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list
+  curl --retry 3 --retry-all-errors -sS https://dl.yarnpkg.com/debian/pubkey.gpg | sudo gpg --dearmor -o /usr/share/keyrings/yarn-archive-keyring.gpg
+  echo "deb [signed-by=/usr/share/keyrings/yarn-archive-keyring.gpg] https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list
 
   apt-get update
   apt-get install -y --no-install-recommends yarn

.ci/docker/common/install_linter.sh

@@ -13,7 +13,3 @@ source "$(dirname "${BASH_SOURCE[0]}")/utils.sh"
 # NB: Install all linter dependencies, the caching of lintrunner init could be
 # done after Executorch becomes public
 pip_install -r requirements-lintrunner.txt
-
-# Install google-java-format
-curl -L --retry 3 https://github.com/google/google-java-format/releases/download/v1.23.0/google-java-format_linux-x86-64 > /opt/google-java-format
-chmod +x /opt/google-java-format

.ci/docker/common/install_pytorch.sh

@@ -17,24 +17,6 @@ install_domains() {
 }
 
 install_pytorch_and_domains() {
-  if [ "${TORCH_CHANNEL}" != "nightly" ]; then
-    # Test/release: install the published wheels directly. The specs and URL
-    # are passed in as docker build args (computed from torch_pin.py by
-    # .ci/docker/build.sh). RC wheels at /whl/test/ get re-uploaded under the
-    # same version, so use --no-cache-dir there to avoid stale cache hits.
-    local cache_flag=""
-    if [ "${TORCH_CHANNEL}" = "test" ]; then
-      cache_flag="--no-cache-dir"
-    fi
-    pip_install --force-reinstall ${cache_flag} \
-      "${TORCH_SPEC}" "${TORCHVISION_SPEC}" "${TORCHAUDIO_SPEC}" \
-      --index-url "${TORCH_INDEX_URL}/cpu"
-    return
-  fi
-
-  # Nightly: build pytorch from source against the pinned SHA in pytorch.txt
-  # so we catch upstream regressions, then install audio/vision from the
-  # commits that pytorch itself pins.
   git clone https://github.com/pytorch/pytorch.git
 
   # Fetch the target commit
@@ -45,19 +27,20 @@ install_pytorch_and_domains() {
   chown -R ci-user .
 
   export _GLIBCXX_USE_CXX11_ABI=1
-  # PyTorch's FindARM.cmake hard-fails when the SVE+BF16 compile probe
-  # doesn't pass — gcc-11 in this image is too old to accept the combined
-  # NEON/SVE/bfloat16 intrinsics the probe exercises. Executorch's aarch64
-  # runtime targets (phones, embedded) don't use SVE, so bypass the check.
-  export BUILD_IGNORE_SVE_UNAVAILABLE=1
+  if [[ "$(uname -m)" == "aarch64" ]]; then
+    export BUILD_IGNORE_SVE_UNAVAILABLE=1
+  fi
+  if [[ -n "${PYTORCH_BUILD_MAX_JOBS:-}" ]]; then
+    export MAX_JOBS="${PYTORCH_BUILD_MAX_JOBS}"
+  fi
   # Then build and install PyTorch
   conda_run python setup.py bdist_wheel
   pip_install "$(echo dist/*.whl)"
 
-  # Defer to PyTorch's own pinned audio/vision commits.
-  TORCHAUDIO_VERSION=$(cat .github/ci_commit_pins/audio.txt)
+  # Grab the pinned audio and vision commits from PyTorch
+  TORCHAUDIO_VERSION=release/2.11
   export TORCHAUDIO_VERSION
-  TORCHVISION_VERSION=$(cat .github/ci_commit_pins/vision.txt)
+  TORCHVISION_VERSION=release/0.27
   export TORCHVISION_VERSION
 
   install_domains

.ci/docker/common/install_user.sh

@@ -7,6 +7,11 @@
 
 set -ex
 
+# On Ubuntu 24.04, there is a `ubuntu` user with id=1000
+if id ubuntu >/dev/null && [[ "$(id -u ubuntu)" == 1000 ]]; then
+    sudo userdel --remove ubuntu;
+fi
+
 # Same as ec2-user
 echo "ci-user:x:1000:1000::/var/lib/ci-user:" >> /etc/passwd
 echo "ci-user:x:1000:" >> /etc/group