Replace Pyenv with Anaconda / ms-ensureconda in pip-wheels CI

.pipelines/install-scripts/install-blis.sh

@@ -8,6 +8,9 @@ CFLAGS=${4:-"-fPIC -O3"}
 
 # Download BLIS v2.0
 echo "Downloading BLIS ${BLIS_VERSION}..."
+# Clean up any leftover state from a previous (possibly failed) attempt on
+# this self-hosted agent — the workspace persists across builds and retries.
+rm -rf blis blis-${BLIS_VERSION} blis.zip
 export BLIS_CHECKSUM=40134f6570d5539609c6328252ad1530c010931bb96f4e249e08279fd978da7a
 wget -q https://github.com/flame/blis/archive/refs/tags/${BLIS_VERSION}.zip -O blis.zip
 echo "${BLIS_CHECKSUM}  blis.zip" | shasum -a 256 -c || exit 1

.pipelines/install-scripts/install-libflame.sh

@@ -19,6 +19,9 @@ fi
 
 # Download libflame
 echo "Downloading libflame ${LIBFLAME_VERSION}..."
+# Clean up any leftover state from a previous (possibly failed) attempt on
+# this self-hosted agent — the workspace persists across builds and retries.
+rm -rf libflame libflame-${LIBFLAME_VERSION} libflame.zip
 export LIBFLAME_CHECKSUM=e120f559758c21392448f45301918f45760f5ab59d246e4d144079c664d5b64b
 wget -q https://github.com/flame/libflame/archive/refs/tags/${LIBFLAME_VERSION}.zip -O libflame.zip
 echo "${LIBFLAME_CHECKSUM}  libflame.zip" | shasum -a 256 -c || exit 1

.pipelines/pip-scripts/build-pip-wheels.sh

@@ -10,8 +10,7 @@ CMAKE_VERSION=${6:-3.28.3}
 HDF5_VERSION=${7:-1.13.0}
 BLIS_VERSION=${8:-2.0}
 LIBFLAME_VERSION=${9:-5.2.0}
-PYENV_VERSION=${10:-2.6.31}
-MAC_BUILD=${11:-OFF}
+MAC_BUILD=${10:-OFF}
 
 export CFLAGS="-fPIC -Os"
 if [ "$MAC_BUILD" == "OFF" ]; then # Build/install Linux dependencies
@@ -55,6 +54,7 @@ if [ "$MAC_BUILD" == "OFF" ]; then # Build/install Linux dependencies
         python3-dev \
         python3-pip \
         python3-pybind11 \
+        python3-venv \
         tk-dev \
         unzip \
         wget \
@@ -83,8 +83,6 @@ if [ "$MAC_BUILD" == "OFF" ]; then # Build/install Linux dependencies
 
     echo "Downloading and installing libflame..."
     bash .pipelines/install-scripts/install-libflame.sh /usr/local ${MARCH} ${LIBFLAME_VERSION} "${CFLAGS}"
-
-    export PYENV_ROOT="/workspace/.pyenv"
 elif [ "$MAC_BUILD" == "ON" ]; then
     arch -arm64 brew update
     arch -arm64 brew upgrade
@@ -97,12 +95,17 @@ elif [ "$MAC_BUILD" == "ON" ]; then
         ncurses \
         ninja \
         pybind11 \
+        python \
         wget
     export CMAKE_PREFIX_PATH="/opt/homebrew"
-    export PYENV_ROOT="$PWD/.pyenv"
+    # Make sure Homebrew's python3 is preferred when bootstrapping conda.
+    export PATH="/opt/homebrew/bin:$PATH"
 fi
 
 echo "Downloading HDF5 $HDF5_VERSION..."
+# Clean up any leftover state from a previous (possibly failed) attempt on
+# this self-hosted agent — the workspace persists across builds and retries.
+rm -rf hdf5 hdf5-${HDF5_VERSION} hdf5-${HDF5_VERSION}.tar.bz2
 export HDF5_CHECKSUM=1826e198df8dac679f0d3dc703aba02af4c614fd6b7ec936cf4a55e6aa0646ec
 wget -q -nc --no-check-certificate https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.13/hdf5-${HDF5_VERSION}/src/hdf5-${HDF5_VERSION}.tar.bz2
 echo "${HDF5_CHECKSUM}  hdf5-${HDF5_VERSION}.tar.bz2" | shasum -a 256 -c || exit 1
@@ -114,20 +117,115 @@ echo "HDF5 $HDF5_VERSION downloaded and extracted successfully"
 echo "Installing HDF5..."
 bash .pipelines/install-scripts/install-hdf5.sh /usr/local ${BUILD_TYPE} ${PWD} "${CFLAGS}" ${MAC_BUILD}
 
-# Install pyenv to use non-system python3 versions
-# pyenv is used in place of a venv to prevent any collisions with the system Python
-# when building with a non-system Python version.
-echo "Installing pyenv ${PYENV_VERSION}..."
-PYENV_CHECKSUM=7435b8c1481043e48c838ba40d5c8bc724c23d4c94e531adc283a7c121757ad4
-wget -q https://github.com/pyenv/pyenv/archive/refs/tags/v${PYENV_VERSION}.zip -O pyenv.zip
-echo "${PYENV_CHECKSUM}  pyenv.zip" | shasum -a 256 -c || exit 1
-unzip -q pyenv.zip
-mv pyenv-${PYENV_VERSION} "$PYENV_ROOT"
-rm pyenv.zip
-"$PYENV_ROOT/bin/pyenv" install ${PYTHON_VERSION}
-"$PYENV_ROOT/bin/pyenv" global ${PYTHON_VERSION}
-export PATH="$PYENV_ROOT/versions/${PYTHON_VERSION}/bin:$PATH"
-export PATH="$PYENV_ROOT/shims:$PATH"
+# =============================================================================
+# Bootstrap Anaconda's `conda` for Python ${PYTHON_VERSION}.
+# =============================================================================
+#
+# Anaconda is the officially approved Python distribution for Microsoft CI
+# builds (Azure Pipelines / OneBranch); see
+#   https://eng.ms/docs/more/languages-at-microsoft/python/articles/anaconda/install
+# (section "Setting up Conda in CI builds"). The MS-vetted bootstrapper is
+# `ms-ensureconda`.
+#
+# !!! HEADS UP: ms-ensureconda has incomplete platform coverage.            !!!
+# !!! As of writing it only ships `manylinux_2_7_x86_64` and `win_amd64`    !!!
+# !!! wheels — there are NO wheels for macOS (any arch) or Linux aarch64.   !!!
+# !!! See:                                                                  !!!
+# !!!   https://pkgs.dev.azure.com/ms-azurequantum/AzureQuantum/_packaging/quantum-apps-dependencies/pypi/simple/ms-ensureconda/
+# !!! On those platforms `pip install ms-ensureconda` aborts with           !!!
+# !!! "No matching distribution found".                                     !!!
+# !!!                                                                       !!!
+# !!! Until ms-ensureconda publishes broader wheel coverage we fall back to !!!
+# !!! the upstream public `ensureconda` package (pure-Python `py3-none-any`,!!!
+# !!! identical `python -m ensureconda --envfile` CLI). It is fetched from  !!!
+# !!! the same Azure Artifacts feed via its public PyPI upstream so it      !!!
+# !!! still flows through CFS.                                              !!!
+# !!!                                                                       !!!
+# !!! TODO: revert to ms-ensureconda on every platform once arm64 / macOS   !!!
+# !!! wheels are published. File via python@microsoft.com if needed.        !!!
+#
+# Prereq: the AzureQuantum/quantum-apps-dependencies feed must have
+# azure-feed://mseng/Anaconda@Published configured as an upstream so that
+# ms-ensureconda resolves on Linux x86_64. PIP_INDEX_URL is set by
+# PipAuthenticate@1 at job level; on Linux it's forwarded into the docker
+# container via -e PIP_INDEX_URL.
+case "$(uname -s):$(uname -m)" in
+    Linux:x86_64)
+        ENSURECONDA_PKG="ms-ensureconda"
+        ;;
+    *)
+        # macOS (arm64) and Linux aarch64 — see HEADS UP block above.
+        ENSURECONDA_PKG="ensureconda"
+        ;;
+esac
+
+echo "Installing ${ENSURECONDA_PKG} (on $(uname -s):$(uname -m)) and bootstrapping conda..."
+# Use a throwaway venv so we don't fight PEP 668 (Homebrew Python on macOS
+# and apt Python on Ubuntu 24.04 are both externally managed).
+python3 -m venv /tmp/bootstrap-venv
+# shellcheck disable=SC1091
+. /tmp/bootstrap-venv/bin/activate
+python3 -m pip install --upgrade pip
+python3 -m pip install "${ENSURECONDA_PKG}"
+
+if [ "$ENSURECONDA_PKG" = "ms-ensureconda" ]; then
+    # ms-ensureconda's --envfile flag dumps CONDA_BASH_HOOK + friends to a
+    # dotenv file we then source to wire up conda for this shell.
+    python3 -m ensureconda --envfile /tmp/ensureconda.env
+    deactivate
+    set -a; . /tmp/ensureconda.env; set +a
+    # shellcheck disable=SC1090
+    . "$CONDA_BASH_HOOK"
+else
+    # Public ensureconda has no --envfile (it's an ms-ensureconda extension);
+    # it just prints the discovered / installed conda binary path on stdout.
+    # Force it to install conda-standalone (default would be micromamba, which
+    # uses a different CLI: `micromamba shell hook --shell bash` instead of
+    # `conda shell.bash hook`, plus mamba-specific `create`/`activate`).
+    CONDA_EXE=$(python3 -m ensureconda --no-mamba --no-micromamba --conda --conda-exe)
+    deactivate
+    export CONDA_EXE
+    # shellcheck disable=SC1090
+    eval "$("$CONDA_EXE" shell.bash hook)"
+fi
+
+echo "Creating conda environment with Python ${PYTHON_VERSION}..."
+# Explicitly include `pip` — fresh conda envs created with conda-standalone do
+# not include pip by default, which would break `python3 -m pip ...` below.
+#
+# On Linux x86_64 (ms-ensureconda path) we are running under 1ES network
+# isolation (CFSClean): public conda channels (conda.anaconda.org,
+# repo.anaconda.com) are blocked. Force conda to install everything from the
+# Azure Artifacts feed's Conda channel (proxied through the
+# azure-feed://mseng/Anaconda@Published upstream).
+#
+# The feed exposes its upstream conda channels as named subpaths under
+# /Conda/repo/<channel>/ (the feed root /Conda/repo/ itself returns 404 — it
+# is not a channel). We use `main` (the Anaconda defaults channel that hosts
+# python+pip) and `conda-forge` as a fallback for any package that defaults
+# wouldn't carry.
+#
+# Auth: the conda install shipped by ms-ensureconda has a pre-registered
+# azure_artifacts_conda_auth plugin that injects auth on every Azure
+# Artifacts HTTPS request by reading $ARTIFACTS_CONDA_TOKEN. We just need to
+# set that env var (to the pipeline's System.AccessToken). The plugin then
+# handles auth; the channel URL itself must NOT inline the token (the plugin
+# crashes with UnboundLocalError if the var is missing, even when basic-auth
+# creds are present in the URL).
+if [ "$ENSURECONDA_PKG" = "ms-ensureconda" ]; then
+    : "${SYSTEM_ACCESSTOKEN:?SYSTEM_ACCESSTOKEN must be set when bootstrapping conda from the Azure Artifacts feed}"
+    { set +x; } 2>/dev/null
+    export ARTIFACTS_CONDA_TOKEN="${SYSTEM_ACCESSTOKEN}"
+    set -x
+    CONDA_FEED_ROOT="https://pkgs.dev.azure.com/ms-azurequantum/AzureQuantum/_packaging/quantum-apps-dependencies/Conda/repo"
+    conda create --override-channels \
+                 --channel "${CONDA_FEED_ROOT}/main" \
+                 --channel "${CONDA_FEED_ROOT}/conda-forge" \
+                 --yes --quiet --name buildenv "python=${PYTHON_VERSION}" pip
+else
+    conda create --yes --quiet --name buildenv "python=${PYTHON_VERSION}" pip
+fi
+conda activate buildenv
 
 python3 --version
 

.pipelines/pip-scripts/test-pip-wheels.sh

@@ -2,7 +2,6 @@
 set -ex
 PYTHON_VERSION=${1:-3.11}
 MAC_BUILD=${2:-OFF}
-PYENV_VERSION=${3:-2.6.31}
 export MAC_BUILD
 
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
@@ -15,10 +14,8 @@ else
 fi
 
 if [ "$MAC_BUILD" == "OFF" ] && [ -d "/workspace" ]; then
-    export PYENV_ROOT="/workspace/.pyenv"
     VENV_DIR="/workspace/test_wheel_env"
 else
-    export PYENV_ROOT="$REPO_ROOT/.pyenv"
     VENV_DIR="$REPO_ROOT/.test_wheel_env"
 fi
 
@@ -49,6 +46,9 @@ if [ "$MAC_BUILD" == "OFF" ]; then
         libxml2-dev \
         libxmlsec1-dev \
         make \
+        python3 \
+        python3-pip \
+        python3-venv \
         tk-dev \
         unzip \
         wget \
@@ -61,38 +61,125 @@ elif [ "$MAC_BUILD" == "ON" ]; then
     arch -arm64 brew install \
         curl \
         ncurses \
+        python \
         unzip \
         wget
+    # Make sure Homebrew's python3 is preferred when bootstrapping conda.
+    export PATH="/opt/homebrew/bin:$PATH"
 fi
 
-# Install pyenv to use non-system python3 versions
-if [ ! -d "$PYENV_ROOT" ]; then
-    echo "Installing pyenv ${PYENV_VERSION}..."
-    PYENV_CHECKSUM=7435b8c1481043e48c838ba40d5c8bc724c23d4c94e531adc283a7c121757ad4
-    wget -q https://github.com/pyenv/pyenv/archive/refs/tags/v${PYENV_VERSION}.zip -O pyenv.zip
-    echo "${PYENV_CHECKSUM}  pyenv.zip" | shasum -a 256 -c || exit 1
-    unzip -q pyenv.zip
-    mv pyenv-${PYENV_VERSION} "$PYENV_ROOT"
-    rm pyenv.zip
-    "$PYENV_ROOT/bin/pyenv" install ${PYTHON_VERSION}
-    "$PYENV_ROOT/bin/pyenv" global ${PYTHON_VERSION}
-    export PATH="$PYENV_ROOT/versions/${PYTHON_VERSION}/bin:$PATH"
-    export PATH="$PYENV_ROOT/shims:$PATH"
+# =============================================================================
+# Bootstrap Anaconda's `conda` for Python ${PYTHON_VERSION}.
+# =============================================================================
+#
+# Anaconda is the officially approved Python distribution for Microsoft CI
+# builds (Azure Pipelines / OneBranch); see
+#   https://eng.ms/docs/more/languages-at-microsoft/python/articles/anaconda/install
+# (section "Setting up Conda in CI builds"). The MS-vetted bootstrapper is
+# `ms-ensureconda`.
+#
+# !!! HEADS UP: ms-ensureconda has incomplete platform coverage.            !!!
+# !!! As of writing it only ships `manylinux_2_7_x86_64` and `win_amd64`    !!!
+# !!! wheels — there are NO wheels for macOS (any arch) or Linux aarch64.   !!!
+# !!! See:                                                                  !!!
+# !!!   https://pkgs.dev.azure.com/ms-azurequantum/AzureQuantum/_packaging/quantum-apps-dependencies/pypi/simple/ms-ensureconda/
+# !!! On those platforms `pip install ms-ensureconda` aborts with           !!!
+# !!! "No matching distribution found".                                     !!!
+# !!!                                                                       !!!
+# !!! Until ms-ensureconda publishes broader wheel coverage we fall back to !!!
+# !!! the upstream public `ensureconda` package (pure-Python `py3-none-any`,!!!
+# !!! identical `python -m ensureconda --envfile` CLI). It is fetched from  !!!
+# !!! the same Azure Artifacts feed via its public PyPI upstream so it      !!!
+# !!! still flows through CFS.                                              !!!
+# !!!                                                                       !!!
+# !!! TODO: revert to ms-ensureconda on every platform once arm64 / macOS   !!!
+# !!! wheels are published. File via python@microsoft.com if needed.        !!!
+#
+# Prereq: the AzureQuantum/quantum-apps-dependencies feed must have
+# azure-feed://mseng/Anaconda@Published configured as an upstream so that
+# ms-ensureconda resolves on Linux x86_64. PIP_INDEX_URL is set by
+# PipAuthenticate@1 at job level; on Linux it's forwarded into the docker
+# container via -e PIP_INDEX_URL.
+case "$(uname -s):$(uname -m)" in
+    Linux:x86_64)
+        ENSURECONDA_PKG="ms-ensureconda"
+        ;;
+    *)
+        # macOS (arm64) and Linux aarch64 — see HEADS UP block above.
+        ENSURECONDA_PKG="ensureconda"
+        ;;
+esac
+
+echo "Installing ${ENSURECONDA_PKG} (on $(uname -s):$(uname -m)) and bootstrapping conda..."
+# Use a throwaway venv so we don't fight PEP 668 (Homebrew Python on macOS
+# and apt Python on Ubuntu 24.04 are both externally managed).
+python3 -m venv /tmp/bootstrap-venv
+# shellcheck disable=SC1091
+. /tmp/bootstrap-venv/bin/activate
+python3 -m pip install --upgrade pip
+python3 -m pip install "${ENSURECONDA_PKG}"
+
+if [ "$ENSURECONDA_PKG" = "ms-ensureconda" ]; then
+    # ms-ensureconda's --envfile flag dumps CONDA_BASH_HOOK + friends to a
+    # dotenv file we then source to wire up conda for this shell.
+    python3 -m ensureconda --envfile /tmp/ensureconda.env
+    deactivate
+    set -a; . /tmp/ensureconda.env; set +a
+    # shellcheck disable=SC1090
+    . "$CONDA_BASH_HOOK"
+else
+    # Public ensureconda has no --envfile (it's an ms-ensureconda extension);
+    # it just prints the discovered / installed conda binary path on stdout.
+    # Force it to install conda-standalone (default would be micromamba, which
+    # uses a different CLI: `micromamba shell hook --shell bash` instead of
+    # `conda shell.bash hook`, plus mamba-specific `create`/`activate`).
+    CONDA_EXE=$(python3 -m ensureconda --no-mamba --no-micromamba --conda --conda-exe)
+    deactivate
+    export CONDA_EXE
+    # shellcheck disable=SC1090
+    eval "$("$CONDA_EXE" shell.bash hook)"
 fi
 
-# Install and activate the specific Python version
-"$PYENV_ROOT/bin/pyenv" install $PYTHON_VERSION --skip-existing
-"$PYENV_ROOT/bin/pyenv" global $PYTHON_VERSION
-export PATH="$PYENV_ROOT/versions/$PYTHON_VERSION/bin:$PATH"
-export PATH="$PYENV_ROOT/shims:$PATH"
+echo "Creating fresh conda environment for wheel test with Python ${PYTHON_VERSION}..."
+# Explicitly include `pip` — fresh conda envs created with conda-standalone do
+# not include pip by default, which would break `python3 -m pip ...` below.
+#
+# On Linux x86_64 (ms-ensureconda path) we are running under 1ES network
+# isolation (CFSClean): public conda channels (conda.anaconda.org,
+# repo.anaconda.com) are blocked. Force conda to install everything from the
+# Azure Artifacts feed's Conda channel (proxied through the
+# azure-feed://mseng/Anaconda@Published upstream).
+#
+# The feed exposes its upstream conda channels as named subpaths under
+# /Conda/repo/<channel>/ (the feed root /Conda/repo/ itself returns 404 — it
+# is not a channel). We use `main` (the Anaconda defaults channel that hosts
+# python+pip) and `conda-forge` as a fallback for any package that defaults
+# wouldn't carry.
+#
+# Auth: the conda install shipped by ms-ensureconda has a pre-registered
+# azure_artifacts_conda_auth plugin that injects auth on every Azure
+# Artifacts HTTPS request by reading $ARTIFACTS_CONDA_TOKEN. We just need to
+# set that env var (to the pipeline's System.AccessToken). The plugin then
+# handles auth; the channel URL itself must NOT inline the token (the plugin
+# crashes with UnboundLocalError if the var is missing, even when basic-auth
+# creds are present in the URL).
+if [ "$ENSURECONDA_PKG" = "ms-ensureconda" ]; then
+    : "${SYSTEM_ACCESSTOKEN:?SYSTEM_ACCESSTOKEN must be set when bootstrapping conda from the Azure Artifacts feed}"
+    { set +x; } 2>/dev/null
+    export ARTIFACTS_CONDA_TOKEN="${SYSTEM_ACCESSTOKEN}"
+    set -x
+    CONDA_FEED_ROOT="https://pkgs.dev.azure.com/ms-azurequantum/AzureQuantum/_packaging/quantum-apps-dependencies/Conda/repo"
+    conda create --override-channels \
+                 --channel "${CONDA_FEED_ROOT}/main" \
+                 --channel "${CONDA_FEED_ROOT}/conda-forge" \
+                 --yes --quiet --name testenv "python=${PYTHON_VERSION}" pip
+else
+    conda create --yes --quiet --name testenv "python=${PYTHON_VERSION}" pip
+fi
+conda activate testenv
 
 python3 --version
 
-# Create a clean virtual environment for testing the wheel
-rm -rf "$VENV_DIR"
-python3 -m venv "$VENV_DIR"
-. "$VENV_DIR/bin/activate"
-
 python3 -m pip install --upgrade pip
 
 # Install the wheel in the clean environment
@@ -117,5 +204,3 @@ export QSHARP_PYTHON_TELEMETRY=false
 # Run pytest suite
 echo '=== Running pytest suite ==='
 python3 -m pytest -v ./tests
-
-deactivate