diff --git a/.pipelines/install-scripts/install-blis.sh b/.pipelines/install-scripts/install-blis.sh index 83d6e34c3..e9a4d1c42 100644 --- a/.pipelines/install-scripts/install-blis.sh +++ b/.pipelines/install-scripts/install-blis.sh @@ -8,6 +8,9 @@ CFLAGS=${4:-"-fPIC -O3"} # Download BLIS v2.0 echo "Downloading BLIS ${BLIS_VERSION}..." +# Clean up any leftover state from a previous (possibly failed) attempt on +# this self-hosted agent — the workspace persists across builds and retries. +rm -rf blis blis-${BLIS_VERSION} blis.zip export BLIS_CHECKSUM=40134f6570d5539609c6328252ad1530c010931bb96f4e249e08279fd978da7a wget -q https://github.com/flame/blis/archive/refs/tags/${BLIS_VERSION}.zip -O blis.zip echo "${BLIS_CHECKSUM} blis.zip" | shasum -a 256 -c || exit 1 diff --git a/.pipelines/install-scripts/install-libflame.sh b/.pipelines/install-scripts/install-libflame.sh index 7c5d27324..4792b3f5e 100644 --- a/.pipelines/install-scripts/install-libflame.sh +++ b/.pipelines/install-scripts/install-libflame.sh @@ -19,6 +19,9 @@ fi # Download libflame echo "Downloading libflame ${LIBFLAME_VERSION}..." +# Clean up any leftover state from a previous (possibly failed) attempt on +# this self-hosted agent — the workspace persists across builds and retries. +rm -rf libflame libflame-${LIBFLAME_VERSION} libflame.zip export LIBFLAME_CHECKSUM=e120f559758c21392448f45301918f45760f5ab59d246e4d144079c664d5b64b wget -q https://github.com/flame/libflame/archive/refs/tags/${LIBFLAME_VERSION}.zip -O libflame.zip echo "${LIBFLAME_CHECKSUM} libflame.zip" | shasum -a 256 -c || exit 1 diff --git a/.pipelines/pip-scripts/build-pip-wheels.sh b/.pipelines/pip-scripts/build-pip-wheels.sh index babb3e72c..e48e137ce 100644 --- a/.pipelines/pip-scripts/build-pip-wheels.sh +++ b/.pipelines/pip-scripts/build-pip-wheels.sh @@ -10,8 +10,7 @@ CMAKE_VERSION=${6:-3.28.3} HDF5_VERSION=${7:-1.13.0} BLIS_VERSION=${8:-2.0} LIBFLAME_VERSION=${9:-5.2.0} -PYENV_VERSION=${10:-2.6.31} -MAC_BUILD=${11:-OFF} +MAC_BUILD=${10:-OFF} export CFLAGS="-fPIC -Os" if [ "$MAC_BUILD" == "OFF" ]; then # Build/install Linux dependencies @@ -55,6 +54,7 @@ if [ "$MAC_BUILD" == "OFF" ]; then # Build/install Linux dependencies python3-dev \ python3-pip \ python3-pybind11 \ + python3-venv \ tk-dev \ unzip \ wget \ @@ -83,8 +83,6 @@ if [ "$MAC_BUILD" == "OFF" ]; then # Build/install Linux dependencies echo "Downloading and installing libflame..." bash .pipelines/install-scripts/install-libflame.sh /usr/local ${MARCH} ${LIBFLAME_VERSION} "${CFLAGS}" - - export PYENV_ROOT="/workspace/.pyenv" elif [ "$MAC_BUILD" == "ON" ]; then arch -arm64 brew update arch -arm64 brew upgrade @@ -97,12 +95,17 @@ elif [ "$MAC_BUILD" == "ON" ]; then ncurses \ ninja \ pybind11 \ + python \ wget export CMAKE_PREFIX_PATH="/opt/homebrew" - export PYENV_ROOT="$PWD/.pyenv" + # Make sure Homebrew's python3 is preferred when bootstrapping conda. + export PATH="/opt/homebrew/bin:$PATH" fi echo "Downloading HDF5 $HDF5_VERSION..." +# Clean up any leftover state from a previous (possibly failed) attempt on +# this self-hosted agent — the workspace persists across builds and retries. +rm -rf hdf5 hdf5-${HDF5_VERSION} hdf5-${HDF5_VERSION}.tar.bz2 export HDF5_CHECKSUM=1826e198df8dac679f0d3dc703aba02af4c614fd6b7ec936cf4a55e6aa0646ec wget -q -nc --no-check-certificate https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.13/hdf5-${HDF5_VERSION}/src/hdf5-${HDF5_VERSION}.tar.bz2 echo "${HDF5_CHECKSUM} hdf5-${HDF5_VERSION}.tar.bz2" | shasum -a 256 -c || exit 1 @@ -114,20 +117,115 @@ echo "HDF5 $HDF5_VERSION downloaded and extracted successfully" echo "Installing HDF5..." bash .pipelines/install-scripts/install-hdf5.sh /usr/local ${BUILD_TYPE} ${PWD} "${CFLAGS}" ${MAC_BUILD} -# Install pyenv to use non-system python3 versions -# pyenv is used in place of a venv to prevent any collisions with the system Python -# when building with a non-system Python version. -echo "Installing pyenv ${PYENV_VERSION}..." -PYENV_CHECKSUM=7435b8c1481043e48c838ba40d5c8bc724c23d4c94e531adc283a7c121757ad4 -wget -q https://github.com/pyenv/pyenv/archive/refs/tags/v${PYENV_VERSION}.zip -O pyenv.zip -echo "${PYENV_CHECKSUM} pyenv.zip" | shasum -a 256 -c || exit 1 -unzip -q pyenv.zip -mv pyenv-${PYENV_VERSION} "$PYENV_ROOT" -rm pyenv.zip -"$PYENV_ROOT/bin/pyenv" install ${PYTHON_VERSION} -"$PYENV_ROOT/bin/pyenv" global ${PYTHON_VERSION} -export PATH="$PYENV_ROOT/versions/${PYTHON_VERSION}/bin:$PATH" -export PATH="$PYENV_ROOT/shims:$PATH" +# ============================================================================= +# Bootstrap Anaconda's `conda` for Python ${PYTHON_VERSION}. +# ============================================================================= +# +# Anaconda is the officially approved Python distribution for Microsoft CI +# builds (Azure Pipelines / OneBranch); see +# https://eng.ms/docs/more/languages-at-microsoft/python/articles/anaconda/install +# (section "Setting up Conda in CI builds"). The MS-vetted bootstrapper is +# `ms-ensureconda`. +# +# !!! HEADS UP: ms-ensureconda has incomplete platform coverage. !!! +# !!! As of writing it only ships `manylinux_2_7_x86_64` and `win_amd64` !!! +# !!! wheels — there are NO wheels for macOS (any arch) or Linux aarch64. !!! +# !!! See: !!! +# !!! https://pkgs.dev.azure.com/ms-azurequantum/AzureQuantum/_packaging/quantum-apps-dependencies/pypi/simple/ms-ensureconda/ +# !!! On those platforms `pip install ms-ensureconda` aborts with !!! +# !!! "No matching distribution found". !!! +# !!! !!! +# !!! Until ms-ensureconda publishes broader wheel coverage we fall back to !!! +# !!! the upstream public `ensureconda` package (pure-Python `py3-none-any`,!!! +# !!! identical `python -m ensureconda --envfile` CLI). It is fetched from !!! +# !!! the same Azure Artifacts feed via its public PyPI upstream so it !!! +# !!! still flows through CFS. !!! +# !!! !!! +# !!! TODO: revert to ms-ensureconda on every platform once arm64 / macOS !!! +# !!! wheels are published. File via python@microsoft.com if needed. !!! +# +# Prereq: the AzureQuantum/quantum-apps-dependencies feed must have +# azure-feed://mseng/Anaconda@Published configured as an upstream so that +# ms-ensureconda resolves on Linux x86_64. PIP_INDEX_URL is set by +# PipAuthenticate@1 at job level; on Linux it's forwarded into the docker +# container via -e PIP_INDEX_URL. +case "$(uname -s):$(uname -m)" in + Linux:x86_64) + ENSURECONDA_PKG="ms-ensureconda" + ;; + *) + # macOS (arm64) and Linux aarch64 — see HEADS UP block above. + ENSURECONDA_PKG="ensureconda" + ;; +esac + +echo "Installing ${ENSURECONDA_PKG} (on $(uname -s):$(uname -m)) and bootstrapping conda..." +# Use a throwaway venv so we don't fight PEP 668 (Homebrew Python on macOS +# and apt Python on Ubuntu 24.04 are both externally managed). +python3 -m venv /tmp/bootstrap-venv +# shellcheck disable=SC1091 +. /tmp/bootstrap-venv/bin/activate +python3 -m pip install --upgrade pip +python3 -m pip install "${ENSURECONDA_PKG}" + +if [ "$ENSURECONDA_PKG" = "ms-ensureconda" ]; then + # ms-ensureconda's --envfile flag dumps CONDA_BASH_HOOK + friends to a + # dotenv file we then source to wire up conda for this shell. + python3 -m ensureconda --envfile /tmp/ensureconda.env + deactivate + set -a; . /tmp/ensureconda.env; set +a + # shellcheck disable=SC1090 + . "$CONDA_BASH_HOOK" +else + # Public ensureconda has no --envfile (it's an ms-ensureconda extension); + # it just prints the discovered / installed conda binary path on stdout. + # Force it to install conda-standalone (default would be micromamba, which + # uses a different CLI: `micromamba shell hook --shell bash` instead of + # `conda shell.bash hook`, plus mamba-specific `create`/`activate`). + CONDA_EXE=$(python3 -m ensureconda --no-mamba --no-micromamba --conda --conda-exe) + deactivate + export CONDA_EXE + # shellcheck disable=SC1090 + eval "$("$CONDA_EXE" shell.bash hook)" +fi + +echo "Creating conda environment with Python ${PYTHON_VERSION}..." +# Explicitly include `pip` — fresh conda envs created with conda-standalone do +# not include pip by default, which would break `python3 -m pip ...` below. +# +# On Linux x86_64 (ms-ensureconda path) we are running under 1ES network +# isolation (CFSClean): public conda channels (conda.anaconda.org, +# repo.anaconda.com) are blocked. Force conda to install everything from the +# Azure Artifacts feed's Conda channel (proxied through the +# azure-feed://mseng/Anaconda@Published upstream). +# +# The feed exposes its upstream conda channels as named subpaths under +# /Conda/repo// (the feed root /Conda/repo/ itself returns 404 — it +# is not a channel). We use `main` (the Anaconda defaults channel that hosts +# python+pip) and `conda-forge` as a fallback for any package that defaults +# wouldn't carry. +# +# Auth: the conda install shipped by ms-ensureconda has a pre-registered +# azure_artifacts_conda_auth plugin that injects auth on every Azure +# Artifacts HTTPS request by reading $ARTIFACTS_CONDA_TOKEN. We just need to +# set that env var (to the pipeline's System.AccessToken). The plugin then +# handles auth; the channel URL itself must NOT inline the token (the plugin +# crashes with UnboundLocalError if the var is missing, even when basic-auth +# creds are present in the URL). +if [ "$ENSURECONDA_PKG" = "ms-ensureconda" ]; then + : "${SYSTEM_ACCESSTOKEN:?SYSTEM_ACCESSTOKEN must be set when bootstrapping conda from the Azure Artifacts feed}" + { set +x; } 2>/dev/null + export ARTIFACTS_CONDA_TOKEN="${SYSTEM_ACCESSTOKEN}" + set -x + CONDA_FEED_ROOT="https://pkgs.dev.azure.com/ms-azurequantum/AzureQuantum/_packaging/quantum-apps-dependencies/Conda/repo" + conda create --override-channels \ + --channel "${CONDA_FEED_ROOT}/main" \ + --channel "${CONDA_FEED_ROOT}/conda-forge" \ + --yes --quiet --name buildenv "python=${PYTHON_VERSION}" pip +else + conda create --yes --quiet --name buildenv "python=${PYTHON_VERSION}" pip +fi +conda activate buildenv python3 --version diff --git a/.pipelines/pip-scripts/test-pip-wheels.sh b/.pipelines/pip-scripts/test-pip-wheels.sh index dd2488ff3..a3a51009d 100644 --- a/.pipelines/pip-scripts/test-pip-wheels.sh +++ b/.pipelines/pip-scripts/test-pip-wheels.sh @@ -2,7 +2,6 @@ set -ex PYTHON_VERSION=${1:-3.11} MAC_BUILD=${2:-OFF} -PYENV_VERSION=${3:-2.6.31} export MAC_BUILD SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -15,10 +14,8 @@ else fi if [ "$MAC_BUILD" == "OFF" ] && [ -d "/workspace" ]; then - export PYENV_ROOT="/workspace/.pyenv" VENV_DIR="/workspace/test_wheel_env" else - export PYENV_ROOT="$REPO_ROOT/.pyenv" VENV_DIR="$REPO_ROOT/.test_wheel_env" fi @@ -49,6 +46,9 @@ if [ "$MAC_BUILD" == "OFF" ]; then libxml2-dev \ libxmlsec1-dev \ make \ + python3 \ + python3-pip \ + python3-venv \ tk-dev \ unzip \ wget \ @@ -61,38 +61,125 @@ elif [ "$MAC_BUILD" == "ON" ]; then arch -arm64 brew install \ curl \ ncurses \ + python \ unzip \ wget + # Make sure Homebrew's python3 is preferred when bootstrapping conda. + export PATH="/opt/homebrew/bin:$PATH" fi -# Install pyenv to use non-system python3 versions -if [ ! -d "$PYENV_ROOT" ]; then - echo "Installing pyenv ${PYENV_VERSION}..." - PYENV_CHECKSUM=7435b8c1481043e48c838ba40d5c8bc724c23d4c94e531adc283a7c121757ad4 - wget -q https://github.com/pyenv/pyenv/archive/refs/tags/v${PYENV_VERSION}.zip -O pyenv.zip - echo "${PYENV_CHECKSUM} pyenv.zip" | shasum -a 256 -c || exit 1 - unzip -q pyenv.zip - mv pyenv-${PYENV_VERSION} "$PYENV_ROOT" - rm pyenv.zip - "$PYENV_ROOT/bin/pyenv" install ${PYTHON_VERSION} - "$PYENV_ROOT/bin/pyenv" global ${PYTHON_VERSION} - export PATH="$PYENV_ROOT/versions/${PYTHON_VERSION}/bin:$PATH" - export PATH="$PYENV_ROOT/shims:$PATH" +# ============================================================================= +# Bootstrap Anaconda's `conda` for Python ${PYTHON_VERSION}. +# ============================================================================= +# +# Anaconda is the officially approved Python distribution for Microsoft CI +# builds (Azure Pipelines / OneBranch); see +# https://eng.ms/docs/more/languages-at-microsoft/python/articles/anaconda/install +# (section "Setting up Conda in CI builds"). The MS-vetted bootstrapper is +# `ms-ensureconda`. +# +# !!! HEADS UP: ms-ensureconda has incomplete platform coverage. !!! +# !!! As of writing it only ships `manylinux_2_7_x86_64` and `win_amd64` !!! +# !!! wheels — there are NO wheels for macOS (any arch) or Linux aarch64. !!! +# !!! See: !!! +# !!! https://pkgs.dev.azure.com/ms-azurequantum/AzureQuantum/_packaging/quantum-apps-dependencies/pypi/simple/ms-ensureconda/ +# !!! On those platforms `pip install ms-ensureconda` aborts with !!! +# !!! "No matching distribution found". !!! +# !!! !!! +# !!! Until ms-ensureconda publishes broader wheel coverage we fall back to !!! +# !!! the upstream public `ensureconda` package (pure-Python `py3-none-any`,!!! +# !!! identical `python -m ensureconda --envfile` CLI). It is fetched from !!! +# !!! the same Azure Artifacts feed via its public PyPI upstream so it !!! +# !!! still flows through CFS. !!! +# !!! !!! +# !!! TODO: revert to ms-ensureconda on every platform once arm64 / macOS !!! +# !!! wheels are published. File via python@microsoft.com if needed. !!! +# +# Prereq: the AzureQuantum/quantum-apps-dependencies feed must have +# azure-feed://mseng/Anaconda@Published configured as an upstream so that +# ms-ensureconda resolves on Linux x86_64. PIP_INDEX_URL is set by +# PipAuthenticate@1 at job level; on Linux it's forwarded into the docker +# container via -e PIP_INDEX_URL. +case "$(uname -s):$(uname -m)" in + Linux:x86_64) + ENSURECONDA_PKG="ms-ensureconda" + ;; + *) + # macOS (arm64) and Linux aarch64 — see HEADS UP block above. + ENSURECONDA_PKG="ensureconda" + ;; +esac + +echo "Installing ${ENSURECONDA_PKG} (on $(uname -s):$(uname -m)) and bootstrapping conda..." +# Use a throwaway venv so we don't fight PEP 668 (Homebrew Python on macOS +# and apt Python on Ubuntu 24.04 are both externally managed). +python3 -m venv /tmp/bootstrap-venv +# shellcheck disable=SC1091 +. /tmp/bootstrap-venv/bin/activate +python3 -m pip install --upgrade pip +python3 -m pip install "${ENSURECONDA_PKG}" + +if [ "$ENSURECONDA_PKG" = "ms-ensureconda" ]; then + # ms-ensureconda's --envfile flag dumps CONDA_BASH_HOOK + friends to a + # dotenv file we then source to wire up conda for this shell. + python3 -m ensureconda --envfile /tmp/ensureconda.env + deactivate + set -a; . /tmp/ensureconda.env; set +a + # shellcheck disable=SC1090 + . "$CONDA_BASH_HOOK" +else + # Public ensureconda has no --envfile (it's an ms-ensureconda extension); + # it just prints the discovered / installed conda binary path on stdout. + # Force it to install conda-standalone (default would be micromamba, which + # uses a different CLI: `micromamba shell hook --shell bash` instead of + # `conda shell.bash hook`, plus mamba-specific `create`/`activate`). + CONDA_EXE=$(python3 -m ensureconda --no-mamba --no-micromamba --conda --conda-exe) + deactivate + export CONDA_EXE + # shellcheck disable=SC1090 + eval "$("$CONDA_EXE" shell.bash hook)" fi -# Install and activate the specific Python version -"$PYENV_ROOT/bin/pyenv" install $PYTHON_VERSION --skip-existing -"$PYENV_ROOT/bin/pyenv" global $PYTHON_VERSION -export PATH="$PYENV_ROOT/versions/$PYTHON_VERSION/bin:$PATH" -export PATH="$PYENV_ROOT/shims:$PATH" +echo "Creating fresh conda environment for wheel test with Python ${PYTHON_VERSION}..." +# Explicitly include `pip` — fresh conda envs created with conda-standalone do +# not include pip by default, which would break `python3 -m pip ...` below. +# +# On Linux x86_64 (ms-ensureconda path) we are running under 1ES network +# isolation (CFSClean): public conda channels (conda.anaconda.org, +# repo.anaconda.com) are blocked. Force conda to install everything from the +# Azure Artifacts feed's Conda channel (proxied through the +# azure-feed://mseng/Anaconda@Published upstream). +# +# The feed exposes its upstream conda channels as named subpaths under +# /Conda/repo// (the feed root /Conda/repo/ itself returns 404 — it +# is not a channel). We use `main` (the Anaconda defaults channel that hosts +# python+pip) and `conda-forge` as a fallback for any package that defaults +# wouldn't carry. +# +# Auth: the conda install shipped by ms-ensureconda has a pre-registered +# azure_artifacts_conda_auth plugin that injects auth on every Azure +# Artifacts HTTPS request by reading $ARTIFACTS_CONDA_TOKEN. We just need to +# set that env var (to the pipeline's System.AccessToken). The plugin then +# handles auth; the channel URL itself must NOT inline the token (the plugin +# crashes with UnboundLocalError if the var is missing, even when basic-auth +# creds are present in the URL). +if [ "$ENSURECONDA_PKG" = "ms-ensureconda" ]; then + : "${SYSTEM_ACCESSTOKEN:?SYSTEM_ACCESSTOKEN must be set when bootstrapping conda from the Azure Artifacts feed}" + { set +x; } 2>/dev/null + export ARTIFACTS_CONDA_TOKEN="${SYSTEM_ACCESSTOKEN}" + set -x + CONDA_FEED_ROOT="https://pkgs.dev.azure.com/ms-azurequantum/AzureQuantum/_packaging/quantum-apps-dependencies/Conda/repo" + conda create --override-channels \ + --channel "${CONDA_FEED_ROOT}/main" \ + --channel "${CONDA_FEED_ROOT}/conda-forge" \ + --yes --quiet --name testenv "python=${PYTHON_VERSION}" pip +else + conda create --yes --quiet --name testenv "python=${PYTHON_VERSION}" pip +fi +conda activate testenv python3 --version -# Create a clean virtual environment for testing the wheel -rm -rf "$VENV_DIR" -python3 -m venv "$VENV_DIR" -. "$VENV_DIR/bin/activate" - python3 -m pip install --upgrade pip # Install the wheel in the clean environment @@ -117,5 +204,3 @@ export QSHARP_PYTHON_TELEMETRY=false # Run pytest suite echo '=== Running pytest suite ===' python3 -m pytest -v ./tests - -deactivate diff --git a/.pipelines/templates/build-pip-wheels.yml b/.pipelines/templates/build-pip-wheels.yml index d1ab7e536..7b8de2355 100644 --- a/.pipelines/templates/build-pip-wheels.yml +++ b/.pipelines/templates/build-pip-wheels.yml @@ -35,9 +35,6 @@ parameters: - name: libflameVersion type: string default: 5.2.0 -- name: pyenvVersion - type: string - default: 2.6.31 - name: macBuild type: string default: OFF @@ -95,6 +92,7 @@ steps: --platform ${{ parameters.imageTag }} \ -v ${{ parameters.agentBuildDirectory }}:/workspace:rshared \ -e PIP_INDEX_URL \ + -e SYSTEM_ACCESSTOKEN \ -e QDK_UARCH=${{ parameters.march }} \ -w /workspace/qdk-chemistry \ ${{ parameters.dockerImage }} \ @@ -107,10 +105,11 @@ steps: ${{ parameters.hdf5Version }} \ ${{ parameters.blisVersion }} \ ${{ parameters.libflameVersion }} \ - ${{ parameters.pyenvVersion }} \ ${{ parameters.macBuild }} displayName: Build wheel (Linux) retryCountOnTaskFailure: 2 + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) - ${{ if eq(parameters.macBuild, 'ON') }}: - script: | @@ -126,6 +125,5 @@ steps: ${{ parameters.hdf5Version }} \ ${{ parameters.blisVersion }} \ ${{ parameters.libflameVersion }} \ - ${{ parameters.pyenvVersion }} \ ${{ parameters.macBuild }} displayName: Build wheel (macOS native) diff --git a/.pipelines/templates/test-pip-wheels.yml b/.pipelines/templates/test-pip-wheels.yml index 92f6dc387..236dab2c1 100644 --- a/.pipelines/templates/test-pip-wheels.yml +++ b/.pipelines/templates/test-pip-wheels.yml @@ -14,9 +14,6 @@ parameters: - name: macBuild type: string default: OFF -- name: pyenvVersion - type: string - default: 2.6.31 steps: - ${{ if eq(parameters.macBuild, 'OFF') }}: @@ -25,20 +22,21 @@ steps: docker run --rm \ --platform ${{ parameters.imageTag }} \ -e PIP_INDEX_URL \ + -e SYSTEM_ACCESSTOKEN \ -e QSHARP_PYTHON_TELEMETRY=false \ -v ${{ parameters.agentBuildDirectory }}:/workspace \ -w /workspace/qdk-chemistry/python \ ${{ parameters.dockerImage }} \ bash ../.pipelines/pip-scripts/test-pip-wheels.sh ${{ parameters.pythonVersion }} \ - ${{ parameters.macBuild }} \ - ${{ parameters.pyenvVersion }} + ${{ parameters.macBuild }} displayName: Test wheel (Docker) + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) - ${{ if eq(parameters.macBuild, 'ON') }}: - script: | echo "Running wheel tests (native)..." cd ${{ parameters.agentBuildDirectory }}/qdk-chemistry/python bash ../.pipelines/pip-scripts/test-pip-wheels.sh ${{ parameters.pythonVersion }} \ - ${{ parameters.macBuild }} \ - ${{ parameters.pyenvVersion }} + ${{ parameters.macBuild }} displayName: Test wheel (native)