diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 47854f54..d86e5960 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -19,28 +19,28 @@ on: - release-candidate workflow_dispatch: inputs: - rebuild_deps: + REBUILD_DEPS: description: 'Force a rebuild of the dependencies image before building forcing' required: false default: false type: boolean - deps_base_repo: - description: 'Base image to compile the dependencies from (default rockylinux)' + DEPS_BASE_REPO: + description: 'Base image to compile the dependencies from (default python)' required: false type: string - deps_base_tag: - description: 'Dependencies base image tag (default 8)' + DEPS_BASE_TAG: + description: 'Dependencies base image tag (default 3.12-slim-bookworm)' required: false type: string - deps_base_suffix: - description: 'Override the published deps name suffix (default derived, e.g. rocky8)' + DEPS_BASE_SUFFIX: + description: 'Override the published deps name suffix (default derived, e.g. bookworm)' required: false type: string - BMI_BASE_REPO: - description: 'Deps image repo to build forcing FROM when not rebuilding deps (default ghcr.io//ngen-dependencies-rocky8)' + DEPS_IMAGE_REPO: + description: 'Deps image repo to build forcing FROM when not rebuilding deps (default ghcr.io//ngen-dependencies-bookworm)' required: false type: string - BMI_BASE_TAG: + DEPS_IMAGE_TAG: description: 'Deps image tag to build forcing FROM when not rebuilding deps (default latest)' required: false type: string @@ -81,22 +81,6 @@ on: description: 'MSW_MGR_REF' required: false type: string - SZIP_VERSION: - description: 'Deps image: SZIP version (blank = Dockerfile default; forces a deps rebuild)' - required: false - type: string - HDF5_VERSION: - description: 'Deps image: HDF5 version (blank = Dockerfile default; forces a deps rebuild)' - required: false - type: string - NETCDF_C_VERSION: - description: 'Deps image: NetCDF-C version (blank = Dockerfile default; forces a deps rebuild)' - required: false - type: string - NETCDF_FORTRAN_VERSION: - description: 'Deps image: NetCDF-Fortran version (blank = Dockerfile default; forces a deps rebuild)' - required: false - type: string BOOST_VERSION: description: 'Deps image: Boost version (blank = Dockerfile default; forces a deps rebuild)' required: false @@ -105,18 +89,6 @@ on: description: 'Deps image: ESMF version (blank = Dockerfile default; forces a deps rebuild)' required: false type: string - GDAL_VERSION: - description: 'Deps image: GDAL version (blank = Dockerfile default; forces a deps rebuild)' - required: false - type: string - ECFLOW_VERSION: - description: 'Deps image: ecFlow version (blank = Dockerfile default; forces a deps rebuild)' - required: false - type: string - ECBUILD_VERSION: - description: 'Deps image: ecbuild version (blank = Dockerfile default; forces a deps rebuild)' - required: false - type: string permissions: contents: read @@ -256,7 +228,7 @@ jobs: run: | # if any dependency version input was provided, force a deps rebuild so # the override actually takes effect - if [ -n "${{ inputs.SZIP_VERSION }}${{ inputs.HDF5_VERSION }}${{ inputs.NETCDF_C_VERSION }}${{ inputs.NETCDF_FORTRAN_VERSION }}${{ inputs.BOOST_VERSION }}${{ inputs.ESMF_VERSION }}${{ inputs.GDAL_VERSION }}${{ inputs.ECFLOW_VERSION }}${{ inputs.ECBUILD_VERSION }}" ]; then + if [ -n "${{ inputs.BOOST_VERSION }}${{ inputs.ESMF_VERSION }}" ]; then echo "set=true" >> "$GITHUB_OUTPUT" fi @@ -266,22 +238,15 @@ jobs: dependencies: name: dependencies needs: changes - if: ${{ needs.changes.outputs.deps == 'true' || inputs.rebuild_deps || needs.changes.outputs.versions == 'true' }} + if: ${{ needs.changes.outputs.deps == 'true' || inputs.REBUILD_DEPS || needs.changes.outputs.versions == 'true' }} uses: ./.github/workflows/dependencies.yml with: - deps_base_repo: ${{ inputs.deps_base_repo }} - deps_base_tag: ${{ inputs.deps_base_tag }} - deps_base_suffix: ${{ inputs.deps_base_suffix }} - ghcr_org: ${{ inputs.GHCR_ORG }} - szip_version: ${{ inputs.SZIP_VERSION }} - hdf5_version: ${{ inputs.HDF5_VERSION }} - netcdf_c_version: ${{ inputs.NETCDF_C_VERSION }} - netcdf_fortran_version: ${{ inputs.NETCDF_FORTRAN_VERSION }} - boost_version: ${{ inputs.BOOST_VERSION }} - esmf_version: ${{ inputs.ESMF_VERSION }} - gdal_version: ${{ inputs.GDAL_VERSION }} - ecflow_version: ${{ inputs.ECFLOW_VERSION }} - ecbuild_version: ${{ inputs.ECBUILD_VERSION }} + DEPS_BASE_REPO: ${{ inputs.DEPS_BASE_REPO || 'python' }} + DEPS_BASE_TAG: ${{ inputs.DEPS_BASE_TAG || '3.12-slim-bookworm' }} + DEPS_BASE_SUFFIX: ${{ inputs.DEPS_BASE_SUFFIX || 'bookworm' }} + GHCR_ORG: ${{ inputs.GHCR_ORG }} + BOOST_VERSION: ${{ inputs.BOOST_VERSION }} + ESMF_VERSION: ${{ inputs.ESMF_VERSION }} # CodeQL scan codeql-scan: @@ -346,9 +311,9 @@ jobs: # that; otherwise use the override inputs, else this org's deps :latest REPO="${{ needs.dependencies.outputs.image }}" TAG="${{ needs.dependencies.outputs.tag }}" - [ -n "$REPO" ] || REPO="${{ inputs.BMI_BASE_REPO }}" - [ -n "$REPO" ] || REPO="${REGISTRY}/${{ needs.setup.outputs.org }}/ngen-dependencies-rocky8" - [ -n "$TAG" ] || TAG="${{ inputs.BMI_BASE_TAG }}" + [ -n "$REPO" ] || REPO="${{ inputs.DEPS_IMAGE_REPO }}" + [ -n "$REPO" ] || REPO="${REGISTRY}/${{ needs.setup.outputs.org }}/ngen-dependencies-bookworm" + [ -n "$TAG" ] || TAG="${{ inputs.DEPS_IMAGE_TAG }}" [ -n "$TAG" ] || TAG="latest" # inspect the base for the OCI base.* labels on the forcing image @@ -379,10 +344,9 @@ jobs: push: true tags: ${{ needs.setup.outputs.bmi_image }}:${{ needs.setup.outputs.test_image_tag }} build-args: | - BMI_BASE_REPO=${{ steps.base.outputs.repo }} - BMI_BASE_TAG=${{ steps.base.outputs.tag }} - BMI_BASE_DIGEST=${{ steps.base.outputs.digest }} - BMI_BASE_REVISION=${{ steps.base.outputs.revision }} + DEPS_IMAGE=${{ steps.base.outputs.repo }}:${{ steps.base.outputs.tag }} + DEPS_IMAGE_DIGEST=${{ steps.base.outputs.digest }} + DEPS_IMAGE_REVISION=${{ steps.base.outputs.revision }} EWTS_ORG=${{ inputs.EWTS_ORG || github.repository_owner }} EWTS_REF=${{ inputs.EWTS_REF || needs.setup.outputs.default_ref }} EWTS_REVISION=${{ needs.setup.outputs.ewts_revision }} @@ -412,7 +376,7 @@ jobs: - name: Unit Test BMI run: | docker run --rm ${{ needs.setup.outputs.bmi_image }}:${{ needs.setup.outputs.test_image_tag }} \ - echo "Running BMI Forcing Unit Tests..." + -c 'echo "Running BMI Forcing Unit Tests..."' - name: Install Trivy uses: aquasecurity/setup-trivy@v0.2.6 with: diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml index cc944f91..f2025163 100644 --- a/.github/workflows/dependencies.yml +++ b/.github/workflows/dependencies.yml @@ -6,9 +6,11 @@ name: Dependencies Image # forcing build can order itself after a base rebuild with `needs`, and the base # is only rebuilt when the dependency files change. # -# The published image name follows the base image, so rockylinux:8 publishes -# ngen-dependencies-rocky8. Change deps_base_repo / deps_base_tag to move off -# Rocky and the name follows; set deps_base_suffix to force a specific name. +# The published image name follows the base image. The current base is +# python:3.12-slim-bookworm, published as ngen-dependencies-bookworm via the +# explicit DEPS_BASE_SUFFIX=bookworm override (the suffix auto-derived from +# python:3.12-slim-bookworm would not be "bookworm"). Change DEPS_BASE_REPO / +# DEPS_BASE_TAG to change the base; set DEPS_BASE_SUFFIX to control the name. # # Outputs the published repo and tag so the caller can build FROM exactly what # was just built. @@ -16,71 +18,36 @@ name: Dependencies Image on: workflow_call: inputs: - deps_base_repo: - description: 'Base image to compile the dependencies from (default rockylinux)' + DEPS_BASE_REPO: + description: 'Base image to compile the dependencies from (default python)' required: false type: string default: '' - deps_base_tag: - description: 'Base image tag (default 8)' + DEPS_BASE_TAG: + description: 'Base image tag (default 3.12-slim-bookworm)' required: false type: string default: '' - deps_base_suffix: - description: 'Override the published name suffix (default derived from the base, e.g. rocky8)' + DEPS_BASE_SUFFIX: + description: 'Override the published name suffix (default derived from the base, e.g. bookworm)' required: false type: string default: '' - ghcr_org: + GHCR_ORG: description: 'GHCR org (default repo owner, lowercased)' required: false type: string default: '' - szip_version: - description: 'SZIP version (blank = Dockerfile.dependencies default)' - required: false - type: string - default: '' - hdf5_version: - description: 'HDF5 version (blank = Dockerfile.dependencies default)' - required: false - type: string - default: '' - netcdf_c_version: - description: 'NetCDF-C version (blank = Dockerfile.dependencies default)' - required: false - type: string - default: '' - netcdf_fortran_version: - description: 'NetCDF-Fortran version (blank = Dockerfile.dependencies default)' - required: false - type: string - default: '' - boost_version: + BOOST_VERSION: description: 'Boost version (blank = Dockerfile.dependencies default)' required: false type: string default: '' - esmf_version: + ESMF_VERSION: description: 'ESMF version (blank = Dockerfile.dependencies default)' required: false type: string default: '' - gdal_version: - description: 'GDAL version (blank = Dockerfile.dependencies default)' - required: false - type: string - default: '' - ecflow_version: - description: 'ecFlow version (blank = Dockerfile.dependencies default)' - required: false - type: string - default: '' - ecbuild_version: - description: 'ecbuild version (blank = Dockerfile.dependencies default)' - required: false - type: string - default: '' outputs: image: description: 'Published deps image repo (no tag)' @@ -114,7 +81,7 @@ jobs: set -euo pipefail # org (lowercased for GHCR); ghcr_org input overrides the repo owner - ORG="$(echo "${{ inputs.ghcr_org || github.repository_owner }}" | tr '[:upper:]' '[:lower:]')" + ORG="$(echo "${{ inputs.GHCR_ORG || github.repository_owner }}" | tr '[:upper:]' '[:lower:]')" # one datetime for all time variables NOW=$(date -u +'%Y-%m-%d %H:%M:%S') @@ -133,13 +100,13 @@ jobs: # base image the dependencies are compiled FROM. inputs may arrive empty # (the caller passes them through on every event), so fall back here. - DEPS_BASE_REPO="${{ inputs.deps_base_repo }}" - [ -n "$DEPS_BASE_REPO" ] || DEPS_BASE_REPO="rockylinux" - DEPS_BASE_TAG="${{ inputs.deps_base_tag }}" - [ -n "$DEPS_BASE_TAG" ] || DEPS_BASE_TAG="8" + DEPS_BASE_REPO="${{ inputs.DEPS_BASE_REPO }}" + [ -n "$DEPS_BASE_REPO" ] || DEPS_BASE_REPO="python" + DEPS_BASE_TAG="${{ inputs.DEPS_BASE_TAG }}" + [ -n "$DEPS_BASE_TAG" ] || DEPS_BASE_TAG="3.12-slim-bookworm" # resolve the base image's manifest digest for the OCI base.digest - # label (passed through as the DEPS_BASE_DIGEST build-arg below). skopeo + # label (passed through as the BASE_IMAGE_DIGEST build-arg below). skopeo # inspect reads the manifest without pulling; --override-os/--override-arch # pin the amd64 digest (the platform this image builds for) so the value # does not vary with the runner's own architecture. @@ -151,12 +118,13 @@ jobs: DEPS_BASE_DIGEST=$(echo "$DEPS_BASE_INSPECT" | jq -r '.Digest // "unknown"') # derive the published name suffix from the base image (unless overridden) - # so the package name reflects its base (rockylinux:8 -> rocky8): + # so the package name reflects its base (e.g. rockylinux:8 derives rocky8; + # the current python:3.12-slim-bookworm base instead sets DEPS_BASE_SUFFIX=bookworm explicitly): # - keep only the last path segment (drop any registry/namespace) # - drop the literal "linux" (rockylinux -> rocky) # - append the tag reduced to alphanumerics (8 -> 8, 24.04 -> 2404) # - lowercase - SUFFIX_INPUT="${{ inputs.deps_base_suffix }}" + SUFFIX_INPUT="${{ inputs.DEPS_BASE_SUFFIX }}" if [ -n "$SUFFIX_INPUT" ]; then BASE_SUFFIX="$SUFFIX_INPUT" else @@ -186,22 +154,15 @@ jobs: # workflow only overrides when asked). ARGS=( "IMAGE_SOURCE=https://github.com/${{ github.repository }}" - "DEPS_BASE_REPO=${DEPS_BASE_REPO}" - "DEPS_BASE_TAG=${DEPS_BASE_TAG}" - "DEPS_BASE_DIGEST=${DEPS_BASE_DIGEST}" + "IMAGE_REVISION=${REAL_SHA}" + "BASE_IMAGE=${DEPS_BASE_REPO}:${DEPS_BASE_TAG}" + "BASE_IMAGE_DIGEST=${DEPS_BASE_DIGEST}" ) # add NAME VALUE -> appends "NAME=VALUE" to ARGS when VALUE is non-empty # (the "|| true" keeps the skipped case from tripping "set -e"). add() { [ -n "$2" ] && ARGS+=("$1=$2") || true; } - add SZIP_VERSION "${{ inputs.szip_version }}" - add HDF5_VERSION "${{ inputs.hdf5_version }}" - add NETCDF_C_VERSION "${{ inputs.netcdf_c_version }}" - add NETCDF_FORTRAN_VERSION "${{ inputs.netcdf_fortran_version }}" - add BOOST_VERSION "${{ inputs.boost_version }}" - add ESMF_VERSION "${{ inputs.esmf_version }}" - add GDAL_VERSION "${{ inputs.gdal_version }}" - add ECFLOW_VERSION "${{ inputs.ecflow_version }}" - add ECBUILD_VERSION "${{ inputs.ecbuild_version }}" + add BOOST_VERSION "${{ inputs.BOOST_VERSION }}" + add ESMF_VERSION "${{ inputs.ESMF_VERSION }}" # Publish step outputs. $GITHUB_OUTPUT is a FILE the runner provides; # every line appended to it becomes `steps.vars.outputs.`, diff --git a/DOCKER.md b/DOCKER.md index 1f7c47f0..9f024705 100644 --- a/DOCKER.md +++ b/DOCKER.md @@ -31,10 +31,10 @@ docker run -it ngen-bmi-forcing ``` This will drop you to a bash prompt inside the container. -You will next need to activate the required conda environment: -``` -conda activate /ngen-app/conda -``` +The Python virtual environment (`/ngen-app/ngen-python`) is already on the `PATH`, +so `python` runs the container's interpreter with all dependencies installed. No +activation step is needed. + All the ngen-forcing scripts are located at `/ngen-app/ngen-forcing/`. @@ -46,10 +46,11 @@ docker run -it ngen-lumped-forcing ``` This will drop you to a bash prompt inside the container. -You will next need to activate the required conda environment: -``` -conda activate /ngen-app/conda -``` +The `ngen_lumped_forcings_driver` conda environment is already on the `PATH`, so +`python` runs with its dependencies installed. The image also ships the +`ngen_forcing_extraction` and `ngen_streamflow` environments; switch between them +with `conda activate `. + All the ngen-forcing scripts are located at `/ngen-app/ngen-forcing/`. ## Troubleshooting diff --git a/Dockerfile.bmi-forcings b/Dockerfile.bmi-forcings index 8c6a4d96..500be873 100644 --- a/Dockerfile.bmi-forcings +++ b/Dockerfile.bmi-forcings @@ -19,22 +19,22 @@ ARG EWTS_REF=development ############################################################################ # Use the compiled dependencies image. This already includes the slow-moving -# Rocky-based FOSS/compiled stack: WGRIB2, ESMF, SZIP, HDF5, NetCDF-C, -# NetCDF-Fortran, Boost, and GDAL. +# Bookworm-based FOSS/compiled stack: WGRIB2, ESMF, ESMPy, HDF5, NetCDF-C, +# NetCDF-Fortran, Boost, GDAL, UDUNITS2, and stable Python dependencies. # # Default build: # docker build -t ngen-bmi-forcing . # # Build from a different published dependency image: # docker build \ -# --build-arg DEPS_IMAGE=ghcr.io/ngwpc/ngen-dependencies-rocky \ +# --build-arg DEPS_IMAGE=ghcr.io/ngwpc/ngen-dependencies-bookworm \ # -t ngen-bmi-forcing . # # Build from a locally built dependency image: # docker build \ -# --build-arg DEPS_IMAGE=ngen-dependencies-rocky \ +# --build-arg DEPS_IMAGE=ngen-dependencies-bookworm \ # -t ngen-bmi-forcing . -ARG DEPS_IMAGE=ghcr.io/${GHCR_ORG}/ngen-dependencies-rocky8 +ARG DEPS_IMAGE=ghcr.io/${GHCR_ORG}/ngen-dependencies-bookworm FROM ${DEPS_IMAGE} @@ -45,6 +45,7 @@ ARG IMAGE_NAMESPACE ARG USE_EWTS=ON ARG EWTS_ORG ARG EWTS_REF +ARG DEPS_IMAGE # Optional cache-bust args. # Leave this at 0 for normal builds. Override only when you intentionally want @@ -52,8 +53,8 @@ ARG EWTS_REF ARG EWTS_CACHE_BUST=0 # OCI Metadata Arguments -ARG DEPS_IMAGE -ARG DEPS_IMAGE_NAME="${DEPS_IMAGE}" +# +# DEPS_IMAGE_* refers to the dependency image this image is built FROM. ARG DEPS_IMAGE_DIGEST="unknown" ARG DEPS_IMAGE_REVISION="unknown" ARG IMAGE_SOURCE="unknown" @@ -63,18 +64,18 @@ ARG IMAGE_REVISION="unknown" ARG EWTS_REVISION="unknown" # Image Labels: OCI-spec annotations followed by custom source-repo metadata. -LABEL org.opencontainers.image.base.name="${DEPS_IMAGE_NAME}" \ - org.opencontainers.image.base.digest="${DEPS_IMAGE_DIGEST}" \ - org.opencontainers.image.source="${IMAGE_SOURCE}" \ - org.opencontainers.image.vendor="${IMAGE_VENDOR}" \ - org.opencontainers.image.version="${IMAGE_VERSION}" \ - org.opencontainers.image.revision="${IMAGE_REVISION}" \ - org.opencontainers.image.title="NGEN BMI Forcing" \ - org.opencontainers.image.description="Docker image for the NGEN BMI Forcings application" \ - io.${IMAGE_NAMESPACE}.image.base.revision="${DEPS_IMAGE_REVISION}" \ - io.${IMAGE_NAMESPACE}.ewts.org="${EWTS_ORG}" \ - io.${IMAGE_NAMESPACE}.ewts.ref="${EWTS_REF}" \ - io.${IMAGE_NAMESPACE}.ewts.revision="${EWTS_REVISION}" +LABEL org.opencontainers.image.base.name="${DEPS_IMAGE}" \ + org.opencontainers.image.base.digest="${DEPS_IMAGE_DIGEST}" \ + org.opencontainers.image.source="${IMAGE_SOURCE}" \ + org.opencontainers.image.vendor="${IMAGE_VENDOR}" \ + org.opencontainers.image.version="${IMAGE_VERSION}" \ + org.opencontainers.image.revision="${IMAGE_REVISION}" \ + org.opencontainers.image.title="NGEN BMI Forcing" \ + org.opencontainers.image.description="Docker image for the NGEN BMI Forcings application" \ + io.${IMAGE_NAMESPACE}.image.base.revision="${DEPS_IMAGE_REVISION}" \ + io.${IMAGE_NAMESPACE}.ewts.org="${EWTS_ORG}" \ + io.${IMAGE_NAMESPACE}.ewts.ref="${EWTS_REF}" \ + io.${IMAGE_NAMESPACE}.ewts.revision="${EWTS_REVISION}" ############################################################################ # Python environment inherited from dependencies image @@ -84,9 +85,7 @@ LABEL org.opencontainers.image.base.name="${DEPS_IMAGE_NAME}" \ # image. Do not recreate it here; forcing, ngen, and downstream images should # all install into and reuse the same venv. ENV VIRTUAL_ENV="/ngen-app/ngen-python" -ENV PATH="${VIRTUAL_ENV}/bin:${PATH}" \ - PYTHONPATH="${VIRTUAL_ENV}/lib/python3.11/site-packages:/usr/local/lib64/python3.11/site-packages:${PYTHONPATH}" - +ENV PATH="${VIRTUAL_ENV}/bin:${PATH}" ############################################################################ # Optional EWTS Python package @@ -103,7 +102,7 @@ SHELL ["/bin/bash", "-c"] # ON, YES, TRUE, 1 # # Everything else is treated as OFF. -RUN --mount=type=cache,target=/root/.cache/pip,id=pip-cache-rocky \ +RUN --mount=type=cache,target=/root/.cache/pip,id=pip-cache-bookworm \ set -eux && \ USE_EWTS="${USE_EWTS:-ON}" && \ echo "USE_EWTS=${USE_EWTS}; EWTS org: ${EWTS_ORG}; ref: ${EWTS_REF}; cache bust: ${EWTS_CACHE_BUST}" && \ @@ -145,13 +144,15 @@ RUN --mount=type=cache,target=/root/.cache/pip,id=pip-cache-rocky \ COPY . /ngen-app/ngen-forcing/ WORKDIR /ngen-app/ngen-forcing -RUN --mount=type=cache,target=/root/.cache/pip,id=pip-cache-rocky \ +RUN --mount=type=cache,target=/root/.cache/pip,id=pip-cache-bookworm \ set -eux; \ python -m pip install . ARG CI_COMMIT_REF_NAME RUN set -eux; \ + # Ensure local tag metadata includes all remote tags before creating git_info. + git fetch --force --tags origin '+refs/tags/*:refs/tags/*' && \ repo_url=$(git config --get remote.origin.url); \ key=${repo_url##*/}; \ key=${key%.git}; \ diff --git a/Dockerfile.dependencies b/Dockerfile.dependencies index 6da58a79..eae6f9f3 100644 --- a/Dockerfile.dependencies +++ b/Dockerfile.dependencies @@ -1,136 +1,149 @@ # syntax=docker/dockerfile:1.4 ############################################################################ -# Rocky compiled dependency base image for ngen-bmi-forcing / ngen +# Bookworm dependency base image for ngen-bmi-forcing / ngen # -# Contains slow-moving compiled dependencies: -# - WGRIB2 -# - ESMF -# - SZIP +# Uses Debian packages where practical: # - HDF5 # - NetCDF-C # - NetCDF-Fortran -# - Boost # - GDAL # - UDUNITS2 +# +# Builds from source where still needed: +# - WGRIB2 +# - Boost +# - ESMF + ESMPy # - ecFlow Python client ############################################################################ -ARG BASE_IMAGE=rockylinux:8 +ARG BASE_IMAGE=python:3.12-slim-bookworm FROM ${BASE_IMAGE} -# Re-expose the base args after FROM (an ARG declared before FROM is only in -# scope for the FROM line) so they can populate the OCI base.* image labels. -# DEPS_BASE_NAME is composed here; DEPS_BASE_DIGEST defaults to "unknown" and is -# overridden by the build-arg dependencies.yml resolves with skopeo at build time. ARG BASE_IMAGE ARG BASE_IMAGE_NAME="${BASE_IMAGE}" ARG BASE_IMAGE_DIGEST="unknown" ARG IMAGE_SOURCE="unknown" +ARG IMAGE_REVISION="unknown" -ARG SZIP_VERSION=2.1.1 -ARG HDF5_VERSION=1.10.11 -ARG NETCDF_C_VERSION=4.7.4 -ARG NETCDF_FORTRAN_VERSION=4.5.4 -ARG BOOST_VERSION=1.86.0 ARG ESMF_VERSION=8.8.0 -ARG GDAL_VERSION=3.7.3 +ARG BOOST_VERSION=1.86.0 ARG ECFLOW_VERSION=5.15.2 ARG ECBUILD_VERSION=3.14.2 -# Image Labels: OCI-spec base annotations first, then the compiled library -# versions baked into this image. LABEL org.opencontainers.image.base.name="${BASE_IMAGE_NAME}" \ - org.opencontainers.image.base.digest="${BASE_IMAGE_DIGEST}" \ - org.opencontainers.image.source="${IMAGE_SOURCE}" \ - io.ngwpc.szip.version="${SZIP_VERSION}" \ - io.ngwpc.hdf5.version="${HDF5_VERSION}" \ - io.ngwpc.netcdf-c.version="${NETCDF_C_VERSION}" \ - io.ngwpc.netcdf-fortran.version="${NETCDF_FORTRAN_VERSION}" \ - io.ngwpc.boost.version="${BOOST_VERSION}" \ - io.ngwpc.esmf.version="${ESMF_VERSION}" \ - io.ngwpc.gdal.version="${GDAL_VERSION}" \ - io.ngwpc.ecflow.version="${ECFLOW_VERSION}" \ - io.ngwpc.ecbuild.version="${ECBUILD_VERSION}" - -ENV LANG="C.UTF-8" -ENV PATH="/usr/local/bin:${PATH}" + org.opencontainers.image.base.digest="${BASE_IMAGE_DIGEST}" \ + org.opencontainers.image.source="${IMAGE_SOURCE}" \ + org.opencontainers.image.revision="${IMAGE_REVISION}" \ + io.ngwpc.boost.version="${BOOST_VERSION}" \ + io.ngwpc.esmf.version="${ESMF_VERSION}" \ + io.ngwpc.ecflow.version="${ECFLOW_VERSION}" \ + io.ngwpc.ecbuild.version="${ECBUILD_VERSION}" + +ENV LANG="C.UTF-8" \ + PATH="/usr/local/bin:${PATH}" ############################################################################ # System/FOSS dependencies ############################################################################ -RUN set -eux; \ - dnf install -y dnf-plugins-core; \ - dnf install -y epel-release; \ - dnf config-manager --set-enabled powertools; \ - dnf install -y \ - proj proj-devel \ - sqlite sqlite-devel \ +RUN --mount=type=cache,target=/var/cache/apt,id=apt-cache-bookworm,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,id=apt-lib-bookworm,sharing=locked \ + set -eux && \ + apt-get update && \ + apt-get install -y --no-install-recommends \ autoconf \ automake \ - bzip2 bzip2-devel \ + build-essential \ + bzip2 \ + ca-certificates \ + ccache \ cmake \ - curl curl-devel \ - file \ - findutils \ + curl \ + gfortran \ git \ jq \ - gcc-toolset-10 \ - gcc-toolset-10-libasan-devel \ - gcc-toolset-10-gcc-gfortran \ - jasper-libs jasper-devel \ - libaec libaec-devel \ - libasan6 \ - libffi libffi-devel \ - libpng libpng-devel \ + libaec-dev \ + libbz2-dev \ + libcurl4-openssl-dev \ + libffi-dev \ + libgdal-dev \ + libhdf5-dev \ + libnetcdf-dev \ + libnetcdff-dev \ + libopenjp2-7-dev \ + libopenmpi-dev \ + libpng-dev \ + libproj-dev \ + libsqlite3-dev \ + libssl-dev \ + libudunits2-dev \ m4 \ - openmpi openmpi-devel \ - openssl openssl-devel \ - python3.11 \ - python3.11-devel \ - python3.11-pip \ - python3.11-pyyaml \ - rsync \ - udunits2 udunits2-devel \ - uuid uuid-devel \ - which \ - xz xz-devel \ - zlib zlib-devel \ - hdf5-devel \ - netcdf-devel \ - netcdf-fortran-devel \ - ; \ - dnf clean all; \ - rm -rf /var/cache/dnf - -SHELL ["/usr/bin/scl", "enable", "gcc-toolset-10"] - -ENV PATH="${PATH}:/usr/lib64/openmpi/bin/" - -# Create the shared Python virtual environment early so any Python packages -# built by compiled dependencies, including ESMPy, install into the same venv -# reused later by forcing, ngen, and cal-mgr. + netcdf-bin \ + openmpi-bin \ + pkg-config \ + proj-bin \ + proj-data \ + uuid-dev \ + xz-utils \ + zlib1g-dev && \ + rm -rf /var/lib/apt/lists/* + +SHELL ["/bin/bash", "-c"] + +RUN update-ccache-symlinks + +ENV PATH="/usr/lib/ccache:${PATH}:/usr/lib/x86_64-linux-gnu/openmpi/bin" \ + CCACHE_DIR="/root/.cache/ccache" \ + CCACHE_MAXSIZE="5G" \ + CCACHE_COMPRESS="true" + +# Create the shared Python virtual environment early so Python packages built by +# compiled dependencies, including ESMPy and mpi4py, install into the same venv +# reused later by forcing, ngen, and downstream images. +# +# These must be separate ENV instructions. Docker expands references in an ENV +# instruction using values defined before that instruction, so VIRTUAL_ENV is not +# available when expanding PATH if both variables are first defined together. +# Defining VIRTUAL_ENV first ensures PATH includes the venv's bin directory and +# subsequent `python` and `pip` commands use the shared virtual environment. ENV VIRTUAL_ENV="/ngen-app/ngen-python" -ENV PATH="${VIRTUAL_ENV}/bin:/usr/local/bin:${PATH}" -ENV PYTHONPATH="${VIRTUAL_ENV}/lib/python3.11/site-packages:/usr/local/lib64/python3.11/site-packages" +ENV PATH="${VIRTUAL_ENV}/bin:${PATH}" RUN set -eux; \ mkdir -p /ngen-app; \ - python3.11 -m venv --system-site-packages "${VIRTUAL_ENV}"; \ - ln -sf "${VIRTUAL_ENV}/bin/python" /usr/local/bin/python; \ - python --version; \ - python -c "import sys; print(sys.executable)" + python -m venv "${VIRTUAL_ENV}" + +# Install the shared Python packaging and build tools before building ESMPy or +# installing any other Python packages. Python 3.12 virtual environments do not +# include setuptools by default, and source-based package installations require +# current versions of setuptools, wheel, and the PEP 517 build tooling. +# +# These tools are installed into the shared virtual environment so they are also +# available to the forcing, ngen, and downstream images that inherit it. +RUN --mount=type=cache,target=/root/.cache/pip,id=pip-cache-bookworm \ + set -eux; \ + python -m pip install --upgrade \ + pip \ + setuptools \ + wheel \ + build \ + pyproject_hooks \ + packaging + +ENV LD_LIBRARY_PATH="/usr/lib/x86_64-linux-gnu/openmpi/lib:/usr/local/lib:/usr/local/lib64" +ENV OMPI_MCA_btl_vader_single_copy_mechanism=none +ENV OMPI_MCA_btl=^openib,ofi +ENV OMPI_MCA_pml=ob1 +ENV OMPI_MCA_btl_base_warn_component_unused=0 ############################################################################ # WGRIB2 -# -# Unlike the other compiled dependencies, WGRIB2 is not currently version-pinned. -# It is built from NOAA's current wgrib2.tgz distribution when this image is rebuilt. ############################################################################ -RUN set -eux; \ +RUN --mount=type=cache,target=/root/.cache/ccache,id=ccache-bookworm \ + set -eux; \ mkdir -p /tmp/build-wgrib2; \ cd /tmp/build-wgrib2; \ curl --location --output wgrib2.tgz \ @@ -144,8 +157,8 @@ RUN set -eux; \ USE_IPOLATES=3 \ USE_PNG=1 \ USE_AEC=0 \ - USE_JASPER=1 \ - USE_OPENJPEG=0 \ + USE_JASPER=0 \ + USE_OPENJPEG=1 \ USE_G2CLIB=0 \ PREFIX=/usr/local; \ make lib; \ @@ -155,17 +168,30 @@ RUN set -eux; \ ENV WGRIB2=/usr/local/bin/wgrib2 +# Prevent pip installations in this image and downstream images from replacing +# NumPy 1.26.4 while resolving transitive dependencies. +RUN printf '%s\n' \ + 'numpy==1.26.4' \ + > /ngen-app/python-constraints.txt + + +# Apply the NumPy pin automatically to subsequent pip installations in this +# image and downstream images. A downstream installation that requires an +# incompatible NumPy version will fail rather than replace NumPy 1.26.4. +ENV PIP_CONSTRAINT="/ngen-app/python-constraints.txt" ############################################################################ -# ESMF +# ESMF + ESMPy # -# ESMF is built before the source-built SZIP/HDF5/NetCDF stack below so it uses -# the Rocky-provided hdf5/netcdf/netcdf-fortran packages. This matches the -# original ngen-bmi-forcing Dockerfile behavior and avoids ESMF linking against -# the later /usr/local NetCDF install. +# ESMF is the one large scientific library still built from source in this +# Bookworm dependency image. Unlike the Rocky transition image, ESMF is built +# against Bookworm's packaged HDF5/NetCDF stack, and that same stack remains +# available at runtime. This avoids mixing distro NetCDF at build time with +# /usr/local NetCDF at runtime. ############################################################################ -RUN set -eux; \ +RUN --mount=type=cache,target=/root/.cache/ccache,id=ccache-bookworm \ + set -eux; \ mkdir -p /tmp/build-esmf; \ cd /tmp/build-esmf; \ curl --location --output "esmf-${ESMF_VERSION}.tar.gz" \ @@ -177,123 +203,42 @@ RUN set -eux; \ export ESMF_COMPILER=gfortran; \ export ESMF_COMM=openmpi; \ export ESMF_NETCDF="nc-config"; \ - export ESMF_F90COMPILEPATHS="-I/usr/lib64/gfortran/modules"; \ export ESMF_BOPT=O; \ make all; \ - make install - -ENV ESMFMKFILE=/usr/local/esmf/lib/libO/Linux.gfortran.64.openmpi.default/esmf.mk - -############################################################################ -# ESMPy install -############################################################################ - -RUN set -eux; \ - cd "/tmp/build-esmf/esmf-${ESMF_VERSION}"; \ - export ESMF_DIR="$(pwd)"; \ - export ESMFMKFILE=/usr/local/esmf/lib/libO/Linux.gfortran.64.openmpi.default/esmf.mk; \ - cd src/addon/esmpy; \ - python -m pip install . - -RUN rm -rf /tmp/build-esmf - - -############################################################################ -# SZIP / HDF5 / NetCDF-C / NetCDF-Fortran -# -# These are source-built after ESMF for compatibility with the original ngen -# dependency image, where ngen and several submodules expected the compiled -# dependency stack under /usr/local. -############################################################################ - -############################################################################ -# SZIP -############################################################################ - -RUN set -eux; \ - mkdir -p /tmp/build-szip; \ - cd /tmp/build-szip; \ - curl --location --output szip.tar.gz \ - "https://docs.hdfgroup.org/archive/support/ftp/lib-external/szip/${SZIP_VERSION%%[a-z]*}/src/szip-${SZIP_VERSION}.tar.gz"; \ - mkdir szip; \ - tar --extract --directory szip --strip-components=1 --file szip.tar.gz; \ - cd szip; \ - ./configure --prefix=/usr/local/; \ - make -j "$(nproc)"; \ - make install; \ - strip --strip-debug /usr/local/lib/libsz*.so.* || true; \ - rm -rf /tmp/build-szip - -############################################################################ -# HDF5 -############################################################################ - -RUN set -eux; \ - mkdir -p /tmp/build-hdf5; \ - cd /tmp/build-hdf5; \ - curl --location --output hdf5.tar.gz \ - "https://github.com/HDFGroup/hdf5/archive/refs/tags/hdf5-${HDF5_VERSION//./_}.tar.gz"; \ - mkdir hdf5; \ - tar --extract --directory hdf5 --strip-components=1 --file hdf5.tar.gz; \ - cd hdf5; \ - ./configure --prefix=/usr/local/ --with-szlib=/usr/local/; \ - make -j "$(nproc)"; \ make install; \ - strip --strip-debug /usr/local/lib/libhdf5*.so.* || true; \ - rm -f /usr/local/lib/libhdf5*.a; \ - rm -rf /tmp/build-hdf5 - -############################################################################ -# NetCDF-C -############################################################################ - -RUN set -eux; \ - mkdir -p /tmp/build-netcdf-c; \ - cd /tmp/build-netcdf-c; \ - curl --location --output netcdf-c.tar.gz \ - "https://github.com/Unidata/netcdf-c/archive/refs/tags/v${NETCDF_C_VERSION%%[a-z]*}.tar.gz"; \ - mkdir netcdf-c; \ - tar --extract --directory netcdf-c --strip-components=1 --file netcdf-c.tar.gz; \ - cd netcdf-c; \ - cmake -B cmake_build -S . \ - -DCMAKE_BUILD_TYPE=Release \ - -DBUILD_SHARED_LIBS=ON \ - -DENABLE_TESTS=OFF; \ - cmake --build cmake_build --parallel "$(nproc)"; \ - cmake --build cmake_build --target install; \ - ldconfig; \ - rm -rf /tmp/build-netcdf-c - -############################################################################ -# NetCDF-Fortran -############################################################################ + rm -rf /tmp/build-esmf -RUN set -eux; \ - mkdir -p /tmp/build-netcdf-fortran; \ - cd /tmp/build-netcdf-fortran; \ - curl --location --output netcdf-fortran.tar.gz \ - "https://github.com/Unidata/netcdf-fortran/archive/refs/tags/v${NETCDF_FORTRAN_VERSION%%[a-z]*}.tar.gz"; \ - mkdir netcdf-fortran; \ - tar --extract --directory netcdf-fortran --strip-components=1 --file netcdf-fortran.tar.gz; \ - cd netcdf-fortran; \ - cmake -B cmake_build -S . \ - -DCMAKE_BUILD_TYPE=Release \ - -DBUILD_SHARED_LIBS=ON \ - -DENABLE_TESTS=OFF \ - -DCMAKE_Fortran_COMPILER=/opt/rh/gcc-toolset-10/root/usr/bin/gfortran; \ - cmake --build cmake_build --parallel "$(nproc)"; \ - cmake --build cmake_build --target install; \ - ldconfig; \ - rm -rf /tmp/build-netcdf-fortran +ENV ESMFMKFILE=/usr/local/esmf/lib/libO/Linux.gfortran.64.openmpi.default/esmf.mk +# Install ESMPy from the git tag, matching the original forcing Dockerfile. +# The source tarball can build the same code, but ESMPy's package versioning +# depends on git metadata and may otherwise report a beta-looking version. +RUN --mount=type=cache,target=/root/.cache/pip,id=pip-cache-bookworm \ + set -eux; \ + python -m pip install \ + setuptools-git-versioning; \ + git clone --depth 1 --branch "v${ESMF_VERSION}" \ + https://github.com/esmf-org/esmf.git /tmp/esmf; \ + cd /tmp/esmf/src/addon/esmpy; \ + python -m pip install \ + .; \ + cd /; \ + rm -rf /tmp/esmf ############################################################################ # Boost +# +# Bookworm provides Boost 1.74, but LASAM requires Boost >= 1.79. +# Build Boost from source and install a copy of the source tree under /opt/boost +# so downstream CMake builds can use -DBOOST_ROOT=/opt/boost. +# +# The sed call removes Boost.NumPy. The ecFlow Python client needs +# Boost.Python but does not need Boost.NumPy, which can be problematic here. +# Boost.Python is required by the ecFlow Python client. ############################################################################ -# The sed call removes Boost.NumPy. ecFlow Python client does need --with-python (does need Boost.Python) but does not need Boost.Numpy, and including numpy here can be problematic. -# Boost Python is required by the ecFlow Python client. -RUN set -eux; \ +RUN --mount=type=cache,target=/root/.cache/ccache,id=ccache-bookworm \ + set -eux; \ mkdir -p /tmp/build-boost; \ cd /tmp/build-boost; \ curl --location --output boost.tar.gz \ @@ -303,7 +248,8 @@ RUN set -eux; \ cd boost; \ ./bootstrap.sh; \ sed -i 's/boost-install boost_python boost_numpy/boost-install boost_python/' libs/python/build/Jamfile; \ - CPLUS_INCLUDE_PATH="/usr/include/python3.11" ./b2 install --prefix=/usr/local; \ + CPLUS_INCLUDE_PATH="/usr/local/include/python3.12" \ + ./b2 install --prefix=/usr/local; \ rm -rf /opt/boost; \ mkdir --parents /opt/boost; \ cp -a /tmp/build-boost/boost/. /opt/boost/; \ @@ -312,35 +258,45 @@ RUN set -eux; \ rm -rf /tmp/build-boost ############################################################################ -# GDAL +# Stable Python dependencies +# +# mpi4py is built from source so it links against the same Bookworm OpenMPI +# installation used by ESMF, ESMPy, and ngen. ############################################################################ -RUN set -eux && \ - mkdir -p /tmp/build-gdal && \ - cd /tmp/build-gdal && \ - curl --location --output gdal.tar.gz \ - "https://github.com/OSGeo/gdal/releases/download/v${GDAL_VERSION}/gdal-${GDAL_VERSION}.tar.gz" && \ - mkdir gdal && \ - tar --extract --directory gdal --strip-components=1 --file gdal.tar.gz && \ - cd gdal && \ - cmake -B build -S . \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX=/usr/local && \ - cmake --build build --parallel "$(nproc)" && \ - cmake --build build --target install && \ - ldconfig && \ - rm -rf /tmp/build-gdal +# pybind11 is required to build the ecFlow Python client +RUN --mount=type=cache,target=/root/.cache/pip,id=pip-cache-bookworm \ + set -eux; \ + python -m pip install \ + "numpy==1.26.4" \ + netcdf4 \ + bmipy \ + pandas; \ + python -m pip install \ + --no-binary=mpi4py \ + mpi4py; \ + python -m pip install \ + --index-url https://download.pytorch.org/whl/cpu \ + torch \ + torchvision ############################################################################ -# ecFLow python client (requires build from source) +# ecFlow Python client (requires build from source) ############################################################################ -RUN mkdir -p /tmp/build-ecflow && cd /tmp/build-ecflow && \ - python -m pip install "pybind11>=2.10.3" && \ - git clone --depth 1 --branch "${ECFLOW_VERSION}" https://github.com/ecmwf/ecflow.git && \ - git clone --depth 1 --branch "${ECBUILD_VERSION}" https://github.com/ecmwf/ecbuild.git && \ - PYBIND11_CMAKE_DIR="$(python -c 'import pybind11; print(pybind11.get_cmake_dir())')" && \ +RUN --mount=type=cache,target=/root/.cache/ccache,id=ccache-bookworm \ + --mount=type=cache,target=/root/.cache/pip,id=pip-cache-bookworm \ + set -eux; \ + mkdir -p /tmp/build-ecflow; \ + python -m pip install "pybind11>=2.10.3"; \ + cd /tmp/build-ecflow; \ + git clone --depth 1 --branch "${ECFLOW_VERSION}" \ + https://github.com/ecmwf/ecflow.git; \ + git clone --depth 1 --branch "${ECBUILD_VERSION}" \ + https://github.com/ecmwf/ecbuild.git; \ + PYBIND11_CMAKE_DIR="$(python -c \ + 'import pybind11; print(pybind11.get_cmake_dir())')"; \ cmake -B ecflow/build -S ecflow \ -DCMAKE_INSTALL_PREFIX="${VIRTUAL_ENV}" \ -DCMAKE_BUILD_TYPE=Release \ @@ -354,23 +310,33 @@ RUN mkdir -p /tmp/build-ecflow && cd /tmp/build-ecflow && \ -DENABLE_STATIC_BOOST_LIBS=OFF \ -DBOOST_ROOT=/usr/local \ -DBoost_NO_SYSTEM_PATHS=on \ - -DPython3_EXECUTABLE="$(which python)" && \ - cmake --build ecflow/build --parallel "$(( $(nproc) < 4 ? $(nproc) : 4 ))" && \ - cmake --install ecflow/build && \ - python -m pip uninstall -y pybind11 && \ - python -c "import ecflow; print(f'ecflow Python client version {ecflow.__version__} installed successfully')" && \ - rm -r /tmp/build-ecflow + -DPython3_EXECUTABLE="$(which python)"; \ + cmake --build ecflow/build \ + --parallel "$(( $(nproc) < 4 ? $(nproc) : 4 ))"; \ + cmake --install ecflow/build; \ + python -m pip uninstall -y pybind11; \ + python -c "import ecflow; print(f'ecFlow Python client version {ecflow.__version__} installed successfully')"; \ + rm -rf /tmp/build-ecflow -############################################################################ -# Runtime environment -############################################################################ +WORKDIR / -ENV LD_LIBRARY_PATH="/usr/lib64/openmpi/lib:/usr/local/lib:/usr/local/lib64" -ENV OMPI_MCA_btl_vader_single_copy_mechanism=none -ENV OMPI_MCA_btl=^openib,ofi -ENV OMPI_MCA_pml=ob1 -ENV OMPI_MCA_btl_base_warn_component_unused=0 + +# Verify that all installed Python packages have compatible dependencies and +# that later installations did not replace NumPy 1.26.4. +RUN set -eux; \ + test "$(command -v python)" = "${VIRTUAL_ENV}/bin/python"; \ + test "${PIP_CONSTRAINT}" = "/ngen-app/python-constraints.txt"; \ + grep -qx 'numpy==1.26.4' "${PIP_CONSTRAINT}"; \ + python -m pip check; \ + python -c "import sys, setuptools, wheel, build, pyproject_hooks, packaging; \ +assert sys.executable == '${VIRTUAL_ENV}/bin/python', sys.executable; \ +print('Python:', sys.executable); \ +print('setuptools:', setuptools.__version__)"; \ + python -c "import numpy as np; \ +assert np.__version__ == '1.26.4', np.__version__; \ +np.linalg.svd(np.eye(8)); \ +print('NumPy version:', np.__version__)" WORKDIR / -ENTRYPOINT ["/bin/bash"] \ No newline at end of file +ENTRYPOINT ["/bin/bash"] diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/mpi_utils.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/mpi_utils.py index 7f37b920..f8157f27 100644 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/mpi_utils.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/mpi_utils.py @@ -17,15 +17,10 @@ def get_new_broadcasted_uid() -> str: if MPI.COMM_WORLD.rank == 0: rng = np.random.default_rng() rand_uint64 = rng.integers(0, 2**64, dtype=np.uint64) - else: - rand_uint64 = None rand_uint64 = MPI.COMM_WORLD.bcast(rand_uint64, root=0) - # Convert the NumPy uint64 to a built-in Python int. Python 3.14's uuid - # implementation expects a native int internally, while this remains fully - # compatible with earlier Python versions. + # uuid.UUID expects a built-in Python int. Convert the NumPy uint64 uid_64bit_hex = uuid.UUID(int=int(rand_uint64)).hex assert len(uid_64bit_hex) == 32 - uid64 = uid_64bit_hex[16:] - return uid64 + return uid_64bit_hex[16:]