Peter aws migration

.github/workflows/cicd.yml

@@ -19,28 +19,28 @@ on:
       - release-candidate
   workflow_dispatch:
     inputs:
-      rebuild_deps:
+      REBUILD_DEPS:
         description: 'Force a rebuild of the dependencies image before building forcing'
         required: false
         default: false
         type: boolean
-      deps_base_repo:
-        description: 'Base image to compile the dependencies from (default rockylinux)'
+      DEPS_BASE_REPO:
+        description: 'Base image to compile the dependencies from (default python)'
         required: false
         type: string
-      deps_base_tag:
-        description: 'Dependencies base image tag (default 8)'
+      DEPS_BASE_TAG:
+        description: 'Dependencies base image tag (default 3.12-slim-bookworm)'
         required: false
         type: string
-      deps_base_suffix:
-        description: 'Override the published deps name suffix (default derived, e.g. rocky8)'
+      DEPS_BASE_SUFFIX:
+        description: 'Override the published deps name suffix (default derived, e.g. bookworm)'
         required: false
         type: string
-      BMI_BASE_REPO:
-        description: 'Deps image repo to build forcing FROM when not rebuilding deps (default ghcr.io/<org>/ngen-dependencies-rocky8)'
+      DEPS_IMAGE_REPO:
+        description: 'Deps image repo to build forcing FROM when not rebuilding deps (default ghcr.io/<org>/ngen-dependencies-bookworm)'
         required: false
         type: string
-      BMI_BASE_TAG:
+      DEPS_IMAGE_TAG:
         description: 'Deps image tag to build forcing FROM when not rebuilding deps (default latest)'
         required: false
         type: string
@@ -81,22 +81,6 @@ on:
         description: 'MSW_MGR_REF'
         required: false
         type: string
-      SZIP_VERSION:
-        description: 'Deps image: SZIP version (blank = Dockerfile default; forces a deps rebuild)'
-        required: false
-        type: string
-      HDF5_VERSION:
-        description: 'Deps image: HDF5 version (blank = Dockerfile default; forces a deps rebuild)'
-        required: false
-        type: string
-      NETCDF_C_VERSION:
-        description: 'Deps image: NetCDF-C version (blank = Dockerfile default; forces a deps rebuild)'
-        required: false
-        type: string
-      NETCDF_FORTRAN_VERSION:
-        description: 'Deps image: NetCDF-Fortran version (blank = Dockerfile default; forces a deps rebuild)'
-        required: false
-        type: string
       BOOST_VERSION:
         description: 'Deps image: Boost version (blank = Dockerfile default; forces a deps rebuild)'
         required: false
@@ -105,18 +89,6 @@ on:
         description: 'Deps image: ESMF version (blank = Dockerfile default; forces a deps rebuild)'
         required: false
         type: string
-      GDAL_VERSION:
-        description: 'Deps image: GDAL version (blank = Dockerfile default; forces a deps rebuild)'
-        required: false
-        type: string
-      ECFLOW_VERSION:
-        description: 'Deps image: ecFlow version (blank = Dockerfile default; forces a deps rebuild)'
-        required: false
-        type: string
-      ECBUILD_VERSION:
-        description: 'Deps image: ecbuild version (blank = Dockerfile default; forces a deps rebuild)'
-        required: false
-        type: string
 
 permissions:
   contents: read
@@ -256,7 +228,7 @@ jobs:
         run: |
           # if any dependency version input was provided, force a deps rebuild so
           # the override actually takes effect
-          if [ -n "${{ inputs.SZIP_VERSION }}${{ inputs.HDF5_VERSION }}${{ inputs.NETCDF_C_VERSION }}${{ inputs.NETCDF_FORTRAN_VERSION }}${{ inputs.BOOST_VERSION }}${{ inputs.ESMF_VERSION }}${{ inputs.GDAL_VERSION }}${{ inputs.ECFLOW_VERSION }}${{ inputs.ECBUILD_VERSION }}" ]; then
+          if [ -n "${{ inputs.BOOST_VERSION }}${{ inputs.ESMF_VERSION }}" ]; then
             echo "set=true" >> "$GITHUB_OUTPUT"
           fi
 
@@ -266,22 +238,15 @@ jobs:
   dependencies:
     name: dependencies
     needs: changes
-    if: ${{ needs.changes.outputs.deps == 'true' || inputs.rebuild_deps || needs.changes.outputs.versions == 'true' }}
+    if: ${{ needs.changes.outputs.deps == 'true' || inputs.REBUILD_DEPS || needs.changes.outputs.versions == 'true' }}
     uses: ./.github/workflows/dependencies.yml
     with:
-      deps_base_repo: ${{ inputs.deps_base_repo }}
-      deps_base_tag: ${{ inputs.deps_base_tag }}
-      deps_base_suffix: ${{ inputs.deps_base_suffix }}
-      ghcr_org: ${{ inputs.GHCR_ORG }}
-      szip_version: ${{ inputs.SZIP_VERSION }}
-      hdf5_version: ${{ inputs.HDF5_VERSION }}
-      netcdf_c_version: ${{ inputs.NETCDF_C_VERSION }}
-      netcdf_fortran_version: ${{ inputs.NETCDF_FORTRAN_VERSION }}
-      boost_version: ${{ inputs.BOOST_VERSION }}
-      esmf_version: ${{ inputs.ESMF_VERSION }}
-      gdal_version: ${{ inputs.GDAL_VERSION }}
-      ecflow_version: ${{ inputs.ECFLOW_VERSION }}
-      ecbuild_version: ${{ inputs.ECBUILD_VERSION }}
+      DEPS_BASE_REPO: ${{ inputs.DEPS_BASE_REPO || 'python' }}
+      DEPS_BASE_TAG: ${{ inputs.DEPS_BASE_TAG || '3.12-slim-bookworm' }}
+      DEPS_BASE_SUFFIX: ${{ inputs.DEPS_BASE_SUFFIX || 'bookworm' }}
+      GHCR_ORG: ${{ inputs.GHCR_ORG }}
+      BOOST_VERSION: ${{ inputs.BOOST_VERSION }}
+      ESMF_VERSION: ${{ inputs.ESMF_VERSION }}
 
   # CodeQL scan
   codeql-scan:
@@ -346,9 +311,9 @@ jobs:
           # that; otherwise use the override inputs, else this org's deps :latest
           REPO="${{ needs.dependencies.outputs.image }}"
           TAG="${{ needs.dependencies.outputs.tag }}"
-          [ -n "$REPO" ] || REPO="${{ inputs.BMI_BASE_REPO }}"
-          [ -n "$REPO" ] || REPO="${REGISTRY}/${{ needs.setup.outputs.org }}/ngen-dependencies-rocky8"
-          [ -n "$TAG" ] || TAG="${{ inputs.BMI_BASE_TAG }}"
+          [ -n "$REPO" ] || REPO="${{ inputs.DEPS_IMAGE_REPO }}"
+          [ -n "$REPO" ] || REPO="${REGISTRY}/${{ needs.setup.outputs.org }}/ngen-dependencies-bookworm"
+          [ -n "$TAG" ] || TAG="${{ inputs.DEPS_IMAGE_TAG }}"
           [ -n "$TAG" ] || TAG="latest"
 
           # inspect the base for the OCI base.* labels on the forcing image
@@ -379,10 +344,9 @@ jobs:
           push: true
           tags: ${{ needs.setup.outputs.bmi_image }}:${{ needs.setup.outputs.test_image_tag }}
           build-args: |
-            BMI_BASE_REPO=${{ steps.base.outputs.repo }}
-            BMI_BASE_TAG=${{ steps.base.outputs.tag }}
-            BMI_BASE_DIGEST=${{ steps.base.outputs.digest }}
-            BMI_BASE_REVISION=${{ steps.base.outputs.revision }}
+            DEPS_IMAGE=${{ steps.base.outputs.repo }}:${{ steps.base.outputs.tag }}
+            DEPS_IMAGE_DIGEST=${{ steps.base.outputs.digest }}
+            DEPS_IMAGE_REVISION=${{ steps.base.outputs.revision }}
             EWTS_ORG=${{ inputs.EWTS_ORG || github.repository_owner }}
             EWTS_REF=${{ inputs.EWTS_REF || needs.setup.outputs.default_ref }}
             EWTS_REVISION=${{ needs.setup.outputs.ewts_revision }}
@@ -412,7 +376,7 @@ jobs:
       - name: Unit Test BMI
         run: |
           docker run --rm ${{ needs.setup.outputs.bmi_image }}:${{ needs.setup.outputs.test_image_tag }} \
-          echo "Running BMI Forcing Unit Tests..."
+          -c 'echo "Running BMI Forcing Unit Tests..."'
       - name: Install Trivy
         uses: aquasecurity/setup-trivy@v0.2.6
         with:

.github/workflows/dependencies.yml

@@ -6,81 +6,48 @@ name: Dependencies Image
 # forcing build can order itself after a base rebuild with `needs`, and the base
 # is only rebuilt when the dependency files change.
 #
-# The published image name follows the base image, so rockylinux:8 publishes
-# ngen-dependencies-rocky8. Change deps_base_repo / deps_base_tag to move off
-# Rocky and the name follows; set deps_base_suffix to force a specific name.
+# The published image name follows the base image. The current base is
+# python:3.12-slim-bookworm, published as ngen-dependencies-bookworm via the
+# explicit DEPS_BASE_SUFFIX=bookworm override (the suffix auto-derived from
+# python:3.12-slim-bookworm would not be "bookworm"). Change DEPS_BASE_REPO /
+# DEPS_BASE_TAG to change the base; set DEPS_BASE_SUFFIX to control the name.
 #
 # Outputs the published repo and tag so the caller can build FROM exactly what
 # was just built.
 
 on:
   workflow_call:
     inputs:
-      deps_base_repo:
-        description: 'Base image to compile the dependencies from (default rockylinux)'
+      DEPS_BASE_REPO:
+        description: 'Base image to compile the dependencies from (default python)'
         required: false
         type: string
         default: ''
-      deps_base_tag:
-        description: 'Base image tag (default 8)'
+      DEPS_BASE_TAG:
+        description: 'Base image tag (default 3.12-slim-bookworm)'
         required: false
         type: string
         default: ''
-      deps_base_suffix:
-        description: 'Override the published name suffix (default derived from the base, e.g. rocky8)'
+      DEPS_BASE_SUFFIX:
+        description: 'Override the published name suffix (default derived from the base, e.g. bookworm)'
         required: false
         type: string
         default: ''
-      ghcr_org:
+      GHCR_ORG:
         description: 'GHCR org (default repo owner, lowercased)'
         required: false
         type: string
         default: ''
-      szip_version:
-        description: 'SZIP version (blank = Dockerfile.dependencies default)'
-        required: false
-        type: string
-        default: ''
-      hdf5_version:
-        description: 'HDF5 version (blank = Dockerfile.dependencies default)'
-        required: false
-        type: string
-        default: ''
-      netcdf_c_version:
-        description: 'NetCDF-C version (blank = Dockerfile.dependencies default)'
-        required: false
-        type: string
-        default: ''
-      netcdf_fortran_version:
-        description: 'NetCDF-Fortran version (blank = Dockerfile.dependencies default)'
-        required: false
-        type: string
-        default: ''
-      boost_version:
+      BOOST_VERSION:
         description: 'Boost version (blank = Dockerfile.dependencies default)'
         required: false
         type: string
         default: ''
-      esmf_version:
+      ESMF_VERSION:
         description: 'ESMF version (blank = Dockerfile.dependencies default)'
         required: false
         type: string
         default: ''
-      gdal_version:
-        description: 'GDAL version (blank = Dockerfile.dependencies default)'
-        required: false
-        type: string
-        default: ''
-      ecflow_version:
-        description: 'ecFlow version (blank = Dockerfile.dependencies default)'
-        required: false
-        type: string
-        default: ''
-      ecbuild_version:
-        description: 'ecbuild version (blank = Dockerfile.dependencies default)'
-        required: false
-        type: string
-        default: ''
     outputs:
       image:
         description: 'Published deps image repo (no tag)'
@@ -114,7 +81,7 @@ jobs:
           set -euo pipefail
 
           # org (lowercased for GHCR); ghcr_org input overrides the repo owner
-          ORG="$(echo "${{ inputs.ghcr_org || github.repository_owner }}" | tr '[:upper:]' '[:lower:]')"
+          ORG="$(echo "${{ inputs.GHCR_ORG || github.repository_owner }}" | tr '[:upper:]' '[:lower:]')"
 
           # one datetime for all time variables
           NOW=$(date -u +'%Y-%m-%d %H:%M:%S')
@@ -133,13 +100,13 @@ jobs:
 
           # base image the dependencies are compiled FROM. inputs may arrive empty
           # (the caller passes them through on every event), so fall back here.
-          DEPS_BASE_REPO="${{ inputs.deps_base_repo }}"
-          [ -n "$DEPS_BASE_REPO" ] || DEPS_BASE_REPO="rockylinux"
-          DEPS_BASE_TAG="${{ inputs.deps_base_tag }}"
-          [ -n "$DEPS_BASE_TAG" ] || DEPS_BASE_TAG="8"
+          DEPS_BASE_REPO="${{ inputs.DEPS_BASE_REPO }}"
+          [ -n "$DEPS_BASE_REPO" ] || DEPS_BASE_REPO="python"
+          DEPS_BASE_TAG="${{ inputs.DEPS_BASE_TAG }}"
+          [ -n "$DEPS_BASE_TAG" ] || DEPS_BASE_TAG="3.12-slim-bookworm"
 
           # resolve the base image's manifest digest for the OCI base.digest
-          # label (passed through as the DEPS_BASE_DIGEST build-arg below). skopeo
+          # label (passed through as the BASE_IMAGE_DIGEST build-arg below). skopeo
           # inspect reads the manifest without pulling; --override-os/--override-arch
           # pin the amd64 digest (the platform this image builds for) so the value
           # does not vary with the runner's own architecture.
@@ -151,12 +118,13 @@ jobs:
           DEPS_BASE_DIGEST=$(echo "$DEPS_BASE_INSPECT" | jq -r '.Digest // "unknown"')
 
           # derive the published name suffix from the base image (unless overridden)
-          # so the package name reflects its base (rockylinux:8 -> rocky8):
+          # so the package name reflects its base (e.g. rockylinux:8 derives rocky8;
+          # the current python:3.12-slim-bookworm base instead sets DEPS_BASE_SUFFIX=bookworm explicitly):
           #   - keep only the last path segment (drop any registry/namespace)
           #   - drop the literal "linux" (rockylinux -> rocky)
           #   - append the tag reduced to alphanumerics (8 -> 8, 24.04 -> 2404)
           #   - lowercase
-          SUFFIX_INPUT="${{ inputs.deps_base_suffix }}"
+          SUFFIX_INPUT="${{ inputs.DEPS_BASE_SUFFIX }}"
           if [ -n "$SUFFIX_INPUT" ]; then
             BASE_SUFFIX="$SUFFIX_INPUT"
           else
@@ -186,22 +154,15 @@ jobs:
           # workflow only overrides when asked).
           ARGS=(
             "IMAGE_SOURCE=https://github.com/${{ github.repository }}"
-            "DEPS_BASE_REPO=${DEPS_BASE_REPO}"
-            "DEPS_BASE_TAG=${DEPS_BASE_TAG}"
-            "DEPS_BASE_DIGEST=${DEPS_BASE_DIGEST}"
+            "IMAGE_REVISION=${REAL_SHA}"
+            "BASE_IMAGE=${DEPS_BASE_REPO}:${DEPS_BASE_TAG}"
+            "BASE_IMAGE_DIGEST=${DEPS_BASE_DIGEST}"
           )
           # add NAME VALUE -> appends "NAME=VALUE" to ARGS when VALUE is non-empty
           # (the "|| true" keeps the skipped case from tripping "set -e").
           add() { [ -n "$2" ] && ARGS+=("$1=$2") || true; }
-          add SZIP_VERSION "${{ inputs.szip_version }}"
-          add HDF5_VERSION "${{ inputs.hdf5_version }}"
-          add NETCDF_C_VERSION "${{ inputs.netcdf_c_version }}"
-          add NETCDF_FORTRAN_VERSION "${{ inputs.netcdf_fortran_version }}"
-          add BOOST_VERSION "${{ inputs.boost_version }}"
-          add ESMF_VERSION "${{ inputs.esmf_version }}"
-          add GDAL_VERSION "${{ inputs.gdal_version }}"
-          add ECFLOW_VERSION "${{ inputs.ecflow_version }}"
-          add ECBUILD_VERSION "${{ inputs.ecbuild_version }}"
+          add BOOST_VERSION "${{ inputs.BOOST_VERSION }}"
+          add ESMF_VERSION "${{ inputs.ESMF_VERSION }}"
 
           # Publish step outputs. $GITHUB_OUTPUT is a FILE the runner provides;
           # every line appended to it becomes `steps.vars.outputs.<name>`,

DOCKER.md

@@ -31,10 +31,10 @@ docker run -it ngen-bmi-forcing
 ```
 This will drop you to a bash prompt inside the container.
 
-You will next need to activate the required conda environment:
-```
-conda activate /ngen-app/conda
-```
+The Python virtual environment (`/ngen-app/ngen-python`) is already on the `PATH`,
+so `python` runs the container's interpreter with all dependencies installed. No
+activation step is needed.
+
 All the ngen-forcing scripts are located at `/ngen-app/ngen-forcing/`.
 
 
@@ -46,10 +46,11 @@ docker run -it ngen-lumped-forcing
 ```
 This will drop you to a bash prompt inside the container.
 
-You will next need to activate the required conda environment:
-```
-conda activate /ngen-app/conda
-```
+The `ngen_lumped_forcings_driver` conda environment is already on the `PATH`, so
+`python` runs with its dependencies installed. The image also ships the
+`ngen_forcing_extraction` and `ngen_streamflow` environments; switch between them
+with `conda activate <env-name>`.
+
 All the ngen-forcing scripts are located at `/ngen-app/ngen-forcing/`.
 
 ## Troubleshooting