Modernize, minimize, and cleanup containers

.github/workflows/build_docker_layers.yaml

@@ -0,0 +1,140 @@
+name: Build microservices docker images
+on:
+  push:
+    branches:
+      - develop
+    tags:
+      - 'v*'
+  pull_request:
+    branches:
+      - develop
+    paths:
+      - 'dockerfiles/requirements.txt'
+      - 'dockerfiles/Dockerfile.dependencies'
+      - 'dockerfiles/Dockerfile'
+      - '.github/workflows/build_docker_layers.yaml'
+  workflow_dispatch:
+    inputs:
+      force_rebuild_dependencies:
+        description: 'Force rebuild of dependencies image'
+        required: false
+        type: boolean
+        default: false
+jobs:
+  build-dependencies:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    # Only build dependencies:
+    #   if manually triggered with force flag
+    #   on tag push
+    #   on push to default branch AND dependency files changed
+    if: |
+      (github.event_name == 'workflow_dispatch' && github.event.inputs.force_rebuild_dependencies == 'true') ||
+      github.ref_type == 'tag' ||
+      (github.event_name == 'push' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch) &&
+        github.event.head_commit != null &&
+        (
+          contains(github.event.head_commit.modified, 'dockerfiles/requirements.txt') ||
+          contains(github.event.head_commit.added, 'dockerfiles/requirements.txt') ||
+          contains(github.event.head_commit.modified, 'dockerfiles/Dockerfile.dependencies') ||
+          contains(github.event.head_commit.added, 'dockerfiles/Dockerfile.dependencies') ||
+          contains(github.event.head_commit.modified, 'dockerfiles/Dockerfile') ||
+          contains(github.event.head_commit.added, 'dockerfiles/Dockerfile')
+        ))
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Log in to GHCR
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Extract Docker metadata for dependencies
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ghcr.io/DUNE-DAQ/microservices_dependencies
+          tags: |
+            type=raw,value=latest,enable={{is_default_branch}}
+            type=ref,event=branch
+            type=ref,event=tag
+            type=sha,format=short
+      - name: Build and push dependencies image
+        uses: docker/build-push-action@v6
+        with:
+          context: ./dockerfiles
+          file: ./dockerfiles/Dockerfile.dependencies
+          platforms: linux/amd64
+          push: ${{ github.event_name != 'pull_request' }}
+          provenance: true
+          sbom: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+  build-microservices:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    # Always run, but depend on dependencies job if it ran
+    needs: [build-dependencies]
+    if: |
+      always() &&
+      (needs.build-dependencies.result == 'success' || needs.build-dependencies.result == 'skipped')
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+      - name: Get git refs
+        id: git_refs
+        run: |
+          echo "short_sha=$(git rev-parse --short HEAD)" >> "${GITHUB_OUTPUT}"
+          echo "full_sha=$(git rev-parse HEAD)" >> "${GITHUB_OUTPUT}"
+      - name: Find microservices_dependency tag
+        id: find_dep_tag
+        run: |
+          if [[ "${{ needs.build-dependencies.result }}" == "success" ]]; then
+            # Dependencies image was rebuilt for this commit, so use current short SHA
+            echo "dep_tag=$(git rev-parse --short HEAD)" >> "${GITHUB_OUTPUT}"
+          else
+            # Fallback: use 'latest' if no SHA-like tag was found or API call failed
+            echo "Warning: Could not determine SHA-based dependency tag, falling back to 'latest'" >&2
+            echo "dep_tag=latest" >> "${GITHUB_OUTPUT}"
+          fi
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Log in to GHCR
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Extract Docker metadata for microservices
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ghcr.io/DUNE-DAQ/microservices
+          tags: |
+            type=raw,value=latest,enable={{is_default_branch}}
+            type=ref,event=branch
+            type=ref,event=tag
+            type=sha,format=short
+      - name: Build and push microservices image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ./dockerfiles/Dockerfile
+          platforms: linux/amd64
+          push: ${{ github.event_name != 'pull_request' }}
+          provenance: true
+          sbom: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          build-args: |
+            DEPENDENCY_TAG=${{ steps.find_dep_tag.outputs.dep_tag }}
+            MICROSERVICES_VERSION=${{ steps.git_refs.outputs.full_sha }}

dockerfiles/Dockerfile

@@ -0,0 +1,13 @@
+# Must define DEPENDENCY_TAG before it is used
+ARG DEPENDENCY_TAG=latest
+FROM ghcr.io/dune-daq/microservices_dependencies:$DEPENDENCY_TAG
+
+ARG MICROSERVICES_VERSION=develop
+RUN : "${APP_ROOT:?APP_ROOT variable is required}" \
+    && git clone -b ${MICROSERVICES_VERSION} https://github.com/DUNE-DAQ/microservices.git \
+    && cp microservices/entrypoint.sh /
+
+WORKDIR ${APP_ROOT}/microservices
+
+USER 1234
+ENTRYPOINT ["/entrypoint.sh"]

dockerfiles/Dockerfile.dependencies

@@ -0,0 +1,70 @@
+FROM docker.io/almalinux:10
+
+ARG ERSVERSION=coredaq-v5.5.0         # For issue.proto from ers
+ARG ERSKAFKAVERSION=coredaq-v5.5.0    # For ERSSubscriber.py from erskafka
+ARG OPMONLIBVERSION=coredaq-v5.5.0    # For opmon_entry.proto from opmonlib
+ARG KAFKAOPMONVERSION=coredaq-v5.5.0  # For OpMonSubscriber.py from kafkaopmon
+
+ARG VENV_PATH=/opt/venv
+ENV \
+    APP_ROOT=/opt/app                 \
+    APP_DATA=/opt/data                \
+    HOME=/opt/app                     \
+    PYTHONPYCACHEPREFIX=/tmp/pycache  \
+    PYTHONUNBUFFERED=1                \
+    PIP_NO_CACHE_DIR=1
+
+ENV PATH="${VENV_PATH}/bin:$PATH"
+
+RUN mkdir -p ${APP_ROOT} ${APP_DATA} ${VENV_PATH} && chmod 1777 ${APP_DATA}
+WORKDIR ${APP_ROOT}
+
+# Install base python bits
+# Install required devel bits
+RUN yum clean expire-cache \
+    && yum -y install curl git python3-pip python3-pip-wheel \
+    && yum -y install make gcc python3-devel protobuf-compiler protobuf-devel krb5-devel libffi-devel libpq-devel postgresql \
+    && yum clean all
+
+# setup venv
+RUN python3 -m venv ${VENV_PATH} \
+    && ${VENV_PATH}/bin/pip install --no-cache-dir --upgrade pip \
+    && rm -rf /root/.cache ${HOME}/.cache ${VENV_PATH}/pip-selfcheck.json
+
+COPY requirements.txt ${VENV_PATH}/
+RUN ${VENV_PATH}/bin/pip install --no-cache-dir -r ${VENV_PATH}/requirements.txt \
+    && rm -rf /root/.cache ${HOME}/.cache ${VENV_PATH}/pip-selfcheck.json ${VENV_PATH}/requirements.txt
+
+# setup protobuf schemas
+RUN echo "Installing https://raw.githubusercontent.com/DUNE-DAQ/ers/${ERSVERSION}/schema/ers/issue.proto" \
+    && curl -fSL -O https://raw.githubusercontent.com/DUNE-DAQ/ers/$ERSVERSION/schema/ers/issue.proto \
+    && mkdir -p ${VENV_PATH}/ers \
+    && touch ${VENV_PATH}/ers/__init__.py \
+    && protoc --python_out=${VENV_PATH}/ers issue.proto \
+    && rm -f issue.proto \
+    && echo "Installing https://raw.githubusercontent.com/DUNE-DAQ/opmonlib/${OPMONLIBVERSION}/schema/opmonlib/opmon_entry.proto" \
+    && curl -fSL -O https://raw.githubusercontent.com/DUNE-DAQ/opmonlib/$OPMONLIBVERSION/schema/opmonlib/opmon_entry.proto \
+    && mkdir -p ${VENV_PATH}/opmonlib \
+    && touch ${VENV_PATH}/opmonlib/__init__.py \
+    && protoc --python_out=${VENV_PATH}/opmonlib -I${APP_ROOT} opmon_entry.proto \
+    && rm -f opmon_entry.proto
+
+# fetch ERS python bindings
+RUN mkdir -p ${VENV_PATH}/erskafka \
+    && touch ${VENV_PATH}/erskafka/__init__.py \
+    && echo "Installing https://raw.githubusercontent.com/DUNE-DAQ/erskafka/$ERSKAFKAVERSION/python/erskafka/ERSSubscriber.py" \
+    && curl -fSL https://raw.githubusercontent.com/DUNE-DAQ/erskafka/$ERSKAFKAVERSION/python/erskafka/ERSSubscriber.py -o ${VENV_PATH}/erskafka/ERSSubscriber.py \
+    && mkdir -p ${VENV_PATH}/kafkaopmon \
+    && touch ${VENV_PATH}/kafkaopmon/__init__.py \
+    && echo "Installing https://raw.githubusercontent.com/DUNE-DAQ/kafkaopmon/${KAFKAOPMONVERSION}/python/kafkaopmon/OpMonSubscriber.py" \
+    && curl -fSL https://raw.githubusercontent.com/DUNE-DAQ/kafkaopmon/$KAFKAOPMONVERSION/python/kafkaopmon/OpMonSubscriber.py -o ${VENV_PATH}/kafkaopmon/OpMonSubscriber.py
+
+# elisa_client_api and CERN kerberos needed by the logbook microservice at NP04
+COPY cern.repo /etc/yum.repos.d/
+RUN yum clean expire-cache \
+    && yum -y install krb5-workstation cern-krb5-conf \
+    && yum clean all
+
+RUN git clone --depth 1 -b develop https://github.com/DUNE-DAQ/elisa_client_api.git \
+    && ${VENV_PATH}/bin/pip install --no-cache-dir ./elisa_client_api \
+    && rm -rf /root/.cache ${HOME}/.cache ${VENV_PATH}/pip-selfcheck.json elisa_client_api

dockerfiles/cern.repo

@@ -0,0 +1,20 @@
+[cern]
+name=AlmaLinux $releasever - CERN
+baseurl=https://linuxsoft.cern.ch/cern/alma/$releasever/CERN/$basearch
+enabled=1
+gpgcheck=1
+gpgkey=https://gitlab.cern.ch/linuxsupport/rpms/cern-gpg-keys/-/raw/main/src/RPM-GPG-KEY-kojiv3
+
+[cern-testing]
+name=AlmaLinux $releasever - CERN - testing
+baseurl=https://linuxsoft.cern.ch/cern/alma/$releasever-testing/CERN/$basearch
+enabled=0
+gpgcheck=1
+gpgkey=https://gitlab.cern.ch/linuxsupport/rpms/cern-gpg-keys/-/raw/main/src/RPM-GPG-KEY-kojiv3
+
+[cern-source]
+name=AlmaLinux $releasever - CERN Source
+baseurl=https://linuxsoft.cern.ch/cern/alma/$releasever/CERN/Source/
+enabled=0
+gpgcheck=1
+gpgkey=https://gitlab.cern.ch/linuxsupport/rpms/cern-gpg-keys/-/raw/main/src/RPM-GPG-KEY-kojiv3