Dev

.github/workflows/ci-base-image.yml

@@ -2,6 +2,11 @@ name: CI - Build base image
 
 on:
   workflow_dispatch:
+    inputs:
+      version:
+        description: 'Version tag for base image (e.g., v1.0.0, latest)'
+        required: true
+        default: 'latest'
 
 jobs:
   build-base:
@@ -20,8 +25,17 @@ jobs:
           docker build . \
             --file docker/Dockerfile \
             --target emfos-base \
-            -t ${{ vars.DOCKER_USERNAME }}/emfos-base:latest
-          docker push ${{ vars.DOCKER_USERNAME}}/emfos-base:latest
+            -t ${{ vars.DOCKER_USERNAME }}/emfos-base:${{ github.event.inputs.version }}
+          docker push ${{ vars.DOCKER_USERNAME }}/emfos-base:${{ github.event.inputs.version }}
+
+      - name: Conditionally push as latest
+        shell: bash
+        run: |
+          if [[ "${{ github.event.inputs.version }}" == v* ]]; then
+            docker tag ${{ vars.DOCKER_USERNAME }}/emfos-base:${{ github.event.inputs.version }} \
+                      ${{ vars.DOCKER_USERNAME }}/emfos-base:latest
+            docker push ${{ vars.DOCKER_USERNAME }}/emfos-base:latest
+          fi
 
       - name: Display Success Message
-        run: echo "emfos-base image has been built and pushed manually."
+        run: echo "emfos-base:${{ github.event.inputs.version }} image has been built and pushed manually."

.github/workflows/ci-workers.yml

@@ -16,6 +16,10 @@ on:
       version:
         description: 'Version tag for the release (e.g., v1.0.0)'
         required: true
+      base_image_tag:
+        description: 'Base image tag to use (e.g., latest, v1.0.0)'
+        required: true
+        default: 'latest'
 
 jobs:
   build_workers:
@@ -41,6 +45,7 @@ jobs:
           # echo "IMAGE_TAG=$(date +%Y%m%d%H%M%S)"        >> $GITHUB_ENV
           echo "IMAGE_NAME=${{ vars.DOCKER_USERNAME }}/${{ matrix.worker }}" >> $GITHUB_ENV
           echo "LATEST_TAG=latest"                      >> $GITHUB_ENV
+          echo "BASE_IMAGE_TAG=${{ github.event.inputs.base_image_tag || 'latest' }}" >> $GITHUB_ENV
 
       - name: Compute IMAGE_TAG
         id: meta
@@ -58,33 +63,26 @@ jobs:
           fi
 
           echo "IMAGE_TAG=$TAG" >> "$GITHUB_ENV"
-          echo "Computed IMAGE_TAG=$TAG"
+          echo "Computed IMAGE_TAG=$TAG with base image tag=${{ env.BASE_IMAGE_TAG }}"
 
-      - name: Pull base image or fallback to build
-        run: |
-          docker pull ${{ vars.DOCKER_USERNAME }}/emfos-base:latest || \
-          docker build . \
-            --file docker/Dockerfile \
-            --target emfos-base \
-            -t ${{ vars.DOCKER_USERNAME }}/emfos-base:main
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ vars.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
 
-      - name: Build worker image
+      - name: Build worker image with specified base image
         run: |
           docker build . \
             --file docker/Dockerfile \
             --target ${{ env.WORKER_NAME }} \
+            --build-arg BASE_IMAGE=${{ vars.DOCKER_USERNAME }}/emfos-base:${{ env.BASE_IMAGE_TAG }} \
             -t ${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }}
 
       - name: Tag image as latest
         run: |
           docker tag ${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }} ${{ env.IMAGE_NAME }}:${{ env.LATEST_TAG }}
 
-      - name: Log in to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ vars.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-
       - name: Push versioned image
         run: |
           docker push ${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }}

config/integrations/minio.properties

@@ -4,4 +4,5 @@ MINIO_USERNAME = None
 MINIO_PASSWORD = None
 TOKEN_EXPIRATION = 86400
 TOKEN_RENEW_MARGIN = 120
-MAXSIZE = 50
+MAXSIZE = 50
+SLEEP_DURATION = 2

docker/Dockerfile

@@ -1,3 +1,6 @@
+# Global argument for base image for workers, can be overridden when building with --build-arg
+ARG BASE_IMAGE=balticrcc/emfos-base:latest
+
 # Define base container image (platform has to be defined if using with ARM64 architecture)
 FROM python:3.11-slim as emfos-base
 #FROM --platform=linux/amd64 python:3.11-slim as emfos-base
@@ -23,7 +26,7 @@
 # Set to use interpreter from venv (if venv is used)
 #ENV PATH="/.venv/bin:$PATH"
 
-FROM balticrcc/emfos-base:latest as emfos-model-retriever
+FROM ${BASE_IMAGE} as emfos-model-retriever
 
 # Set working directory
 WORKDIR /src
@@ -38,7 +41,7 @@
 ENTRYPOINT ["python", "emf/model_retriever/opdm_worker.py"]
 #ENTRYPOINT ["/bin/bash"]
 
-FROM balticrcc/emfos-base:latest as emfos-local-model-retriever
+FROM ${BASE_IMAGE} as emfos-local-model-retriever
 
 # Set working directory
 WORKDIR /src
@@ -52,7 +55,7 @@
 ENV PYTHONPATH="${PYTHONPATH}:/src"
 ENTRYPOINT ["python", "emf/model_retriever/local_worker.py"]
 
-FROM balticrcc/emfos-base:latest as emfos-model-validator
+FROM ${BASE_IMAGE} as emfos-model-validator
 
 # Set working directory
 WORKDIR /src
@@ -67,7 +70,7 @@
 ENTRYPOINT ["python", "emf/model_validator/worker.py"]
 #ENTRYPOINT ["/bin/bash"]
 
-FROM balticrcc/emfos-base:latest as emfos-model-merger
+FROM ${BASE_IMAGE} as emfos-model-merger
 
 # Set working directory
 WORKDIR /src
@@ -82,7 +85,7 @@
 ENTRYPOINT ["python", "emf/model_merger/worker.py"]
 #ENTRYPOINT ["/bin/bash"]
 
-FROM balticrcc/emfos-base:latest as emfos-task-generator
+FROM ${BASE_IMAGE} as emfos-task-generator
 
 # Set working directory
 WORKDIR /src
@@ -97,14 +100,16 @@
 ENTRYPOINT ["python", "emf/task_generator/worker.py"]
 #ENTRYPOINT ["/bin/bash"]
 
-# Model Quality image
-FROM balticrcc/emfos-base:latest as emfos-model-quality
+FROM ${BASE_IMAGE} as emfos-model-quality
 
+# Set working directory
 WORKDIR /src
 
+# Copy files to container
 COPY emf/model_quality/ ./emf/model_quality/
 COPY emf/common/ ./emf/common/
 COPY config/ ./config/
 
+# Start process
 ENV PYTHONPATH="${PYTHONPATH}:/src"
 ENTRYPOINT ["python", "emf/model_quality/worker.py"]

emf/common/integrations/minio_api.py

@@ -7,6 +7,7 @@
 import sys
 import mimetypes
 import re
+import time
 import logging
 import config
 import functools
@@ -64,7 +65,7 @@
                                   http_client=self.http_client,
                                   )
 
     def _get_credentials(self, action: str = "AssumeRoleWithLDAPIdentity", version: str = "2011-06-15"):
         """
         Method to get temporary credentials for LDAP user
         :param action: string of action
@@ -77,19 +78,84 @@
             "LDAPUsername": self.username,
             "LDAPPassword": self.password,
             "Version": version,
-            "DurationSeconds": TOKEN_EXPIRATION,
+            "DurationSeconds": '3600',
         }
 
-        # Sending request for temporary credentials and parsing it out from returned xml
-        response = requests.post(f"https://{self.server}", params=params, verify=False).content
-        credentials = {}
-        root = etree.fromstring(response)
-        et = root.find("{*}AssumeRoleWithLDAPIdentityResult/{*}Credentials")
-        for element in et:
-            _, _, tag = element.tag.rpartition("}")
-            credentials[tag] = element.text
+        last_error = None
 
-        return credentials
+        for attempt in range(1, 4):
+            response = None
+            try:
+                response = requests.post(f"https://{self.server}", params=params, verify=False, timeout=30)
+                logger.debug(f"MinIO auth response received, attempt {attempt}, HTTP status={response.status_code}")
+
+                # Parse received response content
+                root = etree.fromstring(response.content)
+
+                # Handle MinIO error response
+                err = root.find(".//{*}Error")
+                if err is not None:
+                    code = err.findtext("{*}Code")
+                    message = err.findtext("{*}Message")
+                    request_id = root.findtext(".//{*}RequestId")
+
+                    last_error = (
+                        f"STS error: Code={code}, "
+                        f"Message={message}, "
+                        f"RequestId={request_id}"
+                    )
+
+                    logger.warning(f"MinIO auth failed on attempt {attempt}: {last_error}")
+
+                    if attempt < 3:
+                        logger.info(f"Retrying credential request in 1 second...")
+                        time.sleep(int(SLEEP_DURATION))
+                        continue
+
+                    raise RuntimeError(last_error)
+
+                # Handle non-200 HTTP response without STS Error XML
+                if response.status_code != 200:
+                    last_error = (
+                        f"HTTP error: status={response.status_code}, "
+                        f"body={response.content!r}"
+                    )
+
+                    logger.warning(f"MinIO HTTP error on attempt {attempt}: {last_error}")
+
+                    if attempt < 3:
+                        logger.info(f"Retrying credential request in 1 second...")
+                        time.sleep(int(SLEEP_DURATION))
+                        continue
+
+                    raise RuntimeError(last_error)
+
+                # Extract credentials
+                credentials = {}
+                et = root.find("{*}AssumeRoleWithLDAPIdentityResult/{*}Credentials")
+                if et is None:
+                    raise RuntimeError(f"Credentials not found in response: {response.content!r}")
+
+                for element in et:
+                    tag = etree.QName(element).localname
+                    credentials[tag] = element.text
+
+                return credentials
+
+            except requests.RequestException as e:
+                last_error = f"Request failed: {e}"
+                raise RuntimeError(last_error) from e
+
+            except etree.XMLSyntaxError as e:
+                body = response.content if response is not None else b""
+                last_error = (
+                    f"Failed to parse XML response: {e}. "
+                    f"Response body: {body!r}"
+                )
+
+                raise RuntimeError(last_error) from e
+
+        raise RuntimeError(last_error or "Failed to get credentials")
 
     @staticmethod
     def dict_to_tags(tags: dict):
@@ -151,7 +217,7 @@
             return file_data.read()
 
         except minio.error.S3Error as err:
             logger.error(err)
 
     @renew_authentication_token
     def object_exists(self, object_name: str, bucket_name: str) -> bool:
@@ -178,7 +244,7 @@
             objects = self.client.list_objects(bucket_name, prefix, recursive, start_after, include_user_meta, include_version)
             return objects
         except minio.error.S3Error as err:
             logger.error(err, exc_info=True)
 
     @renew_authentication_token
     def query_objects(self, bucket_name: str, metadata: dict = None, prefix: str = None, use_regex: bool = False):