diff --git a/.github/workflows/ci-base-image.yml b/.github/workflows/ci-base-image.yml index f2143b10..d23ff8ee 100644 --- a/.github/workflows/ci-base-image.yml +++ b/.github/workflows/ci-base-image.yml @@ -2,6 +2,11 @@ name: CI - Build base image on: workflow_dispatch: + inputs: + version: + description: 'Version tag for base image (e.g., v1.0.0, latest)' + required: true + default: 'latest' jobs: build-base: @@ -20,8 +25,17 @@ jobs: docker build . \ --file docker/Dockerfile \ --target emfos-base \ - -t ${{ vars.DOCKER_USERNAME }}/emfos-base:latest - docker push ${{ vars.DOCKER_USERNAME}}/emfos-base:latest + -t ${{ vars.DOCKER_USERNAME }}/emfos-base:${{ github.event.inputs.version }} + docker push ${{ vars.DOCKER_USERNAME }}/emfos-base:${{ github.event.inputs.version }} + + - name: Conditionally push as latest + shell: bash + run: | + if [[ "${{ github.event.inputs.version }}" == v* ]]; then + docker tag ${{ vars.DOCKER_USERNAME }}/emfos-base:${{ github.event.inputs.version }} \ + ${{ vars.DOCKER_USERNAME }}/emfos-base:latest + docker push ${{ vars.DOCKER_USERNAME }}/emfos-base:latest + fi - name: Display Success Message - run: echo "emfos-base image has been built and pushed manually." + run: echo "emfos-base:${{ github.event.inputs.version }} image has been built and pushed manually." diff --git a/.github/workflows/ci-workers.yml b/.github/workflows/ci-workers.yml index de320ae0..6c789bf8 100644 --- a/.github/workflows/ci-workers.yml +++ b/.github/workflows/ci-workers.yml @@ -16,6 +16,10 @@ on: version: description: 'Version tag for the release (e.g., v1.0.0)' required: true + base_image_tag: + description: 'Base image tag to use (e.g., latest, v1.0.0)' + required: true + default: 'latest' jobs: build_workers: @@ -41,6 +45,7 @@ jobs: # echo "IMAGE_TAG=$(date +%Y%m%d%H%M%S)" >> $GITHUB_ENV echo "IMAGE_NAME=${{ vars.DOCKER_USERNAME }}/${{ matrix.worker }}" >> $GITHUB_ENV echo "LATEST_TAG=latest" >> $GITHUB_ENV + echo "BASE_IMAGE_TAG=${{ github.event.inputs.base_image_tag || 'latest' }}" >> $GITHUB_ENV - name: Compute IMAGE_TAG id: meta @@ -58,33 +63,26 @@ jobs: fi echo "IMAGE_TAG=$TAG" >> "$GITHUB_ENV" - echo "Computed IMAGE_TAG=$TAG" + echo "Computed IMAGE_TAG=$TAG with base image tag=${{ env.BASE_IMAGE_TAG }}" - - name: Pull base image or fallback to build - run: | - docker pull ${{ vars.DOCKER_USERNAME }}/emfos-base:latest || \ - docker build . \ - --file docker/Dockerfile \ - --target emfos-base \ - -t ${{ vars.DOCKER_USERNAME }}/emfos-base:main + - name: Log in to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ vars.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} - - name: Build worker image + - name: Build worker image with specified base image run: | docker build . \ --file docker/Dockerfile \ --target ${{ env.WORKER_NAME }} \ + --build-arg BASE_IMAGE=${{ vars.DOCKER_USERNAME }}/emfos-base:${{ env.BASE_IMAGE_TAG }} \ -t ${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }} - name: Tag image as latest run: | docker tag ${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }} ${{ env.IMAGE_NAME }}:${{ env.LATEST_TAG }} - - name: Log in to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ vars.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - name: Push versioned image run: | docker push ${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }} diff --git a/config/integrations/minio.properties b/config/integrations/minio.properties index 9a1fd757..3d800362 100644 --- a/config/integrations/minio.properties +++ b/config/integrations/minio.properties @@ -4,4 +4,5 @@ MINIO_USERNAME = None MINIO_PASSWORD = None TOKEN_EXPIRATION = 86400 TOKEN_RENEW_MARGIN = 120 -MAXSIZE = 50 \ No newline at end of file +MAXSIZE = 50 +SLEEP_DURATION = 2 \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile index 470b11a2..2797f3f3 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,3 +1,6 @@ +# Global argument for base image for workers, can be overridden when building with --build-arg +ARG BASE_IMAGE=balticrcc/emfos-base:latest + # Define base container image (platform has to be defined if using with ARM64 architecture) FROM python:3.11-slim as emfos-base #FROM --platform=linux/amd64 python:3.11-slim as emfos-base @@ -23,7 +26,7 @@ COPY licenses/ /licenses/ # Set to use interpreter from venv (if venv is used) #ENV PATH="/.venv/bin:$PATH" -FROM balticrcc/emfos-base:latest as emfos-model-retriever +FROM ${BASE_IMAGE} as emfos-model-retriever # Set working directory WORKDIR /src @@ -38,7 +41,7 @@ ENV PYTHONPATH="${PYTHONPATH}:/src" ENTRYPOINT ["python", "emf/model_retriever/opdm_worker.py"] #ENTRYPOINT ["/bin/bash"] -FROM balticrcc/emfos-base:latest as emfos-local-model-retriever +FROM ${BASE_IMAGE} as emfos-local-model-retriever # Set working directory WORKDIR /src @@ -52,7 +55,7 @@ COPY config/ ./config/ ENV PYTHONPATH="${PYTHONPATH}:/src" ENTRYPOINT ["python", "emf/model_retriever/local_worker.py"] -FROM balticrcc/emfos-base:latest as emfos-model-validator +FROM ${BASE_IMAGE} as emfos-model-validator # Set working directory WORKDIR /src @@ -67,7 +70,7 @@ ENV PYTHONPATH="${PYTHONPATH}:/src" ENTRYPOINT ["python", "emf/model_validator/worker.py"] #ENTRYPOINT ["/bin/bash"] -FROM balticrcc/emfos-base:latest as emfos-model-merger +FROM ${BASE_IMAGE} as emfos-model-merger # Set working directory WORKDIR /src @@ -82,7 +85,7 @@ ENV PYTHONPATH="${PYTHONPATH}:/src" ENTRYPOINT ["python", "emf/model_merger/worker.py"] #ENTRYPOINT ["/bin/bash"] -FROM balticrcc/emfos-base:latest as emfos-task-generator +FROM ${BASE_IMAGE} as emfos-task-generator # Set working directory WORKDIR /src @@ -97,14 +100,16 @@ ENV PYTHONPATH="${PYTHONPATH}:/src" ENTRYPOINT ["python", "emf/task_generator/worker.py"] #ENTRYPOINT ["/bin/bash"] -# Model Quality image -FROM balticrcc/emfos-base:latest as emfos-model-quality +FROM ${BASE_IMAGE} as emfos-model-quality +# Set working directory WORKDIR /src +# Copy files to container COPY emf/model_quality/ ./emf/model_quality/ COPY emf/common/ ./emf/common/ COPY config/ ./config/ +# Start process ENV PYTHONPATH="${PYTHONPATH}:/src" ENTRYPOINT ["python", "emf/model_quality/worker.py"] diff --git a/emf/common/integrations/minio_api.py b/emf/common/integrations/minio_api.py index 3e01732e..e63d0a38 100644 --- a/emf/common/integrations/minio_api.py +++ b/emf/common/integrations/minio_api.py @@ -7,6 +7,7 @@ import sys import mimetypes import re +import time import logging import config import functools @@ -77,19 +78,84 @@ def _get_credentials(self, action: str = "AssumeRoleWithLDAPIdentity", version: "LDAPUsername": self.username, "LDAPPassword": self.password, "Version": version, - "DurationSeconds": TOKEN_EXPIRATION, + "DurationSeconds": '3600', } - # Sending request for temporary credentials and parsing it out from returned xml - response = requests.post(f"https://{self.server}", params=params, verify=False).content - credentials = {} - root = etree.fromstring(response) - et = root.find("{*}AssumeRoleWithLDAPIdentityResult/{*}Credentials") - for element in et: - _, _, tag = element.tag.rpartition("}") - credentials[tag] = element.text + last_error = None - return credentials + for attempt in range(1, 4): + response = None + try: + response = requests.post(f"https://{self.server}", params=params, verify=False, timeout=30) + logger.debug(f"MinIO auth response received, attempt {attempt}, HTTP status={response.status_code}") + + # Parse received response content + root = etree.fromstring(response.content) + + # Handle MinIO error response + err = root.find(".//{*}Error") + if err is not None: + code = err.findtext("{*}Code") + message = err.findtext("{*}Message") + request_id = root.findtext(".//{*}RequestId") + + last_error = ( + f"STS error: Code={code}, " + f"Message={message}, " + f"RequestId={request_id}" + ) + + logger.warning(f"MinIO auth failed on attempt {attempt}: {last_error}") + + if attempt < 3: + logger.info(f"Retrying credential request in 1 second...") + time.sleep(int(SLEEP_DURATION)) + continue + + raise RuntimeError(last_error) + + # Handle non-200 HTTP response without STS Error XML + if response.status_code != 200: + last_error = ( + f"HTTP error: status={response.status_code}, " + f"body={response.content!r}" + ) + + logger.warning(f"MinIO HTTP error on attempt {attempt}: {last_error}") + + if attempt < 3: + logger.info(f"Retrying credential request in 1 second...") + time.sleep(int(SLEEP_DURATION)) + continue + + raise RuntimeError(last_error) + + # Extract credentials + credentials = {} + et = root.find("{*}AssumeRoleWithLDAPIdentityResult/{*}Credentials") + if et is None: + raise RuntimeError(f"Credentials not found in response: {response.content!r}") + + for element in et: + tag = etree.QName(element).localname + credentials[tag] = element.text + + return credentials + + except requests.RequestException as e: + last_error = f"Request failed: {e}" + raise RuntimeError(last_error) from e + + except etree.XMLSyntaxError as e: + body = response.content if response is not None else b"" + last_error = ( + f"Failed to parse XML response: {e}. " + f"Response body: {body!r}" + ) + + raise RuntimeError(last_error) from e + + raise RuntimeError(last_error or "Failed to get credentials") @staticmethod def dict_to_tags(tags: dict):