diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cb999848..177a5b18 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -140,6 +140,12 @@ jobs: - img : dockerhubaneo/armonik_demo_java_worker path: ./java/native/HelloWorld/worker/Dockerfile ctx: java/native/HelloWorld/worker/ + - img : dockerhubaneo/armonik_python_helloworld + path: ./python/hello-world/Dockerfile + ctx: ./python/hello-world/ + - img : dockerhubaneo/armonik_python_subtasking + path: ./python/subtasking/Dockerfile + ctx: ./python/subtasking/ steps: - name: Checkout @@ -193,7 +199,7 @@ jobs: - name: Change version run: | cp tools/parameters.tfvars ${{ github.workspace }}/infra/infrastructure/quick-deploy/localhost/all-in-one/parameters.tfvars - cat ${{ github.workspace }}/infra/versions.tfvars.json | jq --arg version "${{ needs.versionning.outputs.version }}" '.armonik_versions.samples=$version | .armonik_images.samples+=["dockerhubaneo/armonik_demo_helloworld_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_multipleresults_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_subtasking_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_linearsubtasking_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_dynamicsubmission_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_java_worker"]' > .versions.tfvars.json + cat ${{ github.workspace }}/infra/versions.tfvars.json | jq --arg version "${{ needs.versionning.outputs.version }}" '.armonik_versions.samples=$version | .armonik_images.samples+=["dockerhubaneo/armonik_demo_helloworld_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_multipleresults_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_subtasking_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_linearsubtasking_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_dynamicsubmission_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_java_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_python_helloworld"] | .armonik_images.samples+=["dockerhubaneo/armonik_python_subtasking"]' > .versions.tfvars.json mv .versions.tfvars.json ${{ github.workspace }}/infra/versions.tfvars.json - id: deploy @@ -245,7 +251,7 @@ jobs: - name: Change version run: | cp tools/parameters.tfvars ${{ github.workspace }}/infra/infrastructure/quick-deploy/localhost/all-in-one/parameters.tfvars - cat ${{ github.workspace }}/infra/versions.tfvars.json | jq --arg version "${{ needs.versionning.outputs.version }}" '.armonik_versions.samples=$version | .armonik_images.samples+=["dockerhubaneo/armonik_demo_helloworld_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_multipleresults_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_subtasking_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_linearsubtasking_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_dynamicsubmission_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_java_worker"]' > .versions.tfvars.json + cat ${{ github.workspace }}/infra/versions.tfvars.json | jq --arg version "${{ needs.versionning.outputs.version }}" '.armonik_versions.samples=$version | .armonik_images.samples+=["dockerhubaneo/armonik_demo_helloworld_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_multipleresults_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_subtasking_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_linearsubtasking_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_dynamicsubmission_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_java_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_python_helloworld"] | .armonik_images.samples+=["dockerhubaneo/armonik_python_subtasking"]' > .versions.tfvars.json mv .versions.tfvars.json ${{ github.workspace }}/infra/versions.tfvars.json - id: deploy @@ -300,7 +306,7 @@ jobs: - name: Change version run: | cp tools/parameters.tfvars ${{ github.workspace }}/infra/infrastructure/quick-deploy/localhost/all-in-one/parameters.tfvars - cat ${{ github.workspace }}/infra/versions.tfvars.json | jq --arg version "${{ needs.versionning.outputs.version }}" '.armonik_versions.samples=$version | .armonik_images.samples+=["dockerhubaneo/armonik_demo_helloworld_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_multipleresults_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_subtasking_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_linearsubtasking_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_dynamicsubmission_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_java_worker"]' > .versions.tfvars.json + cat ${{ github.workspace }}/infra/versions.tfvars.json | jq --arg version "${{ needs.versionning.outputs.version }}" '.armonik_versions.samples=$version | .armonik_images.samples+=["dockerhubaneo/armonik_demo_helloworld_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_multipleresults_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_subtasking_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_linearsubtasking_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_dynamicsubmission_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_java_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_python_helloworld"] | .armonik_images.samples+=["dockerhubaneo/armonik_python_subtasking"]' > .versions.tfvars.json mv .versions.tfvars.json ${{ github.workspace }}/infra/versions.tfvars.json - id: deploy @@ -349,3 +355,82 @@ jobs: with: working-directory: ${{ github.workspace }}/infra type: localhost + + testPythonHelloWorld: + runs-on: ubuntu-latest + needs: + - versionning + - buildImages + strategy: + fail-fast: false + matrix: + demo: + - helloworld + - subtasking + + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + ref: ${{ github.ref }} + + - name: Install Dependencies + uses: aneoconsulting/ArmoniK.Action.Deploy/dependencies@main + with: + docker: true + terraform: true + k3s: true + aws: true + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12.3' + architecture: 'x64' + + - name: Checkout Infra + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + repository: aneoconsulting/ArmoniK + path: infra + + - name: Change version + run: | + cp tools/parameters.tfvars ${{ github.workspace }}/infra/infrastructure/quick-deploy/localhost/all-in-one/parameters.tfvars + cat ${{ github.workspace }}/infra/versions.tfvars.json | jq --arg version "${{ needs.versionning.outputs.version }}" '.armonik_versions.samples=$version | .armonik_images.samples+=["dockerhubaneo/armonik_demo_helloworld_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_multipleresults_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_subtasking_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_linearsubtasking_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_dynamicsubmission_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_java_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_python_helloworld"] | .armonik_images.samples+=["dockerhubaneo/armonik_python_subtasking"]' > .versions.tfvars.json + mv .versions.tfvars.json ${{ github.workspace }}/infra/versions.tfvars.json + + - id: deploy + name: Deploy + uses: aneoconsulting/ArmoniK.Action.Deploy/deploy@main + with: + working-directory: ${{ github.workspace }}/infra + type: localhost + log-suffix: pythonHelloWorld + + - name: Run Demo Helloworld + timeout-minutes: 10 + run: | + cd python/hello-world/ + python -m venv .venv + . .venv/bin/activate + pip install --upgrade pip setuptools && pip install -r client/client-requirements.txt + export CPIP=$(kubectl get svc control-plane -n armonik -o custom-columns="IP:.spec.clusterIP" --no-headers=true) + export CPPort=$(kubectl get svc control-plane -n armonik -o custom-columns="PORT:.spec.ports[*].port" --no-headers=true) + export Grpc__Endpoint=$CPIP:$CPPort + echo $Grpc__Endpoint + python client/client.py --endpoint $Grpc__Endpoint --partition hellopython + + - name: Run Demo Subtasking + timeout-minutes: 10 + run: | + cd python/subtasking + python -m venv ./armonik + . ./armonik/bin/activate + pip install --upgrade pip + python -m pip install -r requirements.txt + export CPIP=$(kubectl get svc control-plane -n armonik -o custom-columns="IP:.spec.clusterIP" --no-headers=true) + export CPPort=$(kubectl get svc control-plane -n armonik -o custom-columns="PORT:.spec.ports[*].port" --no-headers=true) + export Grpc__Endpoint=$CPIP:$CPPort + python client.py --endpoint $Grpc__Endpoint --partition subtaskingpython 100 + diff --git a/python/subtasking/Dockerfile b/python/subtasking/Dockerfile index 840b569a..1f62d713 100644 --- a/python/subtasking/Dockerfile +++ b/python/subtasking/Dockerfile @@ -1,8 +1,10 @@ -FROM python:3.7-slim +FROM python:3.12.3 WORKDIR /app COPY requirements.txt ./ +RUN python -m venv .venv +RUN . .venv/bin/activate # Install ArmoniK -RUN python -m pip install --no-cache-dir -r requirements.txt +RUN .venv/bin/pip install --no-cache-dir -U pip setuptools && pip install --extra-index-url https://test.pypi.org/simple/ armonik==3.25.1.dev3601 # Setup the user (mandatory) RUN groupadd --gid 5000 armonikuser && useradd --home-dir /home/armonikuser --create-home --uid 5000 --gid 5000 --shell /bin/sh --skel /dev/null armonikuser && mkdir /cache && chown armonikuser: /cache USER armonikuser diff --git a/python/subtasking/requirements.txt b/python/subtasking/requirements.txt index 0bed1370..8bcb7860 100644 --- a/python/subtasking/requirements.txt +++ b/python/subtasking/requirements.txt @@ -1 +1,2 @@ -armonik +grpcio>=1.62.3 +grpcio-tools>=1.62.3 diff --git a/tools/parameters.tfvars b/tools/parameters.tfvars index 43ab8fd7..d58e8437 100644 --- a/tools/parameters.tfvars +++ b/tools/parameters.tfvars @@ -410,6 +410,107 @@ compute_plane = { ] } }, + + # Default partition that uses the C# extension for the worker + hellopython = { + # number of replicas for each deployment of compute plane + replicas = 0 + # ArmoniK polling agent + polling_agent = { + limits = { + cpu = "2000m" + memory = "2048Mi" + } + requests = { + cpu = "50m" + memory = "50Mi" + } + } + # ArmoniK workers + worker = [ + { + image = "dockerhubaneo/armonik_python_helloworld" + limits = { + cpu = "1000m" + memory = "1024Mi" + } + requests = { + cpu = "50m" + memory = "50Mi" + } + } + ] + hpa = { + type = "prometheus" + polling_interval = 15 + cooldown_period = 300 + min_replica_count = 0 + max_replica_count = 5 + behavior = { + restore_to_original_replica_count = true + stabilization_window_seconds = 300 + type = "Percent" + value = 100 + period_seconds = 15 + } + triggers = [ + { + type = "prometheus" + threshold = 2 + }, + ] + } + }, + # Default partition that uses the C# extension for the worker + subtaskingpython = { + # number of replicas for each deployment of compute plane + replicas = 0 + # ArmoniK polling agent + polling_agent = { + limits = { + cpu = "2000m" + memory = "2048Mi" + } + requests = { + cpu = "50m" + memory = "50Mi" + } + } + # ArmoniK workers + worker = [ + { + image = "dockerhubaneo/armonik_python_subtasking" + limits = { + cpu = "1000m" + memory = "1024Mi" + } + requests = { + cpu = "50m" + memory = "50Mi" + } + } + ] + hpa = { + type = "prometheus" + polling_interval = 15 + cooldown_period = 300 + min_replica_count = 0 + max_replica_count = 5 + behavior = { + restore_to_original_replica_count = true + stabilization_window_seconds = 300 + type = "Percent" + value = 100 + period_seconds = 15 + } + triggers = [ + { + type = "prometheus" + threshold = 2 + }, + ] + } + }, # Partition for the bench worker bench = { # number of replicas for each deployment of compute plane