From 6e580bb234f023ccd9c5deeb2989f98a564f2783 Mon Sep 17 00:00:00 2001 From: Achille Bossu Date: Mon, 17 Mar 2025 16:17:04 +0100 Subject: [PATCH 01/27] ci: add job to test python --- .github/workflows/build.yml | 78 +++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index dfc9c6cb..4f40bb35 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -137,6 +137,12 @@ jobs: - img : dockerhubaneo/armonik_demo_java_client path: ./java/native/HelloWorld/client/Dockerfile ctx: java/native/HelloWorld/client/ + - img : armonik_worker + path: ./python/hello-world/Dockerfile + ctx: python/subtasking/ + - img : dockerhubaneo/armonik_test_python:latest + path: ./python/subtasking/Dockerfile + ctx: python/subtasking/ steps: - name: Checkout @@ -344,3 +350,75 @@ jobs: with: working-directory: ${{ github.workspace }}/infra type: localhost + + testPythonHelloWorld: + runs-on: ubuntu-latest + needs: + - versionning + - buildImages + strategy: + fail-fast: false + + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + ref: ${{ github.ref }} + + - name: Install Dependencies + uses: aneoconsulting/ArmoniK.Action.Deploy/dependencies@main + with: + docker: true + terraform: true + k3s: true + aws: true + + - name: Checkout Infra + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + repository: aneoconsulting/ArmoniK + path: infra + + - name: Change version + run: | + cp tools/parameters.tfvars ${{ github.workspace }}/infra/infrastructure/quick-deploy/localhost/all-in-one/parameters.tfvars + cat ${{ github.workspace }}/infra/versions.tfvars.json | jq --arg version "${{ needs.versionning.outputs.version }}" '.armonik_versions.samples=$version | .armonik_images.samples+=["armonik-worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_test_python:latest"]' > .versions.tfvars.json + mv .versions.tfvars.json ${{ github.workspace }}/infra/versions.tfvars.json + + - id: deploy + name: Deploy + uses: aneoconsulting/ArmoniK.Action.Deploy/deploy@main + with: + working-directory: ${{ github.workspace }}/infra + type: localhost + ext-csharp-version: 0.18.0 + core-version: 0.24.4 + log-suffix: pythonHelloWorld + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + architecture: 'x64' + + - name: Run Demo Helloworld + run: | + cd client + python -m venv .venv + source .venv/bin/activate + pip install -r client-requirements.txt + export CPIP=$(kubectl get svc control-plane -n armonik -o custom-columns="IP:.spec.clusterIP" --no-headers=true) + export CPPort=$(kubectl get svc control-plane -n armonik -o custom-columns="PORT:.spec.ports[*].port" --no-headers=true) + export Grpc__Endpoint=http://$CPIP:$CPPort + python client.py --partition helloworld --endpoint $Grpc__Endpoint + + - name: Run Demo Subtasking + run: | + python -m venv ./armonik + source ./armonik/bin/activate + python -m pip install -r requirements.txt + export CPIP=$(kubectl get svc control-plane -n armonik -o custom-columns="IP:.spec.clusterIP" --no-headers=true) + export CPPort=$(kubectl get svc control-plane -n armonik -o custom-columns="PORT:.spec.ports[*].port" --no-headers=true) + export Grpc__Endpoint=http://$CPIP:$CPPort + python client.py --partition testpython --endpoint $Grpc__Endpoint 100 + From d3c06b34807d2c7bfe016efdea078a9423a1245d Mon Sep 17 00:00:00 2001 From: Achille Bossu Date: Mon, 17 Mar 2025 16:36:31 +0100 Subject: [PATCH 02/27] ci: python job to test --- .github/workflows/build.yml | 24 ++++----- tools/parameters.tfvars | 102 ++++++++++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4f40bb35..4a20148a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -137,12 +137,6 @@ jobs: - img : dockerhubaneo/armonik_demo_java_client path: ./java/native/HelloWorld/client/Dockerfile ctx: java/native/HelloWorld/client/ - - img : armonik_worker - path: ./python/hello-world/Dockerfile - ctx: python/subtasking/ - - img : dockerhubaneo/armonik_test_python:latest - path: ./python/subtasking/Dockerfile - ctx: python/subtasking/ steps: - name: Checkout @@ -379,20 +373,23 @@ jobs: repository: aneoconsulting/ArmoniK path: infra + - name: Deploy Docker Build + run: | + docker build -t armonik_python_helloworld:latest -f python/hello-world/Dockerfile python/hello-world/client/ + docker build -t armonik_python_subtasking:latest -f python/subtasking/Dockerfile python/subtasking/ + - name: Change version run: | cp tools/parameters.tfvars ${{ github.workspace }}/infra/infrastructure/quick-deploy/localhost/all-in-one/parameters.tfvars - cat ${{ github.workspace }}/infra/versions.tfvars.json | jq --arg version "${{ needs.versionning.outputs.version }}" '.armonik_versions.samples=$version | .armonik_images.samples+=["armonik-worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_test_python:latest"]' > .versions.tfvars.json + cat ${{ github.workspace }}/infra/versions.tfvars.json | jq --arg version "${{ needs.versionning.outputs.version }}" '.armonik_versions.samples=$version | .armonik_images.samples+=["dockerhubaneo/armonik_demo_helloworld_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_multipleresults_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_subtasking_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_linearsubtasking_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_dynamicsubmission_worker"]' > .versions.tfvars.json mv .versions.tfvars.json ${{ github.workspace }}/infra/versions.tfvars.json - + - id: deploy name: Deploy uses: aneoconsulting/ArmoniK.Action.Deploy/deploy@main with: working-directory: ${{ github.workspace }}/infra type: localhost - ext-csharp-version: 0.18.0 - core-version: 0.24.4 log-suffix: pythonHelloWorld - name: Set up Python @@ -403,22 +400,23 @@ jobs: - name: Run Demo Helloworld run: | - cd client + cd python/hello-world/client python -m venv .venv source .venv/bin/activate pip install -r client-requirements.txt export CPIP=$(kubectl get svc control-plane -n armonik -o custom-columns="IP:.spec.clusterIP" --no-headers=true) export CPPort=$(kubectl get svc control-plane -n armonik -o custom-columns="PORT:.spec.ports[*].port" --no-headers=true) export Grpc__Endpoint=http://$CPIP:$CPPort - python client.py --partition helloworld --endpoint $Grpc__Endpoint + python client.py --partition hellopython --endpoint $Grpc__Endpoint - name: Run Demo Subtasking run: | + cd python/subtasking python -m venv ./armonik source ./armonik/bin/activate python -m pip install -r requirements.txt export CPIP=$(kubectl get svc control-plane -n armonik -o custom-columns="IP:.spec.clusterIP" --no-headers=true) export CPPort=$(kubectl get svc control-plane -n armonik -o custom-columns="PORT:.spec.ports[*].port" --no-headers=true) export Grpc__Endpoint=http://$CPIP:$CPPort - python client.py --partition testpython --endpoint $Grpc__Endpoint 100 + python client.py --partition subtaskingpython --endpoint $Grpc__Endpoint 100 diff --git a/tools/parameters.tfvars b/tools/parameters.tfvars index 0be01158..09667142 100644 --- a/tools/parameters.tfvars +++ b/tools/parameters.tfvars @@ -360,6 +360,108 @@ compute_plane = { ] } }, + # Default partition that uses the C# extension for the worker + hellopython = { + # number of replicas for each deployment of compute plane + replicas = 0 + # ArmoniK polling agent + polling_agent = { + limits = { + cpu = "2000m" + memory = "2048Mi" + } + requests = { + cpu = "50m" + memory = "50Mi" + } + } + # ArmoniK workers + worker = [ + { + image = "armonik_python_helloworld" + tag = "latest" + limits = { + cpu = "1000m" + memory = "1024Mi" + } + requests = { + cpu = "50m" + memory = "50Mi" + } + } + ] + hpa = { + type = "prometheus" + polling_interval = 15 + cooldown_period = 300 + min_replica_count = 0 + max_replica_count = 5 + behavior = { + restore_to_original_replica_count = true + stabilization_window_seconds = 300 + type = "Percent" + value = 100 + period_seconds = 15 + } + triggers = [ + { + type = "prometheus" + threshold = 2 + }, + ] + } + }, + # Default partition that uses the C# extension for the worker + subtaskingpython = { + # number of replicas for each deployment of compute plane + replicas = 0 + # ArmoniK polling agent + polling_agent = { + limits = { + cpu = "2000m" + memory = "2048Mi" + } + requests = { + cpu = "50m" + memory = "50Mi" + } + } + # ArmoniK workers + worker = [ + { + image = "armonik_python_subtasking" + tag = "latest" + limits = { + cpu = "1000m" + memory = "1024Mi" + } + requests = { + cpu = "50m" + memory = "50Mi" + } + } + ] + hpa = { + type = "prometheus" + polling_interval = 15 + cooldown_period = 300 + min_replica_count = 0 + max_replica_count = 5 + behavior = { + restore_to_original_replica_count = true + stabilization_window_seconds = 300 + type = "Percent" + value = 100 + period_seconds = 15 + } + triggers = [ + { + type = "prometheus" + threshold = 2 + }, + ] + } + }, # Partition for the bench worker bench = { # number of replicas for each deployment of compute plane From 5b0d386cfce124cf9f6ddd00d884127e944cd435 Mon Sep 17 00:00:00 2001 From: Achille Bossu Date: Mon, 17 Mar 2025 16:46:05 +0100 Subject: [PATCH 03/27] ci: adjust context for docker image build --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4a20148a..d4edea7c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -375,7 +375,7 @@ jobs: - name: Deploy Docker Build run: | - docker build -t armonik_python_helloworld:latest -f python/hello-world/Dockerfile python/hello-world/client/ + docker build -t armonik_python_helloworld:latest -f python/hello-world/Dockerfile python/hello-world/ docker build -t armonik_python_subtasking:latest -f python/subtasking/Dockerfile python/subtasking/ - name: Change version From cafcdacf82371a27be3066b2582ae3177c0f0c70 Mon Sep 17 00:00:00 2001 From: Achille Bossu Date: Mon, 17 Mar 2025 16:56:13 +0100 Subject: [PATCH 04/27] ci: fix java image --- tools/parameters.tfvars | 50 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/tools/parameters.tfvars b/tools/parameters.tfvars index 09667142..dd4ba247 100644 --- a/tools/parameters.tfvars +++ b/tools/parameters.tfvars @@ -360,6 +360,56 @@ compute_plane = { ] } }, + helloworldjava = { + # number of replicas for each deployment of compute plane + replicas = 0 + socket_type = "tcp" + # ArmoniK polling agent + polling_agent = { + limits = { + cpu = "2000m" + memory = "2048Mi" + } + requests = { + cpu = "50m" + memory = "50Mi" + } + } + # ArmoniK workers + worker = [ + { + image = "dockerhubaneo/armonik_demo_java_worker" + limits = { + cpu = "1000m" + memory = "1024Mi" + } + requests = { + cpu = "50m" + memory = "50Mi" + } + } + ] + hpa = { + type = "prometheus" + polling_interval = 15 + cooldown_period = 300 + min_replica_count = 0 + max_replica_count = 5 + behavior = { + restore_to_original_replica_count = true + stabilization_window_seconds = 300 + type = "Percent" + value = 100 + period_seconds = 15 + } + triggers = [ + { + type = "prometheus" + threshold = 2 + }, + ] + } + }, # Default partition that uses the C# extension for the worker hellopython = { # number of replicas for each deployment of compute plane From 12d858f1051d47d7477c6e2f2893ba5082e082cb Mon Sep 17 00:00:00 2001 From: Achille Bossu Date: Tue, 18 Mar 2025 10:08:09 +0100 Subject: [PATCH 05/27] ci: fix armonik deploy for python tests --- .github/workflows/build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d4edea7c..fd1841f3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -390,6 +390,8 @@ jobs: with: working-directory: ${{ github.workspace }}/infra type: localhost + ext-csharp-version: 0.18.0 + core-version: 0.24.4 log-suffix: pythonHelloWorld - name: Set up Python From 6716f71dbc9b6b84695c911d90e0a5d344a22f46 Mon Sep 17 00:00:00 2001 From: Achille Bossu Date: Tue, 18 Mar 2025 10:16:19 +0100 Subject: [PATCH 06/27] ci: add missing images to version --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a1f681af..3d560fa1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -386,7 +386,7 @@ jobs: - name: Change version run: | cp tools/parameters.tfvars ${{ github.workspace }}/infra/infrastructure/quick-deploy/localhost/all-in-one/parameters.tfvars - cat ${{ github.workspace }}/infra/versions.tfvars.json | jq --arg version "${{ needs.versionning.outputs.version }}" '.armonik_versions.samples=$version | .armonik_images.samples+=["dockerhubaneo/armonik_demo_helloworld_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_multipleresults_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_subtasking_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_linearsubtasking_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_dynamicsubmission_worker"]' > .versions.tfvars.json + cat ${{ github.workspace }}/infra/versions.tfvars.json | jq --arg version "${{ needs.versionning.outputs.version }}" '.armonik_versions.samples=$version | .armonik_images.samples+=["dockerhubaneo/armonik_demo_helloworld_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_multipleresults_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_subtasking_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_linearsubtasking_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_dynamicsubmission_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_java_worker"]' > .versions.tfvars.json mv .versions.tfvars.json ${{ github.workspace }}/infra/versions.tfvars.json - id: deploy From a67756902f9ed2b1e52d180f4d0868c3d6fbe220 Mon Sep 17 00:00:00 2001 From: Achille Bossu Date: Tue, 18 Mar 2025 10:39:19 +0100 Subject: [PATCH 07/27] ci: upgrade pip to resolve dependencies install --- .github/workflows/build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3d560fa1..576c7601 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -410,6 +410,7 @@ jobs: cd python/hello-world/client python -m venv .venv source .venv/bin/activate + pip install --upgrade pip pip install -r client-requirements.txt export CPIP=$(kubectl get svc control-plane -n armonik -o custom-columns="IP:.spec.clusterIP" --no-headers=true) export CPPort=$(kubectl get svc control-plane -n armonik -o custom-columns="PORT:.spec.ports[*].port" --no-headers=true) @@ -421,6 +422,7 @@ jobs: cd python/subtasking python -m venv ./armonik source ./armonik/bin/activate + pip install --upgrade pip python -m pip install -r requirements.txt export CPIP=$(kubectl get svc control-plane -n armonik -o custom-columns="IP:.spec.clusterIP" --no-headers=true) export CPPort=$(kubectl get svc control-plane -n armonik -o custom-columns="PORT:.spec.ports[*].port" --no-headers=true) From e5b70bd2d4bf6d5edf5e65e20b0a725b53b06619 Mon Sep 17 00:00:00 2001 From: Achille Bossu Date: Tue, 18 Mar 2025 10:52:44 +0100 Subject: [PATCH 08/27] ci: adjust requirements for grpcio and grpcio-tools while it is not updated --- .github/workflows/build.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 576c7601..ae72630d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -411,6 +411,8 @@ jobs: python -m venv .venv source .venv/bin/activate pip install --upgrade pip + pip install grpcio==1.62.3 + pip install grpcio-tools==1.62.3 pip install -r client-requirements.txt export CPIP=$(kubectl get svc control-plane -n armonik -o custom-columns="IP:.spec.clusterIP" --no-headers=true) export CPPort=$(kubectl get svc control-plane -n armonik -o custom-columns="PORT:.spec.ports[*].port" --no-headers=true) @@ -423,6 +425,8 @@ jobs: python -m venv ./armonik source ./armonik/bin/activate pip install --upgrade pip + pip install grpcio==1.62.3 + pip install grpcio-tools==1.62.3 python -m pip install -r requirements.txt export CPIP=$(kubectl get svc control-plane -n armonik -o custom-columns="IP:.spec.clusterIP" --no-headers=true) export CPPort=$(kubectl get svc control-plane -n armonik -o custom-columns="PORT:.spec.ports[*].port" --no-headers=true) From 71f058a94eb662926b84792bb3d28b4f2f255fb6 Mon Sep 17 00:00:00 2001 From: Achille Bossu Date: Tue, 18 Mar 2025 11:02:43 +0100 Subject: [PATCH 09/27] ci: adjust python version --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ae72630d..4191b8ec 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -402,7 +402,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.x' + python-version: '3.12.3' architecture: 'x64' - name: Run Demo Helloworld From d6d92904eee544935d6f398af211283392f787f2 Mon Sep 17 00:00:00 2001 From: Achille Bossu Date: Tue, 18 Mar 2025 11:18:06 +0100 Subject: [PATCH 10/27] ci: try to fix wrong endpoint --- .github/workflows/build.yml | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4191b8ec..476897f0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -371,7 +371,13 @@ jobs: terraform: true k3s: true aws: true - + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12.3' + architecture: 'x64' + - name: Checkout Infra uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 with: @@ -398,12 +404,6 @@ jobs: ext-csharp-version: 0.18.0 core-version: 0.24.4 log-suffix: pythonHelloWorld - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.12.3' - architecture: 'x64' - name: Run Demo Helloworld run: | @@ -411,8 +411,6 @@ jobs: python -m venv .venv source .venv/bin/activate pip install --upgrade pip - pip install grpcio==1.62.3 - pip install grpcio-tools==1.62.3 pip install -r client-requirements.txt export CPIP=$(kubectl get svc control-plane -n armonik -o custom-columns="IP:.spec.clusterIP" --no-headers=true) export CPPort=$(kubectl get svc control-plane -n armonik -o custom-columns="PORT:.spec.ports[*].port" --no-headers=true) @@ -425,8 +423,6 @@ jobs: python -m venv ./armonik source ./armonik/bin/activate pip install --upgrade pip - pip install grpcio==1.62.3 - pip install grpcio-tools==1.62.3 python -m pip install -r requirements.txt export CPIP=$(kubectl get svc control-plane -n armonik -o custom-columns="IP:.spec.clusterIP" --no-headers=true) export CPPort=$(kubectl get svc control-plane -n armonik -o custom-columns="PORT:.spec.ports[*].port" --no-headers=true) From 91c3584cf9a8c1bd947f123db3f26c3aefd0bf5a Mon Sep 17 00:00:00 2001 From: Achille Bossu Date: Tue, 18 Mar 2025 11:38:01 +0100 Subject: [PATCH 11/27] ci: try to fix wrong endpoint --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 476897f0..aaceec37 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -415,7 +415,7 @@ jobs: export CPIP=$(kubectl get svc control-plane -n armonik -o custom-columns="IP:.spec.clusterIP" --no-headers=true) export CPPort=$(kubectl get svc control-plane -n armonik -o custom-columns="PORT:.spec.ports[*].port" --no-headers=true) export Grpc__Endpoint=http://$CPIP:$CPPort - python client.py --partition hellopython --endpoint $Grpc__Endpoint + python client.py --endpoint $Grpc__Endpoint --partition hellopython - name: Run Demo Subtasking run: | @@ -427,5 +427,5 @@ jobs: export CPIP=$(kubectl get svc control-plane -n armonik -o custom-columns="IP:.spec.clusterIP" --no-headers=true) export CPPort=$(kubectl get svc control-plane -n armonik -o custom-columns="PORT:.spec.ports[*].port" --no-headers=true) export Grpc__Endpoint=http://$CPIP:$CPPort - python client.py --partition subtaskingpython --endpoint $Grpc__Endpoint 100 + python client.py --endpoint $Grpc__Endpoint --partition subtaskingpython 100 From 68807dd73b30c86aec6af6d913c475ef0e9463f6 Mon Sep 17 00:00:00 2001 From: Achille Bossu Date: Tue, 18 Mar 2025 12:00:28 +0100 Subject: [PATCH 12/27] ci: debugging endpoint --- .github/workflows/build.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index aaceec37..a5d4f3d3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -259,11 +259,14 @@ jobs: - name: Run Demo java Client timeout-minutes: 10 run: | + sleep 30 export CPIP=$(kubectl get svc control-plane -n armonik -o custom-columns="IP:.spec.clusterIP" --no-headers=true) export CPPort=$(kubectl get svc control-plane -n armonik -o custom-columns="PORT:.spec.ports[*].port" --no-headers=true) + echo "CPIP: $CPIP" + echo "CPPort: $CPPort" export Grpc__Endpoint=http://$CPIP:$CPPort docker run --rm dockerhubaneo/armonik_demo_java_client:${{ needs.versionning.outputs.version }} --endpoint $Grpc__Endpoint --partition helloworldjava - + testInfraWorker: needs: - versionning From 75434632f9f42df7b49b6d1ed4a69adbf382a0fe Mon Sep 17 00:00:00 2001 From: Achille Bossu Date: Tue, 18 Mar 2025 13:52:17 +0100 Subject: [PATCH 13/27] ci: debugging endpoint --- .github/workflows/build.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a5d4f3d3..25d99933 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -259,14 +259,11 @@ jobs: - name: Run Demo java Client timeout-minutes: 10 run: | - sleep 30 export CPIP=$(kubectl get svc control-plane -n armonik -o custom-columns="IP:.spec.clusterIP" --no-headers=true) export CPPort=$(kubectl get svc control-plane -n armonik -o custom-columns="PORT:.spec.ports[*].port" --no-headers=true) - echo "CPIP: $CPIP" - echo "CPPort: $CPPort" export Grpc__Endpoint=http://$CPIP:$CPPort docker run --rm dockerhubaneo/armonik_demo_java_client:${{ needs.versionning.outputs.version }} --endpoint $Grpc__Endpoint --partition helloworldjava - + testInfraWorker: needs: - versionning @@ -410,6 +407,7 @@ jobs: - name: Run Demo Helloworld run: | + sleep 30 cd python/hello-world/client python -m venv .venv source .venv/bin/activate @@ -417,6 +415,8 @@ jobs: pip install -r client-requirements.txt export CPIP=$(kubectl get svc control-plane -n armonik -o custom-columns="IP:.spec.clusterIP" --no-headers=true) export CPPort=$(kubectl get svc control-plane -n armonik -o custom-columns="PORT:.spec.ports[*].port" --no-headers=true) + echo $CPIP + echo $CPPort export Grpc__Endpoint=http://$CPIP:$CPPort python client.py --endpoint $Grpc__Endpoint --partition hellopython From 617d3fa8d35383960093f359d53abdb004c99db5 Mon Sep 17 00:00:00 2001 From: Achille Bossu Date: Tue, 18 Mar 2025 14:01:52 +0100 Subject: [PATCH 14/27] ci: debugging endpoint 3 --- .github/workflows/build.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 25d99933..0909479f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -404,8 +404,14 @@ jobs: ext-csharp-version: 0.18.0 core-version: 0.24.4 log-suffix: pythonHelloWorld + run: | + export CPIP=$(kubectl get svc control-plane -n armonik -o custom-columns="IP:.spec.clusterIP" --no-headers=true) + export CPPort=$(kubectl get svc control-plane -n armonik -o custom-columns="PORT:.spec.ports[*].port" --no-headers=true) + echo $CPIP + echo $CPPort - name: Run Demo Helloworld + timeout-minutes: 10 run: | sleep 30 cd python/hello-world/client From b12f231cf821836a271007b5beb517de64b1f103 Mon Sep 17 00:00:00 2001 From: Achille Bossu Date: Tue, 18 Mar 2025 14:17:39 +0100 Subject: [PATCH 15/27] ci: debugging endpoint 4 --- .github/workflows/build.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0909479f..63356c9b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -391,9 +391,9 @@ jobs: - name: Change version run: | - cp tools/parameters.tfvars ${{ github.workspace }}/infra/infrastructure/quick-deploy/localhost/all-in-one/parameters.tfvars - cat ${{ github.workspace }}/infra/versions.tfvars.json | jq --arg version "${{ needs.versionning.outputs.version }}" '.armonik_versions.samples=$version | .armonik_images.samples+=["dockerhubaneo/armonik_demo_helloworld_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_multipleresults_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_subtasking_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_linearsubtasking_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_dynamicsubmission_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_java_worker"]' > .versions.tfvars.json - mv .versions.tfvars.json ${{ github.workspace }}/infra/versions.tfvars.json + cp tools/parameters.tfvars ${{ github.workspace }}/infra/infrastructure/quick-deploy/localhost/all-in-one/parameters.tfvars + cat ${{ github.workspace }}/infra/versions.tfvars.json | jq --arg version "${{ needs.versionning.outputs.version }}" '.armonik_versions.samples=$version | .armonik_images.samples+=["dockerhubaneo/armonik_demo_helloworld_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_multipleresults_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_subtasking_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_linearsubtasking_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_dynamicsubmission_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_java_worker"]' > .versions.tfvars.json + mv .versions.tfvars.json ${{ github.workspace }}/infra/versions.tfvars.json - id: deploy name: Deploy @@ -404,6 +404,8 @@ jobs: ext-csharp-version: 0.18.0 core-version: 0.24.4 log-suffix: pythonHelloWorld + + - name: Debug run: | export CPIP=$(kubectl get svc control-plane -n armonik -o custom-columns="IP:.spec.clusterIP" --no-headers=true) export CPPort=$(kubectl get svc control-plane -n armonik -o custom-columns="PORT:.spec.ports[*].port" --no-headers=true) From 50a74854989ed1d2cd315af8fafb9c60bc30f91b Mon Sep 17 00:00:00 2001 From: Achille Bossu Date: Tue, 18 Mar 2025 14:29:36 +0100 Subject: [PATCH 16/27] ci: debugging endpoint 5 --- .github/workflows/build.yml | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 63356c9b..8004e512 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -404,18 +404,10 @@ jobs: ext-csharp-version: 0.18.0 core-version: 0.24.4 log-suffix: pythonHelloWorld - - - name: Debug - run: | - export CPIP=$(kubectl get svc control-plane -n armonik -o custom-columns="IP:.spec.clusterIP" --no-headers=true) - export CPPort=$(kubectl get svc control-plane -n armonik -o custom-columns="PORT:.spec.ports[*].port" --no-headers=true) - echo $CPIP - echo $CPPort - name: Run Demo Helloworld timeout-minutes: 10 run: | - sleep 30 cd python/hello-world/client python -m venv .venv source .venv/bin/activate @@ -423,9 +415,7 @@ jobs: pip install -r client-requirements.txt export CPIP=$(kubectl get svc control-plane -n armonik -o custom-columns="IP:.spec.clusterIP" --no-headers=true) export CPPort=$(kubectl get svc control-plane -n armonik -o custom-columns="PORT:.spec.ports[*].port" --no-headers=true) - echo $CPIP - echo $CPPort - export Grpc__Endpoint=http://$CPIP:$CPPort + export Grpc__Endpoint=$CPIP:$CPPort python client.py --endpoint $Grpc__Endpoint --partition hellopython - name: Run Demo Subtasking @@ -437,6 +427,6 @@ jobs: python -m pip install -r requirements.txt export CPIP=$(kubectl get svc control-plane -n armonik -o custom-columns="IP:.spec.clusterIP" --no-headers=true) export CPPort=$(kubectl get svc control-plane -n armonik -o custom-columns="PORT:.spec.ports[*].port" --no-headers=true) - export Grpc__Endpoint=http://$CPIP:$CPPort + export Grpc__Endpoint=$CPIP:$CPPort python client.py --endpoint $Grpc__Endpoint --partition subtaskingpython 100 From 0fdcd9ae2b00e1a9a1940e2de9e8b57804620cf9 Mon Sep 17 00:00:00 2001 From: Achille Bossu Date: Tue, 18 Mar 2025 14:45:08 +0100 Subject: [PATCH 17/27] ci: fixing context for running demo hello-world --- .github/workflows/build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8004e512..104b8b23 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -408,15 +408,15 @@ jobs: - name: Run Demo Helloworld timeout-minutes: 10 run: | - cd python/hello-world/client + cd python/hello-world python -m venv .venv source .venv/bin/activate pip install --upgrade pip - pip install -r client-requirements.txt + pip install -r client/client-requirements.txt export CPIP=$(kubectl get svc control-plane -n armonik -o custom-columns="IP:.spec.clusterIP" --no-headers=true) export CPPort=$(kubectl get svc control-plane -n armonik -o custom-columns="PORT:.spec.ports[*].port" --no-headers=true) export Grpc__Endpoint=$CPIP:$CPPort - python client.py --endpoint $Grpc__Endpoint --partition hellopython + python client/client.py --endpoint $Grpc__Endpoint --partition hellopython - name: Run Demo Subtasking run: | From 7db43c675b33639bc03f97093a779ad1777cb666 Mon Sep 17 00:00:00 2001 From: Achille Bossu Date: Tue, 18 Mar 2025 15:00:33 +0100 Subject: [PATCH 18/27] ci: fixing context for running demo hello-world --- .github/workflows/build.yml | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 104b8b23..65862247 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -404,20 +404,7 @@ jobs: ext-csharp-version: 0.18.0 core-version: 0.24.4 log-suffix: pythonHelloWorld - - - name: Run Demo Helloworld - timeout-minutes: 10 - run: | - cd python/hello-world - python -m venv .venv - source .venv/bin/activate - pip install --upgrade pip - pip install -r client/client-requirements.txt - export CPIP=$(kubectl get svc control-plane -n armonik -o custom-columns="IP:.spec.clusterIP" --no-headers=true) - export CPPort=$(kubectl get svc control-plane -n armonik -o custom-columns="PORT:.spec.ports[*].port" --no-headers=true) - export Grpc__Endpoint=$CPIP:$CPPort - python client/client.py --endpoint $Grpc__Endpoint --partition hellopython - + - name: Run Demo Subtasking run: | cd python/subtasking From e28f0532ddc4fdc9a86742632521798aa015d02a Mon Sep 17 00:00:00 2001 From: Achille Bossu Date: Tue, 18 Mar 2025 15:28:18 +0100 Subject: [PATCH 19/27] ci: fix no worker --- .github/workflows/build.yml | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 65862247..2adf7cf9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -384,10 +384,7 @@ jobs: repository: aneoconsulting/ArmoniK path: infra - - name: Deploy Docker Build - run: | - docker build -t armonik_python_helloworld:latest -f python/hello-world/Dockerfile python/hello-world/ - docker build -t armonik_python_subtasking:latest -f python/subtasking/Dockerfile python/subtasking/ + - name: Change version run: | @@ -404,9 +401,26 @@ jobs: ext-csharp-version: 0.18.0 core-version: 0.24.4 log-suffix: pythonHelloWorld - + + - name: Run Demo Helloworld + timeout-minutes: 10 + run: | + docker build -t armonik_python_helloworld:latest -f python/hello-world/Dockerfile python/hello-world/ + cd python/hello-world/ + python -m venv .venv + source .venv/bin/activate + pip install --upgrade pip + pip install -r client/client-requirements.txt + export CPIP=$(kubectl get svc control-plane -n armonik -o custom-columns="IP:.spec.clusterIP" --no-headers=true) + export CPPort=$(kubectl get svc control-plane -n armonik -o custom-columns="PORT:.spec.ports[*].port" --no-headers=true) + export Grpc__Endpoint=$CPIP:$CPPort + echo $Grpc_Endpoint + python client/client.py --endpoint $Grpc__Endpoint --partition hellopython + - name: Run Demo Subtasking + timeout-minutes: 10 run: | + docker build -t armonik_python_subtasking:latest -f python/subtasking/Dockerfile python/subtasking/ cd python/subtasking python -m venv ./armonik source ./armonik/bin/activate From 8843267a86fa177a76ad0271ed5c6999cce42f3f Mon Sep 17 00:00:00 2001 From: Achille Bossu Date: Tue, 18 Mar 2025 15:41:10 +0100 Subject: [PATCH 20/27] ci: fix no worker --- .github/workflows/build.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2adf7cf9..ab1919d3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -384,7 +384,10 @@ jobs: repository: aneoconsulting/ArmoniK path: infra - + - name: Deploy Docker Build + run: | + docker build -t armonik_python_helloworld:latest -f python/hello-world/Dockerfile python/hello-world/ + docker build -t armonik_python_subtasking:latest -f python/subtasking/Dockerfile python/subtasking/ - name: Change version run: | @@ -405,7 +408,6 @@ jobs: - name: Run Demo Helloworld timeout-minutes: 10 run: | - docker build -t armonik_python_helloworld:latest -f python/hello-world/Dockerfile python/hello-world/ cd python/hello-world/ python -m venv .venv source .venv/bin/activate @@ -414,13 +416,12 @@ jobs: export CPIP=$(kubectl get svc control-plane -n armonik -o custom-columns="IP:.spec.clusterIP" --no-headers=true) export CPPort=$(kubectl get svc control-plane -n armonik -o custom-columns="PORT:.spec.ports[*].port" --no-headers=true) export Grpc__Endpoint=$CPIP:$CPPort - echo $Grpc_Endpoint + echo $Grpc__Endpoint python client/client.py --endpoint $Grpc__Endpoint --partition hellopython - name: Run Demo Subtasking timeout-minutes: 10 run: | - docker build -t armonik_python_subtasking:latest -f python/subtasking/Dockerfile python/subtasking/ cd python/subtasking python -m venv ./armonik source ./armonik/bin/activate From e46b48f24fbb3cb29f4377265188dbc196291d09 Mon Sep 17 00:00:00 2001 From: Achille Bossu Date: Tue, 18 Mar 2025 16:16:51 +0100 Subject: [PATCH 21/27] ci: change docker image python build --- .github/workflows/build.yml | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ab1919d3..862ccbac 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -140,6 +140,12 @@ jobs: - img : dockerhubaneo/armonik_demo_java_worker path: ./java/native/HelloWorld/worker/Dockerfile ctx: java/native/HelloWorld/worker/ + - img : armonik_python_helloworld + path: ./python/hello-world/Dockerfile + ctx: ./python/hello-world/ + - img : armonik_python_subtasking + path: ./python/subtasking/Dockerfile + ctx: ./python/subtasking/ steps: - name: Checkout @@ -193,7 +199,7 @@ jobs: - name: Change version run: | cp tools/parameters.tfvars ${{ github.workspace }}/infra/infrastructure/quick-deploy/localhost/all-in-one/parameters.tfvars - cat ${{ github.workspace }}/infra/versions.tfvars.json | jq --arg version "${{ needs.versionning.outputs.version }}" '.armonik_versions.samples=$version | .armonik_images.samples+=["dockerhubaneo/armonik_demo_helloworld_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_multipleresults_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_subtasking_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_linearsubtasking_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_dynamicsubmission_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_java_worker"]' > .versions.tfvars.json + cat ${{ github.workspace }}/infra/versions.tfvars.json | jq --arg version "${{ needs.versionning.outputs.version }}" '.armonik_versions.samples=$version | .armonik_images.samples+=["dockerhubaneo/armonik_demo_helloworld_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_multipleresults_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_subtasking_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_linearsubtasking_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_dynamicsubmission_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_java_worker"] | .armonik_images.samples+=["armonik_python_helloworld"] | .armonik_images.samples+=["armonik_python_subtasking"]' > .versions.tfvars.json mv .versions.tfvars.json ${{ github.workspace }}/infra/versions.tfvars.json - id: deploy @@ -245,7 +251,7 @@ jobs: - name: Change version run: | cp tools/parameters.tfvars ${{ github.workspace }}/infra/infrastructure/quick-deploy/localhost/all-in-one/parameters.tfvars - cat ${{ github.workspace }}/infra/versions.tfvars.json | jq --arg version "${{ needs.versionning.outputs.version }}" '.armonik_versions.samples=$version | .armonik_images.samples+=["dockerhubaneo/armonik_demo_helloworld_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_multipleresults_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_subtasking_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_linearsubtasking_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_dynamicsubmission_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_java_worker"]' > .versions.tfvars.json + cat ${{ github.workspace }}/infra/versions.tfvars.json | jq --arg version "${{ needs.versionning.outputs.version }}" '.armonik_versions.samples=$version | .armonik_images.samples+=["dockerhubaneo/armonik_demo_helloworld_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_multipleresults_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_subtasking_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_linearsubtasking_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_dynamicsubmission_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_java_worker"] | .armonik_images.samples+=["armonik_python_helloworld"] | .armonik_images.samples+=["armonik_python_subtasking"]' > .versions.tfvars.json mv .versions.tfvars.json ${{ github.workspace }}/infra/versions.tfvars.json - id: deploy @@ -300,7 +306,7 @@ jobs: - name: Change version run: | cp tools/parameters.tfvars ${{ github.workspace }}/infra/infrastructure/quick-deploy/localhost/all-in-one/parameters.tfvars - cat ${{ github.workspace }}/infra/versions.tfvars.json | jq --arg version "${{ needs.versionning.outputs.version }}" '.armonik_versions.samples=$version | .armonik_images.samples+=["dockerhubaneo/armonik_demo_helloworld_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_multipleresults_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_subtasking_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_linearsubtasking_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_dynamicsubmission_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_java_worker"]' > .versions.tfvars.json + cat ${{ github.workspace }}/infra/versions.tfvars.json | jq --arg version "${{ needs.versionning.outputs.version }}" '.armonik_versions.samples=$version | .armonik_images.samples+=["dockerhubaneo/armonik_demo_helloworld_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_multipleresults_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_subtasking_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_linearsubtasking_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_dynamicsubmission_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_java_worker"] | .armonik_images.samples+=["armonik_python_helloworld"] | .armonik_images.samples+=["armonik_python_subtasking"]' > .versions.tfvars.json mv .versions.tfvars.json ${{ github.workspace }}/infra/versions.tfvars.json - id: deploy @@ -383,16 +389,11 @@ jobs: with: repository: aneoconsulting/ArmoniK path: infra - - - name: Deploy Docker Build - run: | - docker build -t armonik_python_helloworld:latest -f python/hello-world/Dockerfile python/hello-world/ - docker build -t armonik_python_subtasking:latest -f python/subtasking/Dockerfile python/subtasking/ - name: Change version run: | cp tools/parameters.tfvars ${{ github.workspace }}/infra/infrastructure/quick-deploy/localhost/all-in-one/parameters.tfvars - cat ${{ github.workspace }}/infra/versions.tfvars.json | jq --arg version "${{ needs.versionning.outputs.version }}" '.armonik_versions.samples=$version | .armonik_images.samples+=["dockerhubaneo/armonik_demo_helloworld_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_multipleresults_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_subtasking_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_linearsubtasking_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_dynamicsubmission_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_java_worker"]' > .versions.tfvars.json + cat ${{ github.workspace }}/infra/versions.tfvars.json | jq --arg version "${{ needs.versionning.outputs.version }}" '.armonik_versions.samples=$version | .armonik_images.samples+=["dockerhubaneo/armonik_demo_helloworld_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_multipleresults_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_subtasking_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_linearsubtasking_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_dynamicsubmission_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_java_worker"] | .armonik_images.samples+=["armonik_python_helloworld"] | .armonik_images.samples+=["armonik_python_subtasking"]' > .versions.tfvars.json mv .versions.tfvars.json ${{ github.workspace }}/infra/versions.tfvars.json - id: deploy From 2c2b2710926a485af9d7286d57771ac4ac066f87 Mon Sep 17 00:00:00 2001 From: Achille Bossu Date: Tue, 18 Mar 2025 16:23:16 +0100 Subject: [PATCH 22/27] ci: change names of images --- .github/workflows/build.yml | 12 ++++++------ tools/parameters.tfvars | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 862ccbac..db77bc05 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -140,10 +140,10 @@ jobs: - img : dockerhubaneo/armonik_demo_java_worker path: ./java/native/HelloWorld/worker/Dockerfile ctx: java/native/HelloWorld/worker/ - - img : armonik_python_helloworld + - img : dockerhubaneo/armonik_python_helloworld path: ./python/hello-world/Dockerfile ctx: ./python/hello-world/ - - img : armonik_python_subtasking + - img : dockerhubaneo/armonik_python_subtasking path: ./python/subtasking/Dockerfile ctx: ./python/subtasking/ @@ -199,7 +199,7 @@ jobs: - name: Change version run: | cp tools/parameters.tfvars ${{ github.workspace }}/infra/infrastructure/quick-deploy/localhost/all-in-one/parameters.tfvars - cat ${{ github.workspace }}/infra/versions.tfvars.json | jq --arg version "${{ needs.versionning.outputs.version }}" '.armonik_versions.samples=$version | .armonik_images.samples+=["dockerhubaneo/armonik_demo_helloworld_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_multipleresults_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_subtasking_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_linearsubtasking_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_dynamicsubmission_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_java_worker"] | .armonik_images.samples+=["armonik_python_helloworld"] | .armonik_images.samples+=["armonik_python_subtasking"]' > .versions.tfvars.json + cat ${{ github.workspace }}/infra/versions.tfvars.json | jq --arg version "${{ needs.versionning.outputs.version }}" '.armonik_versions.samples=$version | .armonik_images.samples+=["dockerhubaneo/armonik_demo_helloworld_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_multipleresults_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_subtasking_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_linearsubtasking_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_dynamicsubmission_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_java_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_python_helloworld"] | .armonik_images.samples+=["dockerhubaneo/armonik_python_subtasking"]' > .versions.tfvars.json mv .versions.tfvars.json ${{ github.workspace }}/infra/versions.tfvars.json - id: deploy @@ -251,7 +251,7 @@ jobs: - name: Change version run: | cp tools/parameters.tfvars ${{ github.workspace }}/infra/infrastructure/quick-deploy/localhost/all-in-one/parameters.tfvars - cat ${{ github.workspace }}/infra/versions.tfvars.json | jq --arg version "${{ needs.versionning.outputs.version }}" '.armonik_versions.samples=$version | .armonik_images.samples+=["dockerhubaneo/armonik_demo_helloworld_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_multipleresults_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_subtasking_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_linearsubtasking_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_dynamicsubmission_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_java_worker"] | .armonik_images.samples+=["armonik_python_helloworld"] | .armonik_images.samples+=["armonik_python_subtasking"]' > .versions.tfvars.json + cat ${{ github.workspace }}/infra/versions.tfvars.json | jq --arg version "${{ needs.versionning.outputs.version }}" '.armonik_versions.samples=$version | .armonik_images.samples+=["dockerhubaneo/armonik_demo_helloworld_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_multipleresults_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_subtasking_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_linearsubtasking_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_dynamicsubmission_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_java_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_python_helloworld"] | .armonik_images.samples+=["dockerhubaneo/armonik_python_subtasking"]' > .versions.tfvars.json mv .versions.tfvars.json ${{ github.workspace }}/infra/versions.tfvars.json - id: deploy @@ -306,7 +306,7 @@ jobs: - name: Change version run: | cp tools/parameters.tfvars ${{ github.workspace }}/infra/infrastructure/quick-deploy/localhost/all-in-one/parameters.tfvars - cat ${{ github.workspace }}/infra/versions.tfvars.json | jq --arg version "${{ needs.versionning.outputs.version }}" '.armonik_versions.samples=$version | .armonik_images.samples+=["dockerhubaneo/armonik_demo_helloworld_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_multipleresults_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_subtasking_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_linearsubtasking_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_dynamicsubmission_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_java_worker"] | .armonik_images.samples+=["armonik_python_helloworld"] | .armonik_images.samples+=["armonik_python_subtasking"]' > .versions.tfvars.json + cat ${{ github.workspace }}/infra/versions.tfvars.json | jq --arg version "${{ needs.versionning.outputs.version }}" '.armonik_versions.samples=$version | .armonik_images.samples+=["dockerhubaneo/armonik_demo_helloworld_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_multipleresults_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_subtasking_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_linearsubtasking_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_dynamicsubmission_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_java_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_python_helloworld"] | .armonik_images.samples+=["dockerhubaneo/armonik_python_subtasking"]' > .versions.tfvars.json mv .versions.tfvars.json ${{ github.workspace }}/infra/versions.tfvars.json - id: deploy @@ -393,7 +393,7 @@ jobs: - name: Change version run: | cp tools/parameters.tfvars ${{ github.workspace }}/infra/infrastructure/quick-deploy/localhost/all-in-one/parameters.tfvars - cat ${{ github.workspace }}/infra/versions.tfvars.json | jq --arg version "${{ needs.versionning.outputs.version }}" '.armonik_versions.samples=$version | .armonik_images.samples+=["dockerhubaneo/armonik_demo_helloworld_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_multipleresults_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_subtasking_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_linearsubtasking_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_dynamicsubmission_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_java_worker"] | .armonik_images.samples+=["armonik_python_helloworld"] | .armonik_images.samples+=["armonik_python_subtasking"]' > .versions.tfvars.json + cat ${{ github.workspace }}/infra/versions.tfvars.json | jq --arg version "${{ needs.versionning.outputs.version }}" '.armonik_versions.samples=$version | .armonik_images.samples+=["dockerhubaneo/armonik_demo_helloworld_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_multipleresults_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_subtasking_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_linearsubtasking_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_dynamicsubmission_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_demo_java_worker"] | .armonik_images.samples+=["dockerhubaneo/armonik_python_helloworld"] | .armonik_images.samples+=["dockerhubaneo/armonik_python_subtasking"]' > .versions.tfvars.json mv .versions.tfvars.json ${{ github.workspace }}/infra/versions.tfvars.json - id: deploy diff --git a/tools/parameters.tfvars b/tools/parameters.tfvars index f6f79932..dcc20c83 100644 --- a/tools/parameters.tfvars +++ b/tools/parameters.tfvars @@ -429,7 +429,7 @@ compute_plane = { # ArmoniK workers worker = [ { - image = "armonik_python_helloworld" + image = "dockerhubaneo/armonik_python_helloworld" tag = "latest" limits = { cpu = "1000m" @@ -480,7 +480,7 @@ compute_plane = { # ArmoniK workers worker = [ { - image = "armonik_python_subtasking" + image = "dockerhubaneo/armonik_python_subtasking" tag = "latest" limits = { cpu = "1000m" From c97758f09d8edb73db537a00485022360a78dfb3 Mon Sep 17 00:00:00 2001 From: Achille Bossu Date: Tue, 18 Mar 2025 16:49:56 +0100 Subject: [PATCH 23/27] ci: fix tag --- tools/parameters.tfvars | 2 -- 1 file changed, 2 deletions(-) diff --git a/tools/parameters.tfvars b/tools/parameters.tfvars index dcc20c83..d58e8437 100644 --- a/tools/parameters.tfvars +++ b/tools/parameters.tfvars @@ -430,7 +430,6 @@ compute_plane = { worker = [ { image = "dockerhubaneo/armonik_python_helloworld" - tag = "latest" limits = { cpu = "1000m" memory = "1024Mi" @@ -481,7 +480,6 @@ compute_plane = { worker = [ { image = "dockerhubaneo/armonik_python_subtasking" - tag = "latest" limits = { cpu = "1000m" memory = "1024Mi" From 3d291e1cfa3c5e11b8e2883a1c19b9a805497a0c Mon Sep 17 00:00:00 2001 From: Achille Bossu Date: Fri, 21 Mar 2025 17:09:40 +0100 Subject: [PATCH 24/27] WIP --- .github/workflows/build.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index db77bc05..803d8b21 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -363,6 +363,10 @@ jobs: - buildImages strategy: fail-fast: false + matrix: + demo: + - helloworld + - subtasking steps: - name: Checkout @@ -402,10 +406,16 @@ jobs: with: working-directory: ${{ github.workspace }}/infra type: localhost - ext-csharp-version: 0.18.0 - core-version: 0.24.4 log-suffix: pythonHelloWorld + - name: Run Demo + timeout-minutes: 10 + run: | + cd python/${{ matrix.demo}} + python -m venv .venv + source + + - name: Run Demo Helloworld timeout-minutes: 10 run: | From 61509bde65d5583cd86adba4189c6caca326dd7d Mon Sep 17 00:00:00 2001 From: Achille Bossu Date: Tue, 25 Mar 2025 10:04:00 +0100 Subject: [PATCH 25/27] update correctly armonik version for the worker, but still produce a weird result --- python/subtasking/Dockerfile | 6 ++++-- python/subtasking/requirements.txt | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/python/subtasking/Dockerfile b/python/subtasking/Dockerfile index 840b569a..1f62d713 100644 --- a/python/subtasking/Dockerfile +++ b/python/subtasking/Dockerfile @@ -1,8 +1,10 @@ -FROM python:3.7-slim +FROM python:3.12.3 WORKDIR /app COPY requirements.txt ./ +RUN python -m venv .venv +RUN . .venv/bin/activate # Install ArmoniK -RUN python -m pip install --no-cache-dir -r requirements.txt +RUN .venv/bin/pip install --no-cache-dir -U pip setuptools && pip install --extra-index-url https://test.pypi.org/simple/ armonik==3.25.1.dev3601 # Setup the user (mandatory) RUN groupadd --gid 5000 armonikuser && useradd --home-dir /home/armonikuser --create-home --uid 5000 --gid 5000 --shell /bin/sh --skel /dev/null armonikuser && mkdir /cache && chown armonikuser: /cache USER armonikuser diff --git a/python/subtasking/requirements.txt b/python/subtasking/requirements.txt index 0bed1370..8bcb7860 100644 --- a/python/subtasking/requirements.txt +++ b/python/subtasking/requirements.txt @@ -1 +1,2 @@ -armonik +grpcio>=1.62.3 +grpcio-tools>=1.62.3 From 074c7f2dcad1f72d0e031920e67ab26e5aae8a8d Mon Sep 17 00:00:00 2001 From: Achille Bossu Date: Tue, 25 Mar 2025 11:12:50 +0100 Subject: [PATCH 26/27] fix pipeline --- .github/workflows/build.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 803d8b21..000d288e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -421,9 +421,8 @@ jobs: run: | cd python/hello-world/ python -m venv .venv - source .venv/bin/activate - pip install --upgrade pip - pip install -r client/client-requirements.txt + . .venv/bin/activate + pip install --upgrade pip setuptools && pip install -r client/client-requirements.txt export CPIP=$(kubectl get svc control-plane -n armonik -o custom-columns="IP:.spec.clusterIP" --no-headers=true) export CPPort=$(kubectl get svc control-plane -n armonik -o custom-columns="PORT:.spec.ports[*].port" --no-headers=true) export Grpc__Endpoint=$CPIP:$CPPort @@ -435,7 +434,7 @@ jobs: run: | cd python/subtasking python -m venv ./armonik - source ./armonik/bin/activate + . ./armonik/bin/activate pip install --upgrade pip python -m pip install -r requirements.txt export CPIP=$(kubectl get svc control-plane -n armonik -o custom-columns="IP:.spec.clusterIP" --no-headers=true) From 3db5cc1597f6d016d093e552969c97a6daa1384a Mon Sep 17 00:00:00 2001 From: Achille Bossu Date: Tue, 25 Mar 2025 11:17:04 +0100 Subject: [PATCH 27/27] fix pipeline --- .github/workflows/build.yml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 000d288e..177a5b18 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -407,15 +407,7 @@ jobs: working-directory: ${{ github.workspace }}/infra type: localhost log-suffix: pythonHelloWorld - - - name: Run Demo - timeout-minutes: 10 - run: | - cd python/${{ matrix.demo}} - python -m venv .venv - source - - + - name: Run Demo Helloworld timeout-minutes: 10 run: |