diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml new file mode 100644 index 0000000..f36d051 --- /dev/null +++ b/.github/workflows/python.yml @@ -0,0 +1,111 @@ +name: Python3 + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + runner-job: + runs-on: ubuntu-latest + # Service containers to run with `runner-job` + services: + # label used to access the service container + minio: + image: fclairamb/minio-github-actions + ports: + - 9000:9000 + env: + MINIO_DOMAIN: localhost + postgres: + # docker hub image + image: postgres + # provide the password for postgres + env: + POSTGRES_PASSWORD: rFcLGNkgsNtksg6Pgtn9CumL4xXBQ7 + # set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + # maps tcp port 5432 on service container to the host + - 5432:5432 + colonies: + image: colonyos/colonies:v1.8.2 + env: + LANG: en_US.UTF-8 + LANGUAGE: en_US.UTF-8 + LC_ALL: en_US.UTF-8 + LC_CTYPE: UTF-8 + TZ: Europe/Stockholm + COLONIES_SERVER_TLS: false + COLONIES_SERVER_HOST: localhost + COLONIES_SERVER_PORT: 50080 + COLONIES_MONITOR_PORT: 21120 + COLONIES_MONITOR_INTERVAL: 1 + COLONIES_SERVER_ID: 039231c7644e04b6895471dd5335cf332681c54e27f81fac54f9067b3f2c0103 + COLONIES_SERVER_PRVKEY: fcc79953d8a751bf41db661592dc34d30004b1a651ffa0725b03ac227641499d + COLONIES_DB_HOST: postgres + COLONIES_DB_USER: postgres + COLONIES_DB_PORT: 5432 + COLONIES_DB_PASSWORD: rFcLGNkgsNtksg6Pgtn9CumL4xXBQ7 + COLONIES_COLONY_NAME: test + COLONIES_COLONY_ID: 4787a5071856a4acf702b2ffcea422e3237a679c681314113d86139461290cf4 + COLONIES_COLONY_PRVKEY: ba949fa134981372d6da62b6a56f336ab4d843b22c02a4257dcf7d0d73097514 + COLONIES_EXECUTOR_ID: 3fc05cf3df4b494e95d6a3d297a34f19938f7daa7422ab0d4f794454133341ac + COLONIES_PRVKEY: ddf7f7791208083b6a9ed975a72684f6406a269cfa36f1b1c32045c0a71fff05 + COLONIES_EXECUTOR_TYPE: cli + COLONIES_CRON_CHECKER_PERIOD: 1000 + COLONIES_GENERATOR_CHECKER_PERIOD: 1000 + COLONIES_EXCLUSIVE_ASSIGN: true + COLONIES_INITDB: true + ports: + - 50080:50080 + options: >- + --health-cmd "colonies server alive" + --health-interval 1s + --health-timeout 60s + --health-retries 20 + steps: + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/checkout@v3 + with: + python-version: 3.9 + + - name: Test + env: + LANG: en_US.UTF-8 + LANGUAGE: en_US.UTF-8 + LC_ALL: en_US.UTF-8 + LC_CTYPE: UTF-8 + TZ: Europe/Stockholm + COLONIES_SERVER_TLS: false + COLONIES_DB_HOST: localhost + COLONIES_DB_USER: postgres + COLONIES_DB_PORT: 5432 + COLONIES_DB_PASSWORD: rFcLGNkgsNtksg6Pgtn9CumL4xXBQ7 + COLONIES_SERVER_HOST: localhost + COLONIES_SERVER_PORT: 50080 + COLONIES_SERVER_ID: 039231c7644e04b6895471dd5335cf332681c54e27f81fac54f9067b3f2c0103 + COLONIES_SERVER_PRVKEY: fcc79953d8a751bf41db661592dc34d30004b1a651ffa0725b03ac227641499d + COLONIES_COLONY_NAME: test + COLONIES_COLONY_ID: 4787a5071856a4acf702b2ffcea422e3237a679c681314113d86139461290cf4 + COLONIES_COLONY_PRVKEY: ba949fa134981372d6da62b6a56f336ab4d843b22c02a4257dcf7d0d73097514 + COLONIES_EXECUTOR_ID: 3fc05cf3df4b494e95d6a3d297a34f19938f7daa7422ab0d4f794454133341ac + COLONIES_PRVKEY: ddf7f7791208083b6a9ed975a72684f6406a269cfa36f1b1c32045c0a71fff05 + COLONIES_EXECUTOR_NAME: cli + COLONIES_EXECUTOR_TYPE: cli + CRYPTOLIB: /home/runner/work/pycolonies/pycolonies/lib/libcryptolib.so + CFSLIB: /home/runner/work/pycolonies/pycolonies/lib/libcfslib.so + AWS_S3_ENDPOINT: localhost:9000 + AWS_S3_ACCESSKEY: minioadmin + AWS_S3_SECRETKEY: minioadmin + AWS_S3_TLS: false + AWS_S3_SKIPVERIFY: false + AWS_S3_BUCKET: test + run: make github_test diff --git a/.gitignore b/.gitignore index 688ae53..8ab2a93 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,8 @@ - __pycache__/ - *.swp +pycolonies.egg-info +dist +__pycache__/ +*.swp +build/ +.vscode +libcryptolib.so +libcfslib.so diff --git a/.pypirc b/.pypirc new file mode 100644 index 0000000..a14d84b --- /dev/null +++ b/.pypirc @@ -0,0 +1,8 @@ +[distutils] +index-servers=pypi + +[pypi] +repository: https://upload.pypi.org/legacy/ + +[testpypi] +repository: https://test.pypi.org/legacy/ diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..4c5a0be --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2022 colonyos + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..3423a93 --- /dev/null +++ b/Makefile @@ -0,0 +1,30 @@ +all: build + +.PHONY: build +build: + python3 setup.py sdist bdist_wheel + +.PHONY: test +test: + @python3 ./test/crypto_test.py + @python3 ./test/colonies_test.py + +.PHONY: github_test +github_test: + wget https://github.com/colonyos/colonies/releases/download/v1.8.7/colonies_1.8.7_linux_amd64.tar.gz + tar -xzf colonies_1.8.7_linux_amd64.tar.gz + env + ./colonies database create + ./colonies colony add --name ${COLONIES_COLONY_NAME} --colonyid ${COLONIES_COLONY_ID} + ./colonies executor add --spec ./executor.json --executorid ${COLONIES_EXECUTOR_ID} + ./colonies executor approve --name ${COLONIES_EXECUTOR_NAME} + @pip3 install -r requirements.txt + @python3 ./test/crypto_test.py + @python3 ./test/colonies_test.py + +.PHONY: install +install: + pip3 install dist/pycolonies-1.0.24-py3-none-any.whl --force-reinstall + +publish: + python3 -m twine upload dist/pycolonies-1.0.24-py3-none-any.whl diff --git a/README.md b/README.md new file mode 100644 index 0000000..6594343 --- /dev/null +++ b/README.md @@ -0,0 +1,453 @@ +[![Python3](https://github.com/colonyos/pycolonies/actions/workflows/python.yml/badge.svg)](https://github.com/colonyos/pycolonies/actions/workflows/python.yml) + +# Introduction +This repo contains a Python [Colonies](https://github.com/colonyos/colonies), making it possible to implement Colonies Executors in Python. + +# Installation +Note that the SDK has only be tested on Linux and MacOS. + +```bash +pip3 install pycolonies +``` + +## Starting a Colonies server +You need to have access to a Colonies server. + +```bash +wget https://raw.githubusercontent.com/colonyos/colonies/main/docker-compose.env; +source docker-compose.env; +wget https://raw.githubusercontent.com/colonyos/colonies/main/docker-compose.yml; +docker-compose up +``` + +Press control-c to exit. + +To remove all data, type: + +```bash +docker-compose down --volumes +``` + +See the [Colonies release page](https://github.com/colonyos/colonies/releases) for Windows and Mac binaries. + +## Calling a function +To execute a function, a function specification must be submitted to the Colonies server. Colonies will then wrap the function specification in a process and assign the process to an Executor. + +Below is an example of function specification. +```json +{ + "conditions": { + "executortype": "echo_executor" + }, + "func": "echo", + "args": [ + "helloworld" + ], + "priority": 0, + "maxexectime": 10, + "maxretries": 3, + "maxwaittime": 100, + +} +``` + +A function specification can be submitted using the Colonies CLI. +```console +colonies function submit --spec echo_func_spec.json + +INFO[0000] Process submitted ProcessID=ea398af346db85f45b118bb77ecda9ae25f4700dcafcccb4ba3e4d40eba5205a +``` + +Or using the Python SDK. +```python +spec = FuncSpec.create( + func=sum_nums, + args=["helloworld"], + colonyname=colonyname, + executortype="echo_executor", + priority=0, + maxexectime=10, + maxretries=3, + maxwaittime=100 + ) + +process = colonies.submit_func_spec(spec, executor_prvkey) +``` +See [echo.py](https://github.com/colonyos/pycolonies/blob/main/examples/submit_echo.py) for a full example. + +Now it possible to look up the process using the Colonies CLI. +```console +colonies process get -p fd20992e624aa27dac064ce6853556fc0f1fd68369ca219b0ff483be5c441e62 + +╭───────────────────────────────────────────────────────────────────────────────────────╮ +│ Process │ +├────────────────────┬──────────────────────────────────────────────────────────────────┤ +│ Id │ fd20992e624aa27dac064ce6853556fc0f1fd68369ca219b0ff483be5c441e62 │ +│ IsAssigned │ False │ +│ InitiatorID │ 3fc05cf3df4b494e95d6a3d297a34f19938f7daa7422ab0d4f794454133341ac │ +│ Initiator │ myuser │ +│ AssignedExecutorID │ None │ +│ AssignedExecutorID │ Waiting │ +│ PriorityTime │ 1704037946987524185 │ +│ SubmissionTime │ 2024-07-18 17:52:26 │ +│ StartTime │ 2024-07-18 17:52:26 │ +│ EndTime │ 2024-07-18 17:52:26 │ +│ WaitDeadline │ 2024-07-18 17:54:06 │ +│ ExecDeadline │ 0001-01-01 00:53:28 │ +│ WaitingTime │ 34.060359471s │ +│ ProcessingTime │ 0s │ +│ Retries │ 0 │ +│ Input │ │ +│ Output │ │ +│ Errors │ │ +╰────────────────────┴──────────────────────────────────────────────────────────────────╯ +╭───────────────────────────╮ +│ Function Specification │ +├─────────────┬─────────────┤ +│ Func │ echo │ +│ Args │ helloworld │ +│ KwArgs │ None │ +│ MaxWaitTime │ 100 │ +│ MaxExecTime │ 100 │ +│ MaxRetries │ 3 │ +│ Label │ │ +╰─────────────┴─────────────╯ +╭──────────────────────────────────╮ +│ Conditions │ +├──────────────────┬───────────────┤ +│ Colony │ dev │ +│ ExecutorNames │ None │ +│ ExecutorType │ echo-executor │ +│ Dependencies │ │ +│ Nodes │ 0 │ +│ CPU │ 0m │ +│ Memory │ 0Mi │ +│ Processes │ 0 │ +│ ProcessesPerNode │ 0 │ +│ Storage │ 0Mi │ +│ Walltime │ 0 │ +│ GPUName │ │ +│ GPUs │ 0 │ +│ GPUPerNode │ 0 │ +│ GPUMemory │ 0Mi │ +╰──────────────────┴───────────────╯ + +No attributes found +``` + +The command below shows all waiting processes. Note that the process is just enqueued since we don't yet have an Executor of the type *echo_executor*. +```console +colonies process psw + +╭──────────┬────────────┬────────┬─────────────────────┬───────────────┬───────────────┬───────────┬───────╮ +│ FUNCNAME │ ARGS │ KWARGS │ SUBMSSION TIME │ EXECUTOR NAME │ EXECUTOR TYPE │ INITIATOR │ LABEL │ +├──────────┼────────────┼────────┼─────────────────────┼───────────────┼───────────────┼───────────┼───────┤ +│ echo │ helloworld │ │ 2024-07-18 17:52:26 │ │ echo-executor │ myuser │ │ +╰──────────┴────────────┴────────┴─────────────────────┴───────────────┴───────────────┴───────────┴───────╯ +``` + +## Implementing an Executor in Python +Executors are responsible for executing processes. They connect to the Colonies server and get process assignments. To be able to submit function specifications or get process assignments, an Executor must be a member of a Colony. Only the Colony owner has the authority to add an Executor to a Colony. In order to interact with the Colonies server and other Executors, Executors must authenticate and prove their membership. This security mechanism is implemented through the utilization of public key encryption. + +Since we have access to the Colony private key (see devenv file), we can implement a self-registering Executor. +```python +from pycolonies import Colonies, Crypto, rpc + +colonies = Colonies("localhost", 50080) +colonyname = "4787a5071856a4acf702b2ffcea422e3237a679c681314113d86139461290cf4" +colony_prvkey="ba949fa134981372d6da62b6a56f336ab4d843b22c02a4257dcf7d0d73097514" + +crypto = Crypto() +executor_prvkey = crypto.prvkey() +executorid = crypto.id(executor_prvkey) +executorname = "echo_executor" +executortype = "echo_executor" +colonies.add_executor(executorid, executorname, executortype, colonyname, colony_prvkey) +colonies.approve_executor(executorname, colonyname, colony_prvkey) +``` + +*Optinally:* We also need to register the `echo` function, telling the Colonies server that this executor is capable of executing a function called `echo`. + +```python +colonies.add_function(colonyname, executorid, "echo", executor_prvkey) +``` + +The next step is to connect the Colonies server and get process assignments. Note that the Colonies server never establish connections to the Executors, but rather it the responsibility of the Executors to connects to the Colonies server. In this way, Executors may run behind firewalls without problems. The `assign` function below will block for 10 seconds if there are no suitable process to assign. + +```python +process = colonies.assign(colonyname, 10, executor_prvkey) +if process.spec.funcname == "echo": + assigned_args = process.spec.args + colonies.close(process.processid, [arg], executor_prvkey) +``` + +The *close* method sets the output (same the args in this case) and the process state to *successful*. Only the Executor assigned to a process may alter process information stored on the Colonies server. By setting the *maxexectime* attribute on the function spec, it is possible to specify how long an executor may run a process before it is released back the waiting queue at the Colonies server. This is a very useful feature to implement robust processing pipelines. + +See [echo_executor.py](https://github.com/colonyos/pycolonies/blob/main/examples/echo_executor.py) for a full example. Type the command below to start the *echo Executor*. + +```console +python3 examples/echo_executor.py +``` + +# Code-injection +Python has a built-in `eval()` function that allows execution of any piece of Python code encoded as strings. We are going to use the `eval()` function to implement an executor that can execute arbitrary Python functions. + +Python also has support for introspection, which allows Python code to examine itself. We are going to use that to get the source code of function definitions, e.g. the `echo` function. +```python +def echo(arg) + return arg + +code = inspect.getsource(echo) +``` + +We are now going to base64 encode the obtained code and add it to the function specification, allowing an executor to inject the code and then execute it. +```python +code_bytes = code.encode("ascii") +code_base64_bytes = base64.b64encode(code_bytes) +code_base64 = code_base64_bytes.decode("ascii") + +FuncSpec.create = { + "funcname": "echo", + "args": ["helloworld"], + "priority": 0, + "maxwaittime": -1, + "maxexectime": 200, + "maxretries": 3, + "conditions": { + "colonyname": colonyname, + "executortype": executortype + }, + "env": { + "code": code_base64, + }, +} +``` + +The executor can now obtain the code, inject it, and then execute the specified function. +```python +assigned_process = colonies.assign(colonyname, 10, executor_prvkey) + +code_base64 = assigned_process.spec.env["code"] +code_bytes2 = base64.b64decode(code_base64) +code = code_bytes2.decode("ascii") + +exec(code) +res = eval(funcname)(*tuple(args)) +colonies.close(assigned_process.processid, [res], executor_prvkey) +``` + +We can now create a distributed Python application where parts of the code runs on a remote executor. + +```python +def sum_nums(n1, n2, ctx={}): + return n1 + n2 + +spec = FuncSpec.create( + func=sum_nums, + args=[1, 2], + colonyname=colonyname, + executortype="python_executor", + priority=200, + maxexectime=100, + maxretries=3, + maxwaittime=100 + ) + +submitted_process = colonies.submit_func_spec(spec, executor_prvkey) +completed_process = colonies.wait(submitted_process, 100, executor_prvkey) +``` + +The `wait()` function blocks until the submitted process is completed, either successful or failed. + +See [func_spec_example1.py](https://github.com/colonyos/pycolonies/blob/main/examples/submit_python.py) and [python_executor.py](https://github.com/colonyos/pycolonies/blob/main/examples/python_executor.py) for a full example. Type the commands below to try it out. + +```console +python3 examples/submit_python.py + +Process df7cf3a54af88627ec45b525128ada2bd30c352d58f49f5a59c55fa04c781a8d submitted +3 +``` +And in another terminal: +```console +python3 examples/python_executor.py + +Executor de28c7ca3526a9d6a94e78ead8f44966670389aeeeda86e6f84b5d98b722db30 registered + +Process df7cf3a54af88627ec45b525128ada2bd30c352d58f49f5a59c55fa04c781a8d is assigned to Executor +Executing: sum_nums +``` + +It is also possible to use the Colonies CLI to list registered functions: +```console +colonies function ls + +╭───────────────────────────────────╮ +│ Function: execute │ +├──────────────┬────────────────────┤ +│ ExecutorType │ container-executor │ +│ FunctionName │ execute │ +│ Call counter │ 0 │ +│ MinWaitTime │ 0.000000 s │ +│ MaxWaitTime │ 0.000000 s │ +│ AvgWaitTime │ 0.000000 s │ +│ MinExecTime │ 0.000000 s │ +│ MaxExecTime │ 0.000000 s │ +│ AvgExecTime │ 0.000000 s │ +╰──────────────┴────────────────────╯ +╭────────────────────────────────╮ +│ Function: sum_nums │ +├──────────────┬─────────────────┤ +│ ExecutorType │ python-executor │ +│ FunctionName │ sum_nums │ +│ Call counter │ 1 │ +│ MinWaitTime │ 12.788976 s │ +│ MaxWaitTime │ 12.788976 s │ +│ AvgWaitTime │ 12.788976 s │ +│ MinExecTime │ 0.022253 s │ +│ MaxExecTime │ 0.022253 s │ +│ AvgExecTime │ 0.022253 s │ +╰──────────────┴─────────────────╯ +``` + +# Workflows +Colonies supports creation of computational DAGs (Directed Acyclic Graphs). This makes it possible to create dependencies between several functions, i.e. control the order which functions are called and pass values between function calls, even if they run on different Executors. Since Executors may reside *anywhere* on the Internet, we can create workflows that are executed across platforms and infrastructures, **creating compute continuums**. + +The example below calculates `sum_nums(gen_nums())`. The `gen_nums()` function simply return a tuple containing 1 and 2. The `sum_nums()` function takes two arguments and calculates the sum of them. + +```python +def gen_nums(ctx={}): + return 1, 2 + +def sum_nums(n1, n2, ctx={}): + return n1 + n2 + +wf = Workflow(colonyname=colonyname) +f = FuncSpec.create(func=gen_nums, + args=[], + colonyname=colonyname, + executortype="python-executor", + priority=200, + maxexectime=100, + maxretries=3, + maxwaittime=100) + +wf.functionspecs.append(f) + +f = FuncSpec.create(func=sum_nums, + args=[], + colonyname=colonyname, + executortype="python-executor", + priority=200, + maxexectime=100, + maxretries=3, + maxwaittime=100) + +f.conditions.dependencies.append("gen_nums") + +wf.functionspecs.append(f) + +processgraph = colonies.submit_workflow(wf, prvkey) +``` + +## Dynamic processgraphs + It also possible to dynamically modify a processgraph while it is still active, e.g. a function may submit more function specifications to a workflow while executing. This makes it possible to implement patterns like [MapReduce](https://en.wikipedia.org/wiki/MapReduce). + +The `map()` function below dynamically adds 5 `gen_nums()` functions to the processgraph. + +```python +def map(ctx={}): + code = """def gen_nums(ctx={}): + return 1, 2""" + processgraphid = ctx["process"].processgraphid + map_processid = ctx["process"].processid + executor_prvkey = ctx["executor_prvkey"] + + processgraph = colonies.get_processgraph(processgraphid, executor_prvkey) + + reduce_process = colonies.find_process("reduce", processgraph.processids, executor_prvkey) + reduce_processid = reduce_process.processid + + insert = True + for i in range(1): + f = FuncSpec.create(func="gen_nums", + args=[], + colonyname=ctx["colonyname"], + executortype="python-executor", + priority=200, + maxexectime=100, + maxretries=3, + maxwaittime=100, + code=code) + + + colonies.add_child(processgraphid, map_processid, reduce_processid, f, "gen_nums_" + str(i), insert, executor_prvkey) + + insert = False +``` + +The `reduce()` function takes arbitrary integer arguments and returns the sum of them. + +```python +def reduce(*nums, ctx={}): + total = 0 + for n in nums: + total += n + return total +``` + +We can now create a workflow to calculate: `reduce(gen_nums(), gen_nums(), gen_nums(), gen_nums(), gen_nums())`. The result should be (1+2)*5=15. + +```python +wf = Workflow(colonyname=colonyname) + +f = FuncSpec.create(func=map, + args=[], + colonyname=colonyname, + executortype="python-executor", + priority=200, + maxexectime=100, + maxretries=3, + maxwaittime=100) + +wf.functionspecs.append(f) + +f = FuncSpec.create(func=reduce, + args=[], + colonyname=colonyname, + executortype="python-executor", + priority=200, + maxexectime=100, + maxretries=3, + maxwaittime=100) + +f.conditions.dependencies.append("map") +wf.functionspecs.append(f) + +processgraph = colonies.submit_workflow(wf, prvkey) +``` + +![MapReduce example](docs/images/mapreduce.png) + +# Monadic workflows +The workflow code can be significantly simplified by expressing it as a monad. A good introduction to monads can be found [here](https://brian-candler.medium.com/function-composition-with-bind-4f6e3fdc0e7). The example below is not a complete monad, but illustrated how the Colonies *plumbing* can be removed and create elegant functional expressions. The `>>` operator is usually call the `bind` functions and makes it possible to chain function calls. + +```python +def gen_data(ctx={}): + return 1, 2 + +def process_data(*nums, ctx={}): + total = 0 + for n in nums: + total += n + return total + +gen_data = Function(gen_data, colonyname, executortype="python_executor") +process_data = Function(process_data, colonyname, executortype="python_executor") +echo = Function("echo", colonyname, executortype="echo_executor") + +m = ColoniesMonad("localhost", 50080, colonyname, executor_prvkey) +result = (m >> gen_data >> process_data >> echo).unwrap() +print(result) # prints 3 +``` + +See [colonies_monad.py](https://github.com/colonyos/pycolonies/blob/main/examples/colonies_monad.py) and [monad_example2.py](https://github.com/colonyos/pycolonies/blob/main/examples/monad_example2.py) for a full example. diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..ef4fe08 --- /dev/null +++ b/__init__.py @@ -0,0 +1,20 @@ +__version__ = "1.0.24" +__author__ = 'Johan Kristiansson' +__credits__ = 'ri.se' + +import crypto +import cfs +import model +import rpc +from pycolonies import colonies_client, Colonies +from model import FuncSpec + +__all__ = [ + 'crypto', + 'cfs', + 'model', + 'rpc', + 'colonies_client', + 'Colonies', + 'FuncSpec' +] diff --git a/build/lib/crypto.py b/build/lib/crypto.py new file mode 100644 index 0000000..969057d --- /dev/null +++ b/build/lib/crypto.py @@ -0,0 +1,226 @@ +# This code is based on the code from the following repository (MIT-licensed): +# https://github.com/ethereum/eth-utils +# https://github.com/ethereum/eth-keys + +import ctypes +import os +import hashlib +import hmac +import os +from typing import ( + Any, + Callable, + Tuple, +) + +# SECPK1N +A: int = 0 +N: int = (115792089237316195423570985008687907852837564279074904382605163141518161494337) +Gx: int = (55066263022277343669578718895168534326250603453777594175500187360389116729240) +Gy: int = (32670510020758816978083085130507043184471273380659243275938904335757337482424) +G: Tuple[int, int] = (Gx, Gy) +P: int = 2**256 - 2**32 - 977 + +class Crypto: + def __init__(self, native: bool = False) -> None: + self.native = native + if native: + libname = os.environ.get("CRYPTOLIB") + if libname == None: + libname = "/usr/local/lib/libcryptolib.so" + self.c_lib = ctypes.CDLL(libname) + self.c_lib.prvkey.restype = ctypes.c_char_p + self.c_lib.id.restype = ctypes.c_char_p + self.c_lib.sign.restype = ctypes.c_char_p + self.c_lib.hash.restype = ctypes.c_char_p + self.c_lib.recoverid.restype = ctypes.c_char_p + + def prvkey(self) -> str: + if self.native: + k = self.c_lib.prvkey() + return k.decode("utf-8") + else: + return genkey() + + def id(self, id: str) -> str: + if self.native: + h = self.c_lib.id(id.encode('utf-8')) + return h.decode("utf-8") + else: + return get_id(id) + + def sign(self, data: str, prvkey: str) -> str: + if self.native: + s = self.c_lib.sign(data.encode('utf-8'), prvkey.encode('utf-8')) + return s.decode("utf-8") + else: + return sign(data, prvkey) + +def genkey() -> str: + random_bytes = os.urandom(32) # Generate 32 random bytes + hash_obj = hashlib.sha3_256() # Create a SHA-3 256 hash object + hash_obj.update(random_bytes) # Update the hash object with the random bytes + hash_bytes = hash_obj.digest() # Get the digest of the hash + return hash_bytes.hex() # Return the hexadecimal representation + +def sign(msg: str, prv_hex: str) -> str: + prv_bytes = bytes.fromhex(prv_hex) + + hash = hashlib.sha3_256() + hash.update(msg.encode('utf-8')) + hash_bytes = hash.digest() + + s = ecdsa_raw_sign(hash_bytes, prv_bytes) + vb = int_to_byte(s[0]) + rb = pad32(int_to_big_endian(s[1])) + sb = pad32(int_to_big_endian(s[2])) + sig = b"".join((rb, sb, vb)) + + sig_hex = sig.hex() + return sig_hex + +def get_id(prv_key: str) -> str: + prv_key_bytes = bytes.fromhex(prv_key) + pub = private_key_to_public_key(prv_key_bytes) + pub_hex = "04"+pub.hex() # the prefix "04" denotes that the public key is in uncompressed format + hash = hashlib.sha3_256() + hash.update(pub_hex.encode('utf-8')) + + return hash.hexdigest() + +def pad32(value: bytes) -> bytes: + return value.rjust(32, b"\x00") + +def int_to_byte(value: int) -> bytes: + return bytes([value]) + +def int_to_big_endian(value: int) -> bytes: + return value.to_bytes((value.bit_length() + 7) // 8 or 1, "big") + + +def big_endian_to_int(value: bytes) -> int: + return int.from_bytes(value, "big") + +def encode_raw_public_key(raw_public_key: Tuple[int, int]) -> bytes: + left, right = raw_public_key + return b"".join( + ( + pad32(int_to_big_endian(left)), + pad32(int_to_big_endian(right)), + ) + ) + +def private_key_to_public_key(private_key_bytes: bytes) -> bytes: + private_key_as_num = big_endian_to_int(private_key_bytes) + + if private_key_as_num >= N: + raise Exception("Invalid privkey") + + raw_public_key = fast_multiply(G, private_key_as_num) + public_key_bytes = encode_raw_public_key(raw_public_key) + return public_key_bytes + +def deterministic_generate_k( + msg_hash: bytes, + private_key_bytes: bytes, + digest_fn: Callable[[], Any] = hashlib.sha256, +) -> int: + v_0 = b"\x01" * 32 + k_0 = b"\x00" * 32 + + k_1 = hmac.new( + k_0, v_0 + b"\x00" + private_key_bytes + msg_hash, digest_fn + ).digest() + v_1 = hmac.new(k_1, v_0, digest_fn).digest() + k_2 = hmac.new( + k_1, v_1 + b"\x01" + private_key_bytes + msg_hash, digest_fn + ).digest() + v_2 = hmac.new(k_2, v_1, digest_fn).digest() + + kb = hmac.new(k_2, v_2, digest_fn).digest() + k = big_endian_to_int(kb) + return k + +def ecdsa_raw_sign(msg_hash: bytes, private_key_bytes: bytes) -> Tuple[int, int, int]: + z = big_endian_to_int(msg_hash) + k = deterministic_generate_k(msg_hash, private_key_bytes) + + r, y = fast_multiply(G, k) + s_raw = inv(k, N) * (z + r * big_endian_to_int(private_key_bytes)) % N + + v = 27 + ((y % 2) ^ (0 if s_raw * 2 < N else 1)) + s = s_raw if s_raw * 2 < N else N - s_raw + + return v - 27, r, s + +def fast_multiply(a: Tuple[int, int], n: int) -> Tuple[int, int]: + return from_jacobian(jacobian_multiply(to_jacobian(a), n)) + +def from_jacobian(p: Tuple[int, int, int]) -> Tuple[int, int]: + z = inv(p[2], P) + return ((p[0] * z**2) % P, (p[1] * z**3) % P) + +def to_jacobian(p: Tuple[int, int]) -> Tuple[int, int, int]: + o = (p[0], p[1], 1) + return o + +def inv(a: int, n: int) -> int: + if a == 0: + return 0 + lm, hm = 1, 0 + low, high = a % n, n + while low > 1: + r = high // low + nm, new = hm - lm * r, high - low * r + lm, low, hm, high = nm, new, lm, low + return lm % n + +def jacobian_double(p: Tuple[int, int, int]) -> Tuple[int, int, int]: + if not p[1]: + return (0, 0, 0) + ysq = (p[1] ** 2) % P + S = (4 * p[0] * ysq) % P + M = (3 * p[0] ** 2 + A * p[2] ** 4) % P + nx = (M**2 - 2 * S) % P + ny = (M * (S - nx) - 8 * ysq**2) % P + nz = (2 * p[1] * p[2]) % P + return (nx, ny, nz) + +def jacobian_add( + p: Tuple[int, int, int], q: Tuple[int, int, int] +) -> Tuple[int, int, int]: + if not p[1]: + return q + if not q[1]: + return p + U1 = (p[0] * q[2] ** 2) % P + U2 = (q[0] * p[2] ** 2) % P + S1 = (p[1] * q[2] ** 3) % P + S2 = (q[1] * p[2] ** 3) % P + if U1 == U2: + if S1 != S2: + return (0, 0, 1) + return jacobian_double(p) + H = U2 - U1 + R = S2 - S1 + H2 = (H * H) % P + H3 = (H * H2) % P + U1H2 = (U1 * H2) % P + nx = (R**2 - H3 - 2 * U1H2) % P + ny = (R * (U1H2 - nx) - S1 * H3) % P + nz = (H * p[2] * q[2]) % P + return (nx, ny, nz) + +def jacobian_multiply(a: Tuple[int, int, int], n: int) -> Tuple[int, int, int]: + if a[1] == 0 or n == 0: + return (0, 0, 1) + if n == 1: + return a + if n < 0 or n >= N: + return jacobian_multiply(a, n % N) + if (n % 2) == 0: + return jacobian_double(jacobian_multiply(a, n // 2)) + elif (n % 2) == 1: + return jacobian_add(jacobian_double(jacobian_multiply(a, n // 2)), a) + else: + raise Exception("Invariant: Unreachable code path") diff --git a/build/lib/pycolonies.py b/build/lib/pycolonies.py new file mode 100644 index 0000000..9fef695 --- /dev/null +++ b/build/lib/pycolonies.py @@ -0,0 +1,869 @@ +import requests +import json +from typing import List, Optional, Any, TypeVar, Type +from pydantic import TypeAdapter, ValidationError +import base64 +from websocket import create_connection +import os +import ctypes +import boto3 +import hashlib +import uuid +from botocore.exceptions import ClientError + +from crypto import Crypto +from model import ( + Attribute, Empty, Process, FuncSpec, Workflow, ProcessGraph, S3Object, Reference, File, FileData, Cron, Log, Executor, Colony, + Statistics, Function, Snapshot, Allocations, Generator +) +import rpc + +def colonies_client(native_crypto=False) -> tuple['Colonies', str, str, str, str]: + colonies_server = os.getenv("COLONIES_SERVER_HOST") + colonies_port = os.getenv("COLONIES_SERVER_PORT") + colonies_tls = os.getenv("COLONIES_SERVER_TLS") + colonyname = os.getenv("COLONIES_COLONY_NAME") + colony_prvkey = os.getenv("COLONIES_COLONY_PRVKEY") + executorname = os.getenv("COLONIES_EXECUTOR_NAME") + prvkey = os.getenv("COLONIES_PRVKEY") + + if ( + colonies_server is None or + colonies_port is None or + colonyname is None or + colony_prvkey is None or + executorname is None or + prvkey is None + ): + raise ValueError("Environment variables COLONIES_SERVER_HOST, COLONIES_SERVER_PORT, COLONIES_COLONY_NAME, and COLONIES_COLONY_PRVKEY must be set.") + + client = Colonies(colonies_server, int(colonies_port), colonies_tls == "true", native_crypto=native_crypto) + + return client, colonyname, colony_prvkey, executorname, prvkey + +# Type variable for generic return types +T = TypeVar('T') + +class ColoniesConnectionError(Exception): + pass + +class ColoniesError(Exception): + pass + +class Colonies: + url: str + host: str + port: int + tls: bool + native_crypto: bool + + WAITING = 0 + RUNNING = 1 + SUCCESSFUL = 2 + FAILED = 3 + + def __init__( + self, + host: str, + port: int, + tls: bool = False, + native_crypto: bool = False + ) -> None: + self.host = host + self.port = port + self.native_crypto = native_crypto + self.tls = tls + self.url = ("https://" if self.tls else "http://") + self.host + ":" + str(self.port) + "/api" + + def __rpc( + self, + request_payload: rpc.RequestPayload, + prvkey: str, + response_payload_type: Type[T] + ) -> T: + request_payload_json = request_payload.model_dump_json(by_alias=True) + request_payload_b64 = str(base64.b64encode(request_payload_json.encode('utf-8')), "utf-8") + crypto = Crypto(native=self.native_crypto) + signature = crypto.sign(request_payload_b64, prvkey) + + request = rpc.Request( + payloadtype=request_payload.msgtype, + payload=request_payload_b64, + signature=signature + ) + + request_json = request.model_dump_json() + + try: + http_response = requests.post(url = self.url, data=request_json, verify=True) + http_response.raise_for_status() + response = rpc.Response.model_validate_json(http_response.content) + response_payload_json = base64.b64decode(response.payload) + if response.payloadtype == "error": + error_payload = rpc.ErrorResponse.model_validate_json(response_payload_json) + raise ColoniesConnectionError(error_payload.message) + return TypeAdapter(response_payload_type).validate_json(response_payload_json) + except requests.exceptions.RequestException as err: + raise ColoniesConnectionError(f"Network request failed: {err}") from err + except ValidationError as err: + raise ColoniesError(f"API response validation failed: {err}") from err + except UnicodeDecodeError as err: + raise ColoniesError(f"Failed to decode response payload: {err}") from err + + def wait(self, process: Process, timeout: int, executor_prvkey: str) -> Process: + state = Colonies.SUCCESSFUL + if not process.spec.conditions.colonyname: + raise ValueError("Process must have a colony name set in its conditions") + payload = rpc.SubscribeProcess( + processid=process.processid, + executortype=process.spec.conditions.executortype, + state=state, + timeout=timeout, + colonyname=process.spec.conditions.colonyname, + ) + + payload_b64 = str(base64.b64encode(json.dumps(payload).encode('utf-8')), "utf-8") + request = rpc.Request(payloadtype=payload.msgtype, payload=payload_b64, signature="") + + crypto = Crypto() + request.signature = crypto.sign(request.payload, executor_prvkey) + + if self.tls: + ws = create_connection("wss://" + self.host + ":" + str(self.port) + "/pubsub") + else: + ws = create_connection("ws://" + self.host + ":" + str(self.port) + "/pubsub") + ws.send(json.dumps(request)) + ws.recv() + ws.close() + + return self.get_process(process.processid, executor_prvkey) + + def add_colony(self, colony: Colony, server_prvkey: str) -> Colony: + payload = rpc.AddColony(colony=colony) + return self.__rpc(payload, server_prvkey, Colony) + + def del_colony(self, colonyname: str, server_prvkey: str) -> None: + payload = rpc.RemoveColony(colonyname=colonyname) + self.__rpc(payload, server_prvkey, Colony) + + def list_colonies(self, server_prvkey: str) -> List[Colony]: + payload = rpc.GetColonies() + return self.__rpc(payload, server_prvkey, List[Colony]) + + def get_colony(self, colonyname: str, server_prvkey: str) -> Colony: + payload = rpc.GetColony(colonyname=colonyname) + return self.__rpc(payload, server_prvkey, Colony) + + def add_executor( + self, + executorid: str, + executorname: str, + colonyname: str, + colony_prvkey: str, + executortype: Optional[str] = None, + capabilities: Optional[rpc.Capabilities] = None + ) -> Executor: + payload = rpc.Executor( + executorid=executorid, + executortype=executortype or executorname, + executorname=executorname, + colonyname=colonyname, + capabilities=capabilities + ) + payload = rpc.AddExecutor(executor=payload) + return self.__rpc(payload, colony_prvkey, Executor) + + def report_allocation(self, colonyname: str, executorname: str, allocations: Allocations, executor_prvkey: str) -> None: + """ + Reports resource allocations for an executor. + """ + payload = rpc.ReportAllocations( + colonyname=colonyname, + executorname=executorname, + allocations=allocations + ) + self.__rpc(payload, executor_prvkey, Empty) + + def list_executors(self, colonyname: str, executor_prvkey: str) -> List[Executor]: + payload = rpc.GetExecutors(colonyname=colonyname) + return self.__rpc(payload, executor_prvkey, List[Executor]) + + def approve_executor(self, colonyname: str, executorname: str, colony_prvkey: str) -> None: + payload = rpc.ApproveExecutor(colonyname=colonyname, executorname=executorname) + self.__rpc(payload, colony_prvkey, Empty) + + def reject_executor(self, colonyname: str, executorname: str, colony_prvkey: str) -> None: + payload = rpc.RejectExecutor(colonyname=colonyname, executorname=executorname) + self.__rpc(payload, colony_prvkey, Empty) + + def remove_executor(self, colonyname: str, executorname: str, colony_prvkey: str) -> None: + payload = rpc.RemoveExecutor(colonyname=colonyname, executorname=executorname) + self.__rpc(payload, colony_prvkey, Empty) + + def get_executor(self, colonyname: str, executorname: str, executor_prvkey: str) -> Executor: + payload = rpc.GetExecutor(colonyname=colonyname, executorname=executorname) + return self.__rpc(payload, executor_prvkey, Executor) + + def submit_func_spec(self, spec: FuncSpec, executor_prvkey: str) -> Process: + payload = rpc.SubmitFunctionSpec(spec=spec) + return self.__rpc(payload, executor_prvkey, Process) + + def submit_workflow(self, workflow: Workflow, executor_prvkey: str) -> ProcessGraph: + payload = rpc.SubmitWorkflowSpec(spec=workflow) + return self.__rpc(payload, executor_prvkey, ProcessGraph) + + def assign( + self, + colonyname: str, + timeout: int, + executor_prvkey: str, + available_cpu: str = "1000m", + available_mem: str = "1000Mi" + ) -> Process: + payload = rpc.AssignProcess( + colonyname=colonyname, + timeout=timeout, + availablecpu=available_cpu, + availablemem=available_mem + ) + return self.__rpc(payload, executor_prvkey, Process) + + def list_processes(self, colonyname: str, count: int, state: int, executor_prvkey: str) -> List[Process]: + payload = rpc.GetProcesses( + colonyname=colonyname, + count=count, + state=state + ) + return self.__rpc(payload, executor_prvkey, List[Process]) + + def get_process(self, processid: str, executor_prvkey: str) -> Process: + payload = rpc.GetProcess(processid=processid) + return self.__rpc(payload, executor_prvkey, Process) + + def remove_process(self, processid: str, executor_prvkey: str, all: bool = False) -> None: + payload = rpc.RemoveProcess(processid=processid, all=all) + self.__rpc(payload, executor_prvkey, Empty) + + def close(self, processid: str, output: List[Any], executor_prvkey: str) -> None: + payload = rpc.CloseSuccessful(processid=processid, out=output) + self.__rpc(payload, executor_prvkey, Empty) + + def fail(self, processid: str, errors: List[str], executor_prvkey: str) -> None: + payload = rpc.CloseFailed(processid=processid, errors=errors) + self.__rpc(payload, executor_prvkey, Empty) + + def set_output(self, processid: str, arr: List[Any], executor_prvkey: str) -> None: + payload = rpc.SetOutput(processid=processid, out=arr) + self.__rpc(payload, executor_prvkey, Empty) + + def stats(self, colonyname: str, executor_prvkey: str) -> Statistics: + payload = rpc.GetColonyStatistics(colonyname=colonyname) + return self.__rpc(payload, executor_prvkey, Statistics) + + def add_attribute(self, processid: str, key: str, value: str, executor_prvkey: str) -> Attribute: + attribute = rpc.Attribute( + key=key, + value=value, + targetid=processid, + attributetype=1 + ) + payload = rpc.AddAttribute(attribute=attribute) + return self.__rpc(payload, executor_prvkey, Attribute) + + def get_attribute(self, attributeid: str, executor_prvkey: str) -> Attribute: + payload = rpc.GetAttribute(attributeid=attributeid) + return self.__rpc(payload, executor_prvkey, Attribute) + + def get_processgraph(self, processgraphid: str, executor_prvkey: str) -> ProcessGraph: # TODO: unittest + payload = rpc.GetProcessGraph(processgraphid=processgraphid) + return self.__rpc(payload, executor_prvkey, ProcessGraph) + + def add_function(self, colonyname: str, executorname: str, funcname: str, executor_prvkey: str) -> Function: + func = rpc.Function( + colonyname=colonyname, + executorname=executorname, + funcname=funcname, + ) + payload = rpc.AddFunction(fun=func) + return self.__rpc(payload, executor_prvkey, Function) + + def get_functions_by_executor(self, colonyname: str, executorname: str, executor_prvkey: str) -> List[Function]: + payload = rpc.GetFunctions( + colonyname=colonyname, + executorname=executorname + ) + return self.__rpc(payload, executor_prvkey, List[Function]) + + def get_functions_by_colony(self, colonyname: str, executor_prvkey: str) -> List[Function]: + payload = rpc.GetFunctions(colonyname=colonyname) + return self.__rpc(payload, executor_prvkey, List[Function]) + + def find_process(self, nodename: str, processids: List[str], executor_prvkey: str) -> Optional[Process]: + for processid in processids: + process = self.get_process(processid, executor_prvkey) + if process.spec.nodename == nodename: + return process + return None + + def add_child( + self, + processgraphid: str, + parentprocessid: str, + childprocessid: str, + funcspec: FuncSpec, + nodename: str, + insert: bool, + executor_prvkey: str + ) -> Process: + funcspec.nodename = nodename + payload = rpc.AddChild( + processgraphid=processgraphid, + parentprocessid=parentprocessid, + childprocessid=childprocessid, + insert=insert, + spec=funcspec + ) + return self.__rpc(payload, executor_prvkey, Process) + + def create_snapshot(self, colonyname: str, label: str, name: str, executor_prvkey: str) -> Snapshot: + payload = rpc.CreateSnapshot(colonyname=colonyname, label=label, name=name) + return self.__rpc(payload, executor_prvkey, Snapshot) + + def get_snapshots(self, colonyname: str, executor_prvkey: str) -> List[Snapshot]: + payload = rpc.GetSnapshot(colonyname=colonyname) + return self.__rpc(payload, executor_prvkey, List[Snapshot]) + + def get_snapshot_by_name(self, colonyname: str, name: str, executor_prvkey: str) -> Snapshot: + payload = rpc.GetSnapshot(colonyname=colonyname, name=name) + return self.__rpc(payload, executor_prvkey, Snapshot) + + def get_snapshot_by_id(self, colonyname: str, snapshotid: str, executor_prvkey: str) -> Snapshot: + payload = rpc.GetSnapshot(colonyname=colonyname, snapshotid=snapshotid,) + return self.__rpc(payload, executor_prvkey, Snapshot) + + def add_log(self, processid: str, logmsg: str, executor_prvkey: str) -> None: + payload = rpc.AddLog(processid=processid, message=logmsg) + self.__rpc(payload, executor_prvkey, Empty) + + def get_process_log(self, colonyname: str, processid: str, count: int, since: int, executor_prvkey: str) -> List[Log]: + payload = rpc.GetLogs( + colonyname=colonyname, + executorname="", + processid=processid, + count=count, + since=since + ) + return self.__rpc(payload, executor_prvkey, List[Log]) + + def get_executor_log(self, colonyname: str, executorname: str, count: int, since: int, executor_prvkey: str) -> List[Log]: + payload = rpc.GetLogs( + colonyname=colonyname, + executorname=executorname, + processid="", + count=count, + since=since + ) + return self.__rpc(payload, executor_prvkey, List[Log]) + + def sync(self, dir: str, label: str, keeplocal: bool, colonyname: str, executor_prvkey: str) -> None: + libname = os.environ.get("CFSLIB") + if libname == None: + libname = "/usr/local/lib/libcfslib.so" + c_lib = ctypes.CDLL(libname) + c_lib.sync.restype = ctypes.c_int + + c_host = ctypes.c_char_p(self.host.encode('utf-8')) + c_port = ctypes.c_int(self.port) + c_insecure = ctypes.c_int(self.tls==False) + c_skip_tls_verify = ctypes.c_int(False) + c_dir = ctypes.c_char_p(dir.encode('utf-8')) + c_label = ctypes.c_char_p(label.encode('utf-8')) + c_keeplocal = ctypes.c_int(keeplocal) + c_colonyname = ctypes.c_char_p(colonyname.encode('utf-8')) + c_prvkey = ctypes.c_char_p(executor_prvkey.encode('utf-8')) + + res = c_lib.sync(c_host, c_port, c_insecure, c_skip_tls_verify, c_dir, c_label, c_keeplocal, c_colonyname, c_prvkey) + if res != 0: + raise Exception("failed to sync") + + def get_files(self, label: str, colonyname: str, executor_prvkey: str) -> List[FileData]: + payload = rpc.GetFiles(colonyname=colonyname, label=label) + return self.__rpc(payload, executor_prvkey, List[FileData]) + + def add_cron( + self, + cronname: str, + cronexpr: str, + wait: bool, + workflow: Workflow, + colonyname: str, + executor_prvkey: str, + random:bool=False + ) -> Cron: + cron = rpc.Cron( + name=cronname, + colonyname=colonyname, + interval=-1, # -1 means cron expression is used + waitforprevprocessgraph=wait, + cronexpression=cronexpr, + workflowspec=json.dumps(workflow.model_dump(by_alias=True)), + random=random, + ) + payload = rpc.AddCron(cron=cron) + return self.__rpc(payload, executor_prvkey, Cron) + + def get_cron(self, cronid: str, executor_prvkey: str) -> Cron: + payload = rpc.GetCron(cronid=cronid) + return self.__rpc(payload, executor_prvkey, Cron) + + def get_crons(self, colonyname: str, count: int, executor_prvkey: str) -> List[Cron]: + payload = rpc.GetCrons(colonyname=colonyname, count=count) + return self.__rpc(payload, executor_prvkey, List[Cron]) + + def del_cron(self, cronid: str, executor_prvkey: str, all: bool=False) -> None: + payload = rpc.RemoveCron(all=all, cronid=cronid) + self.__rpc(payload, executor_prvkey, Empty) + + def run_cron(self, cronid: str, executor_prvkey: str) -> Cron: + payload = rpc.RunCron(cronid=cronid) + return self.__rpc(payload, executor_prvkey, Cron) + + def resolve_generator_by_name(self, colonyname: str, generatorname: str, executor_prvkey: str) -> Generator: + """ + Resolves a generator by its name within a colony. + """ + payload = rpc.ResolveGenerator( + colonyname=colonyname, + generatorname=generatorname + ) + return self.__rpc(payload, executor_prvkey, Generator) + + def change_colony_id(self, colonyname: str, new_colony_id: str, server_prvkey: str) -> None: + """ + Changes the ID of a colony. Requires server owner credentials. + """ + payload = rpc.ChangeColonyID( + colonyname=colonyname, + colonyid=new_colony_id + ) + self.__rpc(payload, server_prvkey, Empty) + + def change_executor_id(self, colonyname: str, new_executor_id: str, colony_prvkey: str) -> None: + """ + Changes the ID of an executor. Requires colony owner credentials. + """ + payload = rpc.ChangeExecutorID( + executorid=new_executor_id, + colonyname=colonyname + ) + self.__rpc(payload, colony_prvkey, Empty) + + def change_user_id(self, colonyname: str, new_user_id: str, colony_prvkey: str) -> None: + """ + Changes the ID of a user. Requires colony owner credentials. + """ + payload = rpc.ChangeUserID( + colonyname=colonyname, + userid=new_user_id + ) + self.__rpc(payload, colony_prvkey, Empty) + + def change_server_id(self, new_server_id: str, server_prvkey: str) -> None: + """ + Changes the ID of the server. Requires server owner credentials. + """ + payload = rpc.ChangeServerID(serverid=new_server_id) + self.__rpc(payload, server_prvkey, Empty) + + def __generate_random_id(self) -> str: + random_uuid = uuid.uuid4() + hasher = hashlib.sha256() + hasher.update(random_uuid.bytes) + return hasher.hexdigest() + + def __checksum_file(self, file_path: str) -> str: + try: + with open(file_path, 'rb') as f: + buffer = bytearray(10000) + hasher = hashlib.sha256() + while True: + n = f.readinto(buffer) + if not n: + break + hasher.update(buffer[:n]) + return hasher.hexdigest() + except Exception as e: + raise e + + def __checksum_data(self, file_data: bytes) -> str: + try: + hasher = hashlib.sha256() + hasher.update(file_data) + return hasher.hexdigest() + except Exception as e: + raise e + + def __get_file_size(self, file_path: str) -> Optional[int]: + try: + size = os.path.getsize(file_path) + return size + except OSError as e: + print(f"Error getting file size: {e}") + return None + + def __check_bucket(self, s3_client: Any, bucket_name: str) -> None: + try: + s3_client.head_bucket(Bucket=bucket_name) + except ClientError as e: + error_code = e.response['Error']['Code'] + if error_code == '404': + try: + s3_client.create_bucket(Bucket=bucket_name) + except ClientError as e: + raise Exception(f"Error creating bucket: {e}") + else: + raise Exception(f"Error checking bucket: {e}") + + def upload_file( + self, + colonyname: str, + executor_prvkey: str, + filepath: str, + label: str + ) -> File: + return self.__upload_file(filepath, label, colonyname, executor_prvkey) + + def upload_data( + self, + colonyname: str, + executor_prvkey: str, + filename: str, + data: bytes, + label: str + ) -> File: + return self.__upload_file(filename, label, colonyname, executor_prvkey, file_bytes=data) + + def __upload_file( + self, + filepath: str, + label: str, + colonyname: str, + executor_prvkey: str, + file_bytes: Optional[bytes] = None + ) -> File: + endpoint = os.getenv("AWS_S3_ENDPOINT") + access_key = os.getenv("AWS_S3_ACCESSKEY") + secret_key = os.getenv("AWS_S3_SECRETKEY") + region = os.getenv("AWS_S3_REGION") + use_tls_str = os.getenv("AWS_S3_TLS") + bucket_name = os.getenv("AWS_S3_BUCKET") + skip_verify_str = os.getenv("AWS_S3_SKIPVERIFY") + + if endpoint is None: + raise ValueError("Environment variable AWS_S3_ENDPOINT is not set") + if access_key is None: + raise ValueError("Environment variable AWS_S3_ACCESSKEY is not set") + if secret_key is None: + raise ValueError("Environment variable AWS_S3_SECRETKEY is not set") + if region is None: + raise ValueError("Environment variable AWS_S3_REGION is not set") + if use_tls_str is None: + raise ValueError("Environment variable AWS_S3_TLS is not set") + if bucket_name is None: + raise ValueError("Environment variable AWS_S3_BUCKET is not set") + if skip_verify_str is None: + raise ValueError("Environment variable AWS_S3_SKIPVERIFY is not set") + + object_name = self.__generate_random_id() + if file_bytes is None: + filesize = self.__get_file_size(filepath) + if filesize is None: + raise ValueError(f"Could not get file size of {filepath}") + else: + filesize = len(file_bytes) + + endpoint_parts = endpoint.split(":") + if len(endpoint_parts) == 2: + server = endpoint_parts[0] + port = int(endpoint_parts[1]) + else: + raise Exception("invalid endpoint") + + use_tls = use_tls_str.lower() in ['true', '1', 'yes'] + + if not endpoint.startswith('http://') and not endpoint.startswith('https://'): + endpoint = f"http{'s' if use_tls else ''}://{endpoint}" + + + s3_client = boto3.client( + 's3', + endpoint_url=endpoint, + aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + region_name=region, + use_ssl=use_tls, + verify=skip_verify_str.lower() not in ['true', '1', 'yes'] + ) + + self.__check_bucket(s3_client, bucket_name) + + filename = os.path.basename(filepath) + + try: + if file_bytes is None: + s3_client.upload_file(filepath, bucket_name, object_name) + else: + # Upload byte array + s3_client.put_object(Bucket=bucket_name, Key=object_name, Body=file_bytes) + except Exception as e: + raise e + + if region == None: + region = "" + + if use_tls_str == "true": + tls = True + else: + tls = False + + obj = S3Object( + server=server, + port=port, + tls=tls, + accesskey="", + secretkey="", + region=region, + encryptionkey="", + encryptionalg="", + object=object_name, + bucket=bucket_name + ) + + ref = Reference( + protocol="s3", + s3object=obj + ) + + if file_bytes is None: + checksum = self.__checksum_file(filepath) + else: + checksum = self.__checksum_data(file_bytes) + + f = rpc.File( + colonyname=colonyname, + label=label, + name=filename, + size=filesize, + checksum=checksum, + checksumalg="SHA256", + ref=ref + ) + + payload = rpc.AddFile( + file=f + ) + + return self.__rpc(payload, executor_prvkey, File) + + def get_file( + self, + colonyname: str, + executor_prvkey: str, + label: Optional[str], + filename: Optional[str] = None, + fileid: Optional[str] = None, + latest: bool = True + ) -> List[File]: + if (fileid and filename) or (not fileid and not filename) or (not filename and not label): + raise ValueError("Please provide exactly one of: 'fileid' or ('filename' AND 'label')") + payload = rpc.GetFile( + colonyname=colonyname, + fileid=fileid, + label=label, + name=filename, + latest=latest + ) + return self.__rpc(payload, executor_prvkey, List[File]) + + def __remove_file( + self, + label: Optional[str], + fileid: Optional[str], + filename: Optional[str], + colonyname: str, + executor_prvkey: str + ) -> None: + if (fileid and filename) or (not fileid and not filename) or (not filename and not label): + raise ValueError("Please provide exactly one of: 'fileid' or ('filename' AND 'label')") + + payload = rpc.RemoveFile( + colonyname=colonyname, + fileid=fileid, + label=label, + name=filename + ) + self.__rpc(payload, executor_prvkey, Empty) + + def download_file( + self, + colonyname: str, + executor_prvkey: str, + dst: str, + filename: str, + label: Optional[str] = None, + fileid: Optional[str] = None, + latest: bool = True + ) -> str: + access_key = os.getenv("AWS_S3_ACCESSKEY") + secret_key = os.getenv("AWS_S3_SECRETKEY") + skip_verify_str = os.getenv("AWS_S3_SKIPVERIFY") + + dst = os.path.abspath(dst) + + try: + os.makedirs(dst, exist_ok=True) + except Exception as e: + raise e + + file = self.get_file(colonyname, executor_prvkey, label=label, fileid=fileid, filename=filename, latest=latest) + + if len(file) == 0: + raise Exception("invalid file") + + object_name = file[0].ref.s3object.object + region = file[0].ref.s3object.region + endpoint = file[0].ref.s3object.server + ":" + str(file[0].ref.s3object.port) + use_tls = file[0].ref.s3object.tls + bucket_name = file[0].ref.s3object.bucket + + verify = True + if skip_verify_str: + verify = skip_verify_str.lower() not in ['true', '1', 'yes'] + + if not endpoint.startswith('http://') and not endpoint.startswith('https://'): + endpoint = f"http{'s' if use_tls else ''}://{endpoint}" + + if region == "": + region = None + + s3_client = boto3.client( + 's3', + endpoint_url=endpoint, + aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + region_name=region, + use_ssl=use_tls, + verify=verify + ) + + dst = os.path.join(dst, filename) + + try: + s3_client.download_file(bucket_name, object_name, dst) + return dst + except Exception as e: + raise e + + def download_data( + self, + colonyname: str, + executor_prvkey: str, + label: Optional[str] = None, + filename: Optional[str] = None, + fileid: Optional[str] = None, + latest: bool = True + ) -> bytes: + access_key = os.getenv("AWS_S3_ACCESSKEY") + secret_key = os.getenv("AWS_S3_SECRETKEY") + skip_verify_str = os.getenv("AWS_S3_SKIPVERIFY") + + file = self.get_file(colonyname, executor_prvkey, label=label, fileid=fileid, filename=filename, latest=latest) + + if len(file) == 0: + raise Exception("invalid file") + + object_name = file[0].ref.s3object.object + region = file[0].ref.s3object.region + endpoint = file[0].ref.s3object.server + ":" + str(file[0].ref.s3object.port) + use_tls = file[0].ref.s3object.tls + bucket_name = file[0].ref.s3object.bucket + + verify = True + if skip_verify_str: + verify = skip_verify_str.lower() not in ['true', '1', 'yes'] + + if not endpoint.startswith('http://') and not endpoint.startswith('https://'): + endpoint = f"http{'s' if use_tls else ''}://{endpoint}" + + if region == "": + region = None + + try: + s3_client = boto3.client( + 's3', + endpoint_url=endpoint, + aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + region_name=region, + use_ssl=use_tls, + verify=verify + ) + except Exception as e: + raise e + + try: + response = s3_client.get_object(Bucket=bucket_name, Key=object_name) + data = response['Body'].read() + return data + except Exception as e: + raise e + + def delete_file( + self, + colonyname: str, + executor_prvkey: str, + filename: Optional[str] = None, + label: Optional[str] = None, + fileid: Optional[str] = None + ) -> None: + access_key = os.getenv("AWS_S3_ACCESSKEY") + secret_key = os.getenv("AWS_S3_SECRETKEY") + skip_verify_str = os.getenv("AWS_S3_SKIPVERIFY") + + file = self.get_file(colonyname, executor_prvkey, label=label, fileid=fileid, filename=filename) + + if len(file) == 0: + raise Exception("invalid file") + + object_name = file[0].ref.s3object.object + region = file[0].ref.s3object.region + endpoint = file[0].ref.s3object.server + ":" + str(file[0].ref.s3object.port) + use_tls = file[0].ref.s3object.tls + bucket_name = file[0].ref.s3object.bucket + + verify = True + if skip_verify_str: + verify = skip_verify_str.lower() not in ['true', '1', 'yes'] + + if not endpoint.startswith('http://') and not endpoint.startswith('https://'): + endpoint = f"http{'s' if use_tls else ''}://{endpoint}" + + if region == "": + region = None + + s3_client = boto3.client( + 's3', + endpoint_url=endpoint, + aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + region_name=region, + use_ssl=use_tls, + verify=verify + ) + + try: + s3_client.delete_object(Bucket=bucket_name, Key=object_name) + except Exception as e: + raise e + + self.__remove_file(label, fileid, filename, colonyname, executor_prvkey) diff --git a/build/scripts-3.9/pycolonies.py b/build/scripts-3.9/pycolonies.py new file mode 100755 index 0000000..551293a --- /dev/null +++ b/build/scripts-3.9/pycolonies.py @@ -0,0 +1,357 @@ +import requests +import json +import sys +sys.path.append(".") +from crypto import Crypto +import base64 +from websocket import create_connection +import inspect + +class ColoniesConnectionError(Exception): + pass + +class ColoniesError(Exception): + pass + +def create_func_spec(func, args, colonyname, executortype, priority, maxexectime, maxretries, maxwaittime, code=None): + if isinstance(func, str): + func_spec = { + "nodename": func, + "funcname": func, + "args": args, + "priority": priority, + "maxwaittime": maxwaittime, + "maxexectime": maxexectime, + "maxretries": maxretries, + "conditions": { + "colonyname": colonyname, + "executortype": executortype + }, + "label": "" + } + if code is not None: + code_bytes = code.encode("ascii") + code_base64_bytes = base64.b64encode(code_bytes) + code_base64 = code_base64_bytes.decode("ascii") + func_spec["env"] = {} + func_spec["env"]["code"] = code_base64 + + else: + code = inspect.getsource(func) + code_bytes = code.encode("ascii") + code_base64_bytes = base64.b64encode(code_bytes) + code_base64 = code_base64_bytes.decode("ascii") + + funcname = func.__name__ + args_spec = inspect.getfullargspec(func) + args_spec_str = ','.join(args_spec.args) + + func_spec = { + "nodename": funcname, + "funcname": funcname, + "args": args, + "priority": priority, + "maxwaittime": maxwaittime, + "maxexectime": maxexectime, + "maxretries": maxretries, + "conditions": { + "colonyname": colonyname, + "executortype": executortype + }, + "env": { + "args_spec": args_spec_str, + "code": code_base64, + }, + } + + return func_spec + +class Workflow: + def __init__(self, colonyname): + self.colonyname = colonyname + self.func_specs = [] + + def add(self, func_spec, nodename, dependencies): + func_spec["nodename"] = nodename + func_spec["conditions"]["dependencies"] = dependencies + self.func_specs.append(func_spec) + + def workflow_spec(self): + return { + "colonyname" : self.colonyname, + "functionspecs" : self.func_specs + } + +class Colonies: + WAITING = 0 + RUNNING = 1 + SUCCESSFUL = 2 + FAILED = 3 + + def __init__(self, host, port): + self.url = "http://" + host + ":" + str(port) + "/api" + self.host = host + self.port = port + + def __rpc(self, msg, prvkey): + payload = str(base64.b64encode(json.dumps(msg).encode('utf-8')), "utf-8") + crypto = Crypto() + signature = crypto.sign(payload, prvkey) + + rpc = { + "payloadtype" : msg["msgtype"], + "payload" : payload, + "signature" : signature + } + + rpc_json = json.dumps(rpc) + try: + reply = requests.post(url = self.url, data=rpc_json, verify=False) + reply_msg_json = json.loads(reply.content) + base64_payload = reply_msg_json["payload"] + payload_bytes = base64.b64decode(base64_payload) + payload = json.loads(payload_bytes) + except requests.exceptions.ConnectionError as err: + raise ColoniesConnectionError(err) + except Exception as err: + raise ColoniesConnectionError(err) + + if reply.status_code == 200: + return payload + else: + raise ColoniesError(payload["message"]) + + def wait(self, process, timeout, prvkey): + processid = process["processid"] + executortype = process["spec"]["conditions"]["executortype"] + state = 2 + msg = { + "processid": processid, + "executortype": executortype, + "state": state, + "timeout": timeout, + "msgtype": "subscribeprocessmsg" + } + + rpcmsg = { + "payloadtype": msg["msgtype"], + "payload": "", + "signature": "" + } + + rpcmsg["payload"] = str(base64.b64encode(json.dumps(msg).encode('utf-8')), "utf-8") + crypto = Crypto() + rpcmsg["signature"] = crypto.sign(rpcmsg["payload"], prvkey) + + ws = create_connection("ws://" + self.host + ":" + str(self.port) + "/pubsub") + ws.send(json.dumps(rpcmsg)) + ws.recv() + ws.close() + + return self.get_process(process["processid"], prvkey) + + def add_colony(self, colony, prvkey): + msg = { + "msgtype": "addcolonymsg", + "colony": colony + } + return self.__rpc(msg, prvkey) + + def del_colony(self, colonyname, prvkey): + msg = { + "msgtype": "deletecolonymsg", + "colonyname": colonyname + } + return self.__rpc(msg, prvkey) + + def list_colonies(self, prvkey): + msg = { + "msgtype": "getcoloniesmsg", + } + return self.__rpc(msg, prvkey) + + def get_colony(self, colonyname, prvkey): + msg = { + "msgtype": "getcolonymsg", + "colonyname": colonyname + } + return self.__rpc(msg, prvkey) + + def add_executor(self, executor, prvkey): + msg = { + "msgtype": "addexecutormsg", + "executor": executor + } + return self.__rpc(msg, prvkey) + + def list_executors(self, colonyname, prvkey): + msg = { + "msgtype": "getexecutorsmsg", + "colonyname": colonyname + } + return self.__rpc(msg, prvkey) + + def approve_executor(self, executorid, prvkey): + msg = { + "msgtype": "approveexecutormsg", + "executorid": executorid + } + return self.__rpc(msg, prvkey) + + def reject_executor(self, executorid, prvkey): + msg = { + "msgtype": "rejectexecutormsg", + "executorid": executorid + } + return self.__rpc(msg, prvkey) + + def delete_executor(self, executorid, prvkey): + msg = { + "msgtype": "deleteexecutormsg", + "executorid": executorid + } + return self.__rpc(msg, prvkey) + + def submit(self, spec, prvkey): + if isinstance(spec, Workflow): + msg = { + "msgtype": "submitworkflowspecmsg", + "spec": spec.workflow_spec() + } + return self.__rpc(msg, prvkey) + else: + msg = { + "msgtype": "submitfuncspecmsg", + "spec": spec + } + return self.__rpc(msg, prvkey) + + def assign(self, colonyname, timeout, prvkey): + msg = { + "msgtype": "assignprocessmsg", + "timeout": timeout, + "colonyname": colonyname + } + return self.__rpc(msg, prvkey) + + def list_processes(self, colonyname, count, state, prvkey): + msg = { + "msgtype": "getprocessesmsg", + "colonyname": colonyname, + "count": count, + "state": state + } + return self.__rpc(msg, prvkey) + + def get_process(self, processid, prvkey): + msg = { + "msgtype": "getprocessmsg", + "processid": processid + } + return self.__rpc(msg, prvkey) + + def delete_process(self, processid, prvkey): + msg = { + "msgtype": "deleteprocessmsg", + "processid": processid + } + return self.__rpc(msg, prvkey) + + def close(self, processid, output, prvkey): + msg = { + "msgtype": "closesuccessfulmsg", + "processid": processid, + "out": output + } + + return self.__rpc(msg, prvkey) + + def fail(self, processid, errors, prvkey): + msg = { + "msgtype": "closefailedmsg", + "processid": processid, + "errors": errors + } + + return self.__rpc(msg, prvkey) + + def stats(self, colonyname, prvkey): + msg = { + "msgtype": "getcolonystatsmsg", + "colonyname": colonyname + } + return self.__rpc(msg, prvkey) + + def add_attribute(self, processid, key, value, prvkey): + attribute = {} + attribute["key"] = key + attribute["value"] = value + attribute["targetid"] = processid + attribute["attributetype"] = 1 + + msg = { + "msgtype": "addattributemsg", + "attribute": attribute + } + return self.__rpc(msg, prvkey) + + def get_attribute(self, attributeid, prvkey): + msg = { + "msgtype": "getattributemsg", + "attributeid": attributeid + } + return self.__rpc(msg, prvkey) + + def get_processgraph(self, processgraphid, prvkey): # TODO: unittest + msg = { + "msgtype": "getprocessgraphmsg", + "processgraphid": processgraphid + } + return self.__rpc(msg, prvkey) + + def add_function(self, executorid, colonyname, funcname, args, desc, prvkey): + func = {} + func["executorid"] = executorid + func["colonyname"] = colonyname + func["funcname"] = funcname + func["args"] = args + func["desc"] = desc + + msg = { + "msgtype": "addfunctionmsg", + "fun": func + } + return self.__rpc(msg, prvkey) + + def get_functions_by_executor(self, executorid, prvkey): + msg = { + "msgtype": "getfunctionsmsg", + "executorid": executorid + } + return self.__rpc(msg, prvkey) + + def get_functions_by_colony(self, colonyname, prvkey): + msg = { + "msgtype": "getfunctionsmsg", + "colonyname": colonyname + } + return self.__rpc(msg, prvkey) + + def find_process(self, nodename, processids, prvkey): + for processid in processids: + process = self.get_process(processid, prvkey) + if process["spec"]["nodename"] == nodename: + return process + return None + + def add_child(self, processgraphid, parentprocessid, childprocessid, funcspec, nodename, insert, prvkey): + funcspec["nodename"] = nodename + msg = { + "msgtype": "addchildmsg", + "processgraphid": processgraphid, + "parentprocessid": parentprocessid, + "childprocessid": childprocessid, + "insert": insert, + "spec": funcspec + } + return self.__rpc(msg, prvkey) diff --git a/colonies.py b/colonies.py deleted file mode 100644 index 4158fdf..0000000 --- a/colonies.py +++ /dev/null @@ -1,49 +0,0 @@ -import requests -import json -import sys -sys.path.append(".") -from crypto import Crypto -import base64 - -class Colonies: - - def __init__(self, url): - self.url = "https://10.0.0.240:8080/api" - pass - - def __rpc(self, msg, prvkey): - payload = str(base64.b64encode(json.dumps(msg).encode('utf-8')), "utf-8") - crypto = Crypto() - signature = crypto.sign(payload, prvkey) - - rpc = { - "payloadtype" : msg["msgtype"], - "payload" : payload, - "signature" : signature - } - - rpc_json = json.dumps(rpc) - reply = requests.post(url = self.url, data=rpc_json, verify=False) - reply_msg_json = json.loads(reply.content) - base64_payload = reply_msg_json["payload"] - payload_bytes = base64.b64decode(base64_payload) - payload = json.loads(payload_bytes) - - if reply.status_code == 200: - return payload - else: - raise Exception(payload["message"]) - - def add_colony(self, colony, prvkey): - add_colony_msg = { - "msgtype": "addcolonymsg", - "colony": colony - } - return self.__rpc(add_colony_msg, prvkey) - - def del_colony(self, colonyid, prvkey): - add_colony_msg = { - "msgtype": "deletecolonymsg", - "colonyid": colonyid - } - return self.__rpc(add_colony_msg, prvkey) diff --git a/colonies_test.py b/colonies_test.py deleted file mode 100644 index 3636532..0000000 --- a/colonies_test.py +++ /dev/null @@ -1,34 +0,0 @@ -import unittest -import sys -sys.path.append(".") -from crypto import Crypto -from colonies import Colonies - -class TestColonies(unittest.TestCase): - - def test_add_colony(self): - url = "https://10.0.0.240:8080/api" - client = Colonies(url) - - crypto = Crypto() - prvkey = crypto.prvkey() - colonyid = crypto.id(prvkey) - - colony = { - "colonyid" : colonyid, - "name" : "python_test" - } - - server_prv = "09545df1812e252a2a853cca29d7eace4a3fe2baad334e3b7141a98d43c31e7b" - added_colony = client.add_colony(colony, server_prv) - self.assertEqual(added_colony["colonyid"], colonyid) - - client.del_colony(colonyid, server_prv) - - server_prv = "09545df1812e252a2a853cca29d7eace4a3fe2baad334e3b7141a98d43c31e7c" # note last c, incorrect key - with self.assertRaises(Exception): - client.add_colony(colony, server_prv) - - -if __name__ == '__main__': - unittest.main() diff --git a/crypto.py b/crypto.py index 8f9318a..969057d 100644 --- a/crypto.py +++ b/crypto.py @@ -1,33 +1,226 @@ +# This code is based on the code from the following repository (MIT-licensed): +# https://github.com/ethereum/eth-utils +# https://github.com/ethereum/eth-keys + import ctypes -import pathlib +import os +import hashlib +import hmac +import os +from typing import ( + Any, + Callable, + Tuple, +) + +# SECPK1N +A: int = 0 +N: int = (115792089237316195423570985008687907852837564279074904382605163141518161494337) +Gx: int = (55066263022277343669578718895168534326250603453777594175500187360389116729240) +Gy: int = (32670510020758816978083085130507043184471273380659243275938904335757337482424) +G: Tuple[int, int] = (Gx, Gy) +P: int = 2**256 - 2**32 - 977 class Crypto: + def __init__(self, native: bool = False) -> None: + self.native = native + if native: + libname = os.environ.get("CRYPTOLIB") + if libname == None: + libname = "/usr/local/lib/libcryptolib.so" + self.c_lib = ctypes.CDLL(libname) + self.c_lib.prvkey.restype = ctypes.c_char_p + self.c_lib.id.restype = ctypes.c_char_p + self.c_lib.sign.restype = ctypes.c_char_p + self.c_lib.hash.restype = ctypes.c_char_p + self.c_lib.recoverid.restype = ctypes.c_char_p - def __init__(self): - libname = pathlib.Path().absolute() / "lib/cryptolib.so" - self.c_lib = ctypes.CDLL(libname) - self.c_lib.prvkey.restype = ctypes.c_char_p - self.c_lib.id.restype = ctypes.c_char_p - self.c_lib.sign.restype = ctypes.c_char_p - self.c_lib.hash.restype = ctypes.c_char_p - self.c_lib.recoverid.restype = ctypes.c_char_p - - def prvkey(self): - k = self.c_lib.prvkey() - return k.decode("utf-8") - - def id(self, id): - h = self.c_lib.id(id.encode('utf-8')) - return h.decode("utf-8") + def prvkey(self) -> str: + if self.native: + k = self.c_lib.prvkey() + return k.decode("utf-8") + else: + return genkey() + + def id(self, id: str) -> str: + if self.native: + h = self.c_lib.id(id.encode('utf-8')) + return h.decode("utf-8") + else: + return get_id(id) - def hash(self, data): - h = self.c_lib.hash(data.encode('utf-8')) - return h.decode("utf-8") + def sign(self, data: str, prvkey: str) -> str: + if self.native: + s = self.c_lib.sign(data.encode('utf-8'), prvkey.encode('utf-8')) + return s.decode("utf-8") + else: + return sign(data, prvkey) + +def genkey() -> str: + random_bytes = os.urandom(32) # Generate 32 random bytes + hash_obj = hashlib.sha3_256() # Create a SHA-3 256 hash object + hash_obj.update(random_bytes) # Update the hash object with the random bytes + hash_bytes = hash_obj.digest() # Get the digest of the hash + return hash_bytes.hex() # Return the hexadecimal representation + +def sign(msg: str, prv_hex: str) -> str: + prv_bytes = bytes.fromhex(prv_hex) - def sign(self, data, prvkey): - s = self.c_lib.sign(data.encode('utf-8'), prvkey.encode('utf-8')) - return s.decode("utf-8") + hash = hashlib.sha3_256() + hash.update(msg.encode('utf-8')) + hash_bytes = hash.digest() + + s = ecdsa_raw_sign(hash_bytes, prv_bytes) + vb = int_to_byte(s[0]) + rb = pad32(int_to_big_endian(s[1])) + sb = pad32(int_to_big_endian(s[2])) + sig = b"".join((rb, sb, vb)) + + sig_hex = sig.hex() + return sig_hex + +def get_id(prv_key: str) -> str: + prv_key_bytes = bytes.fromhex(prv_key) + pub = private_key_to_public_key(prv_key_bytes) + pub_hex = "04"+pub.hex() # the prefix "04" denotes that the public key is in uncompressed format + hash = hashlib.sha3_256() + hash.update(pub_hex.encode('utf-8')) - def recoverid(self, data, prvkey): - id = self.c_lib.recoverid(data.encode('utf-8'), prvkey.encode('utf-8')) - return id.decode("utf-8") + return hash.hexdigest() + +def pad32(value: bytes) -> bytes: + return value.rjust(32, b"\x00") + +def int_to_byte(value: int) -> bytes: + return bytes([value]) + +def int_to_big_endian(value: int) -> bytes: + return value.to_bytes((value.bit_length() + 7) // 8 or 1, "big") + + +def big_endian_to_int(value: bytes) -> int: + return int.from_bytes(value, "big") + +def encode_raw_public_key(raw_public_key: Tuple[int, int]) -> bytes: + left, right = raw_public_key + return b"".join( + ( + pad32(int_to_big_endian(left)), + pad32(int_to_big_endian(right)), + ) + ) + +def private_key_to_public_key(private_key_bytes: bytes) -> bytes: + private_key_as_num = big_endian_to_int(private_key_bytes) + + if private_key_as_num >= N: + raise Exception("Invalid privkey") + + raw_public_key = fast_multiply(G, private_key_as_num) + public_key_bytes = encode_raw_public_key(raw_public_key) + return public_key_bytes + +def deterministic_generate_k( + msg_hash: bytes, + private_key_bytes: bytes, + digest_fn: Callable[[], Any] = hashlib.sha256, +) -> int: + v_0 = b"\x01" * 32 + k_0 = b"\x00" * 32 + + k_1 = hmac.new( + k_0, v_0 + b"\x00" + private_key_bytes + msg_hash, digest_fn + ).digest() + v_1 = hmac.new(k_1, v_0, digest_fn).digest() + k_2 = hmac.new( + k_1, v_1 + b"\x01" + private_key_bytes + msg_hash, digest_fn + ).digest() + v_2 = hmac.new(k_2, v_1, digest_fn).digest() + + kb = hmac.new(k_2, v_2, digest_fn).digest() + k = big_endian_to_int(kb) + return k + +def ecdsa_raw_sign(msg_hash: bytes, private_key_bytes: bytes) -> Tuple[int, int, int]: + z = big_endian_to_int(msg_hash) + k = deterministic_generate_k(msg_hash, private_key_bytes) + + r, y = fast_multiply(G, k) + s_raw = inv(k, N) * (z + r * big_endian_to_int(private_key_bytes)) % N + + v = 27 + ((y % 2) ^ (0 if s_raw * 2 < N else 1)) + s = s_raw if s_raw * 2 < N else N - s_raw + + return v - 27, r, s + +def fast_multiply(a: Tuple[int, int], n: int) -> Tuple[int, int]: + return from_jacobian(jacobian_multiply(to_jacobian(a), n)) + +def from_jacobian(p: Tuple[int, int, int]) -> Tuple[int, int]: + z = inv(p[2], P) + return ((p[0] * z**2) % P, (p[1] * z**3) % P) + +def to_jacobian(p: Tuple[int, int]) -> Tuple[int, int, int]: + o = (p[0], p[1], 1) + return o + +def inv(a: int, n: int) -> int: + if a == 0: + return 0 + lm, hm = 1, 0 + low, high = a % n, n + while low > 1: + r = high // low + nm, new = hm - lm * r, high - low * r + lm, low, hm, high = nm, new, lm, low + return lm % n + +def jacobian_double(p: Tuple[int, int, int]) -> Tuple[int, int, int]: + if not p[1]: + return (0, 0, 0) + ysq = (p[1] ** 2) % P + S = (4 * p[0] * ysq) % P + M = (3 * p[0] ** 2 + A * p[2] ** 4) % P + nx = (M**2 - 2 * S) % P + ny = (M * (S - nx) - 8 * ysq**2) % P + nz = (2 * p[1] * p[2]) % P + return (nx, ny, nz) + +def jacobian_add( + p: Tuple[int, int, int], q: Tuple[int, int, int] +) -> Tuple[int, int, int]: + if not p[1]: + return q + if not q[1]: + return p + U1 = (p[0] * q[2] ** 2) % P + U2 = (q[0] * p[2] ** 2) % P + S1 = (p[1] * q[2] ** 3) % P + S2 = (q[1] * p[2] ** 3) % P + if U1 == U2: + if S1 != S2: + return (0, 0, 1) + return jacobian_double(p) + H = U2 - U1 + R = S2 - S1 + H2 = (H * H) % P + H3 = (H * H2) % P + U1H2 = (U1 * H2) % P + nx = (R**2 - H3 - 2 * U1H2) % P + ny = (R * (U1H2 - nx) - S1 * H3) % P + nz = (H * p[2] * q[2]) % P + return (nx, ny, nz) + +def jacobian_multiply(a: Tuple[int, int, int], n: int) -> Tuple[int, int, int]: + if a[1] == 0 or n == 0: + return (0, 0, 1) + if n == 1: + return a + if n < 0 or n >= N: + return jacobian_multiply(a, n % N) + if (n % 2) == 0: + return jacobian_double(jacobian_multiply(a, n // 2)) + elif (n % 2) == 1: + return jacobian_add(jacobian_double(jacobian_multiply(a, n // 2)), a) + else: + raise Exception("Invariant: Unreachable code path") diff --git a/crypto_test.py b/crypto_test.py deleted file mode 100644 index 17941dd..0000000 --- a/crypto_test.py +++ /dev/null @@ -1,41 +0,0 @@ -import unittest -import sys -sys.path.append(".") -from crypto import Crypto - -class TestCrypto(unittest.TestCase): - - def test_prvkey(self): - crypto = Crypto() - prvkey = crypto.prvkey() - self.assertEqual(len(prvkey), 64) - - def test_id(self): - crypto = Crypto() - i = crypto.id("6d2fb6f546bacfd98c68769e61e0b44a697a30596c018a50e28200aa59b01c0a") - self.assertEqual(i, "4fef2b5a82d134d058c1883c72d6d9caf77cd59ca82d73105017590dea3dcb87") - - def test_sign(self): - crypto = Crypto() - prvkey = "d6eb959e9aec2e6fdc44b5862b269e987b8a4d6f2baca542d8acaa97ee5e74f6" - data = "hello" - digest = crypto.hash(data) - sig = crypto.sign(digest, prvkey) - self.assertEqual(len(sig), 130) - - def test_recoverid(self): - crypto = Crypto() - prvkey = "d6eb959e9aec2e6fdc44b5862b269e987b8a4d6f2baca542d8acaa97ee5e74f6" - data = "hello" - digest = crypto.hash(data) - sig = crypto.sign(digest, prvkey) - id = crypto.recoverid(digest, sig) - self.assertEqual(id, "5d6568f883451ae2e407d1a0a7992e414f2a67b69d0e6e9176d353b98f06f696") - - def test_hash(self): - crypto = Crypto() - self.assertEqual(crypto.hash("hello world"), "644bcc7e564373040999aac89e7622f3ca71fba1d972fd94a31c3bfbf24e3938") - - -if __name__ == '__main__': - unittest.main() diff --git a/docs/images/mapreduce.png b/docs/images/mapreduce.png new file mode 100644 index 0000000..7decb44 Binary files /dev/null and b/docs/images/mapreduce.png differ diff --git a/examples/colonies_monad.py b/examples/colonies_monad.py new file mode 100644 index 0000000..7e34062 --- /dev/null +++ b/examples/colonies_monad.py @@ -0,0 +1,71 @@ +from pycolonies import Colonies +from pycolonies import Workflow +from pycolonies import FuncSpec +from typing import Union, Callable, Any + +import copy + +class Function: + def __init__(self, + func: Union[Callable, str], + colonyname: str, + executortype: str, + priority: int = 0, + maxexectime: int = 200, + maxretries: int = 3, + maxwaittime: int = -1) -> None: + self.func_spec = FuncSpec.create(func=func, + args=[], + colonyname=colonyname, + executortype=executortype, + priority=priority, + maxexectime=maxexectime, + maxretries=maxretries, + maxwaittime=maxwaittime) + if isinstance(func, str): + self.name = func + else: + self.name = func.__name__ + + +class ColoniesMonad: + def __init__(self, + colonies: Colonies, + colonyname: str, + executor_prvkey: str) -> None: + self.wf = Workflow(colonyname=colonyname) + self.colonyname = colonyname + self.executor_prvkey = executor_prvkey + self.prev_func = None + self.colonies = colonies + + def __ror__(self, other: 'ColoniesMonad') -> None: + del other + pass + + def __rshift__(self, f: Function) -> 'ColoniesMonad': # bind function + if self.prev_func is None: + self.wf.functionspecs.append(f.func_spec) + self.prev_func = f.name + else: + fs = copy.deepcopy(f.func_spec) + assert fs.conditions, "FunctionSpec must have conditions defined." + fs.conditions.dependencies = [self.prev_func] + self.wf.functionspecs.append(fs) + self.prev_func = f.name + + return self + + def unwrap(self) -> Any: + if self.prev_func is None: + raise RuntimeError("Monad has no functions to execute.") + + processgraph = self.colonies.submit_workflow(self.wf, self.executor_prvkey) + last_process = self.colonies.find_process(self.prev_func, processgraph.processids, self.executor_prvkey) + + if last_process is None: + raise ValueError(f"Process {self.prev_func} not found in the process graph.") + + process = self.colonies.wait(last_process, 100, self.executor_prvkey) + + return process.output[0] if process.output else None diff --git a/examples/cron_example.py b/examples/cron_example.py new file mode 100644 index 0000000..ebc5e44 --- /dev/null +++ b/examples/cron_example.py @@ -0,0 +1,51 @@ +from pycolonies import colonies_client +from pycolonies import Workflow, FuncSpec, Conditions +from model import Gpu +import time + +colonies, colonyname, colony_prvkey, executor_name, prvkey = colonies_client() + +wf = Workflow(colonyname=colonyname or "cron") +f = FuncSpec( + funcname="execute", + kwargs={ + "cmd": "echo hello world", + "docker-image": "ubuntu:20.04" + }, + conditions = Conditions( + colonyname=colonyname, + executortype="container-executor", + executornames=["dev-docker"], + processespernode=1, + nodes=1, + walltime=60, + cpu="1000m", + mem="1Gi", + gpu=Gpu(count=0) + ), + maxexectime=55, + maxretries=3 +) + +f.nodename = "echo" +wf.functionspecs.append(f) + +# Add a cron +cron = colonies.add_cron("echo_cron", "0/1 * * * * *", True, wf, colonyname, prvkey) +print("Adding new cron with id: ", cron.cronid) + +# List all crons, max 10 cron are listed +crons = colonies.get_crons(colonyname, 10, prvkey) + +for cron in crons: + print(cron.cronid) + +# Get a cron by id +cron = colonies.get_cron(cron.cronid, prvkey) +print(cron.cronid) + +# Sleep for 2 seconds to allow the cron to run +time.sleep(2) + +# Delete a cron by id +colonies.del_cron(cron.cronid, prvkey) diff --git a/examples/cross_platform_demo.py b/examples/cross_platform_demo.py new file mode 100644 index 0000000..9a6f1e7 --- /dev/null +++ b/examples/cross_platform_demo.py @@ -0,0 +1,49 @@ +from typing import List + +from pycolonies import colonies_client +from model import Workflow, FuncSpec, Conditions, Gpu + +colonies, colonyname, colony_prvkey, executorid, executor_prvkey = colonies_client() + +def gen_sleep(executorname: str, nodename: str, dependencies: List[str]) -> FuncSpec: + return FuncSpec ( + conditions=Conditions( + executortype="container-executor", + executornames=[ + executorname + ], + nodes=1, + processespernode=1, + mem="500Mi", + cpu="1000m", + gpu=Gpu( + count=0 + ), + walltime=60, + dependencies=dependencies + ), + funcname="execute", + kwargs={ + "cmd": "sleep 8", + "docker-image": "ubuntu:20.04" + }, + maxexectime=55, + maxretries=3, + nodename=nodename + ) + + +wf = Workflow(colonyname=colonyname) +wf.functionspecs.append(gen_sleep("icekube", nodename="ice-0", dependencies=[])) +wf.functionspecs.append(gen_sleep("lumi-std", nodename="lumi-0", dependencies=["ice-0"])) +wf.functionspecs.append(gen_sleep("lumi-std", nodename="lumi-1", dependencies=["ice-0"])) +wf.functionspecs.append(gen_sleep("lumi-std", nodename="lumi-2", dependencies=["ice-0"])) +wf.functionspecs.append(gen_sleep("lumi-std", nodename="lumi-3", dependencies=["ice-0"])) +wf.functionspecs.append(gen_sleep("lumi-std", nodename="lumi-4", dependencies=["ice-0"])) +wf.functionspecs.append(gen_sleep("lumi-std", nodename="lumi-5", dependencies=["ice-0"])) +wf.functionspecs.append(gen_sleep("lumi-std", nodename="lumi-6", dependencies=["ice-0"])) +wf.functionspecs.append(gen_sleep("lumi-std ", nodename="lumi-7", dependencies=["ice-0"])) +wf.functionspecs.append(gen_sleep("leonardo-booster", nodename="leonardo-0", dependencies=["lumi-0", "lumi-1", "lumi-2", "lumi-3", "lumi-4", "lumi-5", "lumi-6", "lumi-7"])) + +wf.functionspecs.append(gen_sleep("icekube", nodename="ice-1", dependencies=["leonardo-0"])) +colonies.submit_workflow(wf, executor_prvkey) diff --git a/examples/des_executor.py b/examples/des_executor.py new file mode 100644 index 0000000..e7a0cec --- /dev/null +++ b/examples/des_executor.py @@ -0,0 +1,82 @@ +from pycolonies import Crypto +from pycolonies import colonies_client +import signal +import os +from typing import Any + +def calc_ndvi(polygon, product, time): + print("Calculation NDVI for polygon", polygon, "product", product, "time", time) + return [0.1, 0.2, 0.3, 0.4, 0.5] + +class PythonExecutor: + def __init__(self) -> None: + colonies, colonyname, colony_prvkey, _, _ = colonies_client() + self.colonies = colonies + self.colonyname = colonyname + self.colony_prvkey = colony_prvkey + self.executorname = "des-executor" + self.executortype = "des-executor" + + crypto = Crypto() + self.executor_prvkey = crypto.prvkey() + self.executorid = crypto.id(self.executor_prvkey) + + self.register() + + def register(self) -> None: + try: + self.colonies.add_executor( + executorname=self.executorname, + executorid=self.executorid, + colonyname=self.colonyname, + executortype=self.executortype, + colony_prvkey=self.colony_prvkey + ) + self.colonies.approve_executor(self.colonyname, self.executorname, self.colony_prvkey) + + self.colonies.add_function(self.colonyname, + self.executorname, + "calc_ts", + self.executor_prvkey) + except Exception as err: + print(err) + os._exit(0) + + print("Executor", self.executorname, "registered") + + def start(self) -> None: + while (True): + try: + process = self.colonies.assign(self.colonyname, 3, self.executor_prvkey) + print("Process", process.processid, "is assigned to executor") + + self.colonies.add_log(process.processid, "Calculating NDVI\n", self.executor_prvkey) + + if process.spec is None or process.spec.kwargs is None or len(process.spec.kwargs) == 0: + print("invalid process") + continue + + polygon = process.spec.kwargs["polygon"] + product = process.spec.kwargs["product"] + time = process.spec.kwargs["time"] + + ndvi_serie = calc_ndvi(polygon, product, time) + + if process.spec.funcname == "calc_ts": + self.colonies.close(process.processid, ndvi_serie, self.executor_prvkey) + except Exception: + pass + + def unregister(self) -> None: + self.colonies.remove_executor(self.colonyname, self.executorname, self.colony_prvkey) + print("Executor", self.executorname, "unregistered") + os._exit(0) + +def sigint_handler(signum: int, frame: Any) -> None: + del signum, frame + executor.unregister() + +if __name__ == '__main__': + signal.signal(signal.SIGINT, sigint_handler) + executor = PythonExecutor() + executor.start() diff --git a/examples/echo_executor.py b/examples/echo_executor.py new file mode 100644 index 0000000..b71c74d --- /dev/null +++ b/examples/echo_executor.py @@ -0,0 +1,76 @@ +from pycolonies import Crypto +from pycolonies import colonies_client +import signal +import os +from typing import Any + +class PythonExecutor: + def __init__(self) -> None: + colonies, colonyname, colony_prvkey, _, _ = colonies_client() + self.colonyname = colonyname + self.colonies = colonies + self.colony_prvkey = colony_prvkey + + crypto = Crypto() + self.executor_prvkey = crypto.prvkey() + self.executorid = crypto.id(self.executor_prvkey) + + self.register() + + def register(self) -> None: + self.executorname = "echo-executor" + + try: + self.colonies.add_executor( + executorid=self.executorid, + executorname="echo-executor", + executortype="echo-executor", + colonyname=self.colonyname, + colony_prvkey=self.colony_prvkey + ) + self.colonies.approve_executor(self.colonyname, self.executorname, self.colony_prvkey) + except Exception as err: + print(err) + print("Executor", self.executorname, "registered") + + try: + self.colonies.add_function(self.colonyname, + self.executorname, + "echo", + self.executor_prvkey) + + except Exception as err: + print(err) + + def start(self) -> None: + while (True): + try: + process = self.colonies.assign(self.colonyname, 10, self.executor_prvkey) + print("Process", process.processid, "is assigned to executor") + if process.spec.funcname == "echo": + # if "in" is defined, it is the output of the parent process, + # use the output from parent process instead of args + if process.input and len(process.input) > 0: + args = process.input + else: + args = process.spec.args + + # just set output to input value + self.colonies.close(process.processid, [args[0]], self.executor_prvkey) + except Exception as err: + print(err) + pass + + def unregister(self) -> None: + self.colonies.remove_executor(self.colonyname, self.executorname, self.colony_prvkey) + print("Executor", self.executorname, "unregistered") + os._exit(0) + +def sigint_handler(signum: int, frame: Any) -> None: + del signum, frame + executor.unregister() + +if __name__ == '__main__': + signal.signal(signal.SIGINT, sigint_handler) + executor = PythonExecutor() + executor.start() diff --git a/examples/fibonacci/Dockerfile b/examples/fibonacci/Dockerfile new file mode 100644 index 0000000..4bcc345 --- /dev/null +++ b/examples/fibonacci/Dockerfile @@ -0,0 +1,7 @@ +FROM ubuntu:22.04 +WORKDIR / +RUN apt-get update; apt-get -y install python3 python3-pip +RUN pip3 install pycolonies +COPY libcryptolib.so /usr/local/lib +COPY fibonacci_executor.py / +CMD ["python3", "/fibonacci_executor.py"] diff --git a/examples/fibonacci/Makefile b/examples/fibonacci/Makefile new file mode 100644 index 0000000..1857724 --- /dev/null +++ b/examples/fibonacci/Makefile @@ -0,0 +1,2 @@ +container: + @docker build -f Dockerfile . -t colonyos/fibexecutor:latest diff --git a/examples/fibonacci/fibonacci.json b/examples/fibonacci/fibonacci.json new file mode 100644 index 0000000..21a8905 --- /dev/null +++ b/examples/fibonacci/fibonacci.json @@ -0,0 +1,9 @@ +{ + "conditions": { + "executortype": "fibonacci-executor" + }, + "funcname": "fib", + "args": [ + 40 + ] +} diff --git a/examples/fibonacci/fibonacci_executor.py b/examples/fibonacci/fibonacci_executor.py new file mode 100644 index 0000000..13a1351 --- /dev/null +++ b/examples/fibonacci/fibonacci_executor.py @@ -0,0 +1,77 @@ +from pycolonies import Crypto +from pycolonies import colonies_client +import signal +import os +import uuid +from typing import Any + +def fib(n: int) -> int: + if n == 0 or n == 1: + return n + return fib(n - 1) + fib(n - 2) + +class Executor: + def __init__(self) -> None: + colonies, colonyname, colony_prvkey, _, _ = colonies_client() + self.colonies = colonies + self.colonyname = colonyname + self.colony_prvkey = colony_prvkey + self.executorname = "fibonacci-executor" + self.executortype = "fibonacci-executor" + + crypto = Crypto() + self.executor_prvkey = crypto.prvkey() + self.executorid = crypto.id(self.executor_prvkey) + + self.register() + + def register(self) -> None: + try: + self.colonies.add_executor( + executorname=self.executorname + str(uuid.uuid4()), + executorid=self.executorid, + colonyname=self.colonyname, + executortype=self.executortype, + colony_prvkey=self.colony_prvkey + ) + self.colonies.approve_executor(self.colonyname, self.executorname, self.colony_prvkey) + except Exception as err: + print(err) + print("Executor", self.executorname, "registered") + + try: + self.colonies.add_function(self.executorid, self.colonyname, "fib", self.executor_prvkey) + + except Exception as err: + print(err) + os._exit(0) + + def start(self) -> None: + while (True): + try: + process = self.colonies.assign(self.colonyname, 10, self.executor_prvkey) + print("Process", process.processid, "is assigned to executor") + if process.spec.funcname == "fib": + arg = process.spec.args[0] + assert isinstance(arg, int), "Argument for fib function must be an integer" + print("Calculating fib(" + str(arg) + ")") + n = fib(arg) + print("Result is ", n) + self.colonies.close(process.processid, [str(n)], self.executor_prvkey) + except Exception as err: + print(err) + pass + + def unregister(self) -> None: + self.colonies.remove_executor(self.colonyname, self.executorname, self.colony_prvkey) + print("Executor", self.executorname, "unregistered") + os._exit(0) + +def sigint_handler(signum: int, frame: Any) -> None: + del signum, frame + executor.unregister() + +if __name__ == '__main__': + signal.signal(signal.SIGINT, sigint_handler) + executor = Executor() + executor.start() diff --git a/examples/fibonacci/libcryptolib.so b/examples/fibonacci/libcryptolib.so new file mode 100644 index 0000000..09876b1 Binary files /dev/null and b/examples/fibonacci/libcryptolib.so differ diff --git a/examples/fibonacci/start.sh b/examples/fibonacci/start.sh new file mode 100755 index 0000000..c9e48fb --- /dev/null +++ b/examples/fibonacci/start.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +docker run -it -e COLONIES_SERVER_HOST=$COLONIES_SERVER_HOST -e COLONIES_SERVER_PORT=$COLONIES_SERVER_PORT -e COLONIES_COLONY_ID=$COLONIES_COLONY_ID -e COLONIES_COLONY_PRVKEY=$COLONIES_COLONY_PRVKEY colonyos/fibexecutor:latest diff --git a/examples/fs_examples.py b/examples/fs_examples.py new file mode 100644 index 0000000..f62adcb --- /dev/null +++ b/examples/fs_examples.py @@ -0,0 +1,48 @@ +from pycolonies import colonies_client + +colonies, colonyname, colony_prvkey, executor_name, prvkey = colonies_client() + +try: + colonies.upload_file(colonyname, prvkey, filepath="./test.txt", label="/testlabel") + print("File uploaded successfully.") +except Exception as e: + print(e) + +try: + data = b"Sample byte data to upload" + colonies.upload_data(colonyname, prvkey, filename="mydata", data=data, label="/testlabel_data") + print("Data uploaded successfully.") +except Exception as e: + print(e) + +files = colonies.get_files("/testlabel", colonyname, prvkey) +file = colonies.get_file(colonyname, prvkey, label="/testlabel", filename="test.txt") + +try: + dst = colonies.download_file(colonyname, prvkey, dst="./testlabel", label="/testlabel", filename="test.txt") + print(f"File downloaded to {dst} successfully.") +except Exception as e: + print(e) + +try: + dst = colonies.download_file(colonyname, prvkey, dst="./testlabel_data", label="/testlabel_data", filename="mydata") + print(f"Data downloaded to {dst} successfully.") +except Exception as e: + print(e) + +try: + data = colonies.download_data(colonyname, prvkey, label="/testlabel_data", filename="mydata") + data_str = data.decode('utf-8') + print(data_str) +except Exception as e: + print(e) + +try: + colonies.delete_file(colonyname, prvkey, label="/testlabel_data", filename="mydata") +except Exception as e: + print(e) + +try: + colonies.delete_file(colonyname, prvkey, label="/testlabel", filename="test.txt") +except Exception as e: + print(e) diff --git a/examples/helloworld_executor.py b/examples/helloworld_executor.py new file mode 100644 index 0000000..cff2542 --- /dev/null +++ b/examples/helloworld_executor.py @@ -0,0 +1,57 @@ +from pycolonies import Crypto +from pycolonies import colonies_client +import signal +import os +from typing import Any + +class PythonExecutor: + def __init__(self) -> None: + colonies, colonyname, colony_prvkey, _, _ = colonies_client() + self.colonies = colonies + self.colonyname = colonyname + self.colony_prvkey = colony_prvkey + self.executorname = "helloworld-executor" + self.executortype = "helloworld-executor" + + crypto = Crypto() + self.executor_prvkey = crypto.prvkey() + self.executorid = crypto.id(self.executor_prvkey) + + self.register() + + def register(self) -> None: + try: + self.colonies.add_executor(self.executorid, self.executorname, self.colonyname, self.colony_prvkey) + self.colonies.approve_executor(self.colonyname, self.executorname, self.colony_prvkey) + self.colonies.add_function(self.colonyname, self.executorname, "helloworld", self.executor_prvkey) + except Exception as err: + print(err) + os._exit(0) + + print("Executor", self.executorname, "registered") + + def start(self) -> None: + while (True): + try: + process = self.colonies.assign(self.colonyname, 10, self.executor_prvkey) + print("Process", process.processid, "is assigned to executor") + self.colonies.add_log(process.processid, "Hello from executor\n", self.executor_prvkey) + if process.spec.funcname == "helloworld": + self.colonies.close(process.processid, ["helloworld"], self.executor_prvkey) + except Exception as err: + print(err) + pass + + def unregister(self) -> None: + self.colonies.remove_executor(self.colonyname, self.executorname, self.colony_prvkey) + print("Executor", self.executorname, "unregistered") + os._exit(0) + +def sigint_handler(signum: int, frame: Any) -> None: + del signum, frame + executor.unregister() + +if __name__ == '__main__': + signal.signal(signal.SIGINT, sigint_handler) + executor = PythonExecutor() + executor.start() diff --git a/examples/kwargs_executor.py b/examples/kwargs_executor.py new file mode 100644 index 0000000..6b160dd --- /dev/null +++ b/examples/kwargs_executor.py @@ -0,0 +1,64 @@ +from pycolonies import Crypto +from pycolonies import colonies_client +import signal +import os +from typing import Any + +class PythonExecutor: + def __init__(self) -> None: + colonies, colonyname, colony_prvkey, _, _ = colonies_client() + self.colonies = colonies + self.colonyname = colonyname + self.colony_prvkey = colony_prvkey + self.executorname = "kwargs-executor" + self.executortype = "kwargs-executor" + + crypto = Crypto() + self.executor_prvkey = crypto.prvkey() + self.executorid = crypto.id(self.executor_prvkey) + + self.register() + + def register(self) -> None: + try: + self.colonies.add_executor( + executorname=self.executorname, + executorid=self.executorid, + colonyname=self.colonyname, + executortype=self.executortype, + colony_prvkey=self.colony_prvkey + ) + self.colonies.approve_executor(self.colonyname, self.executorname, self.colony_prvkey) + except Exception as err: + print(err) + print("Executor", self.executorname, "registered") + + def start(self) -> None: + while (True): + try: + process = self.colonies.assign(self.colonyname, 10, self.executor_prvkey) + print("Process", process.processid, "is assigned to executor") + if process.spec.funcname == "test": + kwargs = process.spec.kwargs + if kwargs is not None: + print(kwargs["arg_kw_1"]) + + # just set output to input value + self.colonies.close(process.processid, [""], self.executor_prvkey) + except Exception as err: + print(err) + pass + + def unregister(self) -> None: + self.colonies.remove_executor(self.colonyname, self.executorname, self.colony_prvkey) + print("Executor", self.executorname, "unregistered") + os._exit(0) + +def sigint_handler(signum: int, frame: Any) -> None: + del signum, frame + executor.unregister() + +if __name__ == '__main__': + signal.signal(signal.SIGINT, sigint_handler) + executor = PythonExecutor() + executor.start() diff --git a/examples/ml/Dockerfile b/examples/ml/Dockerfile new file mode 100644 index 0000000..24038d1 --- /dev/null +++ b/examples/ml/Dockerfile @@ -0,0 +1,13 @@ +FROM colonyos/notebook:v1.0.15 + +RUN apt-get update && apt-get install -y python3 python3-pip wget vim git fish libgl1-mesa-glx libglib2.0-0 +RUN python3 -m pip install --upgrade pip +RUN pip3 install pycolonies opencv-python tqdm Pillow scikit-learn keras matplotlib numpy + +ENV PATH="$PATH:/root/go/bin" +RUN cd /root; wget https://go.dev/dl/go1.21.6.linux-amd64.tar.gz; tar -xzf go1.21.6.linux-amd64.tar.gz;git clone https://github.com/colonyos/colonies.git; cd colonies; make; make install + +COPY ./ml_executor.py /ml/ml_executor.py +COPY ./weights_unet100.h5 /ml/weights_unet100.h5 + +ENTRYPOINT ["python3", "/ml/ml_executor.py"] diff --git a/examples/ml/Makefile b/examples/ml/Makefile new file mode 100644 index 0000000..9b89353 --- /dev/null +++ b/examples/ml/Makefile @@ -0,0 +1,16 @@ +all: build +.PHONY: all build + +BUILD_IMAGE ?= colonyos/mlexecutor +PUSH_IMAGE ?= colonyos/mltestexecutor + +container: + docker build -t $(BUILD_IMAGE) . + +push: + docker tag $(BUILD_IMAGE) $(PUSH_IMAGE) + docker push $(PUSH_IMAGE) + +run: + colonies executor remove --name python-executor; make container; docker run --gpus all -it colonyos/mlexecutor:latest + diff --git a/examples/ml/ml_executor.py b/examples/ml/ml_executor.py new file mode 100644 index 0000000..0eab4b0 --- /dev/null +++ b/examples/ml/ml_executor.py @@ -0,0 +1,249 @@ +from crypto import Crypto +from pycolonies import colonies_client, rpc +from pycolonies import ColoniesConnectionError +import signal +import base64 +import os +import uuid +import sys +import os +import numpy as np +from tqdm import tqdm +import cv2 as cv +from PIL import Image +import matplotlib.pyplot as plt +from sklearn.model_selection import train_test_split +import sys +from typing import Any + +from keras import Model +from keras.callbacks import Callback +from keras.optimizers import Adam +from keras.layers import Input, Conv2D, Conv2DTranspose, MaxPooling2D, concatenate, Dropout +import logging + +handler = logging.StreamHandler(sys.stdout) +handler.flush = sys.stdout.flush # Ensure flush method is called on each log entry + +# Configure the root logger +logging.basicConfig(level=logging.INFO, handlers=[handler]) + +logger = logging.getLogger(__name__) + +SIZE = 128 + +def unet_model(input_layer, start_neurons): + # Contraction path + conv1 = Conv2D(start_neurons, kernel_size=(3, 3), activation="relu", padding="same")(input_layer) + conv1 = Conv2D(start_neurons, kernel_size=(3, 3), activation="relu", padding="same")(conv1) + pool1 = MaxPooling2D((2, 2))(conv1) + pool1 = Dropout(0.25)(pool1) + + conv2 = Conv2D(start_neurons*2, kernel_size=(3, 3), activation="relu", padding="same")(pool1) + conv2 = Conv2D(start_neurons*2, kernel_size=(3, 3), activation="relu", padding="same")(conv2) + pool2 = MaxPooling2D((2, 2))(conv2) + pool2 = Dropout(0.5)(pool2) + + conv3 = Conv2D(start_neurons*4, kernel_size=(3, 3), activation="relu", padding="same")(pool2) + conv3 = Conv2D(start_neurons*4, kernel_size=(3, 3), activation="relu", padding="same")(conv3) + pool3 = MaxPooling2D((2, 2))(conv3) + pool3 = Dropout(0.5)(pool3) + + conv4 = Conv2D(start_neurons*8, kernel_size=(3, 3), activation="relu", padding="same")(pool3) + conv4 = Conv2D(start_neurons*8, kernel_size=(3, 3), activation="relu", padding="same")(conv4) + pool4 = MaxPooling2D((2, 2))(conv4) + pool4 = Dropout(0.5)(pool4) + + # Middle + convm = Conv2D(start_neurons*16, kernel_size=(3, 3), activation="relu", padding="same")(pool4) + convm = Conv2D(start_neurons*16, kernel_size=(3, 3), activation="relu", padding="same")(convm) + + # Expansive path + deconv4 = Conv2DTranspose(start_neurons*8, kernel_size=(3, 3), strides=(2, 2), padding="same")(convm) + uconv4 = concatenate([deconv4, conv4]) + uconv4 = Dropout(0.5)(uconv4) + uconv4 = Conv2D(start_neurons*8, kernel_size=(3, 3), activation="relu", padding="same")(uconv4) + uconv4 = Conv2D(start_neurons*8, kernel_size=(3, 3), activation="relu", padding="same")(uconv4) + + deconv3 = Conv2DTranspose(start_neurons*4, kernel_size=(3, 3), strides=(2, 2), padding="same")(uconv4) + uconv3 = concatenate([deconv3, conv3]) + uconv3 = Dropout(0.5)(uconv3) + uconv3 = Conv2D(start_neurons*4, kernel_size=(3, 3), activation="relu", padding="same")(uconv3) + uconv3 = Conv2D(start_neurons*4, kernel_size=(3, 3), activation="relu", padding="same")(uconv3) + + deconv2 = Conv2DTranspose(start_neurons*2, kernel_size=(3, 3), strides=(2, 2), padding="same")(uconv3) + uconv2 = concatenate([deconv2, conv2]) + uconv2 = Dropout(0.5)(uconv2) + uconv2 = Conv2D(start_neurons*2, kernel_size=(3, 3), activation="relu", padding="same")(uconv2) + uconv2 = Conv2D(start_neurons*2, kernel_size=(3, 3), activation="relu", padding="same")(uconv2) + + deconv1 = Conv2DTranspose(start_neurons*1, kernel_size=(3, 3), strides=(2, 2), padding="same")(uconv2) + uconv1 = concatenate([deconv1, conv1]) + uconv1 = Dropout(0.5)(uconv1) + uconv1 = Conv2D(start_neurons, kernel_size=(3, 3), activation="relu", padding="same")(uconv1) + uconv1 = Conv2D(start_neurons, kernel_size=(3, 3), activation="relu", padding="same")(uconv1) + + # Last conv and output + output_layer = Conv2D(1, (1,1), padding="same", activation="sigmoid")(uconv1) + + return output_layer + +# Compile unet model +input_layer = Input((SIZE, SIZE, 3)) +output_layer = unet_model(input_layer = input_layer, start_neurons = 16) + +model = Model(input_layer, output_layer) +model.compile(loss="binary_crossentropy", optimizer="adam", metrics=["accuracy"]) +model.summary() + +weights_path = '/ml/weights_unet100.h5' +model.load_weights(weights_path) + + +class PythonExecutor: + def __init__(self) -> None: + + colonies_server = os.getenv("COLONIES_SERVER_HOST") + colonies_port = os.getenv("COLONIES_SERVER_PORT") + colonies_tls = os.getenv("COLONIES_SERVER_TLS") + colonyname = os.getenv("COLONIES_COLONY_NAME") + colony_prvkey = os.getenv("COLONIES_COLONY_PRVKEY") + executorname = os.getenv("COLONIES_EXECUTOR_NAME") + prvkey = os.getenv("COLONIES_PRVKEY") + + print("colonies_server", colonies_server) + print("colonies_port", colonies_port) + print("colonies_tls", colonies_tls) + print("colonyname", colonyname) + print("prvkey", prvkey) + + + global colonies + colonies, colonyname, colony_prvkey, _, _ = colonies_client() + crypto = Crypto() + self.colonies = colonies + self.colonyname = colonyname + self.colony_prvkey= colony_prvkey + self.executor_prvkey = crypto.prvkey() + self.executorid = crypto.id(self.executor_prvkey) + self.executorname = "python-executor" + self.executortype = "container-executor" + + print("colonies prvkey:", colony_prvkey) + print("executor id:", self.executorid) + print("executor prvkey:", self.executor_prvkey) + + self.register() + os.mkdir("/images") + os.mkdir("/generated") + + try: + self.colonies.sync("/images", "/eurohpc-summit-demo/images", False, self.colonyname, self.executor_prvkey) + self.colonies.sync("/generated", "/eurohpc-summit-demo/generated-images", False, self.colonyname, self.executor_prvkey) + except Exception as err: + print(err) + + def register(self) -> None: + print("register: ", executor) + + try: + print("adding executor") + self.colonies.add_executor( + executorname=self.executorname, + executorid=self.executorid, + colonyname=self.colonyname, + executortype=self.executortype + colony_prvkey=self.colony_prvkey + ) + print("done adding executor") + self.colonies.approve_executor(self.colonyname, self.executorname, self.colony_prvkey) + print("done approving executor") + + except Exception as err: + print(err) + sys.exit(-1) + + print("Executor", self.executorname, "registered") + + def start(self) -> None: + while (True): + try: + assigned_process = self.colonies.assign(self.colonyname, 10, self.executor_prvkey) + img = assigned_process.spec.env["IMAGE"] + print(img) + self.colonies.sync("/images", "/eurohpc-summit-demo/images", False, self.colonyname, self.executor_prvkey) + print("sync completed") + image_path = '/images/' + img + print(image_path) + image = Image.open(image_path) + image = image.resize((SIZE, SIZE)) + image_array = np.asarray(image).astype('float32') / 255.0 # Normalize pixel values if your model expects this + image_array = np.expand_dims(image_array, axis=0) # Add batch dimension + + # Perform inference + prediction = model.predict(image_array) + print("1") + +# Process the output (for example, in case of segmentation, you might want to apply a threshold) +# This step depends on your specific use case and model output + +# If the output is a segmentation mask, you might want to visualize it + if prediction.shape[-1] == 1: # Assuming the model outputs a single-channel segmentation mask + predicted_mask = prediction[0, :, :, 0] # Remove batch dimension and get the mask + thresholded_mask = (predicted_mask > 0.5).astype(np.uint8) # Apply a threshold to get a binary mask + original_image = Image.open(image_path) + original_image = original_image.resize((SIZE, SIZE)) + original_image_array = np.asarray(original_image).astype('float32') / 255.0 + + # Preprocess the image for the model + image_for_model = np.expand_dims(original_image_array, axis=0) # Add batch dimension + + # Perform inference to get the mask + predicted_mask = model.predict(image_for_model)[0, :, :, 0] # Assuming the model outputs a single-channel mask + + # Apply a threshold to create a binary mask + thresholded_mask = (predicted_mask > 0.5).astype(np.uint8) + + # Create a red mask where the water regions will be colored in red + # The red mask has the same height and width as the original image, but with 3 channels for RGB + red_mask = np.zeros_like(original_image_array) + red_mask[:, :, 0] = 1 # Set the red channel to maximum + + # Apply the red mask on the original image + colored_image = np.where(thresholded_mask[:, :, np.newaxis] == 1, red_mask, original_image_array) + + dpi = 600 # Change the dpi if needed + figsize = (image.size[0]+100 / dpi, image.size[1]+100 / dpi) + + plt.figure(figsize=figsize) # Adjust figure size as needed + plt.imshow(colored_image) + plt.tight_layout(pad=0) + plt.axis('off') # Hide the axis + plt.subplots_adjust(top=1, bottom=0, right=1, left=0, hspace=0, wspace=0) + plt.margins(0, 0) + plt.gca().xaxis.set_major_locator(plt.NullLocator()) + plt.gca().yaxis.set_major_locator(plt.NullLocator()) + + plt.imshow(colored_image) + + plt.tight_layout() + plt.savefig('/generated/' + img) + self.colonies.sync("/generated", "/eurohpc-summit-demo/generated-images", False, self.colonyname, self.executor_prvkey) + self.colonies.close(assigned_process.processid, [], self.executor_prvkey) + + except Exception as err: + print("Failed to execute function:", err) + continue + + def unregister(self) -> None: + self.colonies.remove_executor(self.colonyname, self.executorname, self.colony_prvkey) + print("Executor", self.executorname, "unregistered") + os._exit(0) + +def sigint_handler(signum: int, frame: Any) -> None: + executor.unregister() + +if __name__ == '__main__': + signal.signal(signal.SIGINT, sigint_handler) + executor = PythonExecutor() + executor.start() diff --git a/examples/ml/sav/Dockerfile b/examples/ml/sav/Dockerfile new file mode 100644 index 0000000..47133b6 --- /dev/null +++ b/examples/ml/sav/Dockerfile @@ -0,0 +1,41 @@ +FROM colonyos/notebook:v1.0.15 + +RUN apt-get update && apt-get install -y python3 python3-pip wget vim git fish libgl1-mesa-glx libglib2.0-0 +RUN python3 -m pip install --upgrade pip +RUN pip3 install pycolonies opencv-python tqdm Pillow scikit-learn keras matplotlib numpy + +ENV LANG=en_US.UTF-8 +ENV LANGUAGE=en_US.UTF-8 +ENV LC_ALL=en_US.UTF-8 +ENV LC_CTYPE=UTF-8 +ENV TZ=Europe/Stockholm +ENV COLONYOS_DASHBOARD_URL="https://dashboard.colonyos.io" +ENV COLONIES_TLS="true" +ENV COLONIES_SERVER_TLS="true" +ENV COLONIES_SERVER_HOST="server.colonyos.io" +ENV COLONIES_SERVER_PORT="443" +ENV COLONIES_MONITOR_PORT="21120" +ENV COLONIES_MONITOR_INTERVAL="1" +ENV COLONIES_SERVER_PRVKEY="f7a09f288428bb317047b60a2a9d93d69be34c187b14960cb38ae7892927e344" +ENV COLONIES_DB_HOST="localhost" +ENV COLONIES_DB_USER="postgres" +ENV COLONIES_DB_PORT="50070" +ENV COLONIES_DB_PASSWORD="rFcLGNkgsNtksg6Pgtn9CumL4xXBQ7" +ENV COLONIES_COLONY_PRVKEY="1715123955587d093b759edb4ecd643fffa581911f5e3fe224a3d11ff51f888e" +ENV COLONIES_PRVKEY="9201993f916425ce8d1ee0fe3dd6585469ebb97fd691824150c728d20e048c20" +ENV COLONIES_COLONY_NAME="hpc" +ENV AWS_S3_ENDPOINT="s3.colonyos.io:443" +ENV AWS_S3_ACCESSKEY="ZbyK0MGWH0Xr3P1NXeaV" +ENV AWS_S3_SECRETKEY="v30R89DGTUp7WnMCr36ffvaUjKBowQpL99INyHHd" +ENV AWS_S3_REGION_KEY="" +ENV AWS_S3_BUCKET="hpc" +ENV AWS_S3_TLS="true" +ENV AWS_S3_SKIPVERIFY="false" + +ENV PATH="$PATH:/root/go/bin" +RUN cd /root; wget https://go.dev/dl/go1.21.6.linux-amd64.tar.gz; tar -xzf go1.21.6.linux-amd64.tar.gz;git clone https://github.com/colonyos/colonies.git; cd colonies; make; make install + +COPY ./ml_executor.py /ml/ml_executor.py +COPY ./weights_unet100.h5 /ml/weights_unet100.h5 + +ENTRYPOINT ["python3", "/ml/ml_executor.py"] diff --git a/examples/ml/sav/Makefile b/examples/ml/sav/Makefile new file mode 100644 index 0000000..129f9dd --- /dev/null +++ b/examples/ml/sav/Makefile @@ -0,0 +1,16 @@ +all: build +.PHONY: all build + +BUILD_IMAGE ?= colonyos/mlexecutor +PUSH_IMAGE ?= colonyos/mlexecutor + +container: + docker build -t $(BUILD_IMAGE) . + +push: + docker tag $(BUILD_IMAGE) $(PUSH_IMAGE) + docker push $(PUSH_IMAGE) + +run: + colonies executor remove --name python-executor; make container; docker run --gpus all -it colonyos/mlexecutor:latest + diff --git a/examples/ml/sav/ml_executor.py b/examples/ml/sav/ml_executor.py new file mode 100644 index 0000000..042baf5 --- /dev/null +++ b/examples/ml/sav/ml_executor.py @@ -0,0 +1,224 @@ +from crypto import Crypto +from pycolonies import colonies_client +from pycolonies import ColoniesConnectionError +import signal +import base64 +import os +import uuid +import sys +import os +import numpy as np +from tqdm import tqdm +import cv2 as cv +from PIL import Image +import matplotlib.pyplot as plt +from sklearn.model_selection import train_test_split +import sys +from typing import Any + +from keras import Model +from keras.callbacks import Callback +from keras.optimizers import Adam +from keras.layers import Input, Conv2D, Conv2DTranspose, MaxPooling2D, concatenate, Dropout +import logging + +handler = logging.StreamHandler(sys.stdout) +handler.flush = sys.stdout.flush # Ensure flush method is called on each log entry + +# Configure the root logger +logging.basicConfig(level=logging.INFO, handlers=[handler]) + +logger = logging.getLogger(__name__) + +SIZE = 128 + +def unet_model(input_layer, start_neurons): + # Contraction path + conv1 = Conv2D(start_neurons, kernel_size=(3, 3), activation="relu", padding="same")(input_layer) + conv1 = Conv2D(start_neurons, kernel_size=(3, 3), activation="relu", padding="same")(conv1) + pool1 = MaxPooling2D((2, 2))(conv1) + pool1 = Dropout(0.25)(pool1) + + conv2 = Conv2D(start_neurons*2, kernel_size=(3, 3), activation="relu", padding="same")(pool1) + conv2 = Conv2D(start_neurons*2, kernel_size=(3, 3), activation="relu", padding="same")(conv2) + pool2 = MaxPooling2D((2, 2))(conv2) + pool2 = Dropout(0.5)(pool2) + + conv3 = Conv2D(start_neurons*4, kernel_size=(3, 3), activation="relu", padding="same")(pool2) + conv3 = Conv2D(start_neurons*4, kernel_size=(3, 3), activation="relu", padding="same")(conv3) + pool3 = MaxPooling2D((2, 2))(conv3) + pool3 = Dropout(0.5)(pool3) + + conv4 = Conv2D(start_neurons*8, kernel_size=(3, 3), activation="relu", padding="same")(pool3) + conv4 = Conv2D(start_neurons*8, kernel_size=(3, 3), activation="relu", padding="same")(conv4) + pool4 = MaxPooling2D((2, 2))(conv4) + pool4 = Dropout(0.5)(pool4) + + # Middle + convm = Conv2D(start_neurons*16, kernel_size=(3, 3), activation="relu", padding="same")(pool4) + convm = Conv2D(start_neurons*16, kernel_size=(3, 3), activation="relu", padding="same")(convm) + + # Expansive path + deconv4 = Conv2DTranspose(start_neurons*8, kernel_size=(3, 3), strides=(2, 2), padding="same")(convm) + uconv4 = concatenate([deconv4, conv4]) + uconv4 = Dropout(0.5)(uconv4) + uconv4 = Conv2D(start_neurons*8, kernel_size=(3, 3), activation="relu", padding="same")(uconv4) + uconv4 = Conv2D(start_neurons*8, kernel_size=(3, 3), activation="relu", padding="same")(uconv4) + + deconv3 = Conv2DTranspose(start_neurons*4, kernel_size=(3, 3), strides=(2, 2), padding="same")(uconv4) + uconv3 = concatenate([deconv3, conv3]) + uconv3 = Dropout(0.5)(uconv3) + uconv3 = Conv2D(start_neurons*4, kernel_size=(3, 3), activation="relu", padding="same")(uconv3) + uconv3 = Conv2D(start_neurons*4, kernel_size=(3, 3), activation="relu", padding="same")(uconv3) + + deconv2 = Conv2DTranspose(start_neurons*2, kernel_size=(3, 3), strides=(2, 2), padding="same")(uconv3) + uconv2 = concatenate([deconv2, conv2]) + uconv2 = Dropout(0.5)(uconv2) + uconv2 = Conv2D(start_neurons*2, kernel_size=(3, 3), activation="relu", padding="same")(uconv2) + uconv2 = Conv2D(start_neurons*2, kernel_size=(3, 3), activation="relu", padding="same")(uconv2) + + deconv1 = Conv2DTranspose(start_neurons*1, kernel_size=(3, 3), strides=(2, 2), padding="same")(uconv2) + uconv1 = concatenate([deconv1, conv1]) + uconv1 = Dropout(0.5)(uconv1) + uconv1 = Conv2D(start_neurons, kernel_size=(3, 3), activation="relu", padding="same")(uconv1) + uconv1 = Conv2D(start_neurons, kernel_size=(3, 3), activation="relu", padding="same")(uconv1) + + # Last conv and output + output_layer = Conv2D(1, (1,1), padding="same", activation="sigmoid")(uconv1) + + return output_layer + +# Compile unet model +input_layer = Input((SIZE, SIZE, 3)) +output_layer = unet_model(input_layer = input_layer, start_neurons = 16) + +model = Model(input_layer, output_layer) +model.compile(loss="binary_crossentropy", optimizer="adam", metrics=["accuracy"]) +model.summary() + +weights_path = '/ml/weights_unet100.h5' +model.load_weights(weights_path) + + +class PythonExecutor: + def __init__(self) -> None: + global colonies + colonies, colonyname, colony_prvkey, _, _ = colonies_client() + crypto = Crypto() + self.colonies = colonies + self.colonyname = colonyname + self.colony_prvkey= colony_prvkey + self.executor_prvkey = crypto.prvkey() + self.executorid = crypto.id(self.executor_prvkey) + self.executorname = "python-executor" + self.executortype = "container-executor" + + self.register() + os.mkdir("/images") + os.mkdir("/generated") + + try: + self.colonies.sync("/images", "/eurohpc-summit-demo/images", False, self.colonyname, self.executor_prvkey) + self.colonies.sync("/generated", "/eurohpc-summit-demo/generated-images", False, self.colonyname, self.executor_prvkey) + except Exception as err: + print(err) + + def register(self) -> None: + try: + self.colonies.add_executor( + self.executorid, + self.executorname, + self.executortype, + self.colonyname, + self.colony_prvkey + ) + self.colonies.approve_executor(self.colonyname, self.executorname, self.colony_prvkey) + except Exception as err: + print(err) + sys.exit(-1) + + print("Executor", self.executorname, "registered") + + def start(self) -> None: + while (True): + try: + assigned_process = self.colonies.assign(self.colonyname, 10, self.executor_prvkey) + img = assigned_process.spec.env["IMAGE"] + print(img) + self.colonies.sync("/images", "/eurohpc-summit-demo/images", False, self.colonyname, self.executor_prvkey) + print("sync completed") + image_path = '/images/' + img + print(image_path) + image = Image.open(image_path) + image = image.resize((SIZE, SIZE)) + image_array = np.asarray(image).astype('float32') / 255.0 # Normalize pixel values if your model expects this + image_array = np.expand_dims(image_array, axis=0) # Add batch dimension + + # Perform inference + prediction = model.predict(image_array) + print("1") + +# Process the output (for example, in case of segmentation, you might want to apply a threshold) +# This step depends on your specific use case and model output + +# If the output is a segmentation mask, you might want to visualize it + if prediction.shape[-1] == 1: # Assuming the model outputs a single-channel segmentation mask + predicted_mask = prediction[0, :, :, 0] # Remove batch dimension and get the mask + thresholded_mask = (predicted_mask > 0.5).astype(np.uint8) # Apply a threshold to get a binary mask + original_image = Image.open(image_path) + original_image = original_image.resize((SIZE, SIZE)) + original_image_array = np.asarray(original_image).astype('float32') / 255.0 + + # Preprocess the image for the model + image_for_model = np.expand_dims(original_image_array, axis=0) # Add batch dimension + + # Perform inference to get the mask + predicted_mask = model.predict(image_for_model)[0, :, :, 0] # Assuming the model outputs a single-channel mask + + # Apply a threshold to create a binary mask + thresholded_mask = (predicted_mask > 0.5).astype(np.uint8) + + # Create a red mask where the water regions will be colored in red + # The red mask has the same height and width as the original image, but with 3 channels for RGB + red_mask = np.zeros_like(original_image_array) + red_mask[:, :, 0] = 1 # Set the red channel to maximum + + # Apply the red mask on the original image + colored_image = np.where(thresholded_mask[:, :, np.newaxis] == 1, red_mask, original_image_array) + + dpi = 600 # Change the dpi if needed + figsize = (image.size[0]+100 / dpi, image.size[1]+100 / dpi) + + plt.figure(figsize=figsize) # Adjust figure size as needed + plt.imshow(colored_image) + plt.tight_layout(pad=0) + plt.axis('off') # Hide the axis + plt.subplots_adjust(top=1, bottom=0, right=1, left=0, hspace=0, wspace=0) + plt.margins(0, 0) + plt.gca().xaxis.set_major_locator(plt.NullLocator()) + plt.gca().yaxis.set_major_locator(plt.NullLocator()) + + plt.imshow(colored_image) + + plt.tight_layout() + plt.savefig('/generated/' + img) + self.colonies.sync("/generated", "/eurohpc-summit-demo/generated-images", False, self.colonyname, self.executor_prvkey) + self.colonies.close(assigned_process.processid, [], self.executor_prvkey) + + except Exception as err: + print("Failed to execute function:", err) + continue + + def unregister(self) -> None: + self.colonies.remove_executor(self.colonyname, self.executorname, self.colony_prvkey) + print("Executor", self.executorname, "unregistered") + os._exit(0) + +def sigint_handler(signum: int, frame: Any) -> None: + del signum, frame + executor.unregister() + +if __name__ == '__main__': + signal.signal(signal.SIGINT, sigint_handler) + executor = PythonExecutor() + executor.start() diff --git a/examples/ml/sav/weights_unet100.h5 b/examples/ml/sav/weights_unet100.h5 new file mode 100644 index 0000000..b259a29 Binary files /dev/null and b/examples/ml/sav/weights_unet100.h5 differ diff --git a/examples/ml/weights_unet100.h5 b/examples/ml/weights_unet100.h5 new file mode 100644 index 0000000..b259a29 Binary files /dev/null and b/examples/ml/weights_unet100.h5 differ diff --git a/examples/monad_example1.py b/examples/monad_example1.py new file mode 100644 index 0000000..31fc046 --- /dev/null +++ b/examples/monad_example1.py @@ -0,0 +1,22 @@ +from pycolonies import colonies_client +from colonies_monad import ColoniesMonad +from colonies_monad import Function +from typing import Dict, Tuple, Any + +colonies, colonyname, colony_prvkey, executor_name, prvkey = colonies_client() + +def gen_data(_: Dict[str, Any] = {}) -> Tuple[int, int]: + return 1, 2 + +def process_data(*nums: int, _: Dict[str, Any] = {}) -> int: + total = 0 + for n in nums: + total += n + return total + +gen_data_fn = Function(gen_data, colonyname, executortype="python-executor") +process_data_fn = Function(process_data, colonyname, executortype="python-executor") + +m = ColoniesMonad(colonies, colonyname, prvkey) +result = (m >> gen_data_fn >> process_data_fn).unwrap() +print(result) # prints 3 diff --git a/examples/monad_example2.py b/examples/monad_example2.py new file mode 100644 index 0000000..10894d0 --- /dev/null +++ b/examples/monad_example2.py @@ -0,0 +1,23 @@ +from pycolonies import colonies_client +from colonies_monad import ColoniesMonad +from colonies_monad import Function +from typing import Dict, Tuple, Any + +colonies, colonyname, colony_prvkey, executor_name, prvkey = colonies_client() + +def gen_data(_: Dict[str, Any] = {}) -> Tuple[int, int]: + return 1, 2 + +def process_data(*nums: int, _: Dict[str, Any] = {}) -> int: + total = 0 + for n in nums: + total += n + return total + +gen_data_fn = Function(gen_data, colonyname, executortype="python-executor") +process_data_fn = Function(process_data, colonyname, executortype="python-executor") +echo = Function("echo", colonyname, executortype="echo-executor") + +m = ColoniesMonad(colonies, colonyname, prvkey) +result = (m >> gen_data_fn >> process_data_fn >> echo).unwrap() +print(result) # prints 3 diff --git a/examples/monad_example3.py b/examples/monad_example3.py new file mode 100644 index 0000000..43bc03d --- /dev/null +++ b/examples/monad_example3.py @@ -0,0 +1,28 @@ +import sys +sys.path.append(".") +from pycolonies import colonies_client +from colonies_monad import ColoniesMonad +from colonies_monad import Function +from typing import Dict, Tuple, Any + +colonies, colonyname, colony_prvkey, executor_name, prvkey = colonies_client() + +def gen_data(_: Dict[str, Any] = {}) -> Tuple[int, int]: + return 1, 2 + +def process_data(*nums: int, _: Dict[str, Any] = {}) -> int: + total = 0 + for n in nums: + total += n + return total + +def print_result(total: int, _: Dict[str, Any] = {}) -> int: + print("total=", total) + return total + +gen_data_fn = Function(gen_data, colonyname, executortype="python-executor") +process_data_fn = Function(process_data, colonyname, executortype="python-executor") +print_fn = Function(print_result, colonyname, executortype="python-executor") + +m = ColoniesMonad(colonies, colonyname, prvkey) +(m >> gen_data_fn >> process_data_fn >> print_fn).unwrap() diff --git a/examples/python_executor.py b/examples/python_executor.py new file mode 100644 index 0000000..0db4523 --- /dev/null +++ b/examples/python_executor.py @@ -0,0 +1,126 @@ +from crypto import Crypto +from pycolonies import colonies_client +from pycolonies import ColoniesConnectionError +import signal +import base64 +import os +import sys +from typing import Any, List + +class PythonExecutor: + def __init__(self) -> None: + global colonies + colonies, colonyname, colony_prvkey, _, _ = colonies_client() + crypto = Crypto() + self.colonies = colonies + self.colonyname = colonyname + self.colony_prvkey= colony_prvkey + self.executor_prvkey = crypto.prvkey() + self.executorid = crypto.id(self.executor_prvkey) + self.executorname = "python-executor" + self.executortype = "python-executor" + + self.register() + + def register(self) -> None: + try: + self.colonies.add_executor( + executorname=self.executorname, + executorid=self.executorid, + colonyname=self.colonyname, + executortype=self.executortype, + colony_prvkey=self.colony_prvkey + ) + self.colonies.approve_executor(self.colonyname, self.executorname, self.colony_prvkey) + except Exception as err: + print(err) + sys.exit(-1) + + print("Executor", self.executorname, "registered") + + def start(self) -> None: + while (True): + try: + # try to get a process from the colonies server, the call will block for max 10 seconds + # an exception will be raised if no processes can be assigned, and we will restart start the while loop + assigned_process = self.colonies.assign(self.colonyname, 10, self.executor_prvkey) + print() + print("Process", assigned_process.processid, "is assigned to Executor") + + # ok, executor was assigned a process, extract the function code to run + code_base64 = assigned_process.spec.env["code"] + code_bytes2 = base64.b64decode(code_base64) + code = code_bytes2.decode("ascii") + + # add the function to the global scope + exec(code) + + # extract args and call the function code we just injected + funcspec = assigned_process.spec + funcname = funcspec.funcname + assert funcname is not None, "Function name is None" + try: + self.colonies.add_function(self.colonyname, + self.executorname, + funcname, + self.executor_prvkey) + except Exception as err: + print(err) + + args: List[Any] = [] + try: + # if "input" is defined, it is the output of the parent process, + # use the output from parent process instead of args + + if assigned_process.input is not None and len(assigned_process.input)>0: + args = assigned_process.input + else: + args = funcspec.args + except Exception as err: + print(err) + + print("Executing:", funcspec.funcname) + + # call the injected function + try: + ctx = {"process": assigned_process, + "colonyname": self.colonyname, + "executorid": self.executorid, + "executor_prvkey": self.executor_prvkey} + + res = eval(funcname)(*tuple(args), ctx=ctx) + + if res is not None: + if type(res) is tuple: + res_arr = list(res) + else: + res_arr = [res] + else: + res_arr = [] + except Exception as err: + print("Failed to execute function:", err) + self.colonies.fail(assigned_process.processid, ["Failed to execute function"], self.executor_prvkey) + continue + + print("done") + # close the process as successful + self.colonies.close(assigned_process.processid, res_arr, self.executor_prvkey) + except ColoniesConnectionError as err: + print(err) + sys.exit(-1) + except Exception as err: + pass + + def unregister(self) -> None: + self.colonies.remove_executor(self.colonyname, self.executorname, self.colony_prvkey) + print("Executor", self.executorname, "unregistered") + os._exit(0) + +def sigint_handler(signum: int, frame: Any) -> None: + del signum, frame + executor.unregister() + +if __name__ == '__main__': + signal.signal(signal.SIGINT, sigint_handler) + executor = PythonExecutor() + executor.start() diff --git a/examples/submit_echo.py b/examples/submit_echo.py new file mode 100644 index 0000000..9c2e7b5 --- /dev/null +++ b/examples/submit_echo.py @@ -0,0 +1,20 @@ +from pycolonies import colonies_client +from pycolonies import FuncSpec + +colonies, colonyname, colony_prvkey, executor_name, prvkey = colonies_client() + +f = FuncSpec.create(func="echo", + args=["helloworld"], + colonyname=colonyname, + executortype="echo-executor", + priority=200, + maxexectime=100, + maxretries=3, + maxwaittime=100) + +process = colonies.submit_func_spec(f, prvkey) +print("Process", process.processid, "submitted") +process = colonies.wait(process, 100, prvkey) +if process and process.output: + print(process.output[0]) + diff --git a/examples/submit_helloworld.py b/examples/submit_helloworld.py new file mode 100644 index 0000000..4bcd947 --- /dev/null +++ b/examples/submit_helloworld.py @@ -0,0 +1,22 @@ +from pycolonies import FuncSpec +from pycolonies import colonies_client + +colonies, colonyname, colony_prvkey, executor_name, prvkey = colonies_client() + +func_spec = FuncSpec.create(func="helloworld", + args=[], + colonyname=colonyname, + executortype="helloworld-executor", + priority=200, + maxexectime=10, + maxretries=3, + maxwaittime=100) + +# submit the function spec to the colonies server +process = colonies.submit_func_spec(func_spec, prvkey) +print("Process", process.processid, "submitted") + +# wait for the process to be executed +process = colonies.wait(process, 10, prvkey) +if process and process.output: + print(process.output[0]) diff --git a/examples/submit_helloworld_wf.py b/examples/submit_helloworld_wf.py new file mode 100644 index 0000000..411cfa3 --- /dev/null +++ b/examples/submit_helloworld_wf.py @@ -0,0 +1,39 @@ +from pycolonies import colonies_client +from pycolonies import FuncSpec +from pycolonies import Workflow + +colonies, colonyname, colony_prvkey, executorid, executor_prvkey = colonies_client() + +fs1 = FuncSpec.create(func="hello1", + args=[], + colonyname=colonyname, + executortype="helloworld-executor", + maxexectime=10, + maxretries=3, + maxwaittime=100) + +fs2 = FuncSpec.create(func="hello2", + args=[], + colonyname=colonyname, + executortype="helloworld-executor", + maxexectime=10, + maxretries=3, + maxwaittime=100) + +fs3 = FuncSpec.create(func="hello3", + args=[], + colonyname=colonyname, + executortype="helloworld-executor", + maxexectime=10, + maxretries=3, + maxwaittime=100) + +assert fs2.conditions is not None +assert fs3.conditions is not None + +fs2.conditions.dependencies.append("hello1") +fs3.conditions.dependencies.append("hello2") + +wf = Workflow(colonyname=colonyname, functionspecs=[fs1, fs2, fs3]) + +colonies.submit_workflow(wf, executor_prvkey) diff --git a/examples/submit_kwargs.py b/examples/submit_kwargs.py new file mode 100644 index 0000000..089a830 --- /dev/null +++ b/examples/submit_kwargs.py @@ -0,0 +1,23 @@ +from pycolonies import colonies_client +from pycolonies import FuncSpec + +colonies, colonyname, colony_prvkey, executor_name, prvkey = colonies_client() + +f = FuncSpec.create(func="test", + args=[], + kwargs={ + "arg_kw_1":"arg_1", + "arg_kw_2":"arg_2" + }, + colonyname=colonyname, + executortype="kwargs-executor", + priority=200, + maxexectime=100, + maxretries=3, + maxwaittime=100) + +process = colonies.submit_func_spec(f, prvkey) +print("Process", process.processid, "submitted") +process = colonies.wait(process, 100, prvkey) +if process and process.output: + print(process.output[0]) diff --git a/examples/submit_python.py b/examples/submit_python.py new file mode 100644 index 0000000..313723a --- /dev/null +++ b/examples/submit_python.py @@ -0,0 +1,26 @@ +from pycolonies import colonies_client +from pycolonies import FuncSpec +from typing import Dict, Any + +colonies, colonyname, colony_prvkey, executor_name, prvkey = colonies_client() + +def sum_nums(n1: int, n2: int, _: Dict[str, Any] = {}) -> int: + return n1 + n2 + +func_spec = FuncSpec.create(func=sum_nums, + args=[1, 2], + colonyname=colonyname, + executortype="python-executor", + priority=200, + maxexectime=10, + maxretries=3, + maxwaittime=100) + +# submit the function spec to the colonies server +process = colonies.submit_func_spec(func_spec, prvkey) +print("Process", process.processid, "submitted") + +# wait for the process to be executed +process = colonies.wait(process, 100, prvkey) +if process and process.output: + print(process.output[0]) diff --git a/examples/workflow_example1.py b/examples/workflow_example1.py new file mode 100644 index 0000000..8edec33 --- /dev/null +++ b/examples/workflow_example1.py @@ -0,0 +1,51 @@ +from pycolonies import colonies_client +from pycolonies import FuncSpec +from pycolonies import Workflow +from typing import Dict, Tuple, Any + +colonies, colonyname, colony_prvkey, executor_name, prvkey = colonies_client() + +def gen_nums(ctx: Dict[str, Any] = {}) -> Tuple[int, int]: + del ctx + return 1, 2 + +def sum_nums(n1: int, n2: int, ctx: Dict[str, Any] = {}) -> int: + del ctx + return n1 + n2 + +wf = Workflow(colonyname=colonyname) +f = FuncSpec.create(func=gen_nums, + args=[], + colonyname=colonyname, + executortype="python-executor", + priority=200, + maxexectime=100, + maxretries=3, + maxwaittime=100) + +wf.functionspecs.append(f) + +f = FuncSpec.create(func=sum_nums, + args=[], + colonyname=colonyname, + executortype="python-executor", + priority=200, + maxexectime=100, + maxretries=3, + maxwaittime=100) + +assert f.conditions + +f.conditions.dependencies.append("gen_nums") + +wf.functionspecs.append(f) + +processgraph = colonies.submit_workflow(wf, prvkey) +print("Workflow", processgraph.processgraphid, "submitted") + +# wait for the sum_list process +process = colonies.find_process("sum_nums", processgraph.processids, prvkey) +if process: + completed_process = colonies.wait(process, 100, prvkey) + if completed_process and completed_process.output: + print(completed_process.output[0]) diff --git a/examples/workflow_example2.py b/examples/workflow_example2.py new file mode 100644 index 0000000..571f138 --- /dev/null +++ b/examples/workflow_example2.py @@ -0,0 +1,66 @@ +from pycolonies import colonies_client +from pycolonies import FuncSpec +from pycolonies import Workflow +from typing import Dict, Tuple, Any + +colonies, colonyname, colony_prvkey, executor_name, prvkey = colonies_client() + +def gen_nums(ctx: Dict[str, Any] = {}) -> Tuple[int, int]: + del ctx + return 1, 2 + +def reduce(*nums: int, ctx: Dict[str, Any] = {}) -> int: + del ctx + total = 0 + for n in nums: + total += n + return total + +wf = Workflow(colonyname=colonyname) +f = FuncSpec.create(func=gen_nums, + args=[], + colonyname=colonyname, + executortype="python-executor", + priority=200, + maxexectime=100, + maxretries=3, + maxwaittime=100) + +f.nodename = "gen_nums1" +wf.functionspecs.append(f) + +f = FuncSpec.create(func=gen_nums, + args=[], + colonyname=colonyname, + executortype="python-executor", + priority=200, + maxexectime=100, + maxretries=3, + maxwaittime=100) + +f.nodename = "gen_nums2" +wf.functionspecs.append(f) + +func_spec = FuncSpec.create(func=reduce, + args=[], + colonyname=colonyname, + executortype="python-executor", + priority=200, + maxexectime=100, + maxretries=3, + maxwaittime=100) + +assert func_spec.conditions + +func_spec.conditions.dependencies = ["gen_nums1", "gen_nums2"] +wf.functionspecs.append(func_spec) + +processgraph = colonies.submit_workflow(wf, prvkey) +print("Workflow", processgraph.processgraphid, "submitted") + +# wait for the sum_list process +process = colonies.find_process("reduce", processgraph.processids, prvkey) +if process: + completed_process = colonies.wait(process, 100, prvkey) + if completed_process and completed_process.output: + print(completed_process.output[0]) diff --git a/examples/workflow_example3.py b/examples/workflow_example3.py new file mode 100644 index 0000000..76a3be0 --- /dev/null +++ b/examples/workflow_example3.py @@ -0,0 +1,82 @@ +from pycolonies import colonies_client +from pycolonies import FuncSpec +from pycolonies import Workflow +from typing import Dict, Any + +colonies, colonyname, colony_prvkey, executor_name, prvkey = colonies_client() + +def map(ctx: Dict[str, Any] = {}) -> None: + code = """def gen_nums(ctx={}): + return 1, 2""" + processgraphid = ctx["process"].processgraphid + map_processid = ctx["process"].processid + executor_prvkey = ctx["executor_prvkey"] + + processgraph = colonies.get_processgraph(processgraphid, executor_prvkey) + + reduce_process = colonies.find_process("reduce", processgraph.processids, executor_prvkey) + if reduce_process is None: + raise RuntimeError("Could not find reduce process") + reduce_processid = reduce_process.processid + + insert = True + for i in range(1): + f = FuncSpec.create(func="gen_nums", + args=[], + colonyname=ctx["colonyname"], + executortype="python-executor", + priority=200, + maxexectime=100, + maxretries=3, + maxwaittime=100, + code=code) + + + colonies.add_child(processgraphid, map_processid, reduce_processid, f, "gen_nums_" + str(i), insert, executor_prvkey) + + insert = False + +def reduce(*nums: int, ctx: Dict[str, Any] = {}) -> int: + del ctx + print("REDUCED CALLED") + total = 0 + for n in nums: + total += n + return total + +wf = Workflow(colonyname=colonyname) + +f = FuncSpec.create(func=map, + args=[], + colonyname=colonyname, + executortype="python-executor", + priority=200, + maxexectime=100, + maxretries=3, + maxwaittime=100) + +wf.functionspecs.append(f) + +f = FuncSpec.create(func=reduce, + args=[], + colonyname=colonyname, + executortype="python-executor", + priority=200, + maxexectime=100, + maxretries=3, + maxwaittime=100) + +assert f.conditions, "FunctionSpec must have conditions defined." + +f.conditions.dependencies.append("map") +wf.functionspecs.append(f) + +processgraph = colonies.submit_workflow(wf, prvkey) +print("Workflow", processgraph.processgraphid, "submitted") + +# wait for the sum_list process +process = colonies.find_process("reduce", processgraph.processids, prvkey) +if process: + completed_process = colonies.wait(process, 1000, prvkey) + if completed_process and completed_process.output: + print(completed_process.output[0]) diff --git a/executor.json b/executor.json new file mode 100644 index 0000000..a4554e1 --- /dev/null +++ b/executor.json @@ -0,0 +1,4 @@ +{ + "executorname": "cli", + "executortype": "cli" +} diff --git a/lib/libcfslib.h b/lib/libcfslib.h new file mode 100644 index 0000000..53cd2da --- /dev/null +++ b/lib/libcfslib.h @@ -0,0 +1,81 @@ +/* Code generated by cmd/cgo; DO NOT EDIT. */ + +/* package command-line-arguments */ + + +#line 1 "cgo-builtin-export-prolog" + +#include + +#ifndef GO_CGO_EXPORT_PROLOGUE_H +#define GO_CGO_EXPORT_PROLOGUE_H + +#ifndef GO_CGO_GOSTRING_TYPEDEF +typedef struct { const char *p; ptrdiff_t n; } _GoString_; +#endif + +#endif + +/* Start of preamble from import "C" comments. */ + + + + +/* End of preamble from import "C" comments. */ + + +/* Start of boilerplate cgo prologue. */ +#line 1 "cgo-gcc-export-header-prolog" + +#ifndef GO_CGO_PROLOGUE_H +#define GO_CGO_PROLOGUE_H + +typedef signed char GoInt8; +typedef unsigned char GoUint8; +typedef short GoInt16; +typedef unsigned short GoUint16; +typedef int GoInt32; +typedef unsigned int GoUint32; +typedef long long GoInt64; +typedef unsigned long long GoUint64; +typedef GoInt64 GoInt; +typedef GoUint64 GoUint; +typedef size_t GoUintptr; +typedef float GoFloat32; +typedef double GoFloat64; +#ifdef _MSC_VER +#include +typedef _Fcomplex GoComplex64; +typedef _Dcomplex GoComplex128; +#else +typedef float _Complex GoComplex64; +typedef double _Complex GoComplex128; +#endif + +/* + static assertion to make sure the file is being used on architecture + at least with matching size of GoInt. +*/ +typedef char _check_for_64_bit_pointer_matching_GoInt[sizeof(void*)==64/8 ? 1:-1]; + +#ifndef GO_CGO_GOSTRING_TYPEDEF +typedef _GoString_ GoString; +#endif +typedef void *GoMap; +typedef void *GoChan; +typedef struct { void *t; void *v; } GoInterface; +typedef struct { void *data; GoInt len; GoInt cap; } GoSlice; + +#endif + +/* End of boilerplate cgo prologue. */ + +#ifdef __cplusplus +extern "C" { +#endif + +extern int sync(char* chost, int cport, int cinsecure, int cskiptlsverify, char* cdir, char* clabel, int ckeeplocal, char* ccolonyname, char* cprvkey); + +#ifdef __cplusplus +} +#endif diff --git a/lib/libcfslib.so b/lib/libcfslib.so new file mode 100644 index 0000000..126a3b4 Binary files /dev/null and b/lib/libcfslib.so differ diff --git a/lib/libcryptolib.h b/lib/libcryptolib.h new file mode 100644 index 0000000..135e23b --- /dev/null +++ b/lib/libcryptolib.h @@ -0,0 +1,85 @@ +/* Code generated by cmd/cgo; DO NOT EDIT. */ + +/* package command-line-arguments */ + + +#line 1 "cgo-builtin-export-prolog" + +#include + +#ifndef GO_CGO_EXPORT_PROLOGUE_H +#define GO_CGO_EXPORT_PROLOGUE_H + +#ifndef GO_CGO_GOSTRING_TYPEDEF +typedef struct { const char *p; ptrdiff_t n; } _GoString_; +#endif + +#endif + +/* Start of preamble from import "C" comments. */ + + + + +/* End of preamble from import "C" comments. */ + + +/* Start of boilerplate cgo prologue. */ +#line 1 "cgo-gcc-export-header-prolog" + +#ifndef GO_CGO_PROLOGUE_H +#define GO_CGO_PROLOGUE_H + +typedef signed char GoInt8; +typedef unsigned char GoUint8; +typedef short GoInt16; +typedef unsigned short GoUint16; +typedef int GoInt32; +typedef unsigned int GoUint32; +typedef long long GoInt64; +typedef unsigned long long GoUint64; +typedef GoInt64 GoInt; +typedef GoUint64 GoUint; +typedef size_t GoUintptr; +typedef float GoFloat32; +typedef double GoFloat64; +#ifdef _MSC_VER +#include +typedef _Fcomplex GoComplex64; +typedef _Dcomplex GoComplex128; +#else +typedef float _Complex GoComplex64; +typedef double _Complex GoComplex128; +#endif + +/* + static assertion to make sure the file is being used on architecture + at least with matching size of GoInt. +*/ +typedef char _check_for_64_bit_pointer_matching_GoInt[sizeof(void*)==64/8 ? 1:-1]; + +#ifndef GO_CGO_GOSTRING_TYPEDEF +typedef _GoString_ GoString; +#endif +typedef void *GoMap; +typedef void *GoChan; +typedef struct { void *t; void *v; } GoInterface; +typedef struct { void *data; GoInt len; GoInt cap; } GoSlice; + +#endif + +/* End of boilerplate cgo prologue. */ + +#ifdef __cplusplus +extern "C" { +#endif + +extern char* prvkey(); +extern char* id(char* cprvkey); +extern char* sign(char* cmsg, char* cprvkey); +extern char* hash(char* cmsg); +extern char* recoverid(char* cmsg, char* csig); + +#ifdef __cplusplus +} +#endif diff --git a/lib/libcryptolib.so b/lib/libcryptolib.so new file mode 100644 index 0000000..6d9dba9 Binary files /dev/null and b/lib/libcryptolib.so differ diff --git a/lib/libcryptolib.wasm b/lib/libcryptolib.wasm new file mode 100755 index 0000000..3afc4f9 Binary files /dev/null and b/lib/libcryptolib.wasm differ diff --git a/lib/ubuntu_2020/cryptolib.so b/lib/ubuntu_2020/cryptolib.so new file mode 100644 index 0000000..0a2deaf Binary files /dev/null and b/lib/ubuntu_2020/cryptolib.so differ diff --git a/lib/cryptolib.so b/lib/ubuntu_2020/libcryptolib.so similarity index 61% rename from lib/cryptolib.so rename to lib/ubuntu_2020/libcryptolib.so index c93b208..d70c236 100644 Binary files a/lib/cryptolib.so and b/lib/ubuntu_2020/libcryptolib.so differ diff --git a/model.py b/model.py new file mode 100644 index 0000000..6c10d24 --- /dev/null +++ b/model.py @@ -0,0 +1,387 @@ +from datetime import datetime +import base64 +import inspect + +from typing import List, Dict, Optional, Any, Callable +from pydantic import BaseModel, Field, field_validator, ConfigDict + +# Base model for all RPC request messages +class Model(BaseModel): + model_config = ConfigDict(populate_by_name=True) + + +class Gpu(Model): + name: Optional[str] = None + mem: Optional[str] = None + count: Optional[int] = None + nodecount: Optional[int] = None + + +class Conditions(Model): + colonyname: Optional[str] = None + executornames: Optional[List[str]] = None + executortype: str + dependencies: List[str] = [] + nodes: int = 0 + cpu: Optional[str] = None + processes: int = 0 + processespernode: int = 0 + mem: Optional[str] = None + storage: Optional[str] = None + gpu: Optional[Gpu] = None + walltime: int = 0 + +class Fs(Model): + mount: str + snapshots: Optional[List[str]] + dirs: Optional[List[str]] + + +class FuncSpec(Model): + nodename: Optional[str] = None + funcname: str + args: List[str | int] = [] + kwargs: Dict[str, str | List[str]] = {} + priority: int = 0 + maxwaittime: int = 0 + maxexectime: int = 0 + maxretries: int = 0 + conditions: Optional[Conditions] = None + label: Optional[str] = None + fs: Optional[Fs] = None + env: Dict[str, str] = {} + + @staticmethod + def create( + func: str | Callable, + args: List[str | int], + colonyname: Optional[str] = None, + executortype: Optional[str] = None, + nodename: Optional[str] = None, + executorname: Optional[str] = None, + dependencies: Optional[List[str]] = None, + priority: int = 1, + maxexectime: int = -1, + maxretries: int = -1, + maxwaittime: int = -1, + code: Optional[str] = None, + kwargs: Dict[str, str | List[str]] = {}, + conditions: Optional[Conditions] = None, + fs: Optional[Fs] = None, + env: Dict[str, str] = {}, + ) -> 'FuncSpec': + if conditions is None \ + and colonyname is not None \ + and executortype is not None: + conditions = Conditions( + colonyname=colonyname, + executortype=executortype, + dependencies=dependencies or [] + ) + else: + raise ValueError("Either `conditions` or `colonyname` and `executortype` must be provided.") + + env = env.copy() + if isinstance(func, str): + nodename = nodename or func + funcname = func + if code is not None: + code_bytes = code.encode("ascii") + code_base64_bytes = base64.b64encode(code_bytes) + code_base64 = code_base64_bytes.decode("ascii") + env["code"] = code_base64 + else: + code = inspect.getsource(func) + code_bytes = code.encode("ascii") + code_base64_bytes = base64.b64encode(code_bytes) + code_base64 = code_base64_bytes.decode("ascii") + + funcname = func.__name__ + args_spec = inspect.getfullargspec(func) + args_spec_str = ','.join(args_spec.args) + + nodename = nodename or funcname + env["args_spec"] = args_spec_str + env["code"] = code_base64 + + if executorname is not None: + conditions.executornames = [ executorname ] + + # Create instance with all the prepared data + return FuncSpec( + nodename=nodename, + funcname=funcname, + args=args, + kwargs=kwargs, + priority=priority, + maxwaittime=maxwaittime, + maxexectime=maxexectime, + maxretries=maxretries, + conditions=conditions, + fs=fs, + env=env, + ) + + + +class Attribute(Model): + id: str = Field(..., alias="attributeid") + targetid: str + targetcolonyname: str = Field(..., alias="targetcolonyname") + targetprocessgraphid: str = Field(..., alias="targetprocessgraphid") + state: int + attributetype: int + key: str + value: str + + +class Process(Model): + processid: str + initiatorid: str + initiatorname: str + assignedexecutorid: str + isassigned: bool + state: int + prioritytime: int + submissiontime: datetime + starttime: datetime + endtime: datetime + waitdeadline: datetime + execdeadline: datetime + retries: int + attributes: Optional[List[Attribute]] + spec: FuncSpec + waitforparents: bool = False + parents: List[str] + children: List[str] + processgraphid: str + input: Optional[List[str | int | float]] = Field(alias="in") + output: Optional[List[str | int | float]] = Field(alias="out") + errors: List[str] + + def __init__(self, **data: Any) -> None: + if 'input' in data: + data['in'] = data.pop('input') + if 'output' in data: + data['out'] = data.pop('output') + super().__init__(**data) + + +class Workflow(Model): + colonyname: str + functionspecs: List[FuncSpec] = [] + + +class Position(Model): + x: int + y: int + + +class ProcessNode(Model): + id: str + data: Dict[str, str] = {} + position: Position + type: str + style: Dict[str, str] = {} + + +class ProcessEdge(Model): + id: str + source: str + target: str + animated: bool + + +class ProcessGraph(Model): + processgraphid: str + initiatorid: str + initiatorname: str + colonyname: str + rootprocessids: List[str] + state: int + submissiontime: datetime + starttime: datetime + endtime: datetime + processids: List[str] + nodes: List[ProcessNode] + edges: List[ProcessEdge] + + +class S3Object(Model): + server: str + port: int + tls: bool + accesskey: str = Field(..., alias="accesskey") + secretkey: str = Field(..., alias="secretkey") + region: str = Field(..., alias="region") + encryptionkey: str = Field(..., alias="encryptionkey") + encryptionalg: str = Field(..., alias="encryptionalg") + object: str = Field(..., alias="object") + bucket: str = Field(..., alias="bucket") + +class Reference(Model): + protocol: str + s3object: S3Object = Field(..., alias="s3object") + +class File(Model): + fileid: str = Field(..., alias="fileid") + colonyname: str = Field(..., alias="colonyname") + label: str = Field(..., alias="label") + name: str = Field(..., alias="name") + size: int = Field(..., alias="size") + sequencenr: int = Field(..., alias="sequencenr") + checksum: str = Field(..., alias="checksum") + checksumalg: str = Field(..., alias="checksumalg") + ref: Reference = Field(..., alias="ref") + added: Optional[datetime] = Field(default=None, alias="added") + + @field_validator('label') + def ensure_single_slash(cls, v): + if not v.startswith('/'): + v = '/' + v + v = '/' + v.strip('/') + return v + + +class FileData(Model): + name: str + checksum: str + size: int + s3filename: str + + +class Cron(Model): + cronid: str + initiatorid: str + initiatorname: str + colonyname: str + name: str + cronexpression: str + interval: int + random: bool + nextrun: datetime + lastrun: datetime + workflowspec: str + prevprocessgraphid: str + waitforprevprocessgraph: bool + checkerperiod: int + + +class Log(Model): + processid: str + colonyname: str + executorname: str + message: str + timestamp: int + + +class Location(Model): + long: float + lat: float + desc: str + + +class Hardware(Model): + model: str + nodes: int + cpu: str + mem: str + storage: str + gpu: Gpu + + +class Software(Model): + name: str + type: str + version: str + + +class Capabilities(Model): + hardware: Hardware + software: Software + + +class Allocations(Model): + projects: Optional[Dict[str, Any]] = None + + +class Executor(Model): + executorid: str + executortype: str + name: str = Field(..., alias="executorname") + colonyname: str + state: int + requirefuncreg: bool + commissiontime: datetime + lastheardfromtime: datetime + location: Location + capabilities: Capabilities + allocations: Allocations + + +class Colony(Model): + colonyid: str + name: str + + +class Statistics(Model): + colonies: int + executors: int + waitingprocesses: int + runningprocesses: int + successfulprocesses: int + failedprocesses: int + waitingworkflows: int + runningworkflows: int + successfulworkflows: int + failedworkflows: int + + +class Function(Model): + functionid: str + executorname: str + executortype: str + colonyname: str + funcname: str + counter: int + minwaittime: float + maxwaittime: float + minexectime: float + maxexectime: float + avgwaittime: float + avgexectime: float + + +class Snapshot(Model): + snapshotid: str + colonyname: str + label: str + name: str + fileids: List[str] + added: datetime + + +class Generator(Model): + generatorid: str + initiatorid: str + initiatorname: str + colonyname: str + name: str + workflowspec: str + trigger: int + checksomeneeded: bool + lastchecksum: str + checkerperiod: int + timeout: int + added: datetime + +class User(Model): + colonyname: str + id: str = Field(..., alias="userid") + name: str + email: str + phone: str + +class Empty(Model): + """An empty class used to represent an empty response in RPC calls.""" + pass diff --git a/pycolonies.py b/pycolonies.py new file mode 100644 index 0000000..ceffdcd --- /dev/null +++ b/pycolonies.py @@ -0,0 +1,847 @@ +import requests +import json +from typing import List, Optional, Any, TypeVar, Type +from pydantic import TypeAdapter, ValidationError +import base64 +from websocket import create_connection +import os +import ctypes +import boto3 +import hashlib +import uuid +from botocore.exceptions import ClientError + +from crypto import Crypto +import rpc +from model import ( + Attribute, Empty, Process, FuncSpec, Workflow, ProcessGraph, S3Object, Reference, File, FileData, Cron, Log, Executor, Colony, + Statistics, Function, Snapshot, Allocations, Generator +) + +def colonies_client(native_crypto=False) -> tuple['Colonies', str, str, str, str]: + colonies_server = os.getenv("COLONIES_SERVER_HOST") + colonies_port = os.getenv("COLONIES_SERVER_PORT") + colonies_tls = os.getenv("COLONIES_SERVER_TLS") + colonyname = os.getenv("COLONIES_COLONY_NAME") + colony_prvkey = os.getenv("COLONIES_COLONY_PRVKEY") + executorname = os.getenv("COLONIES_EXECUTOR_NAME") + prvkey = os.getenv("COLONIES_PRVKEY") + + if ( + colonies_server is None or + colonies_port is None or + colonyname is None or + colony_prvkey is None or + executorname is None or + prvkey is None + ): + raise ValueError("Environment variables COLONIES_SERVER_HOST, COLONIES_SERVER_PORT, COLONIES_COLONY_NAME, and COLONIES_COLONY_PRVKEY must be set.") + + client = Colonies(colonies_server, int(colonies_port), colonies_tls == "true", native_crypto=native_crypto) + + return client, colonyname, colony_prvkey, executorname, prvkey + +T = TypeVar('T') + +class ColoniesConnectionError(Exception): + pass + +class ColoniesError(Exception): + pass + +class Colonies: + url: str + host: str + port: int + tls: bool + native_crypto: bool + + WAITING = 0 + RUNNING = 1 + SUCCESSFUL = 2 + FAILED = 3 + + def __init__(self, host: str, port: int, tls: bool = False, native_crypto: bool = False) -> None: + self.host = host + self.port = port + self.native_crypto = native_crypto + self.tls = tls + self.url = ("https://" if self.tls else "http://") + self.host + ":" + str(self.port) + "/api" + + def __rpc( + self, + request_payload: rpc.RequestPayload, + prvkey: str, + response_payload_type: Type[T] + ) -> T: + request_payload_json = request_payload.model_dump_json(by_alias=True) + request_payload_b64 = str(base64.b64encode(request_payload_json.encode('utf-8')), "utf-8") + crypto = Crypto(native=self.native_crypto) + signature = crypto.sign(request_payload_b64, prvkey) + + request = rpc.Request( + payloadtype=request_payload.msgtype, + payload=request_payload_b64, + signature=signature + ) + + request_json = request.model_dump_json() + + try: + http_response = requests.post(url = self.url, data=request_json, verify=True) + http_response.raise_for_status() + response = rpc.Response.model_validate_json(http_response.content) + response_payload_json = base64.b64decode(response.payload) + if response.payloadtype == "error": + error_payload = rpc.ErrorResponse.model_validate_json(response_payload_json) + raise ColoniesConnectionError(error_payload.message) + return TypeAdapter(response_payload_type).validate_json(response_payload_json) + except requests.exceptions.RequestException as err: + raise ColoniesConnectionError(f"Network request failed: {err}") from err + except ValidationError as err: + raise ColoniesError(f"API response validation failed: {err}") from err + except UnicodeDecodeError as err: + raise ColoniesError(f"Failed to decode response payload: {err}") from err + + def wait(self, process: Process, timeout: int, executor_prvkey: str) -> Process: + state = Colonies.SUCCESSFUL + if not process.spec.conditions: + raise ValueError("Process must have conditions set") + if not process.spec.conditions.colonyname: + raise ValueError("Process must have a colony name set in its conditions") + payload = rpc.SubscribeProcess( + processid=process.processid, + executortype=process.spec.conditions.executortype, + state=state, + timeout=timeout, + colonyname=process.spec.conditions.colonyname, + ) + + payload_dict = payload.model_dump(by_alias=True) + payload_b64 = str(base64.b64encode(json.dumps(payload_dict).encode('utf-8')), "utf-8") + request = rpc.Request(payloadtype=payload.msgtype, payload=payload_b64, signature="") + + crypto = Crypto() + request.signature = crypto.sign(request.payload, executor_prvkey) + + ws = create_connection(("wss://" if self.tls else "ws://") + self.host + ":" + str(self.port) + "/pubsub") + ws.send(request.model_dump_json()) + ws.recv() + ws.close() + + return self.get_process(process.processid, executor_prvkey) + + def add_colony(self, colony: Colony, server_prvkey: str) -> Colony: + payload = rpc.AddColony(colony=colony) + return self.__rpc(payload, server_prvkey, Colony) + + def del_colony(self, colonyname: str, server_prvkey: str) -> None: + payload = rpc.RemoveColony(colonyname=colonyname) + self.__rpc(payload, server_prvkey, Colony) + + def list_colonies(self, server_prvkey: str) -> List[Colony]: + payload = rpc.GetColonies() + return self.__rpc(payload, server_prvkey, List[Colony]) + + def get_colony(self, colonyname: str, server_prvkey: str) -> Colony: + payload = rpc.GetColony(colonyname=colonyname) + return self.__rpc(payload, server_prvkey, Colony) + + def add_executor( + self, + executorid: str, + executorname: str, + executortype: str, + colonyname: str, + colony_prvkey: str, + capabilities: Optional[rpc.Capabilities] = None + ) -> Executor: + payload = rpc.Executor( + executorid=executorid, + executortype=executortype, + executorname=executorname, + colonyname=colonyname, + capabilities=capabilities + ) + payload = rpc.AddExecutor(executor=payload) + return self.__rpc(payload, colony_prvkey, Executor) + + def report_allocation(self, colonyname: str, executorname: str, allocations: Allocations, executor_prvkey: str) -> None: + """ + Reports resource allocations for an executor. + """ + payload = rpc.ReportAllocations( + colonyname=colonyname, + executorname=executorname, + allocations=allocations + ) + self.__rpc(payload, executor_prvkey, Empty) + + def list_executors(self, colonyname: str, executor_prvkey: str) -> List[Executor]: + payload = rpc.GetExecutors(colonyname=colonyname) + return self.__rpc(payload, executor_prvkey, List[Executor]) + + def approve_executor(self, colonyname: str, executorname: str, colony_prvkey: str) -> None: + payload = rpc.ApproveExecutor(colonyname=colonyname, executorname=executorname) + self.__rpc(payload, colony_prvkey, Empty) + + def reject_executor(self, colonyname: str, executorname: str, colony_prvkey: str) -> None: + payload = rpc.RejectExecutor(colonyname=colonyname, executorname=executorname) + self.__rpc(payload, colony_prvkey, Empty) + + def remove_executor(self, colonyname: str, executorname: str, colony_prvkey: str) -> None: + payload = rpc.RemoveExecutor(colonyname=colonyname, executorname=executorname) + self.__rpc(payload, colony_prvkey, Empty) + + def get_executor(self, colonyname: str, executorname: str, executor_prvkey: str) -> Executor: + payload = rpc.GetExecutor(colonyname=colonyname, executorname=executorname) + return self.__rpc(payload, executor_prvkey, Executor) + + def submit_func_spec(self, spec: FuncSpec, executor_prvkey: str) -> Process: + payload = rpc.SubmitFunctionSpec(spec=spec) + return self.__rpc(payload, executor_prvkey, Process) + + def submit_workflow(self, workflow: Workflow, executor_prvkey: str) -> ProcessGraph: + payload = rpc.SubmitWorkflowSpec(spec=workflow) + return self.__rpc(payload, executor_prvkey, ProcessGraph) + + def assign( + self, + colonyname: str, + timeout: int, + executor_prvkey: str, + available_cpu: str = "1000m", + available_mem: str = "1000Mi" + ) -> Process: + payload = rpc.AssignProcess( + colonyname=colonyname, + timeout=timeout, + availablecpu=available_cpu, + availablemem=available_mem + ) + return self.__rpc(payload, executor_prvkey, Process) + + def list_processes(self, colonyname: str, count: int, state: int, executor_prvkey: str) -> List[Process]: + payload = rpc.GetProcesses( + colonyname=colonyname, + count=count, + state=state + ) + return self.__rpc(payload, executor_prvkey, List[Process]) + + def get_process(self, processid: str, executor_prvkey: str) -> Process: + payload = rpc.GetProcess(processid=processid) + return self.__rpc(payload, executor_prvkey, Process) + + def remove_process(self, processid: str, executor_prvkey: str, all: bool = False) -> None: + payload = rpc.RemoveProcess(processid=processid, all=all) + self.__rpc(payload, executor_prvkey, Empty) + + def close(self, processid: str, output: List[Any], executor_prvkey: str) -> None: + payload = rpc.CloseSuccessful(processid=processid, out=output) + self.__rpc(payload, executor_prvkey, Empty) + + def fail(self, processid: str, errors: List[str], executor_prvkey: str) -> None: + payload = rpc.CloseFailed(processid=processid, errors=errors) + self.__rpc(payload, executor_prvkey, Empty) + + def set_output(self, processid: str, arr: List[Any], executor_prvkey: str) -> None: + payload = rpc.SetOutput(processid=processid, out=arr) + self.__rpc(payload, executor_prvkey, Empty) + + def stats(self, colonyname: str, executor_prvkey: str) -> Statistics: + payload = rpc.GetColonyStatistics(colonyname=colonyname) + return self.__rpc(payload, executor_prvkey, Statistics) + + def add_attribute(self, processid: str, key: str, value: str, executor_prvkey: str) -> Attribute: + attribute = rpc.Attribute( + key=key, + value=value, + targetid=processid, + attributetype=1 + ) + payload = rpc.AddAttribute(attribute=attribute) + return self.__rpc(payload, executor_prvkey, Attribute) + + def get_attribute(self, attributeid: str, executor_prvkey: str) -> Attribute: + payload = rpc.GetAttribute(attributeid=attributeid) + return self.__rpc(payload, executor_prvkey, Attribute) + + def get_processgraph(self, processgraphid: str, executor_prvkey: str) -> ProcessGraph: # TODO: unittest + payload = rpc.GetProcessGraph(processgraphid=processgraphid) + return self.__rpc(payload, executor_prvkey, ProcessGraph) + + def add_function(self, colonyname: str, executorname: str, funcname: str, executor_prvkey: str) -> Function: + func = rpc.Function( + colonyname=colonyname, + executorname=executorname, + funcname=funcname, + ) + payload = rpc.AddFunction(fun=func) + return self.__rpc(payload, executor_prvkey, Function) + + def get_functions_by_executor(self, colonyname: str, executorname: str, executor_prvkey: str) -> List[Function]: + payload = rpc.GetFunctions( + colonyname=colonyname, + executorname=executorname + ) + return self.__rpc(payload, executor_prvkey, List[Function]) + + def get_functions_by_colony(self, colonyname: str, executor_prvkey: str) -> List[Function]: + payload = rpc.GetFunctions(colonyname=colonyname) + return self.__rpc(payload, executor_prvkey, List[Function]) + + def find_process(self, nodename: str, processids: List[str], executor_prvkey: str) -> Optional[Process]: + for processid in processids: + process = self.get_process(processid, executor_prvkey) + if process.spec.nodename == nodename: + return process + return None + + def add_child( + self, + processgraphid: str, + parentprocessid: str, + childprocessid: str, + funcspec: FuncSpec, + nodename: str, + insert: bool, + executor_prvkey: str + ) -> Process: + funcspec.nodename = nodename + payload = rpc.AddChild( + processgraphid=processgraphid, + parentprocessid=parentprocessid, + childprocessid=childprocessid, + insert=insert, + spec=funcspec + ) + return self.__rpc(payload, executor_prvkey, Process) + + def create_snapshot(self, colonyname: str, label: str, name: str, executor_prvkey: str) -> Snapshot: + payload = rpc.CreateSnapshot(colonyname=colonyname, label=label, name=name) + return self.__rpc(payload, executor_prvkey, Snapshot) + + def get_snapshots(self, colonyname: str, executor_prvkey: str) -> List[Snapshot]: + payload = rpc.GetSnapshot(colonyname=colonyname) + return self.__rpc(payload, executor_prvkey, List[Snapshot]) + + def get_snapshot_by_name(self, colonyname: str, name: str, executor_prvkey: str) -> Snapshot: + payload = rpc.GetSnapshot(colonyname=colonyname, name=name) + return self.__rpc(payload, executor_prvkey, Snapshot) + + def get_snapshot_by_id(self, colonyname: str, snapshotid: str, executor_prvkey: str) -> Snapshot: + payload = rpc.GetSnapshot(colonyname=colonyname, snapshotid=snapshotid,) + return self.__rpc(payload, executor_prvkey, Snapshot) + + def add_log(self, processid: str, logmsg: str, executor_prvkey: str) -> None: + payload = rpc.AddLog(processid=processid, message=logmsg) + self.__rpc(payload, executor_prvkey, Empty) + + def get_process_log(self, colonyname: str, processid: str, count: int, since: int, executor_prvkey: str) -> List[Log]: + payload = rpc.GetLogs( + colonyname=colonyname, + executorname="", + processid=processid, + count=count, + since=since + ) + return self.__rpc(payload, executor_prvkey, List[Log]) + + def get_executor_log(self, colonyname: str, executorname: str, count: int, since: int, executor_prvkey: str) -> List[Log]: + payload = rpc.GetLogs( + colonyname=colonyname, + executorname=executorname, + processid="", + count=count, + since=since + ) + return self.__rpc(payload, executor_prvkey, List[Log]) + + def sync(self, dir: str, label: str, keeplocal: bool, colonyname: str, executor_prvkey: str) -> None: + libname = os.environ.get("CFSLIB") + if libname == None: + libname = "/usr/local/lib/libcfslib.so" + c_lib = ctypes.CDLL(libname) + c_lib.sync.restype = ctypes.c_int + + c_host = ctypes.c_char_p(self.host.encode('utf-8')) + c_port = ctypes.c_int(self.port) + c_insecure = ctypes.c_int(self.tls==False) + c_skip_tls_verify = ctypes.c_int(False) + c_dir = ctypes.c_char_p(dir.encode('utf-8')) + c_label = ctypes.c_char_p(label.encode('utf-8')) + c_keeplocal = ctypes.c_int(keeplocal) + c_colonyname = ctypes.c_char_p(colonyname.encode('utf-8')) + c_prvkey = ctypes.c_char_p(executor_prvkey.encode('utf-8')) + + res = c_lib.sync(c_host, c_port, c_insecure, c_skip_tls_verify, c_dir, c_label, c_keeplocal, c_colonyname, c_prvkey) + if res != 0: + raise Exception("failed to sync") + + def get_files(self, label: str, colonyname: str, executor_prvkey: str) -> List[FileData]: + payload = rpc.GetFiles(colonyname=colonyname, label=label) + return self.__rpc(payload, executor_prvkey, List[FileData]) + + def add_cron( + self, + cronname: str, + cronexpr: str, + wait: bool, + workflow: Workflow, + colonyname: str, + executor_prvkey: str, + random:bool=False + ) -> Cron: + cron = rpc.Cron( + name=cronname, + colonyname=colonyname, + interval=-1, # -1 means cron expression is used + waitforprevprocessgraph=wait, + cronexpression=cronexpr, + workflowspec=json.dumps(workflow.model_dump(by_alias=True)), + random=random, + ) + payload = rpc.AddCron(cron=cron) + return self.__rpc(payload, executor_prvkey, Cron) + + def get_cron(self, cronid: str, executor_prvkey: str) -> Cron: + payload = rpc.GetCron(cronid=cronid) + return self.__rpc(payload, executor_prvkey, Cron) + + def get_crons(self, colonyname: str, count: int, executor_prvkey: str) -> List[Cron]: + payload = rpc.GetCrons(colonyname=colonyname, count=count) + return self.__rpc(payload, executor_prvkey, List[Cron]) + + def del_cron(self, cronid: str, executor_prvkey: str, all: bool=False) -> None: + payload = rpc.RemoveCron(all=all, cronid=cronid) + self.__rpc(payload, executor_prvkey, Empty) + + def run_cron(self, cronid: str, executor_prvkey: str) -> Cron: + payload = rpc.RunCron(cronid=cronid) + return self.__rpc(payload, executor_prvkey, Cron) + + def resolve_generator_by_name(self, colonyname: str, generatorname: str, executor_prvkey: str) -> Generator: + payload = rpc.ResolveGenerator( + colonyname=colonyname, + generatorname=generatorname + ) + return self.__rpc(payload, executor_prvkey, Generator) + + def change_colony_id(self, colonyname: str, new_colony_id: str, server_prvkey: str) -> None: + payload = rpc.ChangeColonyID( + colonyname=colonyname, + colonyid=new_colony_id + ) + self.__rpc(payload, server_prvkey, Empty) + + def change_executor_id(self, colonyname: str, new_executor_id: str, colony_prvkey: str) -> None: + payload = rpc.ChangeExecutorID( + executorid=new_executor_id, + colonyname=colonyname + ) + self.__rpc(payload, colony_prvkey, Empty) + + def change_user_id(self, colonyname: str, new_user_id: str, colony_prvkey: str) -> None: + payload = rpc.ChangeUserID( + colonyname=colonyname, + userid=new_user_id + ) + self.__rpc(payload, colony_prvkey, Empty) + + def change_server_id(self, new_server_id: str, server_prvkey: str) -> None: + payload = rpc.ChangeServerID(serverid=new_server_id) + self.__rpc(payload, server_prvkey, Empty) + + def __generate_random_id(self) -> str: + random_uuid = uuid.uuid4() + hasher = hashlib.sha256() + hasher.update(random_uuid.bytes) + return hasher.hexdigest() + + def __checksum_file(self, file_path: str) -> str: + try: + with open(file_path, 'rb') as f: + buffer = bytearray(10000) + hasher = hashlib.sha256() + while True: + n = f.readinto(buffer) + if not n: + break + hasher.update(buffer[:n]) + return hasher.hexdigest() + except Exception as e: + raise e + + def __checksum_data(self, file_data: bytes) -> str: + try: + hasher = hashlib.sha256() + hasher.update(file_data) + return hasher.hexdigest() + except Exception as e: + raise e + + def __get_file_size(self, file_path: str) -> Optional[int]: + try: + size = os.path.getsize(file_path) + return size + except OSError as e: + print(f"Error getting file size: {e}") + return None + + def __check_bucket(self, s3_client: Any, bucket_name: str) -> None: + try: + s3_client.head_bucket(Bucket=bucket_name) + except ClientError as e: + error_code = e.response['Error']['Code'] + if error_code == '404': + try: + s3_client.create_bucket(Bucket=bucket_name) + except ClientError as e: + raise Exception(f"Error creating bucket: {e}") + else: + raise Exception(f"Error checking bucket: {e}") + + def upload_file( + self, + colonyname: str, + executor_prvkey: str, + filepath: str, + label: str + ) -> File: + return self.__upload_file(filepath, label, colonyname, executor_prvkey) + + def upload_data( + self, + colonyname: str, + executor_prvkey: str, + filename: str, + data: bytes, + label: str + ) -> File: + return self.__upload_file(filename, label, colonyname, executor_prvkey, file_bytes=data) + + def __upload_file( + self, + filepath: str, + label: str, + colonyname: str, + executor_prvkey: str, + file_bytes: Optional[bytes] = None + ) -> File: + endpoint = os.getenv("AWS_S3_ENDPOINT") + access_key = os.getenv("AWS_S3_ACCESSKEY") + secret_key = os.getenv("AWS_S3_SECRETKEY") + region = os.getenv("AWS_S3_REGION") + use_tls_str = os.getenv("AWS_S3_TLS") + bucket_name = os.getenv("AWS_S3_BUCKET") + skip_verify_str = os.getenv("AWS_S3_SKIPVERIFY") + + if endpoint is None: + raise ValueError("Environment variable AWS_S3_ENDPOINT is not set") + if access_key is None: + raise ValueError("Environment variable AWS_S3_ACCESSKEY is not set") + if secret_key is None: + raise ValueError("Environment variable AWS_S3_SECRETKEY is not set") + if region is None: + raise ValueError("Environment variable AWS_S3_REGION is not set") + if use_tls_str is None: + raise ValueError("Environment variable AWS_S3_TLS is not set") + if bucket_name is None: + raise ValueError("Environment variable AWS_S3_BUCKET is not set") + if skip_verify_str is None: + raise ValueError("Environment variable AWS_S3_SKIPVERIFY is not set") + + object_name = self.__generate_random_id() + if file_bytes is None: + filesize = self.__get_file_size(filepath) + if filesize is None: + raise ValueError(f"Could not get file size of {filepath}") + else: + filesize = len(file_bytes) + + endpoint_parts = endpoint.split(":") + if len(endpoint_parts) == 2: + server = endpoint_parts[0] + port = int(endpoint_parts[1]) + else: + raise Exception("invalid endpoint") + + use_tls = use_tls_str.lower() in ['true', '1', 'yes'] + + if not endpoint.startswith('http://') and not endpoint.startswith('https://'): + endpoint = f"http{'s' if use_tls else ''}://{endpoint}" + + + s3_client = boto3.client( + 's3', + endpoint_url=endpoint, + aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + region_name=region, + use_ssl=use_tls, + verify=skip_verify_str.lower() not in ['true', '1', 'yes'] + ) + + self.__check_bucket(s3_client, bucket_name) + + filename = os.path.basename(filepath) + + try: + if file_bytes is None: + s3_client.upload_file(filepath, bucket_name, object_name) + else: + # Upload byte array + s3_client.put_object(Bucket=bucket_name, Key=object_name, Body=file_bytes) + except Exception as e: + raise e + + if region == None: + region = "" + + if use_tls_str == "true": + tls = True + else: + tls = False + + obj = S3Object( + server=server, + port=port, + tls=tls, + accesskey="", + secretkey="", + region=region, + encryptionkey="", + encryptionalg="", + object=object_name, + bucket=bucket_name + ) + + ref = Reference( + protocol="s3", + s3object=obj + ) + + if file_bytes is None: + checksum = self.__checksum_file(filepath) + else: + checksum = self.__checksum_data(file_bytes) + + f = rpc.File( + colonyname=colonyname, + label=label, + name=filename, + size=filesize, + checksum=checksum, + checksumalg="SHA256", + ref=ref + ) + + payload = rpc.AddFile( + file=f + ) + + return self.__rpc(payload, executor_prvkey, File) + + def get_file( + self, + colonyname: str, + executor_prvkey: str, + label: Optional[str], + filename: Optional[str] = None, + fileid: Optional[str] = None, + latest: bool = True + ) -> List[File]: + if (fileid and filename) or (not fileid and not filename) or (not filename and not label): + raise ValueError("Please provide exactly one of: 'fileid' or ('filename' AND 'label')") + payload = rpc.GetFile( + colonyname=colonyname, + fileid=fileid, + label=label, + name=filename, + latest=latest + ) + return self.__rpc(payload, executor_prvkey, List[File]) + + def __remove_file( + self, + label: Optional[str], + fileid: Optional[str], + filename: Optional[str], + colonyname: str, + executor_prvkey: str + ) -> None: + if (fileid and filename) or (not fileid and not filename) or (not filename and not label): + raise ValueError("Please provide exactly one of: 'fileid' or ('filename' AND 'label')") + + payload = rpc.RemoveFile( + colonyname=colonyname, + fileid=fileid, + label=label, + name=filename + ) + self.__rpc(payload, executor_prvkey, Empty) + + def download_file( + self, + colonyname: str, + executor_prvkey: str, + dst: str, + filename: str, + label: Optional[str] = None, + fileid: Optional[str] = None, + latest: bool = True + ) -> str: + access_key = os.getenv("AWS_S3_ACCESSKEY") + secret_key = os.getenv("AWS_S3_SECRETKEY") + skip_verify_str = os.getenv("AWS_S3_SKIPVERIFY") + + dst = os.path.abspath(dst) + + try: + os.makedirs(dst, exist_ok=True) + except Exception as e: + raise e + + file = self.get_file(colonyname, executor_prvkey, label=label, fileid=fileid, filename=filename, latest=latest) + + if len(file) == 0: + raise Exception("invalid file") + + object_name = file[0].ref.s3object.object + region = file[0].ref.s3object.region + endpoint = file[0].ref.s3object.server + ":" + str(file[0].ref.s3object.port) + use_tls = file[0].ref.s3object.tls + bucket_name = file[0].ref.s3object.bucket + + verify = True + if skip_verify_str: + verify = skip_verify_str.lower() not in ['true', '1', 'yes'] + + if not endpoint.startswith('http://') and not endpoint.startswith('https://'): + endpoint = f"http{'s' if use_tls else ''}://{endpoint}" + + if region == "": + region = None + + s3_client = boto3.client( + 's3', + endpoint_url=endpoint, + aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + region_name=region, + use_ssl=use_tls, + verify=verify + ) + + dst = os.path.join(dst, filename) + + try: + s3_client.download_file(bucket_name, object_name, dst) + return dst + except Exception as e: + raise e + + def download_data( + self, + colonyname: str, + executor_prvkey: str, + label: Optional[str] = None, + filename: Optional[str] = None, + fileid: Optional[str] = None, + latest: bool = True + ) -> bytes: + access_key = os.getenv("AWS_S3_ACCESSKEY") + secret_key = os.getenv("AWS_S3_SECRETKEY") + skip_verify_str = os.getenv("AWS_S3_SKIPVERIFY") + + file = self.get_file(colonyname, executor_prvkey, label=label, fileid=fileid, filename=filename, latest=latest) + + if len(file) == 0: + raise Exception("invalid file") + + object_name = file[0].ref.s3object.object + region = file[0].ref.s3object.region + endpoint = file[0].ref.s3object.server + ":" + str(file[0].ref.s3object.port) + use_tls = file[0].ref.s3object.tls + bucket_name = file[0].ref.s3object.bucket + + verify = True + if skip_verify_str: + verify = skip_verify_str.lower() not in ['true', '1', 'yes'] + + if not endpoint.startswith('http://') and not endpoint.startswith('https://'): + endpoint = f"http{'s' if use_tls else ''}://{endpoint}" + + if region == "": + region = None + + try: + s3_client = boto3.client( + 's3', + endpoint_url=endpoint, + aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + region_name=region, + use_ssl=use_tls, + verify=verify + ) + except Exception as e: + raise e + + try: + response = s3_client.get_object(Bucket=bucket_name, Key=object_name) + data = response['Body'].read() + return data + except Exception as e: + raise e + + def delete_file( + self, + colonyname: str, + executor_prvkey: str, + filename: Optional[str] = None, + label: Optional[str] = None, + fileid: Optional[str] = None + ) -> None: + access_key = os.getenv("AWS_S3_ACCESSKEY") + secret_key = os.getenv("AWS_S3_SECRETKEY") + skip_verify_str = os.getenv("AWS_S3_SKIPVERIFY") + + file = self.get_file(colonyname, executor_prvkey, label=label, fileid=fileid, filename=filename) + + if len(file) == 0: + raise Exception("invalid file") + + object_name = file[0].ref.s3object.object + region = file[0].ref.s3object.region + endpoint = file[0].ref.s3object.server + ":" + str(file[0].ref.s3object.port) + use_tls = file[0].ref.s3object.tls + bucket_name = file[0].ref.s3object.bucket + + verify = True + if skip_verify_str: + verify = skip_verify_str.lower() not in ['true', '1', 'yes'] + + if not endpoint.startswith('http://') and not endpoint.startswith('https://'): + endpoint = f"http{'s' if use_tls else ''}://{endpoint}" + + if region == "": + region = None + + s3_client = boto3.client( + 's3', + endpoint_url=endpoint, + aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + region_name=region, + use_ssl=use_tls, + verify=verify + ) + + try: + s3_client.delete_object(Bucket=bucket_name, Key=object_name) + except Exception as e: + raise e + + self.__remove_file(label, fileid, filename, colonyname, executor_prvkey) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..0c1ac44 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +requests>=2.27.1 +websocket-client>=1.3.1 +pydantic>=2.6.4 +boto3>=1.34.136 diff --git a/rpc.py b/rpc.py new file mode 100644 index 0000000..0dc680d --- /dev/null +++ b/rpc.py @@ -0,0 +1,504 @@ +from typing import List, Any, Optional +from pydantic import Field + +from model import ( + Model, Colony, User, FuncSpec, Allocations, Gpu, Software, Workflow, + Reference +) + +# ============================================================================== +# Request and Response messages +# ============================================================================== + +class RequestPayload(Model): + msgtype: str + +class Request(Model): + payloadtype: str + payload: str + signature: str + +class ErrorResponse(Model): + status: str + message: str + +class Response(Model): + payloadtype: str + payload: str + +# ============================================================================== +# Payloads for API Requests (excluding server-managed fields) +# ============================================================================== + +class Hardware(Model): + """Client-provided hardware capabilities.""" + model: Optional[str] = None + nodes: int = 1 + cpu: Optional[str] = None + mem: Optional[str] = None + storage: Optional[str] = None + gpu: Optional[Gpu] = None + +class Capabilities(Model): + """Client-provided executor capabilities.""" + hardware: Optional[Hardware] = None + software: Optional[Software] = None + +class Executor(Model): + """Payload for adding a new executor. Excludes server-managed fields.""" + executorid: str + executortype: str + name: str = Field(..., alias="executorname") + colonyname: str + capabilities: Optional[Capabilities] = None + +class Attribute(Model): + """Payload for adding an attribute. Excludes server-generated fields.""" + targetid: str + attributetype: int + key: str + value: str + +class Function(Model): + """Payload for adding a function. Excludes server-generated fields.""" + executorname: str + colonyname: str + funcname: str + +class Generator(Model): + """Payload for adding a generator. Excludes server-generated fields.""" + colonyname: str + name: str + workflowspec: str + trigger: int + timeout: int + +class Cron(Model): + """Payload for adding a cron job. Excludes server-generated fields.""" + colonyname: str + name: str + cronexpression: str + interval: int + random: bool + workflowspec: str + waitforprevprocessgraph: bool + +class File(Model): + """Payload for adding a file. Excludes server-generated fields.""" + colonyname: str + label: str + name: str + size: int + checksum: str + checksumalg: str + ref: Reference + +# ============================================================================== +# Colony API Requests +# ============================================================================== + +class AddColony(RequestPayload): + colony: Colony + msgtype: str = "addcolonymsg" + +class RemoveColony(RequestPayload): + colonyname: str + msgtype: str = "removecolonymsg" + +class GetColonies(RequestPayload): + msgtype: str = "getcoloniesmsg" + +class GetColony(RequestPayload): + colonyname: str + msgtype: str = "getcolonymsg" + +# ============================================================================== +# Executor API Requests +# ============================================================================== + +class AddExecutor(RequestPayload): + executor: Executor + msgtype: str = "addexecutormsg" + +class GetExecutors(RequestPayload): + colonyname: str + msgtype: str = "getexecutorsmsg" + +class GetExecutor(RequestPayload): + colonyname: str + executorname: str + msgtype: str = "getexecutormsg" + +class ApproveExecutor(RequestPayload): + colonyname: str + executorname: str + msgtype: str = "approveexecutormsg" + +class RejectExecutor(RequestPayload): + colonyname: str + executorname: str + msgtype: str = "rejectexecutormsg" + +class RemoveExecutor(RequestPayload): + colonyname: str + executorname: str + msgtype: str = "removeexecutormsg" + +class ReportAllocations(RequestPayload): + colonyname: str + executorname: str + allocations: Allocations + msgtype: str = "reportallocationmsg" + +# ============================================================================== +# Process API Requests +# ============================================================================== + +class SubmitFunctionSpec(RequestPayload): + spec: FuncSpec + msgtype: str = "submitfuncspecmsg" + +class AssignProcess(RequestPayload): + colonyname: str + timeout: int + availablecpu: str + availablemem: str + msgtype: str = "assignprocessmsg" + +class GetProcesses(RequestPayload): + colonyname: str + count: int + state: int + executortype: Optional[str] = None + label: Optional[str] = None + initiator: Optional[str] = None + msgtype: str = "getprocessesmsg" + +class GetProcessHist(RequestPayload): + colonyname: str + executorid: str + seconds: int + state: int + msgtype: str = "getprocesshistmsg" + +class GetProcess(RequestPayload): + processid: str + msgtype: str = "getprocessmsg" + +class RemoveProcess(RequestPayload): + processid: str + all: bool + msgtype: str = "removeprocessmsg" + +class RemoveAllProcesses(RequestPayload): + colonyname: str + state: int + msgtype: str = "removeallprocessesmsg" + +class CloseSuccessful(RequestPayload): + processid: str + out: List[Any] + msgtype: str = "closesuccessfulmsg" + +class CloseFailed(RequestPayload): + processid: str + errors: List[str] + msgtype: str = "closefailedmsg" + +class SetOutput(RequestPayload): + processid: str + out: List[Any] + msgtype: str = "setoutputmsg" + +class AddAttribute(RequestPayload): + attribute: Attribute + msgtype: str = "addattributemsg" + +class GetAttribute(RequestPayload): + attributeid: str + msgtype: str = "getattributemsg" + +class SubscribeProcesses(RequestPayload): + colonyname: str + executortype: str + state: int + timeout: int + msgtype: str = "subscribeprocessesmsg" + +class SubscribeProcess(RequestPayload): + colonyname: str + processid: str + executortype: str + state: int + timeout: int + msgtype: str = "subscribeprocessmsg" + +# ============================================================================== +# Workflow & Process Graph API Requests +# ============================================================================== + +class SubmitWorkflowSpec(RequestPayload): + spec: Workflow + msgtype: str = "submitworkflowspecmsg" + +class AddChild(RequestPayload): + processgraphid: str + parentprocessid: str + childprocessid: str + spec: FuncSpec + insert: bool + msgtype: str = "addchildmsg" + +class GetProcessGraph(RequestPayload): + processgraphid: str + msgtype: str = "getprocessgraphmsg" + +class GetProcessGraphs(RequestPayload): + colonyname: str + count: int + state: int + msgtype: str = "getprocessgraphsmsg" + +class RemoveProcessGraph(RequestPayload): + processgraphid: str + all: bool + msgtype: str = "removeprocessgraphmsg" + +class RemoveAllProcessGraphs(RequestPayload): + colonyname: str + state: int + msgtype: str = "removeallprocessgraphsmsg" + +# ============================================================================== +# Cron API Requests +# ============================================================================== + +class AddCron(RequestPayload): + cron: Cron + msgtype: str = "addcronmsg" + +class GetCron(RequestPayload): + cronid: str + msgtype: str = "getcronmsg" + +class GetCrons(RequestPayload): + colonyname: str + count: int + msgtype: str = "getcronsmsg" + +class RunCron(RequestPayload): + cronid: str + msgtype: str = "runcronmsg" + +class RemoveCron(RequestPayload): + cronid: str + all: bool + msgtype: str = "removecronmsg" + +# ============================================================================== +# Generator API Requests +# ============================================================================== + +class AddGenerator(RequestPayload): + generator: Generator + msgtype: str = "addgeneratormsg" + +class GetGenerator(RequestPayload): + generatorid: str + msgtype: str = "getgeneratormsg" + +class ResolveGenerator(RequestPayload): + colonyname: str + generatorname: str + msgtype: str = "resolvegeneratormsg" + +class GetGenerators(RequestPayload): + colonyname: str + count: int + msgtype: str = "getgeneratorsmsg" + +class PackGenerator(RequestPayload): + generatorid: str + arg: str + msgtype: str = "packgeneratormsg" + +class RemoveGenerator(RequestPayload): + generatorid: str + all: bool + msgtype: str = "removegeneratormsg" + +# ============================================================================== +# File Management API Requests +# ============================================================================== + +class AddFile(RequestPayload): + file: File + msgtype: str = "addfilemsg" + +class GetFile(RequestPayload): + """ + Get a file by its ID or (label and name). + """ + colonyname: str + fileid: Optional[str] + label: Optional[str] + name: Optional[str] + latest: bool + msgtype: str = "getfilemsg" + +class GetFiles(RequestPayload): + colonyname: str + label: str + msgtype: str = "getfilesmsg" + +class GetFileLabels(RequestPayload): + colonyname: str + name: str + exact: bool + msgtype: str = "getfilelabelsmsg" + +class RemoveFile(RequestPayload): + """ + Remove a file by its ID or (label and name). + """ + colonyname: str + fileid: Optional[str] + label: Optional[str] + name: Optional[str] + msgtype: str = "removefilemsg" + +# ============================================================================== +# Logging API Requests +# ============================================================================== + +class AddLog(RequestPayload): + processid: str + message: str + msgtype: str = "addlogmsg" + +class GetLogs(RequestPayload): + colonyname: str + processid: str + executorname: str + count: int + since: int + msgtype: str = "getlogsmsg" + +class SearchLogs(RequestPayload): + colonyname: str + text: str + days: int + count: int + msgtype: str = "searchlogsmsg" + +# ============================================================================== +# User Management API Requests +# ============================================================================== + +class AddUser(RequestPayload): + user: User + msgtype: str = "addusermsg" + +class GetUser(RequestPayload): + colonyname: str + name: str + msgtype: str = "getusermsg" + +class GetUsers(RequestPayload): + colonyname: str + msgtype: str = "getusersmsg" + +class RemoveUser(RequestPayload): + colonyname: str + name: str + msgtype: str = "removeusermsg" + +# ============================================================================== +# Snapshot API Requests +# ============================================================================== + +class CreateSnapshot(RequestPayload): + colonyname: str + label: str + name: str + msgtype: str = "createsnapshotmsg" + +class GetSnapshot(RequestPayload): + """ + If `snapshotid` is provided, retrieves a specific snapshot. + If `name` is provided, retrieves the latest snapshot with that name. + If neither is provided, retrieves the latest snapshot. + """ + colonyname: str + snapshotid: Optional[str] = None + name: Optional[str] = None + msgtype: str = "getsnapshotmsg" + +class GetSnapshots(RequestPayload): + colonyname: str + msgtype: str = "getsnapshotsmsg" + +class RemoveSnapshot(RequestPayload): + colonyname: str + snapshotid: str + name: str + msgtype: str = "removesnapshotmsg" + +class RemoveAllSnapshots(RequestPayload): + colonyname: str + msgtype: str = "removeallsnapshotsmsg" + +# ============================================================================== +# Server & Miscellaneous API Requests +# ============================================================================== + +class GetCluster(RequestPayload): + msgtype: str = "getclustermsg" + +class Version(RequestPayload): + buildversion: str + buildtime: str + msgtype: str = "versionmsg" + +class GetStatistics(RequestPayload): + msgtype: str = "getstatisticsmsg" + +class GetColonyStatistics(RequestPayload): + colonyname: str + msgtype: str = "getcolonystatsmsg" + +class AddFunction(RequestPayload): + fun: Function + msgtype: str = "addfunctionmsg" + +class GetFunctions(RequestPayload): + colonyname: str + executorname: Optional[str] = None + msgtype: str = "getfunctionsmsg" + +class RemoveFunction(RequestPayload): + functionid: str + msgtype: str = "removefunctionmsg" + +# ============================================================================== +# ID Management Requests +# ============================================================================== + +class ChangeColonyID(RequestPayload): + colonyname: str + colonyid: str + msgtype: str = "changecolonyidmsg" + +class ChangeExecutorID(RequestPayload): + colonyname: str + executorid: str + msgtype: str = "changeexecutoridmsg" + +class ChangeServerID(RequestPayload): + serverid: str + msgtype: str = "changeserveridmsg" + +class ChangeUserID(RequestPayload): + colonyname: str + userid: str + msgtype: str = "changeuseridmsg" diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..443fa6b --- /dev/null +++ b/setup.py @@ -0,0 +1,28 @@ +import setuptools + +with open("README.md", "r") as fh: + long_description = fh.read() + +setuptools.setup( + name="pycolonies", + version="1.0.24", + author="Johan Kristiansson", + author_email="johan.kristiansson@ri.se", + description="Colonies Python SDK", + long_description=long_description, + py_modules=["pycolonies", "crypto", "cfs", "model", "rpc"], + long_description_content_type="text/markdown", + url="https://github.com/colonyos/pycolonies", + packages=setuptools.find_packages(), + classifiers=[ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + ], + install_requires=[ + "requests>=2.27.1", + "websocket-client>=1.3.1", + "boto3>=1.34.136", + "pydantic>=2.6.4" + ] +) diff --git a/test/colonies_test.py b/test/colonies_test.py new file mode 100644 index 0000000..50daee3 --- /dev/null +++ b/test/colonies_test.py @@ -0,0 +1,685 @@ +import unittest +import string +import random +import sys +from typing import Tuple +sys.path.append(".") +from crypto import Crypto +from pycolonies import Colonies +from model import Executor, FuncSpec, Conditions, Workflow, Process, Colony +import os + +test_colony_host = os.environ.get('TEST_COLONY_HOST', 'localhost') +test_colony_prvkey = os.environ.get('TEST_COLONY_PRVKEY', 'fcc79953d8a751bf41db661592dc34d30004b1a651ffa0725b03ac227641499d') + +class TestColonies(unittest.TestCase): + def setUp(self) -> None: + self.colonies = Colonies(test_colony_host, 50080, tls=False, native_crypto=False) + self.crypto = Crypto(native=False) + self.server_prv = test_colony_prvkey + + def ran_prefix(self) -> str: + return "".join(random.choices(string.ascii_uppercase + string.digits, k=10)) + + def add_test_colony(self) -> Tuple[Colony, str, str, str]: + colony_prvkey = self.crypto.prvkey() + colonyid = self.crypto.id(colony_prvkey) + colony = Colony(colonyid=colonyid, name="python-test-" + self.ran_prefix()) + + return ( + self.colonies.add_colony(colony, self.server_prv), + colonyid, + colony.name, + colony_prvkey, + ) + + def add_test_executor(self, colonyname: str, colony_prvkey: str) -> Tuple[Executor, str, str, str]: + executor_prvkey = self.crypto.prvkey() + executorid = self.crypto.id(executor_prvkey) + executorname = "test-executor-" + self.ran_prefix() + + return ( + self.colonies.add_executor( + executorid=executorid, + executorname=executorname, + colonyname=colonyname, + executortype="test-executor-type", + colony_prvkey=colony_prvkey + ), + executorid, + executorname, + executor_prvkey, + ) + + def submit_test_funcspec(self, colonyname: str, executor_prvkey: str) -> Process: + spec = FuncSpec.create( + func="test_func", + colonyname=colonyname, + executortype="test-executor-type", + args=[], + env={"test_key": "test_value2"}, + maxexectime=-1, + maxretries=3, + ) + + return self.colonies.submit_func_spec(spec, executor_prvkey) + + def test_add_colony(self) -> None: + added_colony, _, colonyname, _ = self.add_test_colony() + self.assertEqual(added_colony.name, colonyname) + self.colonies.del_colony(colonyname, self.server_prv) + + def test_del_colony(self) -> None: + _, _, colonyname, _ = self.add_test_colony() + colonies_from_server = self.colonies.list_colonies(self.server_prv) + + found_colony = False + for colony in colonies_from_server: + if colony.name == colonyname: + found_colony = True + self.assertTrue(found_colony) + + self.colonies.del_colony(colonyname, self.server_prv) + colonies_from_server = self.colonies.list_colonies(self.server_prv) + + found_colony = False + for colony in colonies_from_server: + if colony.name == colonyname: + found_colony = True + self.assertFalse(found_colony) + + def test_add_executor(self) -> None: + _, _, colonyname, colony_prvkey = self.add_test_colony() + added_executor, executorid, _, _ = self.add_test_executor( + colonyname, colony_prvkey + ) + + self.assertEqual(executorid, added_executor.executorid) + self.colonies.del_colony(colonyname, self.server_prv) + + def test_approve_executor(self) -> None: + _, _, colonyname, colony_prvkey = self.add_test_colony() + _, _, executorname, executor_prvkey = self.add_test_executor( + colonyname, colony_prvkey + ) + + self.colonies.approve_executor(colonyname, executorname, colony_prvkey) + executors_from_server = self.colonies.list_executors( + colonyname, executor_prvkey + ) + self.assertEqual(executors_from_server[0].state, 1) + + self.colonies.del_colony(colonyname, self.server_prv) + + def test_reject_executor(self) -> None: + _, _, colonyname, colony_prvkey = self.add_test_colony() + _, _, executorname, _ = self.add_test_executor(colonyname, colony_prvkey) + self.colonies.approve_executor(colonyname, executorname, colony_prvkey) + self.colonies.reject_executor(colonyname, executorname, colony_prvkey) + + self.colonies.del_colony(colonyname, self.server_prv) + + def test_list_executors(self) -> None: + _, _, colonyname, colony_prvkey = self.add_test_colony() + _, _, executorname, executor_prvkey = self.add_test_executor( + colonyname, colony_prvkey + ) + self.colonies.approve_executor(colonyname, executorname, colony_prvkey) + executors_from_server = self.colonies.list_executors( + colonyname, executor_prvkey + ) + self.assertEqual(executors_from_server[0].state, 1) + + executors_from_server = self.colonies.list_executors( + colonyname, executor_prvkey + ) + self.assertEqual(executors_from_server[0].name, executorname) + + self.colonies.del_colony(colonyname, self.server_prv) + + def test_remove_executor(self) -> None: + _, _, colonyname, colony_prvkey = self.add_test_colony() + _, _, executorname, _ = self.add_test_executor(colonyname, colony_prvkey) + + self.colonies.remove_executor(colonyname, executorname, colony_prvkey) + + self.colonies.del_colony(colonyname, self.server_prv) + + def test_submit(self) -> None: + _, _, colonyname, colony_prvkey = self.add_test_colony() + _, _, executorname, executor_prvkey = self.add_test_executor( + colonyname, colony_prvkey + ) + self.colonies.approve_executor(colonyname, executorname, colony_prvkey) + process = self.submit_test_funcspec(colonyname, executor_prvkey) + self.assertEqual(process.state, 0) + + self.colonies.del_colony(colonyname, self.server_prv) + + def test_submit_workflow(self) -> None: + _, _, colonyname, colony_prvkey = self.add_test_colony() + _, _, executorname, executor_prvkey = self.add_test_executor( + colonyname, colony_prvkey + ) + self.colonies.approve_executor(colonyname, executorname, colony_prvkey) + workflow = Workflow( + colonyname=colonyname, + functionspecs=[ + FuncSpec.create( + func="test_func", + args=[], + nodename="node1", + colonyname=colonyname, + executortype="test-executor-type", + env={"test_key": "test_value2"}, + maxexectime=-1, + maxretries=3, + ), + FuncSpec.create( + func="test_func", + args=[], + nodename="node2", + colonyname=colonyname, + executortype="test-executor-type", + dependencies=["node1"], + env={"test_key": "test_value2"}, + maxexectime=-1, + maxretries=3, + ), + ], + ) + process = self.colonies.submit_workflow(workflow, executor_prvkey) + self.assertEqual(process.state, 0) + + self.colonies.del_colony(colonyname, self.server_prv) + + def test_assign(self) -> None: + _, _, colonyname, colony_prvkey = self.add_test_colony() + _, _, executorname, executor_prvkey = self.add_test_executor( + colonyname, colony_prvkey + ) + self.colonies.approve_executor(colonyname, executorname, colony_prvkey) + process = self.submit_test_funcspec(colonyname, executor_prvkey) + + assigned_process = self.colonies.assign(colonyname, 10, executor_prvkey) + self.assertEqual(assigned_process.processid, process.processid) + + self.colonies.del_colony(colonyname, self.server_prv) + + def test_list_process(self) -> None: + _, _, colonyname, colony_prvkey = self.add_test_colony() + _, _, executorname, executor_prvkey = self.add_test_executor( + colonyname, colony_prvkey + ) + self.colonies.approve_executor(colonyname, executorname, colony_prvkey) + + self.submit_test_funcspec(colonyname, executor_prvkey) + self.submit_test_funcspec(colonyname, executor_prvkey) + + waiting_processes = self.colonies.list_processes( + colonyname, 2, Colonies.WAITING, executor_prvkey + ) + self.assertEqual(len(waiting_processes), 2) + + self.colonies.del_colony(colonyname, self.server_prv) + + def test_get_process(self) -> None: + _, _, colonyname, colony_prvkey = self.add_test_colony() + _, _, executorname, executor_prvkey = self.add_test_executor( + colonyname, colony_prvkey + ) + self.colonies.approve_executor(colonyname, executorname, colony_prvkey) + + submitted_process = self.submit_test_funcspec(colonyname, executor_prvkey) + + process = self.colonies.get_process( + submitted_process.processid, executor_prvkey + ) + self.assertEqual(process.processid, submitted_process.processid) + + self.colonies.del_colony(colonyname, self.server_prv) + + def test_remove_process(self) -> None: + _, _, colonyname, colony_prvkey = self.add_test_colony() + _, _, executorname, executor_prvkey = self.add_test_executor( + colonyname, colony_prvkey + ) + self.colonies.approve_executor(colonyname, executorname, colony_prvkey) + + submitted_process = self.submit_test_funcspec(colonyname, executor_prvkey) + + self.colonies.remove_process(submitted_process.processid, executor_prvkey) + + with self.assertRaises(Exception): + self.colonies.get_process(submitted_process.processid, executor_prvkey) + + self.colonies.del_colony(colonyname, self.server_prv) + + # + def test_close_process(self) -> None: + _, _, colonyname, colony_prvkey = self.add_test_colony() + _, _, executorname, executor_prvkey = self.add_test_executor( + colonyname, colony_prvkey + ) + self.colonies.approve_executor(colonyname, executorname, colony_prvkey) + + submitted_process1 = self.submit_test_funcspec(colonyname, executor_prvkey) + submitted_process2 = self.submit_test_funcspec(colonyname, executor_prvkey) + self.submit_test_funcspec(colonyname, executor_prvkey) + self.submit_test_funcspec(colonyname, executor_prvkey) + + self.colonies.assign(colonyname, 10, executor_prvkey) + self.colonies.assign(colonyname, 10, executor_prvkey) + self.colonies.assign(colonyname, 10, executor_prvkey) + + self.colonies.close(submitted_process1.processid, [], executor_prvkey) + self.colonies.fail(submitted_process2.processid, [], executor_prvkey) + + waiting_processes = self.colonies.list_processes( + colonyname, 2, Colonies.WAITING, executor_prvkey + ) + running_processes = self.colonies.list_processes( + colonyname, 2, Colonies.RUNNING, executor_prvkey + ) + successful_processes = self.colonies.list_processes( + colonyname, 2, Colonies.SUCCESSFUL, executor_prvkey + ) + failed_processes = self.colonies.list_processes( + colonyname, 2, Colonies.FAILED, executor_prvkey + ) + + self.assertEqual(len(waiting_processes), 1) + self.assertEqual(len(running_processes), 1) + self.assertEqual(len(successful_processes), 1) + self.assertEqual(len(failed_processes), 1) + + self.colonies.del_colony(colonyname, self.server_prv) + + def test_stats(self) -> None: + _, _, colonyname, colony_prvkey = self.add_test_colony() + _, _, executorname, executor_prvkey = self.add_test_executor( + colonyname, colony_prvkey + ) + self.colonies.approve_executor(colonyname, executorname, colony_prvkey) + + submitted_process1 = self.submit_test_funcspec(colonyname, executor_prvkey) + submitted_process2 = self.submit_test_funcspec(colonyname, executor_prvkey) + self.submit_test_funcspec(colonyname, executor_prvkey) + self.submit_test_funcspec(colonyname, executor_prvkey) + + self.colonies.assign(colonyname, 10, executor_prvkey) + self.colonies.assign(colonyname, 10, executor_prvkey) + self.colonies.assign(colonyname, 10, executor_prvkey) + + self.colonies.close(submitted_process1.processid, [], executor_prvkey) + self.colonies.fail(submitted_process2.processid, [], executor_prvkey) + + waiting_processes = self.colonies.list_processes( + colonyname, 2, 0, executor_prvkey + ) + running_processes = self.colonies.list_processes( + colonyname, 2, 1, executor_prvkey + ) + successful_processes = self.colonies.list_processes( + colonyname, 2, 2, executor_prvkey + ) + failed_processes = self.colonies.list_processes( + colonyname, 2, 3, executor_prvkey + ) + + stats = self.colonies.stats(colonyname, executor_prvkey) + + self.assertEqual(len(waiting_processes), stats.waitingprocesses) + self.assertEqual(len(running_processes), stats.runningprocesses) + self.assertEqual(len(successful_processes), stats.successfulprocesses) + self.assertEqual(len(failed_processes), stats.failedprocesses) + + self.colonies.del_colony(colonyname, self.server_prv) + + def test_set_output(self) -> None: + _, _, colonyname, colony_prvkey = self.add_test_colony() + _, _, executorname, executor_prvkey = self.add_test_executor( + colonyname, colony_prvkey + ) + self.colonies.approve_executor(colonyname, executorname, colony_prvkey) + + self.submit_test_funcspec(colonyname, executor_prvkey) + assigned_process = self.colonies.assign(colonyname, 10, executor_prvkey) + self.colonies.set_output( + assigned_process.processid, ["output1", "output2"], executor_prvkey + ) + + process_from_server = self.colonies.get_process( + assigned_process.processid, executor_prvkey + ) + self.assertIsNotNone(process_from_server.output) + if process_from_server.output is not None: + self.assertTrue(len(process_from_server.output) == 2) + self.assertTrue(process_from_server.output[0] == "output1") + self.assertTrue(process_from_server.output[1] == "output2") + + def test_add_attribute(self) -> None: + _, _, colonyname, colony_prvkey = self.add_test_colony() + _, executorname, executorname, executor_prvkey = self.add_test_executor( + colonyname, colony_prvkey + ) + self.colonies.approve_executor(colonyname, executorname, colony_prvkey) + + submitted_process = self.submit_test_funcspec(colonyname, executor_prvkey) + self.colonies.assign(colonyname, 10, executor_prvkey) + + self.colonies.add_attribute( + submitted_process.processid, "py_test_key", "py_test_value", executor_prvkey + ) + + process = self.colonies.get_process( + submitted_process.processid, executor_prvkey + ) + self.assertIsNotNone(process.attributes) + found = False + if process.attributes is not None: + for attr in process.attributes: + if attr.key == "py_test_key" and attr.value == "py_test_value": + found = True + self.assertTrue(found) + + self.colonies.del_colony(colonyname, self.server_prv) + + def test_get_attribute(self) -> None: + _, _, colonyname, colony_prvkey = self.add_test_colony() + _, _, executorname, executor_prvkey = self.add_test_executor( + colonyname, colony_prvkey + ) + self.colonies.approve_executor(colonyname, executorname, colony_prvkey) + + submitted_process = self.submit_test_funcspec(colonyname, executor_prvkey) + self.colonies.assign(colonyname, 10, executor_prvkey) + + attribute = self.colonies.add_attribute( + submitted_process.processid, "py_test_key", "py_test_value", executor_prvkey + ) + attribute_from_server = self.colonies.get_attribute(attribute.id, executor_prvkey) + self.assertEqual(attribute_from_server.id, attribute.id) + + self.colonies.del_colony(colonyname, self.server_prv) + + def test_add_function(self) -> None: + _, _, colonyname, colony_prvkey = self.add_test_colony() + _, _, executorname, executor_prvkey = self.add_test_executor( + colonyname, colony_prvkey + ) + self.colonies.approve_executor(colonyname, executorname, colony_prvkey) + + self.colonies.add_function( + colonyname, executorname, "funcname", executor_prvkey + ) + + self.colonies.del_colony(colonyname, self.server_prv) + + def test_get_functions_by_colony(self) -> None: + _, _, colonyname, colony_prvkey = self.add_test_colony() + _, _, executorname, executor_prvkey = self.add_test_executor( + colonyname, colony_prvkey + ) + self.colonies.approve_executor(colonyname, executorname, colony_prvkey) + + self.colonies.add_function( + colonyname, executorname, "funcname", executor_prvkey + ) + functions = self.colonies.get_functions_by_colony(colonyname, executor_prvkey) + self.assertTrue(len(functions) == 1) + self.assertEqual(functions[0].funcname, "funcname") + + def test_get_functions_by_executor(self) -> None: + _, _, colonyname, colony_prvkey = self.add_test_colony() + _, _, executorname, executor_prvkey = self.add_test_executor( + colonyname, colony_prvkey + ) + self.colonies.approve_executor(colonyname, executorname, colony_prvkey) + + self.colonies.add_function( + colonyname, executorname, "funcname", executor_prvkey + ) + functions = self.colonies.get_functions_by_executor( + colonyname, executorname, executor_prvkey + ) + self.assertTrue(len(functions) == 1) + self.assertEqual(functions[0].funcname, "funcname") + + self.colonies.del_colony(colonyname, self.server_prv) + + def test_create_snapshot(self) -> None: + _, _, colonyname, colony_prvkey = self.add_test_colony() + _, _, executorname, executor_prvkey = self.add_test_executor( + colonyname, colony_prvkey + ) + self.colonies.approve_executor(colonyname, executorname, colony_prvkey) + + self.colonies.create_snapshot( + colonyname, "test_label", "test_name", executor_prvkey + ) + snapshots = self.colonies.get_snapshots(colonyname, executor_prvkey) + self.assertTrue(len(snapshots) == 1) + snapshot = self.colonies.get_snapshot_by_name( + colonyname, "test_name", executor_prvkey + ) + self.assertEqual(snapshot.name, "test_name") + snapshot2 = self.colonies.get_snapshot_by_id( + colonyname, snapshot.snapshotid, executor_prvkey + ) + self.assertEqual(snapshot2.name, "test_name") + + self.colonies.del_colony(colonyname, self.server_prv) + + def test_get_add_logs(self) -> None: + _, _, colonyname, colony_prvkey = self.add_test_colony() + _, _, executorname, executor_prvkey = self.add_test_executor( + colonyname, colony_prvkey + ) + self.colonies.approve_executor(colonyname, executorname, colony_prvkey) + + func_spec = FuncSpec.create( + func="test_func", + args=[], + colonyname=colonyname, + executortype="test-executor-type", + env={"test_key": "test_value2"}, + maxexectime=-1, + maxretries=3, + ) + + self.colonies.submit_func_spec(func_spec, executor_prvkey) + assigned_process = self.colonies.assign(colonyname, 10, executor_prvkey) + + self.colonies.add_log( + assigned_process.processid, "test_log_msg", executor_prvkey + ) + logs = self.colonies.get_process_log( + colonyname, assigned_process.processid, 100, -1, executor_prvkey + ) + self.assertTrue(len(logs) == 1) + self.assertEqual(logs[0].message, "test_log_msg") + + logs = self.colonies.get_executor_log( + colonyname, executorname, 100, -1, executor_prvkey + ) + self.assertTrue(len(logs) == 1) + self.assertEqual(logs[0].message, "test_log_msg") + + self.colonies.del_colony(colonyname, self.server_prv) + + def test_sync(self) -> None: + _, _, colonyname, colony_prvkey = self.add_test_colony() + _, _, executorname, executor_prvkey = self.add_test_executor( + colonyname, colony_prvkey + ) + self.colonies.approve_executor(colonyname, executorname, colony_prvkey) + + testdir = "/tmp/testdir" + str(random.randint(0, 1000000)) + os.system("mkdir -p " + testdir) + os.system("echo hello > " + testdir + "/hello.txt") + + self.colonies.sync(testdir, "/test", False, colonyname, executor_prvkey) + + testdir2 = "/tmp/testdir" + str(random.randint(0, 1000000)) + os.system("mkdir -p " + testdir2) + self.colonies.sync(testdir2, "/test", False, colonyname, executor_prvkey) + + # check if the files are the same + f = open(testdir + "/hello.txt", "r") + hello = f.read() + f.close() + self.assertEqual(hello, "hello\n") + + self.colonies.del_colony(colonyname, self.server_prv) + + def test_get_files(self) -> None: + _, _, colonyname, colony_prvkey = self.add_test_colony() + _, _, executorname, executor_prvkey = self.add_test_executor( + colonyname, colony_prvkey + ) + self.colonies.approve_executor(colonyname, executorname, colony_prvkey) + + testdir = "/tmp/testdir" + str(random.randint(0, 1000000)) + os.system("mkdir -p " + testdir) + os.system("echo hello > " + testdir + "/hello.txt") + + self.colonies.sync(testdir, "/test", False, colonyname, executor_prvkey) + + testdir2 = "/tmp/testdir" + str(random.randint(0, 1000000)) + os.system("mkdir -p " + testdir2) + self.colonies.sync(testdir2, "/test", False, colonyname, executor_prvkey) + + files = self.colonies.get_files("/test", colonyname, executor_prvkey) + assert len(files) == 1 + + self.colonies.del_colony(colonyname, self.server_prv) + + def test_upload_file(self) -> None: + _, _, colonyname, colony_prvkey = self.add_test_colony() + _, _, executorname, executor_prvkey = self.add_test_executor( + colonyname, colony_prvkey + ) + self.colonies.approve_executor(colonyname, executorname, colony_prvkey) + + srcdir = "/tmp/srcdir" + str(random.randint(0, 1000000)) + os.system("mkdir -p " + srcdir) + os.system("echo hello > " + srcdir + "/hello.txt") + + dstdir = "/tmp/dstdir" + str(random.randint(0, 1000000)) + os.system("mkdir -p " + dstdir) + + filepath = srcdir + "/hello.txt" + self.colonies.upload_file(colonyname, executor_prvkey, filepath=filepath, label="/test") + + dst = self.colonies.download_file(colonyname, executor_prvkey, dst=dstdir, label="/test", filename="hello.txt") + assert dst == dstdir + "/hello.txt" + + f = open(dstdir + "/hello.txt", "r") + hello = f.read() + f.close() + self.assertEqual(hello, "hello\n") + + self.colonies.del_colony(colonyname, self.server_prv) + + def test_upload_data(self) -> None: + _, _, colonyname, colony_prvkey = self.add_test_colony() + _, _, executorname, executor_prvkey = self.add_test_executor( + colonyname, colony_prvkey + ) + self.colonies.approve_executor(colonyname, executorname, colony_prvkey) + + data = b"testdata" + self.colonies.upload_data(colonyname, executor_prvkey, filename="data", label="/testdata", data=data) + + data = self.colonies.download_data(colonyname, executor_prvkey, label="/testdata", filename="data") + data_str = data.decode('utf-8') + assert data_str == "testdata" + + self.colonies.del_colony(colonyname, self.server_prv) + + def test_get_file(self) -> None: + _, _, colonyname, colony_prvkey = self.add_test_colony() + _, _, executorname, executor_prvkey = self.add_test_executor( + colonyname, colony_prvkey + ) + self.colonies.approve_executor(colonyname, executorname, colony_prvkey) + + data = b"testdata" + self.colonies.upload_data(colonyname, executor_prvkey, filename="data", label="/testdata", data=data) + + file = self.colonies.get_file(colonyname, executor_prvkey, label="/testdata", filename="data") + assert len(file) == 1 + assert file[0].name == "data" + + self.colonies.del_colony(colonyname, self.server_prv) + + def test_add_cron(self) -> None: + _, _, colonyname, colony_prvkey = self.add_test_colony() + _, _, executorname, executor_prvkey = self.add_test_executor( + colonyname, colony_prvkey + ) + self.colonies.approve_executor(colonyname, executorname, colony_prvkey) + workflow = Workflow( + colonyname=colonyname, + functionspecs=[ + FuncSpec.create( + func="test_func", + args=[], + nodename="node1", + colonyname=colonyname, + executortype="test-executor-type", + env={"test_key": "test_value2"}, + maxexectime=-1, + maxretries=3, + ), + FuncSpec.create( + func="test_func", + args=[], + nodename="node2", + conditions=Conditions( + colonyname=colonyname, + executortype="test-executor-type", + dependencies=["node1"], + ), + env={"test_key": "test_value2"}, + maxexectime=-1, + maxretries=3, + ), + ], + ) + name = "test_cron" + self.ran_prefix() + cron = self.colonies.add_cron(name, "0/1 * * * * *", True, workflow, colonyname, executor_prvkey) + + assert cron.name == name + + cron2 = self.colonies.get_cron(cron.cronid, executor_prvkey) + assert cron2.name == name + + name = "test_cron" + self.ran_prefix() + cron = self.colonies.add_cron(name, "0/1 * * * * *", False, workflow, colonyname, executor_prvkey) + + crons = self.colonies.get_crons(colonyname, 10, executor_prvkey) + assert len(crons) == 2 + + self.colonies.del_cron(cron.cronid, executor_prvkey) + + crons = self.colonies.get_crons(colonyname, 10, executor_prvkey) + assert len(crons) == 1 + + self.colonies.del_colony(colonyname, self.server_prv) + + def test_download_file_raise_value_error_for_conflicting_parameters(self) -> None: + with self.assertRaises(ValueError) as err: + self.colonies.download_file("test", "prvkey", fileid="123", filename="filename", dst="/tmp", label="/test") + + self.assertEqual("Both 'fileid' and 'filename' cannot be set at the same time. Please provide only one.", str(err.exception)) + + def test_download_data_raise_value_error_for_conflicting_parameters(self) -> None: + with self.assertRaises(ValueError) as err: + self.colonies.download_data("test", "prvkey", fileid="123", filename="filename", label="/test") + + self.assertEqual("Both 'fileid' and 'filename' cannot be set at the same time. Please provide only one.", str(err.exception)) + +if __name__ == "__main__": + unittest.main() diff --git a/test/crypto_test.py b/test/crypto_test.py new file mode 100644 index 0000000..07049c0 --- /dev/null +++ b/test/crypto_test.py @@ -0,0 +1,60 @@ +import unittest +import sys +sys.path.append(".") +from crypto import Crypto +from crypto import jacobian_add, jacobian_double, fast_multiply + +class TestCrypto(unittest.TestCase): + def test_prvkey(self) -> None: + crypto = Crypto() + prvkey = crypto.prvkey() + self.assertEqual(len(prvkey), 64) + + def test_id(self) -> None: + crypto = Crypto() + i = crypto.id("6d2fb6f546bacfd98c68769e61e0b44a697a30596c018a50e28200aa59b01c0a") + self.assertEqual(i, "4fef2b5a82d134d058c1883c72d6d9caf77cd59ca82d73105017590dea3dcb87") + + def test_jacobian_add(self) -> None: + p1 = 9145974245324100229099870468775465651310464820817378424695723232290407343942 + p2 = 5726454693002325744504615879224937090641195997533856518133185097441749801032 + p3 = 115714549703150523321131187169862203539915631312738481595605540015431713717331 + p = (p1, p2, p3) + + q1 = 3378859141843082240981311929530924778908494294056496383285600481501351521548 + q2 = 27521306930728475164406447156615413460758360212583572363332152141481614403438 + q3 = 49323301439068515073562494645799725679211443313890051705798536862743810731758 + q = (q1, q2, q3) + + r = jacobian_add(p, q) + + self.assertEqual(r, (3839523019051154503084769099381507415584753837414379863264960425500703565923, 20189644703747003499840421980750524561734367487063517298774288662171391490014, 60937054099961058364101483468792603644143119999146283853926532658309628838553)) + + def test_jacobian_double(self) -> None: + p1 = 9145974245324100229099870468775465651310464820817378424695723232290407343942 + p2 = 5726454693002325744504615879224937090641195997533856518133185097441749801032 + p3 = 115714549703150523321131187169862203539915631312738481595605540015431713717331 + p = (p1, p2, p3) + + r = jacobian_double(p) + self.assertEqual(r, (47799865997534219673197337605336645889814818209350362461488238402445197905424, 60263407694755846743636806277114588321762727181244820843669538143699156517749, 98392693047863901823354926635876248059211737341107593029860323683906341241571)) + + def test_fast_multiply(self) -> None: + p1 = 9145974245324100229099870468775465651310464820817378424695723232290407343942 + p2 = 5726454693002325744504615879224937090641195997533856518133185097441749801032 + p = (p1, p2) + n = 49323301439068515073562494645799725679211443313890051705798536862743810731758 + r = fast_multiply(p, n) + self.assertEqual(r, (55168891259068323847970500732782990269643885682720201005538882429359294222592, 24653118739118393505255051840680624663656725984701285210882487021736401159116)) + + def test_sign(self) -> None: + crypto = Crypto() + prvkey = "d6eb959e9aec2e6fdc44b5862b269e987b8a4d6f2baca542d8acaa97ee5e74f6" + signature_hex = "e713a1bb015fecabb5a084b0fe6d6e7271fca6f79525a634183cfdb175fe69241f4da161779d8e6b761200e1cf93766010a19072fa778f9643363e2cfadd640900" + data = "hello" + sig = crypto.sign(data, prvkey) + self.assertEqual(len(sig), 130) + self.assertEqual(sig, signature_hex) + +if __name__ == '__main__': + unittest.main() diff --git a/test/func_spec_test.py b/test/func_spec_test.py new file mode 100644 index 0000000..d089cd5 --- /dev/null +++ b/test/func_spec_test.py @@ -0,0 +1,43 @@ +import unittest + +from pycolonies import FuncSpec + + +class TestFuncSpec(unittest.TestCase): + def test_func_spec_sets_simple_properties(self) -> None: + + kwargs = { + "one": "1", + "two": "2", + "three": ["3", "4"] + } + + spec = FuncSpec.create( + func="sum_nums", + args=["helloworld"], + colonyname="colonyname", + executortype="echo_executor", + executorname="exec_name", + priority=0, + maxexectime=10, + maxretries=3, + maxwaittime=100, + kwargs=kwargs + ) + + self.assertEqual(spec.nodename, "sum_nums") + self.assertEqual(spec.funcname, "sum_nums") + self.assertEqual(spec.args, ["helloworld"]) + + assert spec.conditions is not None + self.assertEqual(spec.conditions.colonyname, "colonyname") + self.assertEqual(spec.conditions.executortype, "echo_executor") + if spec.conditions.executornames is not None: + self.assertEqual(spec.conditions.executornames[0], "exec_name") + + self.assertEqual(spec.priority, 0) + self.assertEqual(spec.maxexectime, 10) + self.assertEqual(spec.maxretries, 3) + self.assertEqual(spec.maxwaittime, 100) + + self.assertEqual(spec.kwargs, kwargs) diff --git a/test/model_test.py b/test/model_test.py new file mode 100644 index 0000000..5b4f69d --- /dev/null +++ b/test/model_test.py @@ -0,0 +1,21 @@ +import unittest + +from pycolonies import File, Reference, S3Object + + +class TestModel(unittest.TestCase): + def test_file_ensures_single_slash(self) -> None: + object = S3Object(server="server", port=123, tls=False, accesskey="key", secretkey="secret", + region="region", encryptionkey="enckey", encryptionalg="alg", object="object", bucket="bucket") + + reference = Reference(protocol="proto", s3object=object) + + file_with_auto_appended_slash = File(fileid="id", colonyname="testcolony", label="filelabel", + name="filename", size=100, sequencenr=1, checksum="cheksum", checksumalg="alg", ref=reference, added=None) + + assert "/filelabel" == file_with_auto_appended_slash.label + + file_with_single_leading_slash = file_with_auto_appended_slash = File(fileid="id", colonyname="testcolony", label="///filelabel", + name="filename", size=100, sequencenr=1, checksum="cheksum", checksumalg="alg", ref=reference, added=None) + + assert "/filelabel" == file_with_single_leading_slash.label