diff --git a/.gitignore b/.gitignore index 06f3522e..4cbba462 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,46 @@ -# Python files +# {# pkglts, github +# Byte-compiled / optimized / DLL files +__pycache__/ *.py[cod] +# pkglts files +.pkglts/info.log* + +# Packages +*.egg +*.egg-info +.eggs +.Python +*.pth +dist/ +build/ +env/ +downloads/ +eggs/ +parts/ +bin/ +var/ +sdist/ +develop-eggs/ +.installed.cfg +lib/ +lib64/ + +# editors +.idea/ +.vscode/ + +# Vim files +*.swp +*.*~ + +# Mr Developer +.mr.developer.cfg +.project +.pydevproject +.settings + + # C extensions *.so *.dll @@ -14,54 +54,45 @@ # Compiled Object files *.os -# Packages -*.egg -*.egg-info -dist -build -build-scons -eggs -parts -bin -var -sdist -develop-eggs -.installed.cfg -lib -lib64 -__pycache__ -*.pth +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec # Installer logs pip-log.txt +pip-delete-this-directory.txt .amlog .sconsign.dblite -# Unit test / coverage reports -.coverage -.tox -nosetests.xml - -# Designer files -*visualea/src/visualea/ui_* - # Translations *.mo +*.pot -# Vim files -*.swp -*.*~ +# Django stuff: +*.log -# Mr Developer -.mr.developer.cfg -.project -.pydevproject -.settings -.idea +# PyBuilder +target/ + +# jupyter notebooks +.ipynb_checkpoints/ # svn .svn -# temporary files -*._icon.png +# coverage +.coverage + +# sphinx autogen file +doc/_dvlpt/ + + +# #} + +# user custom filters +# ignore cassandra files +.cassandraWorkbench* +.vscode/ \ No newline at end of file diff --git a/.pkglts/pkg_cfg.json b/.pkglts/pkg_cfg.json index b06c16db..e7cbdcee 100644 --- a/.pkglts/pkg_cfg.json +++ b/.pkglts/pkg_cfg.json @@ -1,81 +1,65 @@ { "_pkglts": { - "auto_install": true, - "install_front_end": "stdout", - "use_prompts": false, - "version": 11 - }, + "auto_install": true, + "install_front_end": "stdout", + "use_prompts": false, + "version": 3 + }, "base": { "authors": [ [ - "Christophe Pradal", + "Christophe Pradal", "christophe dot pradal at cirad dot fr" - ], + ], [ - "Samuel Dufour-Kowalski", + "Samuel Dufour-Kowalski", "dufourko at cirad dot fr" - ], + ], [ - "revesansparole", + "revesansparole", "revesansparole@gmail.com" - ], - [ - "Guillaume Baty", - "" ] - ], - "namespace": "openalea", - "namespace_method": "pkg_util", - "pkgname": "core", - "url": "https://github.com/openalea/core" - }, - "coverage": {}, + ], + "namespace": "openalea", + "pkgname": "core", + "url": null + }, + "coverage": {}, "doc": { - "description": "OpenAlea.Core is able to discover and manage packages and logical components, build and evaluate dataflows and Generate final applications", - "fmt": "rst", + "description": "OpenAlea.Core is able to discover and manage packages and logical components, build and evaluate dataflows and Generate final applications", "keywords": [ "openalea" - ], - "contributors": "OpenAlea", - "authors": "Christophe Pradal" - }, - "git": {}, + ] + }, "github": { - "owner": "{{ base.authors[0][0] }}", - "project": "{{ base.pkgname }}", + "owner": "openalea", + "project": "{{ base.pkgname }}", "url": "https://github.com/{{ github.owner }}/{{ github.project }}" - }, + }, "license": { - "name": "cecill-c", - "organization": "CIRAD/INRIA", - "project": "{{ base.pkgname }}", + "name": "cecill-c", + "organization": "CIRAD/INRIA", + "project": "{{ base.pkgname }}", "year": 2015 - }, + }, "pysetup": { "intended_versions": [ "27" - ], - "require": [ - { - "name": "openalea/deploy", - "pkg_mng": "git" - } - ] - }, + ], + "require": [["git", "openalea/deploy"], ["git", "openalea/provenance"]] + }, "readthedocs": { "project": "openalea-core" - }, + }, "sphinx": { - "autodoc_dvlpt": true, - "build_dir": "build/sphinx", + "autodoc_dvlpt": true, "theme": "default" - }, - "test": { - "suite_name": "pytest" - }, + }, + "test": {}, + "travis": {}, "version": { - "major": 2, - "minor": 0, - "post": 2 + "major": 1, + "minor": 4, + "post": 0 } } \ No newline at end of file diff --git a/.pkglts/pkg_hash.json b/.pkglts/pkg_hash.json index 58b6c48a..2c5f3c21 100644 --- a/.pkglts/pkg_hash.json +++ b/.pkglts/pkg_hash.json @@ -1,66 +1,66 @@ { ".coveragerc": { "coverage": "fJOlJPmBiCjF4ydI8eATEwZLJXAhQR/S/fMnVio8FMbz4iTTM+1JD+8J0wJ+Je7t0Fi7sY+cxlRjpumCigbtVg==" - }, + }, ".gitignore": { - "git": "BajiufpLax6rf5dmemG7nvqJTLIyPShHtj2YTKyJ7vgPbEZrmsUMebpVX4WoruGBcRQwPOqS5fn5D/PjPmQErw==" - }, + "github": "wVVlsQPP65I5Gg6bW37JENFtYxO6Naq7x+mO0DaDGFVbMPk2o5QJ3DUwMoYfkOfou9lcR/6DC0MXr7hnrTWc9g==" + }, ".travis.yml": { - "travis": "p39laCS5PHN7gbwAtgU28Z+UXHRifGIORlali3ZkK7OQKjjRhn1Dbwscp76SZeocjKUEBtl3fI1857Ri37wPWQ==", - "travis.addons": "z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==", - "travis.after": "z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==", - "travis.install": "uieq8tZxbKnpbCZFJeBMPIUo7cheWgDZyiVg50+CQR2yN9/eejR+Go6jNh+0S89LtB5LgmlQ0Ao3KTB3xaN8Xg==", - "travis.script": "DuMVu20DK3x0WVdVZxihdiYnEkfDqmOlMUuCHohyYvi4fv3Ew8OeAca6AdKM/ZCSZvVS9tywx+PhPrmDexl6hg==" - }, + "travis": "p39laCS5PHN7gbwAtgU28Z+UXHRifGIORlali3ZkK7OQKjjRhn1Dbwscp76SZeocjKUEBtl3fI1857Ri37wPWQ==", + "travis.addons": "z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==", + "travis.after": "z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==", + "travis.install": "Ay/BrD6JoeZi+JszHb0sHpFfzVEp9qaeFsWBBBJG0t2xwgF3tinQxifkNFwuJdlyCjy6U9k901dZnqt6yrzzxQ==", + "travis.script": "STjpu3u3YoOeTz4mBIflZ58OodPdFRPFtLt6c8CUciNwe/fYhCTxlKTMB/S7VL1ippUsQVIfrFGl+OQ/ALWmPQ==" + }, "AUTHORS.rst": { - "doc": "mr4dDvb808KCdcLSRWe9brxcIBamJp3DMpFzoEhHCeSLmr7Q0QfrNRZ79dnneEyWHgcs2srtibLIZlGOq1S0rQ==" - }, - "CONTRIBUTING.rst": {}, - "HISTORY.rst": {}, + "doc": "HvXVe1vS1t/xzXuerP756vvyf3I4V8xEOqvNIEGejKpuzVRZr6wR4C24rm8O9OnBsYedYHvDxQRF25urtQCDMQ==" + }, + "CONTRIBUTING.rst": {}, + "HISTORY.rst": {}, "LICENSE": { "license": "21ExMhSzB2uXQuU1VoH7++nrb4u6TbesqGtJDpM39OLPWft9SJ2tJ956R6foYmkI9VuJay7SUrhFLPxxMEHLxg==" - }, + }, "MANIFEST.in": { "pysetup": "vzgcfdwg7fafu0G3HEmFsZnC8jB8sa4zeSFQ2ReZO+pUfGeynOcw4+CIzP/ZV7n5nmpGGI7xqcdcPtOFfoivdQ==" - }, + }, "README.rst": { - "doc": "TgYu4NHXwUZvrQ1Qz0L2XXLNIO5XdbfOzfd9SLpaFYHkcByYGk0nzosJrGveh1tzoGHf8mHbIh3j4hJuZA1YAQ==" - }, + "doc": "324FZvzs5aJAtcREXT4nIIv/zjrfc0EZj+qjxyrZ9EP+UXQERzKyDx+4u1FDMMfzP2WJhOkPf9QQXzV4T+92GA==" + }, "doc/Makefile": { "sphinx": "C58jxdu8XJNOoaWOw7VWy4LwFmS8lTQt/JeZLt5JozMSJ503plOZchucx+vAEzELbx/Czs/h9pshe9s+9HeOeQ==" - }, - "doc/_static/nonempty.txt": {}, - "doc/authors.rst": {}, + }, + "doc/_static/nonempty.txt": {}, + "doc/authors.rst": {}, "doc/conf.py": { - "sphinx": "2tQ8D7EDdBCKug4c/mp+T6BWPLejyfJxv7mvT+e+YeHWxWEf+x3rqpFdHax/2g7Kx6mFSJJjD4OZtD3SFQ0sug==" - }, - "doc/contributing.rst": {}, - "doc/history.rst": {}, - "doc/index.rst": {}, - "doc/installation.rst": {}, + "sphinx": "vfmn2Bw5s7XSpWirr5eXEjxifmpqNg4YCiNCojKEmRaCHDw3F/yzd6ghqanWEdjEkbs7Dn9Z9U3jdHBM6/j3oA==" + }, + "doc/contributing.rst": {}, + "doc/history.rst": {}, + "doc/index.rst": {}, + "doc/installation.rst": {}, "doc/make.bat": { "sphinx": "oA67PK7JSR3w9u1V0gOXFZAb8qgfgUJ0wQ46cVXu7M3iJyAbLf76WkEqw6Y1r5Q6EvHAogcjmImdGsOpiYZ22g==" - }, - "doc/readme.rst": {}, - "doc/usage.rst": {}, + }, + "doc/readme.rst": {}, + "doc/usage.rst": {}, "dvlpt_requirements.txt": { - "pysetup": "ONHd7KJUHI6bXimIhJ5wQNBLkFxjrrx8QaUiAWX5VuehOEm8oScMVo6MBE1ir0kydnZQMPfVfE2xEsCIx0OQKA==" - }, + "pysetup": "XWPKeIfT2RoNPeDhPDgWWOssKAjkq/OXoEfIpSs8vJrYkPahRrEs1qXFvqxzqoV09MjmnVg0zrG9m/wRXfrRxw==" + }, "requirements.txt": { - "pysetup": "Wv2Gn+Cy4Ukfy3XXDV3p9fubX0FVqMuS8E22TTvSMcFdhxdVqQIZP6vTgFVAFbdZ4MDsP9xN7IVXQr177j00hQ==" - }, + "pysetup": "ROpjsUQMjMbn87UPzWB8XD7JnQAUNptjTBj99UwCmDUKNGQNqmlZD0gN+ukVdk4E87brwVEgqOZ47nid1EGpww==" + }, "setup.cfg": { - "pysetup": "60E8EHReCNkZG5fVvzln5jA58snyNhSP2LiCl0ngVMVo9R1qT6G/JRnGQpx8+c6trba9O5teM0eZ4PUNXh/QEw==" - }, + "pysetup": "QBedkKDNLW8WmznOFdyIyCL2sTDHYcmudSzjx+7QUcRI1vVxIHnNm82aiMvaMeEAYpOCr+8+Nj4ZTr9P0dSxSg==" + }, "setup.py": { - "pysetup.call": "2wHuP94SpWKZpqOzExkQGTxnhGPgBHK3qOdPn0Ncstj5FkZ7FBGGM/inqAXaI4QLKQtTNlOUYRGSkYUWAUcf1w==", - "pysetup.kwds": "lvC2rA9j9gbWQdPVGNGAhw6qs2S8QktXHUGLpZ4GhykYRbGFqqlC9B9Wdzktgqspie28hxjP+yCQorsxi38PUg==" - }, + "pysetup.call": "2wHuP94SpWKZpqOzExkQGTxnhGPgBHK3qOdPn0Ncstj5FkZ7FBGGM/inqAXaI4QLKQtTNlOUYRGSkYUWAUcf1w==", + "pysetup.kwds": "dlgrvck1HbacSgA60B/51tvoA65q1tPJ/9ZWW0mrNh0uCq4ceY5GW2McnmDVal/PjP19xCfhoyUBuElz9HvuSA==" + }, "src/openalea/core/__init__.py": { "base": "gR33dW0qqYmsV9NSNB+DD8XmuxnC2t0mKjnMoU5728qh97fSER6MbX+3QKxpZDLByZToaAay4xhx8acxketJmA==" - }, + }, "src/openalea/core/version.py": { - "version": "xMbInCdDTk9EH4B/UZg9ywcImWAvphx9GNRPd1pqfQuDuBxOkF60Y3if3LGty+BFNAXheBp782+M8vMWyY2Uqw==" - }, + "version": "uGFlizzcSxlY4JEVKPFfVPXszFKJiUw9c8Qi6ykA2LAe63uOU3NiKVlpwCmmmHHR+TSxwUnoSGLg+QuZ5yV//g==" + }, "test/__init__.py": {} } \ No newline at end of file diff --git a/.travis.yml b/.travis.yml deleted file mode 120000 index 95c89cfd..00000000 --- a/.travis.yml +++ /dev/null @@ -1 +0,0 @@ -travis.yml \ No newline at end of file diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..ef39fc18 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,58 @@ +# {# pkglts, travis +# Config file for automatic testing at travis-ci.org + +language: python + +python: + - "2.7" + +# #} + +addons: + apt: + packages: +# {# pkglts, travis.addons + + #} + +install: +# {# pkglts, travis.install + # We do this conditionally because it saves us some downloading if the + # version is the same. + - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then + wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh; + else + wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; + fi + - bash miniconda.sh -b -p $HOME/miniconda + - export PATH="$HOME/miniconda/bin:$PATH" + - hash -r + - conda config --set always_yes yes --set changeps1 no + - conda update -q conda + # Useful for debugging any issues with conda + - conda info -a + + - conda create -q -n myenv python=$TRAVIS_PYTHON_VERSION + - source activate myenv + + + - pip install git+https://github.com/openalea/deploy + - pip install git+https://github.com/openalea/provenance + - conda install coverage + - conda install mock + - conda install nose + - conda install sphinx + + - python setup.py install +# #} + +script: + - python setup.py install +# {# pkglts, travis.script + - nosetests +# #} + +after_success: +# {# pkglts, travis.after + +# #} diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 00000000..88d21055 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,6 @@ +{ + "python.pythonPath": "/home/gaetan/miniconda2/envs/visualea/bin/python", + "python.linting.pylintEnabled": false, + "python.linting.banditEnabled": true, + "python.linting.enabled": true +} \ No newline at end of file diff --git a/AUTHORS.rst b/AUTHORS.rst index 598449c8..e8409b7c 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -8,7 +8,6 @@ Development Lead .. {# pkglts, doc -* openalea, * Christophe Pradal, * Samuel Dufour-Kowalski, * revesansparole, diff --git a/README.rst b/README.rst index 078fe66c..a425789d 100644 --- a/README.rst +++ b/README.rst @@ -9,6 +9,11 @@ openalea.core :alt: Documentation status :target: https://openalea-core.readthedocs.io/en/latest/?badge=latest + +.. image:: https://travis-ci.org/openalea/core.svg?branch=master + :alt: Travis build status + :target: https://travis-ci.org/openalea/core + .. #} OpenAlea.Core is able to discover and manage packages and logical components, build and evaluate dataflows and Generate final applications diff --git a/appveyor.yml b/appveyor.yml deleted file mode 100644 index 3a98c636..00000000 --- a/appveyor.yml +++ /dev/null @@ -1,55 +0,0 @@ -platform: - - x86 - - x64 - -environment: - matrix: - - CONDA_RECIPE: conda - CONDA_VERSION: 2 - - CONDA_RECIPE: conda - CONDA_VERSION: 3 - -# matrix: -# allow_failures: -# - platform: x86 -# CONDA_RECIPE: conda -# CONDA_VERSION: 3 -# - platform: x64 -# CONDA_RECIPE: conda -# CONDA_VERSION: 3 - -install: - - git clone https://github.com/OpenAlea/appveyor-ci.git - - cd appveyor-ci - - call install.bat - -before_build: - - call before_build.bat - -build_script: - - call build_script.bat - -after_build: - - call after_build.bat - -deploy: - provider: Script - -before_deploy: - - call before_deploy.bat - -deploy_script: - - call deploy_script.bat - -after_deploy: - - call after_deploy.bat - -on_success: - - call on_success.bat - -on_failure: - - call on_failure.bat - -on_finish: - - call on_finish.bat - diff --git a/conda/meta.yaml b/conda/meta.yaml deleted file mode 100644 index 3b0f398c..00000000 --- a/conda/meta.yaml +++ /dev/null @@ -1,38 +0,0 @@ -{% set version = "2.0.1" %} - -package: - name: openalea.core - version: {{ version }} - -source: - path: .. - -build: - preserve_egg_dir: True - number: 1 - script: python setup.py install --prefix=$PREFIX - -requirements: - build: - - openalea.deploy #==2.0.0 - run: - - python - - ipykernel - -test: - imports: - - openalea.core - requires: - - nose - source_files: - - test/ - - test/*.py - - commands: - - nosetests -v -I test_compositenode.py -I test_alias.py -I test_data.py -I test_eval.py -I test_package.py - -about: - home: http://github.com/openalea/core - license: Cecill-c License - summary: OpenAlea Core component and Scientific Workflow platform. - diff --git a/doc/conf.py b/doc/conf.py index 2fbd8f06..13cac438 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -2,7 +2,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- # -# Package core documentation build configuration file, created by +# Package (Undefined, u'core') documentation build configuration file, created by # sphinx-quickstart on Tue Jul 9 22:26:36 2013. # # This file is execfile()d with the current directory set to its @@ -247,7 +247,7 @@ latex_documents = [ ('index', 'core.tex', u'core Documentation', - u'openalea', 'manual'), + u'Christophe Pradal', 'manual'), ] # The name of an image file (relative to this directory) to place at @@ -293,7 +293,7 @@ texinfo_documents = [ ('index', 'core', u'core Documentation', - u'openalea', + u'Christophe Pradal', 'core', 'OpenAlea.Core is able to discover and manage packages and logical components, build and evaluate dataflows and Generate final applications', 'Miscellaneous'), @@ -315,16 +315,30 @@ # use apidoc to generate developer doc import os from os import path +from sphinx.apidoc import create_modules_toc_file, recurse_tree -from sphinx.apidoc import main +class Opt(object): + pass -rootpath = path.abspath(path.join(project_root, "src")) -destdir = path.abspath(path.join(project_root, "doc", "_dvlpt")) - -if not path.isdir(destdir): - os.makedirs(destdir) -main(['-e', '-o', destdir, '-d', '4', '-s', source_suffix[1:], '--force', rootpath]) +rootpath = path.abspath(path.join(project_root, "src")) +opts = Opt() +opts.modulefirst = None +opts.separatemodules = None +opts.noheadings = None +opts.destdir = path.abspath(path.join(project_root, "doc", "_dvlpt")) +opts.suffix = source_suffix[1:] +opts.dryrun = None +opts.force = None +opts.header = 'src' +opts.maxdepth = 4 +opts.includeprivate = False + +if not path.isdir(opts.destdir): + os.makedirs(opts.destdir) + +modules = recurse_tree(rootpath, [], opts) +create_modules_toc_file(modules, opts) # #} diff --git a/dvlpt_requirements.txt b/dvlpt_requirements.txt index a0df7b57..ecac567c 100644 --- a/dvlpt_requirements.txt +++ b/dvlpt_requirements.txt @@ -1,8 +1,7 @@ # {# pkglts, pysetup -coverage # pip install coverage -pytest # pip install pytest -pytest-cov # pip install pytest-cov -pytest-mock # pip install pytest-mock -sphinx # pip install sphinx +coverage +mock +nose +sphinx # #} diff --git a/requirements.txt b/requirements.txt index 982a059d..7657a191 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,8 @@ # edit .pkglts/pkg_cfg instead # section pysetup -#openalea/deploy # pip install git+git +# openalea/deploy +# openalea/provenance # #} diff --git a/setup.cfg b/setup.cfg index 8d67cc6c..3b75b164 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,18 +5,19 @@ [wheel] universal = 1 -# {# pkglts, pysetup -[wheel] -universal = 1 -[tool:pytest] -addopts = --maxfail=2 -rf --cov=openalea.core +[nosetests] +verbosity=1 +detailed-errors=1 -[aliases] -test=pytest +with-coverage=1 +cover-erase=1 +# cover-inclusive=1 +cover-package=openalea.core -[build_sphinx] -build-dir=build/sphinx +# debug=nose.loader +# pdb=1 +# pdb-failures=1 diff --git a/setup.py b/setup.py index 354dc240..2197bafc 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ short_descr = "OpenAlea.Core is able to discover and manage packages and logical components, build and evaluate dataflows and Generate final applications" readme = open('README.rst').read() -history = open('HISTORY.rst').read() +history = open('HISTORY.rst').read().replace('.. :changelog:', '') # find version number in src/openalea/core/version.py @@ -17,40 +17,32 @@ with open("src/openalea/core/version.py") as fp: exec(fp.read(), version) -# find packages -pkgs = find_packages('src') - - setup_kwds = dict( name='openalea.core', version=version["__version__"], description=short_descr, long_description=readme + '\n\n' + history, - author="Christophe Pradal", - author_email="christophe dot pradal at cirad dot fr", + author="Christophe Pradal, Samuel Dufour-Kowalski, revesansparole, ", + author_email="christophe dot pradal at cirad dot fr, dufourko at cirad dot fr, revesansparole@gmail.com, ", url='https://github.com/openalea/core', license='cecill-c', zip_safe=False, - packages=pkgs, - namespace_packages=['openalea'], + packages=find_packages('src'), package_dir={'': 'src'}, - setup_requires=[ - "pytest-runner", - ], install_requires=[ ], tests_require=[ "coverage", - "pytest", - "pytest-cov", - "pytest-mock", + "mock", + "nose", "sphinx", ], entry_points={}, keywords='openalea', - ) + test_suite='nose.collector', +) # #} # change setup_kwds below before the next pkglts tag diff --git a/src/openalea/core/algo/dataflow_evaluation.py b/src/openalea/core/algo/dataflow_evaluation.py index f3cf618b..0dc1d33e 100644 --- a/src/openalea/core/algo/dataflow_evaluation.py +++ b/src/openalea/core/algo/dataflow_evaluation.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- # -*- python -*- # # OpenAlea.Core @@ -22,133 +23,37 @@ import sys from time import clock import traceback as tb -from openalea.core import ScriptLibrary +from openalea.provenance.simple_dict import Provenance as RVProvenance +from openalea.core import ScriptLibrary from openalea.core.dataflow import SubDataflow from openalea.core.interface import IFunction - -PROVENANCE = False - -# Implement provenance in OpenAlea -db_conn = None - -import sqlite3 from openalea.core.path import path from openalea.core import settings -def db_create(cursor): - cur = cursor - #-prospective provenance-# - #User table creation - cur.execute("CREATE TABLE IF NOT EXISTS User (userid INTEGER,createtime DATETIME,name varchar (25), firstname varchar (25), email varchar (25), password varchar (25),PRIMARY KEY(userid))") - - # CompositeNode table creation - cur.execute("CREATE TABLE IF NOT EXISTS CompositeNode (CompositeNodeid INTEGER, creatime DATETIME, name varchar (25), description varchar (25),userid INTEGER,PRIMARY KEY(CompositeNodeid),FOREIGN KEY(userid) references User)") - #Cr?ation de la table Node - cur.execute("CREATE TABLE IF NOT EXISTS Node (Nodeid INTEGER, createtime DATETIME, name varchar (25), NodeFactory varchar (25),CompositeNodeid INTEGER,PRIMARY KEY(Nodeid),FOREIGN KEY(CompositeNodeid) references CompsiteNode)") - #Cr?ation de la table Input - cur.execute("CREATE TABLE IF NOT EXISTS Input (Inputid INTEGER, createtime DATETIME, name varchar (25), typedata varchar (25), InputPort INTEGER,PRIMARY KEY (Inputid))") - #Cr?ation de la table Output - cur.execute("CREATE TABLE IF NOT EXISTS Output (Outputid INTEGER, createtime DATETIME, name varchar (25), typedata varchar (25), OutputPort INTEGER,PRIMARY KEY (Outputid))") - #Cr?ation de la table elt_connection - cur.execute("CREATE TABLE IF NOT EXISTS elt_connection (elt_connectionid INTEGER, createtime DATETIME,srcNodeid INTEGER, srcNodeOutputPortid INTEGER, targetNodeid INTEGER, targetNodeInputPortid INTEGER ,PRIMARY KEY (elt_connectionid))") - - #- retrospective provenance -# - #- CompositeNodeExec table creation - cur.execute("CREATE TABLE IF NOT EXISTS CompositeNodeExec (CompositeNodeExecid INTEGER, createtime DATETIME, endtime DATETIME,userid INTEGER,CompositeNodeid INTEGER,PRIMARY KEY(CompositeNodeExecid),FOREIGN KEY(CompositeNodeid) references CompositeNode,FOREIGN KEY(userid) references User)") - #- NodeExec - cur.execute("CREATE TABLE IF NOT EXISTS NodeExec (NodeExecid INTEGER, createtime DATETIME, endtime DATETIME,Nodeid INTEGER,CompositeNodeExecid INTEGER,dataid INTEGER,PRIMARY KEY(NodeExecid),FOREIGN KEY(Nodeid) references Node, FOREIGN KEY (CompositeNodeExecid) references CompositeNodeExec, FOREIGN KEY (dataid) references Data)") - #- History - cur.execute("CREATE TABLE IF NOT EXISTS Histoire (Histoireid INTEGER, createtime DATETIME, name varchar (25), description varchar (25),userid INTEGER,CompositeNodeExecid INTEGER,PRIMARY KEY (Histoireid), FOREIGN KEY(Userid) references User, FOREIGN KEY(CompositeNodeExecid) references CompositeNodeExec)") - #- Data - cur.execute("CREATE TABLE IF NOT EXISTS Data (dataid INTEGER, createtime DATETIME,NodeExecid INTEGER, PRIMARY KEY(dataid),FOREIGN KEY(NodeExecid) references NodeExec)") - #- Tag - cur.execute("CREATE TABLE IF NOT EXISTS Tag (CompositeNodeExecid INTEGER, createtime DATETIME, name varchar(25),userid INTEGER,PRIMARY KEY(CompositeNodeExecid),FOREIGN KEY(userid) references User)") - return cur - -def get_database_name(): - db_fn = path(settings.get_openalea_home_dir())/'provenance.sq3' - return db_fn - -def db_connexion(): - """ Return a curso on the database. - - If the database does not exists, create it. - """ - global db_conn - if db_conn is None: - db_fn = get_database_name() - if not db_fn.exists(): - db_conn=sqlite3.connect(db_fn) - cur = db_conn.cursor() - cur = db_create(cur) - return cur - else: - cur = db_conn.cursor() - return cur - -class Provenance(object): - def __init__(self, workflow): - self.clear() - self.workflow = workflow - - def edges(self): - cn = self.workflow - edges= list(cn.edges()) - sources=map(cn.source,edges) - targets = map(cn.target,edges) - source_ports=[cn.local_id(cn.source_port(eid)) for eid in edges] - target_ports=[cn.local_id(cn.target_port(eid)) for eid in edges] - _edges = dict(zip(edges,zip(sources,source_ports,targets, target_ports))) - return _edges - - def clear(self): - self.nodes = [] +import os +import time +import json - def start_time(self): - pass - def end_time(self): - pass - def workflow_exec(self, *args): - pass - def node_exec(self, vid, node, start_time, end_time, *args): - pass - def write(self): - """ Write the provenance in db """ +from openalea.distributed.data.data_manager import load_data, check_data_to_load, write_data -class PrintProvenance(Provenance): - def workflow_exec(self, *args): - print 'Workflow execution ', self.workflow.factory.name - def node_exec(self, vid, node, start_time, end_time, *args): - provenance(vid, node, start_time, end_time) +from openalea.distributed.provenance.provenanceDB import start_provdb +from openalea.distributed.index.indexDB import start_index +from openalea.distributed.index.graph_id import Task_UID_graph +# TODO: remove this id method - used in fragment evaluation - to get a general one +from openalea.distributed.index.id import get_id -def provenance(vid, node, start_time, end_time): - #from service import db - #conn = db.connect() - - - if PROVENANCE: - cur = db_connexion() - - pname = node.factory.package.name - name = node.factory.name - - print "Provenance Process:" - print "instance ID ", vid, "Package Name: ",pname, "Name: ", name - print "start time :", start_time, "end_time: ", end_time, "duration : ", end_time-start_time - print 'Inputs : ', node.inputs - print 'outputs : ', node.outputs - -# print the evaluation time # This variable has to be retrieve by the settings quantify = False +# get the prov when evaluating +provenance = False __evaluators__ = [] -class EvaluationException(Exception): +class EvaluationException(Exception): def __init__(self, vid, node, exception, exc_info): Exception.__init__(self) self.vid = vid @@ -180,8 +85,8 @@ def cmp_posx(x, y): """todo""" (xpid, xvid, xactor) = x (ypid, yvid, yactor) = y - #px = xactor.internal_data.get('posx', 0) - #py = yactor.internal_data.get('posx', 0) + # px = xactor.internal_data.get('posx', 0) + # py = yactor.internal_data.get('posx', 0) px = xactor.get_ad_hoc_dict().get_metadata('position')[0] py = yactor.get_ad_hoc_dict().get_metadata('position')[0] @@ -196,17 +101,41 @@ def cmp_posx(x, y): """ Abstract evaluation algorithm """ -class AbstractEvaluation(object): - def __init__(self, dataflow): +class AbstractEvaluation(object): + def __init__(self, dataflow, *args, **kwargs): """ :param dataflow: to be done """ self._dataflow = dataflow - if PROVENANCE: - self.provenance = PrintProvenance(dataflow) - def eval(self, *args): + if kwargs.get("record_provenance"): + self._prov = RVProvenance() + self._provdb = start_provdb(provenance_config=kwargs.get('provenance_config', None), + provenance_type=kwargs.get('provenance_type', "Files")) + else: + self._prov = None + self._provdb = None + + if kwargs.get('use_index'): + #  Connect to the index db + # self._indexdb = start_index(index_config=kwargs.get('index_config', None), + # index_type=kwargs.get('index_type', "Cassandra")) + # Eval the workflow with a fake evaluation to get the tasks ids of each task + real_eval_algo = dataflow.eval_algo + dataflow.eval_algo= "FakeEvaluation" + dataflow.eval() + tid = dataflow.node(1).get_output("task_ids") + print tid + dataflow.eval_algo= real_eval_algo + # self._index = + # self._index + else: + self._index = None + self._indexdb = None + + + def eval(self, *args, **kwargs): """todo""" raise NotImplementedError() @@ -214,7 +143,7 @@ def is_stopped(self, vid, actor): """ Return True if evaluation must be stop at this vertex. """ return actor.block - def eval_vertex_code(self, vid): + def eval_vertex_code(self, vid, *args, **kwargs): """ Evaluate the vertex vid. Can raise an exception if evaluation failed. @@ -223,14 +152,21 @@ def eval_vertex_code(self, vid): node = self._dataflow.actor(vid) try: + + if self._prov is not None: + self._prov.before_eval(self._dataflow, vid) + t0 = clock() ret = node.eval() - t1 = clock() - if PROVENANCE: - self.provenance.node_exec(vid, node, t0,t1) - #provenance(vid, node, t0,t1) - + dt = clock() - t0 + if self._prov is not None: + taskitem = self._prov.after_eval(self._dataflow, vid, dt) + if self._provdb and taskitem: + self._provdb.add_task_item(taskitem) + # if self._index is not None: + # self._index.add + # When an exception is raised, a flag is set. # So we remove it when evaluation is ok. node.raise_exception = False @@ -252,8 +188,7 @@ def eval_vertex_code(self, vid): node.raise_exception = True node.notify_listeners(('data_modified', None, None)) raise EvaluationException(vid, node, e, \ - tb.format_tb(sys.exc_info()[2])) - + tb.format_tb(sys.exc_info()[2])) def get_parent_nodes(self, pid): """ @@ -266,7 +201,7 @@ def get_parent_nodes(self, pid): # For each connected node npids = [(npid, df.vertex(npid), df.actor(df.vertex(npid))) \ - for npid in df.connected_ports(pid)] + for npid in df.connected_ports(pid)] npids.sort(cmp=cmp_posx) return npids @@ -274,13 +209,14 @@ def get_parent_nodes(self, pid): def set_provenance(self, provenance): self.provenance = provenance + class BrutEvaluation(AbstractEvaluation): """ Basic evaluation algorithm """ __evaluators__.append("BrutEvaluation") - def __init__(self, dataflow): + def __init__(self, dataflow, *args, **kwargs): - AbstractEvaluation.__init__(self, dataflow) + AbstractEvaluation.__init__(self, dataflow, *args, **kwargs) # a property to specify if the node has already been evaluated self._evaluated = set() @@ -294,7 +230,8 @@ def is_stopped(self, vid, actor): if actor.block: status = True n = actor.get_nb_output() - outputs = [i for i in range(n) if actor.get_output(i) is not None ] + outputs = [i for i in range(n) if + actor.get_output(i) is not None] if not outputs: status = False return status @@ -302,7 +239,7 @@ def is_stopped(self, vid, actor): pass return False - def eval_vertex(self, vid, *args): + def eval_vertex(self, vid, *args, **kwargs): """ Evaluate the vertex vid """ df = self._dataflow @@ -332,21 +269,35 @@ def eval_vertex(self, vid, *args): # Eval the node self.eval_vertex_code(vid) - def eval(self, *args): + def eval(self, *args, **kwargs): """ Evaluate the whole dataflow starting from leaves""" - t0 = clock() + + t0 = time.time() df = self._dataflow + if self._prov is not None: + self._prov.init(df) + self._prov.time_init = t0 # Unvalidate all the nodes self._evaluated.clear() # Eval from the leaf - for vid in (vid for vid in df.vertices() if df.nb_out_edges(vid)==0): + for vid in (vid for vid in df.vertices() if df.nb_out_edges(vid) == 0): self.eval_vertex(vid) - t1 = clock() + t1 = time.time() + + if self._prov is not None: + self._prov.time_end = t1 + wfitem = self._prov.as_wlformat() + if self._provdb is not None: + self._provdb.add_wf_item(wfitem) + + # close remote connections + self._provdb.close() + if quantify: - print "Evaluation time: %s"%(t1-t0) + print "Evaluation time: %s" % (t1 - t0) class PriorityEvaluation(BrutEvaluation): @@ -357,7 +308,8 @@ def eval(self, vtx_id=None, *args, **kwds): """todo""" t0 = clock() - is_subdataflow = False if not kwds else kwds.get('is_subdataflow', False) + is_subdataflow = False if not kwds else kwds.get('is_subdataflow', + False) df = self._dataflow # Unvalidate all the nodes if is_subdataflow: @@ -370,7 +322,7 @@ def eval(self, vtx_id=None, *args, **kwds): # Select the leaves (list of (vid, actor)) leaves = [(vid, df.actor(vid)) - for vid in df.vertices() if df.nb_out_edges(vid)==0] + for vid in df.vertices() if df.nb_out_edges(vid) == 0] leaves.sort(cmp_priority) @@ -380,19 +332,19 @@ def eval(self, vtx_id=None, *args, **kwds): t1 = clock() if quantify: - print "Evaluation time: %s"%(t1-t0) + print "Evaluation time: %s" % (t1 - t0) class GeneratorEvaluation(AbstractEvaluation): """ Evaluation algorithm with generator / priority and selection""" __evaluators__.append("GeneratorEvaluation") - def __init__(self, dataflow): + def __init__(self, dataflow, *args, **kwargs): - AbstractEvaluation.__init__(self, dataflow) + AbstractEvaluation.__init__(self, dataflow, *args, **kwargs) # a property to specify if the node has already been evaluated self._evaluated = set() - self.reeval = False # Flag to force reevaluation (for generator) + self.reeval = False # Flag to force reevaluation (for generator) def is_stopped(self, vid, actor): """ Return True if evaluation must be stop at this vertex """ @@ -454,7 +406,7 @@ def eval(self, vtx_id=None, step=False): else: # Select the leafs (list of (vid, actor)) leafs = [(vid, df.actor(vid)) - for vid in df.vertices() if df.nb_out_edges(vid)==0] + for vid in df.vertices() if df.nb_out_edges(vid) == 0] leafs.sort(cmp_priority) @@ -462,25 +414,24 @@ def eval(self, vtx_id=None, step=False): for vid, actor in leafs: if not self.is_stopped(vid, actor): self.reeval = True - while(self.reeval): + while (self.reeval): self.clear() self.eval_vertex(vid) t1 = clock() if quantify: - print "Evaluation time: %s"%(t1-t0) + print "Evaluation time: %s" % (t1 - t0) return False - class LambdaEvaluation(PriorityEvaluation): """ Evaluation algorithm with support of lambda / priority and selection""" __evaluators__.append("LambdaEvaluation") - def __init__(self, dataflow): - PriorityEvaluation.__init__(self, dataflow) + def __init__(self, dataflow, *args, **kwargs): + PriorityEvaluation.__init__(self, dataflow, *args, **kwargs) - self.lambda_value = {} # lambda resolution dictionary + self.lambda_value = {} # lambda resolution dictionary self._resolution_node = set() def eval_vertex(self, vid, context, lambda_value, *args): @@ -528,7 +479,7 @@ def eval_vertex(self, vid, context, lambda_value, *args): transmit_cxt = context transmit_lambda = lambda_value - cpt = 0 # parent counter + cpt = 0 # parent counter # For each connected node for npid, nvid, nactor in self.get_parent_nodes(pid): @@ -548,7 +499,7 @@ def eval_vertex(self, vid, context, lambda_value, *args): # replace the lambda with value. if (isinstance(outval, SubDataflow) - and interface is not IFunction): + and interface is not IFunction): if (not context and not lambda_value): # we are not in resolution mode @@ -566,7 +517,8 @@ def eval_vertex(self, vid, context, lambda_value, *args): try: lambda_value[outval] = context.pop() except Exception: - raise Exception("The number of lambda variables is insuffisant") + raise Exception( + "The number of lambda variables is insuffisant") # We replace the value with a context value outval = lambda_value[outval] @@ -597,9 +549,13 @@ def eval(self, vtx_id=None, context=None, is_subdataflow=False, step=False): :param context: list a value to assign to lambda variables """ t0 = clock() - if PROVENANCE and (not is_subdataflow): - self.provenance.workflow_exec() - self.provenance.start_time() + if self._prov is not None: + self._prov.init(self._dataflow) + self._prov.time_init = t0 + + # if PROVENANCE and (not is_subdataflow): + # self.provenance.workflow_exec() + # self.provenance.start_time() self.lambda_value.clear() @@ -608,22 +564,31 @@ def eval(self, vtx_id=None, context=None, is_subdataflow=False, step=False): # thus, we have to reverse the arguments to evaluate the function (FIFO). context.reverse() - PriorityEvaluation.eval(self, vtx_id, context, self.lambda_value, is_subdataflow=is_subdataflow) - self.lambda_value.clear() # do not keep context in memory - - if PROVENANCE: - self.provenance.end_time() + PriorityEvaluation.eval(self, vtx_id, context, self.lambda_value, + is_subdataflow=is_subdataflow) + self.lambda_value.clear() # do not keep context in memory t1 = clock() + if self._prov is not None: + self._prov.time_end = t1 + wfitem = self._prov.as_wlformat() + if self._provdb is not None: + self._provdb.add_wf_item(wfitem) + + # close remote connections + self._provdb.close() + if quantify: - print "Evaluation time: %s"%(t1-t0) + print "Evaluation time: %s" % (t1 - t0) if not is_subdataflow: self._resolution_node.clear() DefaultEvaluation = LambdaEvaluation -#DefaultEvaluation = GeneratorEvaluation + + +# DefaultEvaluation = GeneratorEvaluation # from collections import deque @@ -896,11 +861,12 @@ def eval(self, *args, **kwds): # Eval from the leaf script = "" - for vid in (vid for vid in df.vertices() if df.nb_out_edges(vid)==0): + for vid in (vid for vid in df.vertices() if df.nb_out_edges(vid) == 0): script += self.eval_vertex(vid) return script + ############################################################################ # Evaluation with scheduling @@ -915,7 +881,7 @@ def __init__(self, dataflow): AbstractEvaluation.__init__(self, dataflow) # a property to specify if the node has already been evaluated self._evaluated = set() - self.reeval = False # Flag to force reevaluation (for generator) + self.reeval = False # Flag to force reevaluation (for generator) # CPL # At each evaluation of the dataflow, increase the current cycle of @@ -932,7 +898,7 @@ def is_stopped(self, vid, actor): """ Return True if evaluation must be stop at this vertex """ stopped = False try: - if hasattr(actor,'block'): + if hasattr(actor, 'block'): stopped = actor.block stopped = stopped or vid in self._evaluated except: @@ -956,7 +922,7 @@ def next_step(self): def eval_vertex(self, vid): """ Evaluate the vertex vid """ - #print "Step ", self._current_cycle + # print "Step ", self._current_cycle df = self._dataflow actor = df.actor(vid) @@ -998,7 +964,6 @@ def eval_vertex(self, vid): if delay == 0: delay = self.eval_vertex_code(vid) - # Reevaluation flag # TODO: Add the node to the scheduler rather to execute if (delay): @@ -1023,7 +988,7 @@ def eval(self, vtx_id=None, step=False): else: # Select the leafs (list of (vid, actor)) leafs = [(vid, df.actor(vid)) - for vid in df.vertices() if df.nb_out_edges(vid)==0] + for vid in df.vertices() if df.nb_out_edges(vid) == 0] leafs.sort(cmp_priority) @@ -1032,7 +997,7 @@ def eval(self, vtx_id=None, step=False): if not self.is_stopped(vid, actor): self.reeval = True if not step: - while(self.reeval and not self._stop): + while (self.reeval and not self._stop): self.clear() self.eval_vertex(vid) self.next_step() @@ -1046,7 +1011,7 @@ def eval(self, vtx_id=None, step=False): for vid in self._nodes_to_reset: df.actor(vid).reset() - #print 'Run %d times the dataflow'%(self._current_cycle,) + # print 'Run %d times the dataflow'%(self._current_cycle,) # Reset the state if not step: @@ -1055,7 +1020,7 @@ def eval(self, vtx_id=None, step=False): t1 = clock() if quantify: - print "Evaluation time: %s"%(t1-t0) + print "Evaluation time: %s" % (t1 - t0) return False @@ -1083,17 +1048,16 @@ def is_operator(actor): if 'SciFloware' not in factory.package.name: return False elif factory.name in algebra: - return True + return True else: return False - + def scifloware_actors(self): """ Compute the scifloware actors. Only those actors will be evaluated. """ - df = self._dataflow self._scifloware_actors.clear() for vid in df.vertices(): @@ -1101,7 +1065,6 @@ def scifloware_actors(self): if self.is_operator(actor): self._scifloware_actors.add(vid) - def eval_vertex(self, vid): """ Evaluate the vertex vid @@ -1117,7 +1080,7 @@ def eval_vertex(self, vid): """ - #print "Step ", self._current_cycle + # print "Step ", self._current_cycle df = self._dataflow actor = df.actor(vid) @@ -1125,7 +1088,7 @@ def eval_vertex(self, vid): is_op = vid in self._scifloware_actors self._evaluated.add(vid) - #assert self.is_operator(actor) + # assert self.is_operator(actor) # For each inputs # Compute the nodes @@ -1142,17 +1105,18 @@ def eval_vertex(self, vid): out_ports = list(df.connected_ports(pid)) nb_out = len(out_ports) if nb_out > 1: - raise Exception('Too many nodes connected to the SciFloware operator.') + raise Exception( + 'Too many nodes connected to the SciFloware operator.') elif nb_out == 1: out_actor = df.actor(df.vertex(out_ports[0])) - dataflow_name = out_actor.factory.package.name+':'+out_actor.factory.name + dataflow_name = out_actor.factory.package.name + ':' + out_actor.factory.name actor.set_input(df.local_id(pid), dataflow_name) else: cpt = 0 # For each connected node for npid, nvid, nactor in self.get_parent_nodes(pid): # Do no reevaluate the same node - + if not self.is_stopped(nvid, nactor): self.eval_vertex(nvid) @@ -1168,7 +1132,6 @@ def eval_vertex(self, vid): self.eval_vertex_code(vid) - def eval(self, vtx_id=None, **kwds): t0 = clock() @@ -1180,7 +1143,7 @@ def eval(self, vtx_id=None, **kwds): else: # Select the leafs (list of (vid, actor)) leafs = [(vid, df.actor(vid)) - for vid in df.vertices() if df.nb_out_edges(vid)==0] + for vid in df.vertices() if df.nb_out_edges(vid) == 0] leafs.sort(cmp_priority) @@ -1190,6 +1153,273 @@ def eval(self, vtx_id=None, **kwds): t1 = clock() if quantify: - print "Evaluation time: %s"%(t1-t0) + print "Evaluation time: %s" % (t1 - t0) + + return False + + +############################################################ +class FragmentEvaluation(AbstractEvaluation): + """ Evaluation with By fragments """ + __evaluators__.append("FragmentEvaluation") + + def __init__(self, dataflow, *args, **kwargs): + + AbstractEvaluation.__init__(self, dataflow, *args, **kwargs) + # a property to specify if the node has already been evaluated + self._evaluated = set() + self._fragment_infos = kwargs.get("fragment_infos", None) + + # Define the path where execution data is store during execution - delete after + tpath = path(settings.get_openalea_home_dir()) / "execution_data" + print("use ", kwargs.get("tmp_path", tpath), " as temporary file path") + self._tmp_path = kwargs.get("tmp_path", tpath) + + # If the data index is not use - force its init + if self._indexdb is None: + self._indexdb = start_index(index_config=kwargs.get('index_config', None), + index_type=kwargs.get('index_type', "Cassandra")) + + + def is_stopped(self, vid, actor): + """ Return True if evaluation must be stop at this vertex """ + + if vid in self._evaluated: + return True + + try: + if actor.block: + status = True + n = actor.get_nb_output() + outputs = [i for i in range(n) if + actor.get_output(i) is not None] + if not outputs: + status = False + return status + except: + pass + return False + + def eval_vertex(self, vid, *args, **kwargs): + """ Evaluate the vertex vid """ + + df = self._dataflow + actor = df.actor(vid) + + self._evaluated.add(vid) + + # For each inputs + for pid in df.in_ports(vid): + inputs = [] + + # check if the data has to be loaded | and get path + ituple = check_data_to_load(vid, pid, self._fragment_infos) + if ituple: + cpt = 1 + for npid, nvid, nactor in self.get_parent_nodes(pid): + data_id = get_id(ituple[0], ituple[1]) + row = self._indexdb.find_one(data_id=data_id) + inputs.append(load_data(row[0].path[0])) + else: + cpt = 0 + # For each connected node + for npid, nvid, nactor in self.get_parent_nodes(pid): + if not self.is_stopped(nvid, nactor): + self.eval_vertex(nvid) + + + inputs.append(nactor.get_output(df.local_id(npid))) + cpt += 1 + + # set input as a list or a simple value + if (cpt == 1): + inputs = inputs[0] + if (cpt > 0): + actor.set_input(df.local_id(pid), inputs) + + # Eval the node + self.eval_vertex_code(vid) + + def eval(self, *args, **kwargs): + """ Evaluate the whole dataflow starting from leaves""" + print "START fragment evaluation" + t0 = time.time() + + df = self._dataflow + + if self._prov is not None: + self._prov.init(df) + self._prov.time_init = t0 + + + # Unvalidate all the nodes + self._evaluated.clear() + # Start by stoping all parents node of input fragments nodes + if self._fragment_infos: + for ivid, ipid in self._fragment_infos['inputs_vid']: + for npid, nvid, nactor in self.get_parent_nodes(ipid): + self._evaluated.add(nvid) + print "Set : ", nvid, " as EVALUATED" + + + # Eval from the leaf + # for vid in (vid for vid in df.vertices() if df.nb_out_edges(vid) == 0): + # self.eval_vertex(vid) + + # Eval from the outputs node of the fragment: + for ovid in self._fragment_infos['outputs_vid']: + self.eval_vertex(ovid[0]) + + t1 = time.time() + + # Save the outputs of the fragment into file + if not os.path.exists(os.path.dirname(self._tmp_path)): + os.makedirs(self._tmp_path) + for i, vid in enumerate([v[0] for v in self._fragment_infos['outputs_vid']]): + for port in range(df.node(vid).get_nb_output()): + data_id = get_id(vid, port) + write_data(data_id=data_id, data=df.node(vid).get_output(port), path=self._tmp_path) + self._indexdb.add_data(data_id=data_id, path=str(os.path.join(self._tmp_path, data_id)), exec_data=True, cache_data=False) + + + if self._prov is not None: + self._prov.time_end = t1 + wfitem = self._prov.as_wlformat() + if self._provdb is not None: + self._provdb.add_wf_item(wfitem) + + # close remote connections + self._provdb.close() + + if quantify: + print "Evaluation time: %s" % (t1 - t0) + +class FakeEvaluation(AbstractEvaluation): + """ Evaluation to get id of egdes """ + __evaluators__.append("FakeEvaluation") + + # TODO: It doesn't work with provenance + def __init__(self, dataflow, *args, **kwargs): + + AbstractEvaluation.__init__(self, dataflow) + # a property to specify if the node has already been evaluated + self._evaluated = set() + + self._index = Task_UID_graph(dataflow) + + def is_stopped(self, vid, actor): + """ Return True if evaluation must be stop at this vertex """ + + if vid in self._evaluated: + return True + + try: + if actor.block: + status = True + n = actor.get_nb_output() + outputs = [i for i in range(n) if + actor.get_output(i) is not None] + if not outputs: + status = False + return status + except: + pass return False + + def eval_vertex_code(self, vid, *args, **kwargs): + """ + Evaluate the vertex vid. + Can raise an exception if evaluation failed. + """ + + node = self._dataflow.actor(vid) + + try: + + if self._index is not None: + self._index.before_eval(self._dataflow, vid) + + t0 = clock() + ret = 0 + + dt = clock() - t0 + if self._index is not None: + self._index.after_eval(self._dataflow, vid) + + # When an exception is raised, a flag is set. + # So we remove it when evaluation is ok. + node.raise_exception = False + # if hasattr(node, 'raise_exception'): + # del node.raise_exception + node.notify_listeners(('data_modified', None, None)) + return ret + + except EvaluationException, e: + e.vid = vid + e.node = node + # When an exception is raised, a flag is set. + node.raise_exception = True + node.notify_listeners(('data_modified', None, None)) + raise e + + except Exception, e: + # When an exception is raised, a flag is set. + node.raise_exception = True + node.notify_listeners(('data_modified', None, None)) + raise EvaluationException(vid, node, e, \ + tb.format_tb(sys.exc_info()[2])) + + + def eval_vertex(self, vid, *args, **kwargs): + """ Evaluate the vertex vid """ + + df = self._dataflow + actor = df.actor(vid) + self._evaluated.add(vid) + + # For each inputs + for pid in df.in_ports(vid): + inputs = [] + + cpt = 0 + # For each connected node + for npid, nvid, nactor in self.get_parent_nodes(pid): + if not self.is_stopped(nvid, nactor): + self.eval_vertex(nvid) + + inputs.append(nactor.get_output(df.local_id(npid))) + cpt += 1 + + # set input as a list or a simple value + if (cpt == 1): + inputs = inputs[0] + if (cpt > 0): + actor.set_input(df.local_id(pid), inputs) + # Eval the node + self.eval_vertex_code(vid) + + def eval(self, *args, **kwargs): + """ Evaluate the whole dataflow starting from leaves""" + + t0 = time.time() + df = self._dataflow + + # Unvalidate all the nodes + self._evaluated.clear() + + # Eval from the leaf + for vid in (vid for vid in df.vertices() if df.nb_out_edges(vid) == 0): + self.eval_vertex(vid) + + t1 = time.time() + + if self._index is not None: + tid = self._index.as_dict() + + if quantify: + print "Evaluation time: %s" % (t1 - t0) + self._dataflow.node(1).add_input(name="task_ids") + self._dataflow.node(1).set_output("task_ids", tid) + return + \ No newline at end of file diff --git a/src/openalea/core/compositenode.py b/src/openalea/core/compositenode.py index 53366d51..e8d63ad1 100644 --- a/src/openalea/core/compositenode.py +++ b/src/openalea/core/compositenode.py @@ -29,7 +29,7 @@ from openalea.core.node import AbstractFactory, AbstractPort, Node from openalea.core.node import RecursionError -from openalea.core.pkgmanager import PackageManager, protected, UnknownPackageError +from openalea.core.pkgmanager import PackageManager, UnknownPackageError from openalea.core.package import UnknownNodeError from openalea.core.dataflow import DataFlow, InvalidEdge, PortError from openalea.core.settings import Settings @@ -38,13 +38,13 @@ quantify = False + class IncompatibleNodeError(Exception): """todo""" pass class CompositeNodeFactory(AbstractFactory): - mimetype = "openalea/compositenodefactory" """ @@ -88,6 +88,10 @@ def __init__(self, *args, **kargs): self.doc = kargs.get('doc', "") self.__doc__ = self.doc + # Unique ID for the factory - TODO: for now, only built from name + self.uid = str(self.name) + + def is_composite_node(self): return True @@ -120,7 +124,7 @@ def copy(self, **args): for k, v in ret.elt_factory.iteritems(): pkg_id, factory_id = v - if(pkg_id == old_pkg.get_id()): + if (pkg_id == old_pkg.get_id()): pkg_id = new_pkg.get_id() ret.elt_factory[k] = pkg_id, factory_id @@ -153,7 +157,7 @@ def instantiate(self, call_stack=None): new_df.set_caption(self.get_id()) new_df.eval_algo = self.eval_algo - cont_eval = set() # continuous evaluated nodes + cont_eval = set() # continuous evaluated nodes # Instantiate the node with each factory for vid in self.elt_factory: @@ -161,7 +165,7 @@ def instantiate(self, call_stack=None): node = self.instantiate_node(vid, call_stack) # Manage continuous eval - if(node.user_application): + if (node.user_application): cont_eval.add(vid) except (UnknownNodeError, UnknownPackageError): @@ -171,7 +175,7 @@ def instantiate(self, call_stack=None): print "-> Cannot find '%s:%s'" % (pkg, fact) node = self.create_fake_node(vid) node.raise_exception = True - node.notify_listeners(('data_modified', None, None )) + node.notify_listeners(('data_modified', None, None)) new_df.add_node(node, vid, False) @@ -179,10 +183,12 @@ def instantiate(self, call_stack=None): try: self.load_ad_hoc_data(new_df.node(new_df.id_in), copy.deepcopy(self.elt_data["__in__"]), - copy.deepcopy(self.elt_ad_hoc.get("__in__", None))) + copy.deepcopy( + self.elt_ad_hoc.get("__in__", None))) self.load_ad_hoc_data(new_df.node(new_df.id_out), copy.deepcopy(self.elt_data["__out__"]), - copy.deepcopy(self.elt_ad_hoc.get("__out__", None))) + copy.deepcopy( + self.elt_ad_hoc.get("__out__", None))) except: pass @@ -191,23 +197,23 @@ def instantiate(self, call_stack=None): (source_vid, source_port, target_vid, target_port) = link # Replace id for in and out nodes - if(source_vid == '__in__'): + if (source_vid == '__in__'): source_vid = new_df.id_in - if(target_vid == '__out__'): + if (target_vid == '__out__'): target_vid = new_df.id_out new_df.connect(source_vid, source_port, target_vid, target_port) # Set continuous evaluation for vid in cont_eval: - new_df.set_continuous_eval(vid, True) + new_df.set_continuous_eval(vid, True) # Set call stack to its original state call_stack.pop() # Properties new_df.lazy = self.lazy - new_df.graph_modified = False # Graph is not modifyied + new_df.graph_modified = False # Graph is not modifyied return new_df @@ -222,41 +228,40 @@ def create_fake_node(self, vid): for eid, link in self.connections.iteritems(): (source_vid, source_port, target_vid, target_port) = link - if(source_vid == vid): + if (source_vid == vid): outs = max(outs, source_port) - elif(target_vid == vid): + elif (target_vid == vid): ins = max(ins, target_port) node = Node() attributes = copy.deepcopy(self.elt_data[vid]) - ad_hoc = copy.deepcopy(self.elt_ad_hoc.get(vid, None)) + ad_hoc = copy.deepcopy(self.elt_ad_hoc.get(vid, None)) self.load_ad_hoc_data(node, attributes, ad_hoc) # copy node input data if any values = copy.deepcopy(self.elt_value.get(vid, ())) - for p in range(ins+1): - port = node.add_input(name="In"+str(p)) + for p in range(ins + 1): + port = node.add_input(name="In" + str(p)) - for p in range(outs+1): - port = node.add_output(name="Out"+str(p)) + for p in range(outs + 1): + port = node.add_output(name="Out" + str(p)) for vs in values: try: - #the two first elements are the historical - #values : port Id and port value - #beyond that are extensions added by gengraph: - #the ad_hoc_dict representation is third. + # the two first elements are the historical + # values : port Id and port value + # beyond that are extensions added by gengraph: + # the ad_hoc_dict representation is third. port, v = vs[:2] node.set_input(port, eval(v)) - if(len(vs)>2): + if (len(vs) > 2): d = MetaDataDict(vs[2]) node.input_desc[port].get_ad_hoc_dict().update(d) except: continue - return node def paste(self, cnode, data_modifiers=[], call_stack=None, meta=False): @@ -282,8 +287,8 @@ def paste(self, cnode, data_modifiers=[], call_stack=None, meta=False): # Apply modifiers (if callable) for (key, func) in data_modifiers: try: - if(callable(func)): - if(meta): + if (callable(func)): + if (meta): func(n) else: n.internal_data[key] = func(n.internal_data[key]) @@ -309,31 +314,33 @@ def paste(self, cnode, data_modifiers=[], call_stack=None, meta=False): def load_ad_hoc_data(self, node, elt_data, elt_ad_hoc=None): if elt_ad_hoc and len(elt_ad_hoc): - #reading 0.8+ files. + # reading 0.8+ files. d = MetaDataDict(dict=elt_ad_hoc) node.get_ad_hoc_dict().update(d) else: - #extracting ad hoc data from old files. - #we parse the Node class' __ad_hoc_from_old_map__ - #which defines conversions between new ad_hoc_dict keywords - #and old internal_data keywords. - #These dictionnaries are used to extend ad_hoc_dict of a node with the - #data that views expect. See node.initialise_standard_metadata() for an example. + # extracting ad hoc data from old files. + # we parse the Node class' __ad_hoc_from_old_map__ + # which defines conversions between new ad_hoc_dict keywords + # and old internal_data keywords. + # These dictionnaries are used to extend ad_hoc_dict of a node with the + # data that views expect. See node.initialise_standard_metadata() for an example. if hasattr(node, "__ad_hoc_from_old_map__"): for newKey, oldKeys in node.__ad_hoc_from_old_map__.iteritems(): - data = [] #list that stores the new values + data = [] # list that stores the new values _type, default = node.__ad_hoc_slots__.get(newKey) for key in oldKeys: data.append(elt_data.pop(key, None)) - if len(data) == 1 : data = data[0] - if data is None or (isinstance(data, list) and None in data): #? + if len(data) == 1: data = data[0] + if data is None or ( + isinstance(data, list) and None in data): # ? data = default - if data is None : continue + if data is None: continue node.get_ad_hoc_dict().set_metadata(newKey, _type(data)) - #finally put the internal data (elt_data) where it has always been expected. + # finally put the internal data (elt_data) where it has always been expected. node._init_internal_data(elt_data) -# node.internal_data.update(elt_data) + + # node.internal_data.update(elt_data) def instantiate_node(self, vid, call_stack=None): """ Partial instantiation @@ -345,18 +352,11 @@ def instantiate_node(self, vid, call_stack=None): (package_id, factory_id) = self.elt_factory[vid] pkgmanager = PackageManager() pkg = pkgmanager[package_id] - try: - factory = pkg.get_factory(factory_id) - except UnknownNodeError, e: - # Bug when both package_id and protected(package_id) exist - pkg = pkgmanager[protected(package_id)] - factory = pkg.get_factory(factory_id) - - + factory = pkg.get_factory(factory_id) node = factory.instantiate(call_stack) attributes = copy.deepcopy(self.elt_data[vid]) - ad_hoc = copy.deepcopy(self.elt_ad_hoc.get(vid, None)) + ad_hoc = copy.deepcopy(self.elt_ad_hoc.get(vid, None)) self.load_ad_hoc_data(node, attributes, ad_hoc) # copy node input data if any @@ -364,13 +364,14 @@ def instantiate_node(self, vid, call_stack=None): for vs in values: try: - #the two first elements are the historical - #values : port Id and port value - #the values beyond are not used. + # the two first elements are the historical + # values : port Id and port value + # the values beyond are not used. port, v = vs[:2] node.set_input(port, eval(v)) node.input_desc[port].get_ad_hoc_dict().set_metadata("hide", - node.is_port_hidden(port)) + node.is_port_hidden( + port)) except: continue @@ -380,7 +381,7 @@ def instantiate_node(self, vid, call_stack=None): # This shouldn't be here, it is related to visual stuff # ######################################################### def instantiate_widget(self, node=None, parent=None, \ - edit=False, autonomous=False): + edit=False, autonomous=False): """ Return the corresponding widget initialised with node @@ -388,11 +389,11 @@ def instantiate_widget(self, node=None, parent=None, \ widget composed with the node sub widget is returned. """ - if(edit): + if (edit): from openalea.visualea.dataflowview import GraphicalGraph return GraphicalGraph(node).create_view(parent) - if(node == None): + if (node == None): node = self.instantiate() from openalea.visualea.compositenode_widget import DisplayGraphWidget @@ -457,23 +458,23 @@ def set_io(self, inputs, outputs): # I/O ports # Remove node if nb of input has changed - if(self.id_in is not None - and len(inputs) != self.node(self.id_in).get_nb_output()): + if (self.id_in is not None + and len(inputs) != self.node(self.id_in).get_nb_output()): self.remove_node(self.id_in) self.id_in = None - if(self.id_out is not None - and len(outputs) != self.node(self.id_out).get_nb_input()): + if (self.id_out is not None + and len(outputs) != self.node(self.id_out).get_nb_input()): self.remove_node(self.id_out) self.id_out = None # Create new io node if necessary - if(self.id_in is None): + if (self.id_in is None): self.id_in = self.add_node(CompositeNodeInput(inputs)) else: self.node(self.id_in).set_io((), inputs) - if(self.id_out is None): + if (self.id_out is None): self.id_out = self.add_node(CompositeNodeOutput(outputs)) else: self.node(self.id_out).set_io(outputs, ()) @@ -498,28 +499,29 @@ def set_output(self, index_key, val): return self.node(self.id_out).set_output(index_key, val) - def get_eval_algo(self): + def get_eval_algo(self, *args, **kwargs): """ Return the evaluation algo instance """ try: algo_str = self.eval_algo - algo_str = algo_str.strip('"'); + algo_str = algo_str.strip('"') algo_str = algo_str.strip("'") - # import module baseimp = "algo.dataflow_evaluation" module = __import__(baseimp, globals(), locals(), [algo_str]) classobj = module.__dict__[algo_str] - return classobj(self) + return classobj(self, *args, **kwargs) except Exception, e: - from openalea.core.algo.dataflow_evaluation import DefaultEvaluation + from openalea.core.algo.dataflow_evaluation import \ + DefaultEvaluation return DefaultEvaluation(self) return self.eval_algo - def eval_as_expression(self, vtx_id=None, step=False): + def eval_as_expression(self, vtx_id=None, step=False, + *args, **kwargs): """ Evaluate a vtx_id @@ -527,21 +529,25 @@ def eval_as_expression(self, vtx_id=None, step=False): """ import time t0 = time.time() - if(self.evaluating): + if self.evaluating: return - if(vtx_id != None): + if vtx_id is not None: self.node(vtx_id).modified = True - algo = self.get_eval_algo() + algo = self.get_eval_algo(*args, **kwargs) try: self.evaluating = True - algo.eval(vtx_id,step=step) + algo.eval(vtx_id, step=step, *args, **kwargs) finally: self.evaluating = False t1 = time.time() if quantify: - logger.info('Evaluation time: %s'%(t1-t0)) - print 'Evaluation time: %s'%(t1-t0) + logger.info('Evaluation time: %s' % (t1 - t0)) + print 'Evaluation time: %s' % (t1 - t0) + + if kwargs.get("record_provenance"): + return algo._prov + # Functions used by the node evaluator def eval(self, *args, **kwds): @@ -550,26 +556,26 @@ def eval(self, *args, **kwds): Return True if the node need a reevaluation (like generator) """ - self.__call__() + self.__call__(*args, **kwds) self.modified = False self.notify_listeners(("status_modified", self.modified)) return False - def __call__(self, inputs=()): + def __call__(self, inputs=(), *args, **kwds): """ Evaluate the graph """ - if(self.id_out and self.get_nb_output()>0): - self.eval_as_expression(self.id_out) + if (self.id_out and self.get_nb_output() > 0): + self.eval_as_expression(self.id_out, *args, **kwds) else: - self.eval_as_expression(None) + self.eval_as_expression(None, *args, **kwds) return () - def to_script (self) : + def to_script(self): """Translate the dataflow into a python script. """ from algo.dataflow_evaluation import ToScriptEvaluation @@ -593,15 +599,15 @@ def compute_external_io(self, vertex_selection, new_vid): in_edges = \ self._compute_outside_connection(vertex_selection, in_edges, - new_vid, is_input=True) + new_vid, is_input=True) out_edges = \ self._compute_outside_connection(vertex_selection, out_edges, - new_vid, is_input=False) + new_vid, is_input=False) return in_edges + out_edges def _compute_outside_connection(self, vertex_selection, new_connections, - new_vid, is_input = True): + new_vid, is_input=True): """ Return external connections of a composite node with input and output ports. @@ -632,7 +638,7 @@ def _compute_outside_connection(self, vertex_selection, new_connections, for edge in new_connections: if is_input: - if(edge[0] != '__in__'): + if (edge[0] != '__in__'): continue target_id, target_port = edge[2:] if (target_id, target_port) in selected_port: @@ -642,7 +648,7 @@ def _compute_outside_connection(self, vertex_selection, new_connections, port_id = self.local_id(self.source_port(e)) connections.append((vid, port_id, new_vid, edge[1])) else: - if(edge[2] != '__out__'): + if (edge[2] != '__out__'): continue source_id, source_port = edge[0:2] @@ -697,7 +703,7 @@ def _compute_inout_connection(self, vertex_selection, is_input=True): n = self.node(vid) desc = dict(io_desc(n)[pname]) - caption= '(%s)' % (n.get_caption()) + caption = '(%s)' % (n.get_caption()) count = '' name = desc['name'] @@ -726,7 +732,7 @@ def _compute_inout_connection(self, vertex_selection, is_input=True): desc['value'] = desc['interface'].default() connections.append(('__in__', len(nodes), vid, pname)) - else: # output + else: # output connections.append((vid, pname, '__out__', len(nodes))) nodes.append(desc) @@ -741,7 +747,6 @@ def compute_io(self, v_list=None): v_list is a vertex id list """ - ins, in_edges = self._compute_inout_connection(v_list, is_input=True) outs, out_edges = \ self._compute_inout_connection(v_list, is_input=False) @@ -749,7 +754,7 @@ def compute_io(self, v_list=None): return (ins, outs, connections) - def to_factory(self, sgfactory, listid = None, auto_io=False): + def to_factory(self, sgfactory, listid=None, auto_io=False): """ Update CompositeNodeFactory to fit with the graph @@ -764,9 +769,9 @@ def to_factory(self, sgfactory, listid = None, auto_io=False): # Properties sgfactory.lazy = self.lazy sgfactory.eval_algo = self.eval_algo - #print self.eval_algo + # print self.eval_algo # I / O - if(auto_io): + if (auto_io): (ins, outs, sup_connect) = self.compute_io(listid) sgfactory.inputs = ins sgfactory.outputs = outs @@ -784,11 +789,11 @@ def to_factory(self, sgfactory, listid = None, auto_io=False): src = self.source(eid) tgt = self.target(eid) - if((src not in listid) or (tgt not in listid)): + if ((src not in listid) or (tgt not in listid)): continue - if(src == self.id_in): + if (src == self.id_in): src = '__in__' - if(tgt == self.id_out): + if (tgt == self.id_out): tgt = '__out__' source_port = self.local_id(self.source_port(eid)) @@ -807,9 +812,9 @@ def to_factory(self, sgfactory, listid = None, auto_io=False): kdata = node.internal_data # Do not copy In and Out - if(vid == self.id_in): + if (vid == self.id_in): vid = "__in__" - elif(vid == self.id_out): + elif (vid == self.id_out): vid = "__out__" else: pkg_id = node.factory.package.get_id() @@ -822,10 +827,11 @@ def to_factory(self, sgfactory, listid = None, auto_io=False): # We do the exact opposite than in load_ad_hoc_data, have a look there. if hasattr(node, "__ad_hoc_from_old_map__"): for newKey, oldKeys in node.__ad_hoc_from_old_map__.iteritems(): - if len(oldKeys)==0: continue + if len(oldKeys) == 0: continue data = node.get_ad_hoc_dict().get_metadata(newKey) for pos, newKey in enumerate(oldKeys): - sgfactory.elt_data[vid][newKey] = data[pos] if isinstance(data, list) else data + sgfactory.elt_data[vid][newKey] = data[ + pos] if isinstance(data, list) else data # Copy ad_hoc data sgfactory.elt_ad_hoc[vid] = copy.deepcopy(node.get_ad_hoc_dict()) @@ -842,10 +848,10 @@ def to_factory(self, sgfactory, listid = None, auto_io=False): self.graph_modified = False # Set node factory if all node have been exported - if(listid is None): + if (listid is None): self.factory = sgfactory - def add_node(self, node, vid = None, modify=True): + def add_node(self, node, vid=None, modify=True): """ Add a node in the Graph with a particular id if id is None, autogenrate one @@ -870,9 +876,9 @@ def add_node(self, node, vid = None, modify=True): self.set_actor(vid, node) self.notify_vertex_addition(node, vid) - #self.id_cpt += 1 - if(modify): - self.notify_listeners(("graph_modified", )) + # self.id_cpt += 1 + if (modify): + self.notify_listeners(("graph_modified",)) self.graph_modified = True return vid @@ -880,24 +886,30 @@ def add_node(self, node, vid = None, modify=True): def notify_vertex_addition(self, vertex, vid=None): vtype = "vertex" doNotify = True - if(vertex.__class__.__dict__.has_key("__graphitem__")): vtype = "annotation" + if (vertex.__class__.__dict__.has_key("__graphitem__")): + vtype = "annotation" elif isinstance(vertex, CompositeNodeOutput): vtype = "outNode" doNotify = True if len(vertex.input_desc) else False - elif isinstance(vertex, CompositeNodeInput) : + elif isinstance(vertex, CompositeNodeInput): vtype = "inNode" doNotify = True if len(vertex.output_desc) else False - else: pass + else: + pass if doNotify: self.notify_listeners(("vertex_added", (vtype, vertex))) def notify_vertex_removal(self, vertex): vtype = "vertex" doNotify = True - if(vertex.__class__.__dict__.has_key("__graphitem__")): vtype = "annotation" - elif isinstance(vertex, CompositeNodeOutput): vtype = "outNode" - elif isinstance(vertex, CompositeNodeInput) : vtype = "inNode" - else: pass + if (vertex.__class__.__dict__.has_key("__graphitem__")): + vtype = "annotation" + elif isinstance(vertex, CompositeNodeOutput): + vtype = "outNode" + elif isinstance(vertex, CompositeNodeInput): + vtype = "inNode" + else: + pass self.notify_listeners(("vertex_removed", (vtype, vertex))) def remove_node(self, vtx_id): @@ -906,12 +918,14 @@ def remove_node(self, vtx_id): :param vtx_id: element id """ node = self.node(vtx_id) - if vtx_id == self.id_in : self.id_in = None - elif vtx_id == self.id_out : self.id_out = None + if vtx_id == self.id_in: + self.id_in = None + elif vtx_id == self.id_out: + self.id_out = None self.remove_vertex(vtx_id) node.close() self.notify_vertex_removal(node) - self.notify_listeners(("graph_modified", )) + self.notify_listeners(("graph_modified",)) self.graph_modified = True def remove_edge(self, eid): @@ -922,9 +936,9 @@ def remove_edge(self, eid): port = None DataFlow.remove_edge(self, eid) if port: - self.actor(port._vid).set_input_state(port._local_pid, "disconnected") - self.notify_listeners(("edge_removed", ("default",eid) )) - + self.actor(port._vid).set_input_state(port._local_pid, + "disconnected") + self.notify_listeners(("edge_removed", ("default", eid))) def simulate_destruction_notifications(self): """emits messages as if we were adding elements to @@ -938,7 +952,7 @@ def simulate_destruction_notifications(self): for eid in self.edges(): (src_id, dst_id) = self.source(eid), self.target(eid) - etype=None + etype = None src_port_id = self.local_id(self.source_port(eid)) dst_port_id = self.local_id(self.target_port(eid)) @@ -947,10 +961,10 @@ def simulate_destruction_notifications(self): src_port = nodeSrc.output_desc[src_port_id] dst_port = nodeDst.input_desc[dst_port_id] - #don't notify if the edge is connected to the input or - #output nodes. + # don't notify if the edge is connected to the input or + # output nodes. # if(src_id == self.id_in or dst_id == self.id_out): - # continue + # continue edgedata = "default", eid self.notify_listeners(("edge_removed", edgedata)) @@ -969,11 +983,12 @@ def connect(self, src_id, port_src, dst_id, port_dst): target_pid = self.in_port(dst_id, port_dst) eid = DataFlow.connect(self, source_pid, target_pid) except: - logger.error("Enable to create the edge %s %d %d %d %d"%( self.factory.name, src_id, port_src, dst_id, port_dst)) + logger.error("Enable to create the edge %s %d %d %d %d" % ( + self.factory.name, src_id, port_src, dst_id, port_dst)) return self.actor(dst_id).set_input_state(port_dst, "connected") - self.notify_listeners(("connection_modified", )) + self.notify_listeners(("connection_modified",)) self.graph_modified = True self.update_eval_listeners(src_id) @@ -983,8 +998,8 @@ def connect(self, src_id, port_src, dst_id, port_dst): dst_port = nodeDst.input_desc[port_dst] edgedata = "default", eid, src_port, dst_port - #connected ports cannot be hidden: - #nodeSrc.set_port_hidden(port_src, False) + # connected ports cannot be hidden: + # nodeSrc.set_port_hidden(port_src, False) nodeDst.set_port_hidden(port_dst, False) self.notify_listeners(("edge_added", edgedata)) @@ -1003,10 +1018,10 @@ def disconnect(self, src_id, port_src, dst_id, port_dst): for eid in self.connected_edges(source_pid): if self.target_port(eid) == target_pid: - self.notify_listeners(("edge_removed", ("default",eid))) + self.notify_listeners(("edge_removed", ("default", eid))) self.remove_edge(eid) self.actor(dst_id).set_input_state(port_dst, "disconnected") - self.notify_listeners(("connection_modified", )) + self.notify_listeners(("connection_modified",)) self.graph_modified = True self.update_eval_listeners(src_id) @@ -1023,8 +1038,8 @@ def replace_node(self, vid, newnode): # newnode.internal_data.update(oldnode.internal_data) newnode.caption = caption - if(oldnode.get_nb_input() != newnode.get_nb_input() or - oldnode.get_nb_output() != newnode.get_nb_output()): + if (oldnode.get_nb_input() != newnode.get_nb_input() or + oldnode.get_nb_output() != newnode.get_nb_output()): raise IncompatibleNodeError() self.set_actor(vid, newnode) @@ -1036,11 +1051,11 @@ def set_continuous_eval(self, vid, state=True): node = self.actor(vid) - if(not node.user_application and not state): + if (not node.user_application and not state): return # Remove previous listener - if(node.user_application and hasattr(node, 'continuous_listener')): + if (node.user_application and hasattr(node, 'continuous_listener')): listener = node.continuous_listener node.continuous_listener = None if listener: @@ -1048,7 +1063,7 @@ def set_continuous_eval(self, vid, state=True): node.user_application = state - if(state): + if (state): listener = ContinuousEvalListener(self, vid) node.continuous_listener = listener @@ -1074,6 +1089,7 @@ def update_eval_listeners(self, vid): listeners = dst_node.continuous_eval.listeners src_node.continuous_eval.listeners.update(listeners) + from openalea.core.observer import AbstractListener @@ -1116,7 +1132,7 @@ def get_input(self, input_pid): def eval(self): return False - def to_script (self): + def to_script(self): return "" @@ -1145,7 +1161,7 @@ def set_output(self, output_pid, val): def eval(self): return False - def to_script (self): + def to_script(self): return "" @@ -1154,7 +1170,8 @@ class PyCNFactoryWriter(object): sgfactory_template = """ -$NAME = CompositeNodeFactory(name=$PNAME, +$NAME = CompositeNodeFactory(uid=$UID, + name=$PNAME, description=$DESCRIPTION, category=$CATEGORY, doc=$DOC, @@ -1186,15 +1203,19 @@ def __repr__(self): name = f.get_python_name() name = name.replace('.', '_') - result = fstr.safe_substitute(NAME=name, + result = fstr.safe_substitute(UID=self.pprint_repr(f.uid), + NAME=name, PNAME=self.pprint_repr(f.name), - DESCRIPTION=self.pprint_repr(f.description), + DESCRIPTION=self.pprint_repr( + f.description), CATEGORY=self.pprint_repr(f.category), DOC=self.pprint_repr(f.doc), INPUTS=self.pprint_repr(f.inputs), OUTPUTS=self.pprint_repr(f.outputs), - ELT_FACTORY=self.pprint_repr(f.elt_factory), - ELT_CONNECTIONS=self.pprint_repr(f.connections), + ELT_FACTORY=self.pprint_repr( + f.elt_factory), + ELT_CONNECTIONS=self.pprint_repr( + f.connections), ELT_DATA=self.pprint_repr(f.elt_data), ELT_VALUE=self.pprint_repr(f.elt_value), ELT_AD_HOC=self.pprint_repr(f.elt_ad_hoc), @@ -1203,14 +1224,18 @@ def __repr__(self): ) return result + import json -class JSONCNFactoryWriter(PyCNFactoryWriter): + +class JSONCNFactoryWriter(PyCNFactoryWriter): def __repr__(self): f = self.factory - minx = min(f.elt_ad_hoc.itervalues(), key=lambda x: x["position"][0])["position"][0] - miny = min(f.elt_ad_hoc.itervalues(), key=lambda x: x["position"][1])["position"][1] + minx = min(f.elt_ad_hoc.itervalues(), key=lambda x: x["position"][0])[ + "position"][0] + miny = min(f.elt_ad_hoc.itervalues(), key=lambda x: x["position"][1])[ + "position"][1] print minx, miny @@ -1223,12 +1248,12 @@ def __repr__(self): description=f.description, category=f.category, doc=f.doc, - #inputs=f.inputs, - #outputs=f.outputs, - #elt_factory=f.elt_factory, + # inputs=f.inputs, + # outputs=f.outputs, + # elt_factory=f.elt_factory, elt_connections=list(f.connections.itervalues()), - #elt_data=f.elt_data, - #elt_value=f.elt_value, + # elt_data=f.elt_data, + # elt_value=f.elt_value, elt_ad_hoc=f.elt_ad_hoc, lazy=f.lazy, eval_algo=f.eval_algo, diff --git a/src/openalea/core/interface.py b/src/openalea/core/interface.py index fa085ad8..915efed3 100644 --- a/src/openalea/core/interface.py +++ b/src/openalea/core/interface.py @@ -193,6 +193,13 @@ class ICodeStr(IStr): pass +class IRef(IStr): + """Interface for uids + """ + __label__ = u'ref' + pass + + class IFloat(IInterface): """ Float interface """ diff --git a/src/openalea/core/interpreter/__init__.py b/src/openalea/core/interpreter/__init__.py index edb76b7b..1b2ee7d0 100644 --- a/src/openalea/core/interpreter/__init__.py +++ b/src/openalea/core/interpreter/__init__.py @@ -1,11 +1,7 @@ -""" OpenAlea.Core. - -""" def get_interpreter_class(): """ :return: the interpreter class to instantiate the shell """ - Interpreter = None try: from openalea.core.interpreter.ipython import Interpreter except ImportError: @@ -15,12 +11,11 @@ def get_interpreter_class(): adapt_interpreter(Interpreter) return Interpreter +from openalea.core.util import warn_deprecated def get_interpreter(): - from openalea.core.util import warn_deprecated warn_deprecated(__name__ + ".get_interpreter", __name__ + 'interpreter', (2014, 10, 8)) - from openalea.oalab.session.session import Session if Session.instantiated: return Session().interpreter @@ -32,19 +27,16 @@ def get_interpreter(): except(ImportError, NameError): pass if not interpreter_: - interpreter_klass = get_interpreter_class() - if interpreter_klass: - interpreter_ = interpreter_klass() + interpreter_ = get_interpreter_class()() if interpreter_: return interpreter_ def _interpreter_class(): - Interpreter = None try: from openalea.core.interpreter.ipython import Interpreter except ImportError: - from code import InteractiveInterpreter as Interpreter + from code import InteractiveInterpreter return Interpreter @@ -75,11 +67,7 @@ def runsource(self, source=None, filename="", symbol="single"): def runcode(self, source=None): return self.run_code(source) - if not hasattr(ip, 'locals'): - ip.locals = ip.user_ns - if not hasattr(ip, 'user_ns'): - ip.user_ns = ip.locals - + ip.locals = ip.user_ns ip.runcode = runcode ip.runsource = runsource ip.loadcode = loadcode diff --git a/src/openalea/core/metadata/__init__.py b/src/openalea/core/metadata/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/openalea/core/metadata/cloud_info.py b/src/openalea/core/metadata/cloud_info.py new file mode 100644 index 00000000..12e8a2ba --- /dev/null +++ b/src/openalea/core/metadata/cloud_info.py @@ -0,0 +1,20 @@ +# from openalea.core.path import path +# from openalea.core import settings + +# # infos about provenance db +# REMOTE = False + +# # SSH +# PROVDB_SSH_ADDR = "134.158.247.32" +# SSU_USERNAME="ubuntu" +# SSH_PKEY="/home/gaetan/.ssh/id_rsa" +# # REMOTE_DB_ADDR=('127.0.0.1', 27017) + +# # Mongo +# MONGO_ADDR='127.0.0.1' +# MONGO_PORT = 27017 + +# # file infos +# PROVENANCE_PATH = path(settings.get_openalea_home_dir()) / 'provenance' +# TMP_PATH = path(settings.get_openalea_home_dir()) / "execution_data" +# CACHE_PATH = path(settings.get_openalea_home_dir()) / "cached_data" diff --git a/src/openalea/core/metadata/costs.py b/src/openalea/core/metadata/costs.py new file mode 100644 index 00000000..f9b815ab --- /dev/null +++ b/src/openalea/core/metadata/costs.py @@ -0,0 +1,45 @@ +# # function to get the site and minimal cost for a task, with prov + sites +# def minimum_cost_site(vid, provenance, multisites): +# vid = str(vid) +# # if no provenance data for task vid -> NOT TAKEN INTO ACCOUNT +# # TODO: manage if no prov data -> average? best site? random? .... +# if vid not in provenance.prov: +# print("Not enough provenance information on task: " + str(vid)) +# return None, None + +# # find the sites where the input data exist: +# input_site = [] +# for s in multisites.list_sites: +# if multisites.list_sites[s].check_input(vid): +# input_site.append(s) +# # IF No site has the input data -> cannot be computed +# if not input_site: +# print('Input data not found') +# return None, None + +# possible_cost = dict() +# # for each site get the cost +# for s1 in multisites.list_sites: +# # if the Input data is not on s1 -> get the minimal cost to transfer the intput data to the site: +# min_transfer_cost = [] +# for s2 in input_site: +# transfer_cost = multisites.list_sites[s1].transfer_cost[s2] +# # get the input data size | here from provenance, in real cases the scheduler knows it +# input_data_size = provenance.prov[vid].input_size + +# get_input_cost_tmp = transfer_cost * input_data_size +# min_transfer_cost.append(get_input_cost_tmp) +# get_input_cost = min(min_transfer_cost) + +# # the cost to compute the task on this site: +# compute_cost = multisites.list_sites[s1].compute_cost * provenance.prov[vid].exec_time + +# total_cost = get_input_cost + compute_cost +# # print s1, "total_cost : " ,total_cost, 'get_input_cost : ', get_input_cost, 'compute_cost', compute_cost + +# possible_cost[s1] = total_cost + +# best_site = min(possible_cost) +# best_cost = possible_cost[best_site] + +# return best_site, best_cost \ No newline at end of file diff --git a/src/openalea/core/metadatadict.py b/src/openalea/core/metadatadict.py index 41e9d5a9..6a641e0d 100644 --- a/src/openalea/core/metadatadict.py +++ b/src/openalea/core/metadatadict.py @@ -84,6 +84,18 @@ def __repr__(self): def __len__(self): return len(self._metaTypes) + def get(self, key, default=None): + """Subclass of dict.get method + + Args: + key (any): + default (any): + + Returns: + (any) + """ + return self._metaValues.get(key, default) + def add_metadata(self, key, valType, notify=True): """Creates a new entry in the meta data registry. The data to set will be of the given 'valType' type.""" diff --git a/src/openalea/core/model.py b/src/openalea/core/model.py index b980df5a..d30e8226 100644 --- a/src/openalea/core/model.py +++ b/src/openalea/core/model.py @@ -406,7 +406,7 @@ def repr_code(self): return code def set_code(self, code): - from openalea.core.model_inout import parse_docstring, get_docstring, extract_functions + from openalea.oalab.model.parse import parse_docstring, get_docstring, extract_functions self._initial_code = code model, self.inputs_info, self.outputs_info = parse_docstring(code) funcs = extract_functions(code) diff --git a/src/openalea/core/model_inout.py b/src/openalea/core/model_inout.py deleted file mode 100644 index f2ef68d4..00000000 --- a/src/openalea/core/model_inout.py +++ /dev/null @@ -1,436 +0,0 @@ -# -*- python -*- -# -# OpenAlea.Core: Multi-Paradigm GUI -# -# Copyright 2014-2017 INRIA - CIRAD - INRA -# -# File author(s): Christophe Pradal -# -# File contributor(s): -# -# Distributed under the Cecill-C License. -# See accompanying file LICENSE.txt or copy at -# http://www.cecill.info/licences/Licence_CeCILL-C_V1-en.html -# -# OpenAlea WebSite : http://openalea.gforge.inria.fr -# -############################################################################### -""" Definition of Input and Output objects. - -Code to parse the functions. -""" -import ast -import re -from openalea.core import logger -from openalea.core.service.interface import interface_class, guess_interface -import textwrap - - -########################### -# Input and Output Objects -########################### - -class InputObj(object): - """ - Inputs object with: - - an attribute *name*: name of the input obj (str) (mandatory) - - an attribute *interface*: interface/type of the input obj (str) (optional) - - an attribute *default*: default value of the input obj (str) (optional) - - >>> from openalea.oalab.model.parse import InputObj - >>> obj = InputObj('a:float=1') - >>> obj.name - 'a' - >>> obj.default - '1' - >>> obj.interface - IFloat - >>> obj - InputObj('a:IFloat=1') - - :param string: string object with format "input_name:input_type=input_default_value" or "input_name=input_default_value" or "input_name:input_type" or "input_name" - """ - - def __init__(self, string=''): - self.name = None - self.interface = None - self.default = None - if "=" in string: - if ":" in string: - self.name = string.split(":")[0].strip() - interf = ":".join(string.split(":")[1:]) - self.interface = interf.split("=")[0].strip() - self.default = "=".join(string.split("=")[1:]).strip() - else: - self.name = string.split("=")[0].strip() - self.default = "=".join(string.split("=")[1:]).strip() - elif ":" in string: - self.name = string.split(":")[0].strip() - self.interface = ":".join(string.split(":")[1:]).strip() - else: - self.name = string.strip() - - set_interface(self) - - def __str__(self): - return self.__class__.__name__ + ". Name: " + str(self.name) + ". Interface: " + str(self.interface) + ". Default Value: " + str(self.default) + "." - - def repr_code(self): - string = self.name - if self.interface: - string += ":%s" % self.interface - if self.default: - string += "=%s" % self.default - return string - - def __repr__(self): - classname = self.__class__.__name__ - return "%s(%r)" % (classname, self.repr_code()) - - -def set_interface(input_obj): - if input_obj.interface is None: - if isinstance(input_obj.default, str): - try: - default_eval = eval(input_obj.default) - input_obj.interface = guess_interface(default_eval) - except SyntaxError: - input_obj.interface = guess_interface(input_obj.default) - else: - input_obj.interface = guess_interface(input_obj.default) - else: - try: - input_obj.interface = interface_class(input_obj.interface) - except ValueError: - input_obj.interface = guess_interface(input_obj.default) - if input_obj.interface == []: - input_obj.interface = None - elif isinstance(input_obj.interface, list): - input_obj.interface = input_obj.interface[0] - - -class OutputObj(InputObj): - pass - - -########################################################### -# Function to define to infer model parameters from the doc -########################################################### - - -def get_docstring(string): - """ - Get a docstring from a string - """ - M = ast_parse(string) - docstring = ast.get_docstring(M) - if docstring is not None: - return docstring - - return parse_doc_in_code(string) - - -def parse_doc_in_code(string): - """ - Take a lpy string_file, parse it and return only the docstring of the file. - - :param string: string representation of lpy file - :return: docstring of the file if exists (must be a multiline docstring!). If not found, return None. - - :use: - >>> f = open(lpyfilename, "r") # doctest: +SKIP - >>> lpystring = f.read() # doctest: +SKIP - >>> f.close() # doctest: +SKIP - >>> - >>> docstring = parse_lpy(lpystring) # doctest: +SKIP - >>> - >>> from openalea.oalab.model.parse import parse_doc - >>> if docstring is not None: # doctest: +SKIP - ... model, inputs, outputs = parse_doc(docstring) # doctest: +SKIP - ... print "model : ", model # doctest: +SKIP - ... print "inputs : ", inputs # doctest: +SKIP - ... print "outputs : ", outputs # doctest: +SKIP - """ - # TODO: need a code review - begin = None - begintype = None - doclines = string.splitlines() - i = 0 - for docline in doclines: - i += 1 - if docline == '"""': - begin = i - begintype = '"""' - break - elif docline == "'''": - begin = 1 - begintype = "'''" - break - elif docline == '"""': - begin = 2 - begintype = '"""' - break - elif docline == "'''": - begin = 2 - begintype = "'''" - break - - if begin is not None: - end = begin - 1 - for docline in doclines[begin:]: - end += 1 - if docline == begintype: - docstrings = doclines[begin:end] - return "\n".join(docstrings) - return None - -######################################### -# Safe ast parsing -######################################### - - -def ast_parse(string): - logger.debug("Parse code: " + string[:10] + "...") - try: - M = ast.parse(string) - except SyntaxError, e: - #raise e - logger.warning(str(e)) - wraper = textwrap.TextWrapper(width=30) - txt = wraper.wrap(string)[0] # Python 2 - # txt = textwrap.shorten(string, width=30, placeholder="...") # Python 3 - logger.warning("Syntax error when parsing: " + txt + "...") - M = ast.parse("") - return M - - -######################################### -# Detect inputs and outputs in docstring -######################################### -def parse_docstring(string): - """ - parse a string (not a docstring), get the docstring and return information on the model. - - :use: model, inputs, outputs = parse_docstring(multiline_string_to_parse) - - :param string: docstring to parse (string) - :return: model, inputs, outputs - """ - d = get_docstring(string) - model, inputs, outputs = parse_doc(d) - return model, inputs, outputs - - -def parse_doc(docstring): - """ - Parse a docstring. - - :return: model, inputs, outputs - """ - model, inputs, outputs = parse_function(docstring) - - inputs2, outputs2 = parse_input_and_output(docstring) - - # TODO: make a real beautifull merge - if inputs2: - inputs = inputs2 - if outputs2: - outputs = outputs2 - - ret_inputs = [] - ret_outputs = [] - if inputs: - ret_inputs = [InputObj(inp) for inp in inputs] - if outputs: - ret_outputs = [OutputObj(outp) for outp in outputs] - - return model, ret_inputs, ret_outputs - - - -def parse_function(docstring): - """ - Parse a docstring with format: - my_model(a:int=4, b)->r:int - - Unused. - - :return: model, inputs, outputs - """ - inputs = None - outputs = None - model = None - if hasattr(docstring, "splitlines"): - for docline in docstring.splitlines(): - if ("->" in docline): - outputs = docline.split("->")[-1].split(",") - model = docline.split("->")[0].split("(")[0] - inputs = docline.split("(")[-1].split(")")[0].split(",") - return model, inputs, outputs - - - -def parse_input_and_output(docstring): - """ - Parse a docstring with format: - inputs = input_name:input_type=input_default_value, ... - outputs = output_name:output_type, ... - - :use: - >>> from openalea.oalab.model.parse import parse_input_and_output - >>> comment = ''' - ... input = a:int=4, b - ... output = r:float''' - >>> inputs, outputs = parse_input_and_output(comment) - >>> inputs - ['a:int=4', 'b'] - >>> outputs - ['r:float'] - - :return: inputs, outputs - """ - inputs = [] - outputs = [] - if hasattr(docstring, "splitlines"): - docsplit = docstring.splitlines() - for line in docsplit: - line = line.strip() - if re.search('^#*\s*input\s*=', line): - line = "input".join(line.split('input')[1:]) - line = "=".join(line.split('=', 1)[1:]).strip() - inputs = _safe_split(line) - if re.search('^#*\s*output\s*=', line): - line = "output".join(line.split('output')[1:]) - line = line.split('=', 1)[1].strip() - outputs = _safe_split(line) - return inputs, outputs - - -def extract_functions(codestring, filename='tmp'): - """ - parse the code *codestring* and detect what are the functions defined inside - :return: dict name -> code - """ - # TODO: use ast.iter_child_nodes instead of walk to know nested level - # For example, currently this code fails because indent level is wrong - # if True: - # def f(): - # print('hello') - # This code fails because "return is outside of function" - # def f(): - # return 1 - # TODO: support IPython %magic - - funcs = {} - - r = ast_parse(codestring) - for statement in ast.walk(r): - if isinstance(statement, ast.FunctionDef): - wrapped = ast.Interactive(body=statement.body) - try: - code = compile(wrapped, filename, 'single') - except SyntaxError: - logger.debug("Parsing code %s not yet supported" % statement.name) - else: - funcs[statement.name] = code - - return funcs - -################################################# - -def _replace_regex(line, regex, replaced="_"): - """ - Search *regex* inside *line* and replace it by *replaced* - - :return: line replaced - :use: - >>> line = "a=(1,2,3), b=[1,2], c=4, d=([1,2,3],4)" - >>> _replace_regex(line, "\([A-Za-z0-9_,()]*\)", "_") - >>> "a=_______, b=[1,2], c=4, d=___________" - """ - line2 = line - search = re.search(regex, line2) - while search is not None: - start = search.start() - end = search.end() - n = end - start - # Replace found part by n*"_" - line2 = line2[:start] + (n * replaced) + line2[end:] - # Search again - search = re.search(regex, line2) - return line2 - - -def _replace_bracket(line): - # Search something with - # - one opening bracket ( - # - what you want - # - one closing bracket ) - return _replace_regex(line, "\([A-Za-z0-9_,()]*\)") - - -def _replace_square_bracket(line): - # Search something with - # - one opening square bracket [ - # - what you want - # - one closing square bracket ] - return _replace_regex(line, "\[[A-Za-z0-9_,()]*\]") - - -def _replace_quoted(line): - # Search something with - # - one opening quote ' - # - what you want - # - one closing quote ' - return _replace_regex(line, "\'\s*([^\"]*?)\s*\'") - - -def _replace_double_quoted(line): - # Search something with - # - one opening quote " - # - what you want - # - one closing quote " - return _replace_regex(line, "\"\s*([^\"]*?)\s*\"") - - -def _safe_split(line): - """ - Split a text by ",", - Manage case where you have a list, a tuple, a string, ... - - :param line: text line to split (str) - :return: splitted line (list) - - :use: - >>> line = "a=(1,2,3), b=[1,2], c=4, d=([1,2,3],4)" - >>> _safe_split(line) - >>> ["a=(1,2,3)", "b=[1,2]", "c=4", "d=([1,2,3],4)"] - """ - line2 = line - line2 = _replace_bracket(line2) - line2 = _replace_square_bracket(line2) - line2 = _replace_quoted(line2) - line2 = _replace_double_quoted(line2) - - # Resulting object is something without (square) bracket - # "a=[1,2,3], b=(1,2), c=1" become "a=_______, b=_____, c=1" - # Split object that have no special character (no bracket, no square bracket) - line_without_specials_splitted = line2.split(',') - - # Stock places where split occurred - i = 0 - virgule_places = [] - for part in line_without_specials_splitted: - i = i + len(part) + 1 - virgule_places.append(i) - - # Come back to **first object** and split it at the places **virgule_places** - final_lines = [] - old = 0 - for virgule_place in virgule_places: - final_lines.append(line[old:virgule_place - 1]) - old = virgule_place - - # Simple strip - return [x.strip() for x in final_lines] - diff --git a/src/openalea/core/node.py b/src/openalea/core/node.py index f2fb2805..b2ec01db 100644 --- a/src/openalea/core/node.py +++ b/src/openalea/core/node.py @@ -24,13 +24,14 @@ __license__ = "Cecill-C" __revision__ = " $Id$ " +from copy import copy, deepcopy import imp import inspect import os import sys import string import types -from copy import copy, deepcopy +from uuid import uuid1 from weakref import ref, proxy # from signature import get_parameters @@ -39,8 +40,10 @@ from actor import IActor from metadatadict import MetaDataDict, HasAdHoc from interface import TypeNameInterfaceMap + + # Exceptions -class RecursionError (Exception): +class RecursionError(Exception): """todo""" pass @@ -110,7 +113,7 @@ def set_compositenode(self, upper): def set_data(self, key, value, notify=True): """ Set internal node data """ self.internal_data[key] = value - if(notify): + if (notify): self.notify_listeners(("data_modified", key, value)) def close(self): @@ -199,7 +202,7 @@ def get_tip(self, current_value=None): desc = self.get('desc', '') value = self.get('value', None) iname = 'Any' - if(interface): + if (interface): try: iname = interface.__name__ except AttributeError: @@ -213,9 +216,9 @@ def get_tip(self, current_value=None): if len(comment) > 100: comment = comment[:100] + ' ...' - if current_value is None : + if current_value is None: return '%s(%s): %s [default=%s] ' % (name, iname, desc, comment) - else : + else: return '%s(%s): %s' % (name, iname, str(current_value)) @@ -236,6 +239,7 @@ def is_hidden(self): class OutputPort(AbstractPort): """The class describing the output ports """ + def __init__(self, node): AbstractPort.__init__(self, node) @@ -244,11 +248,12 @@ class Annotation(AbstractNode): def __init__(self): AbstractNode.__init__(self) - def to_script (self): + def to_script(self): """Script translation of this node. """ return "" + class Node(AbstractNode): """ It is a callable object with typed inputs and outputs. @@ -293,11 +298,11 @@ def __init__(self, inputs=(), outputs=()): self.modified = True # Internal Data - self.internal_data["caption"] = '' # str(self.__class__.__name__) + self.internal_data["caption"] = '' # str(self.__class__.__name__) self.internal_data["lazy"] = True - self.internal_data["block"] = False # Do not evaluate the node + self.internal_data["block"] = False # Do not evaluate the node self.internal_data["priority"] = 0 - self.internal_data["hide"] = True # hide in composite node widget + self.internal_data["hide"] = True # hide in composite node widget self.internal_data["port_hide_changed"] = set() # Add delay self.internal_data["delay"] = 0 @@ -400,7 +405,8 @@ def get_user_application(self): def set_user_application(self, data): """todo""" self.internal_data["user_application"] = data - self.notify_listeners(("internal_data_changed", "user_application", data)) + self.notify_listeners( + ("internal_data_changed", "user_application", data)) user_application = property(get_user_application, set_user_application) @@ -418,12 +424,12 @@ def get_caption(self): def is_port_hidden(self, index_key): """ Return the hidden state of a port """ index = self.map_index_in[index_key] - s = self.input_desc[index].is_hidden() # get('hide', False) + s = self.input_desc[index].is_hidden() # get('hide', False) changed = self.internal_data["port_hide_changed"] c = index in changed - if(index in changed): + if (index in changed): return not s else: return s @@ -436,7 +442,7 @@ def set_port_hidden(self, index_key, state): :param state: a boolean value. """ index = self.map_index_in[index_key] - s = self.input_desc[index].is_hidden() # get('hide', False) + s = self.input_desc[index].is_hidden() # get('hide', False) changed = self.internal_data["port_hide_changed"] @@ -444,12 +450,11 @@ def set_port_hidden(self, index_key, state): changed.add(index) self.input_desc[index].get_ad_hoc_dict().set_metadata("hide", state) self.notify_listeners(("hiddenPortChange",)) - elif(index in changed): + elif (index in changed): changed.remove(index) self.input_desc[index].get_ad_hoc_dict().set_metadata("hide", state) self.notify_listeners(("hiddenPortChange",)) - # Status def unvalidate_input(self, index_key, notify=True): """ @@ -459,7 +464,7 @@ def unvalidate_input(self, index_key, notify=True): """ self.modified = True index = self.map_index_in[index_key] - if(notify): + if (notify): self.notify_listeners(("input_modified", index)) self.continuous_eval.notify_listeners(("node_modified",)) @@ -473,13 +478,13 @@ def set_io(self, inputs, outputs): """ # # Values - if(inputs is None or len(inputs) != len(self.inputs)): + if (inputs is None or len(inputs) != len(self.inputs)): self.clear_inputs() if inputs: for d in inputs: self.add_input(**d) - if(outputs is None or len(outputs) != len(self.outputs)): + if (outputs is None or len(outputs) != len(self.outputs)): self.clear_outputs() if outputs: for d in outputs: @@ -508,7 +513,6 @@ def clear_outputs(self): self.map_index_out = {} self.notify_listeners(("cleared_output_ports",)) - def add_input(self, **kargs): """ Create an input port """ @@ -517,7 +521,7 @@ def add_input(self, **kargs): interface = kargs.get('interface', None) # default value - if(interface and not kargs.has_key('value')): + if (interface and not kargs.has_key('value')): if isinstance(interface, str): # Create mapping between interface name and interface class from openalea.core.interface import TypeNameInterfaceMap @@ -530,7 +534,7 @@ def add_input(self, **kargs): value = copy(value) - name = str(name) # force to have a string + name = str(name) # force to have a string self.inputs.append(None) port = InputPort(self) @@ -575,14 +579,14 @@ def set_input(self, key, val=None, notify=True): index = self.map_index_in[key] changed = True - if(self.lazy): + if (self.lazy): # Test if the inputs has changed try: changed = (cmp(self.inputs[index], val) != 0) except: pass - if(changed): + if (changed): self.inputs[index] = val self.unvalidate_input(index, notify) @@ -636,7 +640,8 @@ def eval(self): and a timed delay if the node needs a reevaluation at a later time. """ # lazy evaluation - if self.block and self.get_nb_output() != 0 and self.output(0) is not None: + if self.block and self.get_nb_output() != 0 and self.output( + 0) is not None: return False if (self.delay == 0 and self.lazy) and not self.modified: return False @@ -659,9 +664,9 @@ def eval(self): self.output_desc[0].notify_listeners(("tooltip_modified",)) - else: # multi output - if(not isinstance(outlist, tuple) and - not isinstance(outlist, list)): + else: # multi output + if (not isinstance(outlist, tuple) and + not isinstance(outlist, list)): outlist = (outlist,) for i in range(min(len(outlist), len(self.outputs))): @@ -684,7 +689,6 @@ def __getstate__(self): odict['modified'] = True - outputs = range(len(self.outputs)) for i in range(self.get_nb_output()): try: @@ -737,7 +741,7 @@ def reload(self): # if(not connected or self.input_states[i] is "connected"): self.set_input(i, self.input_desc[i].get('value', None)) - if(i > 0): + if (i > 0): self.invalidate() def reset(self): @@ -747,7 +751,7 @@ def reset(self): i = self.get_nb_input() - if(i > 0): + if (i > 0): self.invalidate() def invalidate(self): @@ -758,16 +762,16 @@ def invalidate(self): self.continuous_eval.notify_listeners(("node_modified", self)) -# X @property -# X def outputs(self): -# X return [self.output(i) for i in range(self.get_nb_output())] + # X @property + # X def outputs(self): + # X return [self.output(i) for i in range(self.get_nb_output())] - def to_script (self): + def to_script(self): """Script translation of this node. """ - if self._to_script_func is None : + if self._to_script_func is None: return "#node %s do not define any scripting\n" % self.factory.name - else : + else: return self._to_script_func(self.inputs, self.outputs) @@ -787,7 +791,7 @@ def __init__(self, inputs, outputs, func): def __call__(self, inputs=()): """ Call function. Must be overriden """ - if(self.func): + if (self.func): return self.func(*inputs) def get_process_obj(self): @@ -838,6 +842,7 @@ def __init__(self, Observed.__init__(self) # Factory info + self.uid = kargs.get("uid", uuid1().hex) self.name = name self.description = description self.category = category @@ -853,6 +858,7 @@ def __init__(self, self.delay = delay self.alias = alias self.authors = authors + # Package property def set_pkg(self, port): @@ -862,7 +868,7 @@ def set_pkg(self, port): The package id is the name of the package when the package is the Python object. """ - if(not port): + if (not port): self.__pkg__ = None self.__pkg_id = None else: @@ -873,13 +879,13 @@ def set_pkg(self, port): def get_pkg(self): """todo""" - if(self.__pkg__): + if (self.__pkg__): port = self.__pkg__() else: port = None # Test if pkg has been reloaded # In this case the weakref is not valid anymore - if(not port and self.__pkg_id__): + if (not port and self.__pkg_id__): from openalea.core.pkgmanager import PackageManager port = self.set_pkg(PackageManager()[self.__pkg_id__]) return port @@ -908,7 +914,7 @@ def get_python_name(self): name = self.name - if(not name.isalnum()): + if (not name.isalnum()): name = '_%s' % (id(self)) return name @@ -931,7 +937,6 @@ def get_tip(self, asRst=False): found in its package. """ - if not asRst: return "Name: %s
" % (self.name,) + \ "Category: %s
" % (self.category,) + \ @@ -954,7 +959,7 @@ def instantiate(self, call_stack=[]): raise NotImplementedError() def instantiate_widget(self, node=None, parent=None, edit=False, - autonomous=False): + autonomous=False): """ Return the corresponding widget initialised with node""" raise NotImplementedError() @@ -991,8 +996,8 @@ def is_composite_node(self): return False def __getstate__(self): - odict = self.__dict__.copy() # copy the dict since we change it - odict['__pkg__'] = None # remove weakref reference + odict = self.__dict__.copy() # copy the dict since we change it + odict['__pkg__'] = None # remove weakref reference return odict def __setstate__(self, dict): @@ -1002,7 +1007,7 @@ def __setstate__(self, dict): def Alias(factory, name): """ Create a alias for factory """ - if(factory.alias is None): + if (factory.alias is None): factory.alias = [name] else: factory.alias.append(name) @@ -1063,7 +1068,7 @@ def __init__(self, # Module path, value=0 self.nodemodule_path = None - if(not search_path): + if (not search_path): self.search_path = [] else: self.search_path = search_path @@ -1073,20 +1078,17 @@ def __init__(self, # Context directory # inspect.stack()[1][1] is the caller python module caller_dir = os.path.dirname(os.path.abspath(inspect.stack()[1][1])) - if(not caller_dir in self.search_path): + if (not caller_dir in self.search_path): self.search_path.append(caller_dir) - def is_node(self): return True def get_python_name(self): """ Return a python valid name """ - module_name = self.nodemodule_name - py_name = "%s_%s" % (self.nodemodule_name, self.nodeclass_name) - py_name = py_name.replace('.','_') - return py_name + module_name = self.nodemodule_name.replace('.', '_') + return "%s_%s" % (module_name, self.nodeclass_name) def __getstate__(self): """ Pickle function """ @@ -1095,7 +1097,7 @@ def __getstate__(self): odict['nodemodule'] = None odict['nodeclass'] = None odict['module_cache'] = None - odict['__pkg__'] = None # remove weakref reference + odict['__pkg__'] = None # remove weakref reference return odict @@ -1133,22 +1135,21 @@ def instantiate(self, call_stack=[]): if classobj is None: raise Exception("Cannot instantiate '" + \ - self.nodeclass_name + "' from " + str(module)) + self.nodeclass_name + "' from " + str(module)) # If class is not a Node, embed object in a Node class - if(not hasattr(classobj, 'mro') or not AbstractNode in classobj.mro()): + if (not hasattr(classobj, 'mro') or not AbstractNode in classobj.mro()): # Check inputs and outputs - if(self.inputs is None): + if (self.inputs is None): sign = sgn.Signature(classobj) self.inputs = sign.get_parameters() - if(self.outputs is None): + if (self.outputs is None): self.outputs = (dict(name="out", interface=None),) - # Check and Instantiate if we have a functor class - if((type(classobj) == types.TypeType) - or (type(classobj) == types.ClassType)): + if ((type(classobj) == types.TypeType) + or (type(classobj) == types.ClassType)): _classobj = classobj() if callable(_classobj): @@ -1167,7 +1168,7 @@ def instantiate(self, call_stack=[]): try: node.factory = self node.lazy = self.lazy - if(not node.caption): + if (not node.caption): node.set_caption(self.name) node.delay = self.delay @@ -1175,17 +1176,18 @@ def instantiate(self, call_stack=[]): pass # to script - if self.toscriptclass_name is not None : - node._to_script_func = module.__dict__.get(self.toscriptclass_name, None) + if self.toscriptclass_name is not None: + node._to_script_func = module.__dict__.get(self.toscriptclass_name, + None) return node def instantiate_widget(self, node=None, parent=None, - edit=False, autonomous=False): + edit=False, autonomous=False): """ Return the corresponding widget initialised with node """ # Code Editor - if(edit): + if (edit): from openalea.visualea.code_editor import get_editor w = get_editor()(parent) try: @@ -1200,15 +1202,15 @@ def instantiate_widget(self, node=None, parent=None, return w # Node Widget - if(node == None): + if (node == None): node = self.instantiate() modulename = self.widgetmodule_name - if(not modulename): + if (not modulename): modulename = self.nodemodule_name # if no widget declared, we create a default one - if(not modulename or not self.widgetclass_name): + if (not modulename or not self.widgetclass_name): from openalea.visualea.node_widget import DefaultNodeWidget return DefaultNodeWidget(node, parent, autonomous) @@ -1216,13 +1218,13 @@ def instantiate_widget(self, node=None, parent=None, else: # load module (file, pathname, desc) = imp.find_module(modulename, - self.search_path + sys.path) + self.search_path + sys.path) sys.path.append(os.path.dirname(pathname)) module = imp.load_module(modulename, file, pathname, desc) sys.path.pop() - if(file): + if (file): file.close() widgetclass = module.__dict__[self.widgetclass_name] @@ -1245,8 +1247,8 @@ def get_node_module(self): # Test if the module is already in sys.modules if (self.nodemodule_path and - self.module_cache and - not hasattr(self.module_cache, 'oa_invalidate')): + self.module_cache and + not hasattr(self.module_cache, 'oa_invalidate')): return self.module_cache sav_path = sys.path @@ -1291,7 +1293,6 @@ def get_node_file(self): self.get_node_module() return self.nodemodule_path - def get_node_src(self, cache=True): """ Return a string containing the node src @@ -1300,7 +1301,7 @@ def get_node_src(self, cache=True): """ # Return cached source if any - if(self.src_cache and cache): + if (self.src_cache and cache): return self.src_cache module = self.get_node_module() @@ -1339,20 +1340,19 @@ def save_new_src(self, newsrc): modulesrc = inspect.getsource(module) # Pass if no modications - if(nodesrc == newsrc): + if (nodesrc == newsrc): return # replace old code with new one modulesrc = modulesrc.replace(nodesrc, newsrc) - # write file myfile = open(self.nodemodule_path, 'w') myfile.write(modulesrc) myfile.close() # reload module - if(self.module_cache): + if (self.module_cache): self.module_cache.invalidate_oa = True self.src_cache = None @@ -1362,6 +1362,7 @@ def save_new_src(self, newsrc): # import py_compile # py_compile.compile(self.nodemodule_path) + # Class Factory: Factory = NodeFactory @@ -1371,7 +1372,8 @@ class PyNodeFactoryWriter(object): nodefactory_template = """ -$NAME = Factory(name=$PNAME, +$NAME = Factory(uid=$UID, + name=$PNAME, authors=$AUTHORS, description=$DESCRIPTION, category=$CATEGORY, @@ -1395,7 +1397,8 @@ def __repr__(self): name = f.get_python_name() name = name.replace('.', '_') - result = fstr.safe_substitute(NAME=name, + result = fstr.safe_substitute(UID=repr(f.uid), + NAME=name, AUTHORS=repr(f.get_authors()), PNAME=repr(f.name), DESCRIPTION=repr(f.description), @@ -1405,9 +1408,10 @@ def __repr__(self): LISTIN=repr(f.inputs), LISTOUT=repr(f.outputs), WIDGETMODULE=repr(f.widgetmodule_name), - WIDGETCLASS=repr(f.widgetclass_name),) + WIDGETCLASS=repr(f.widgetclass_name), ) return result + # Utility functions def gen_port_list(size): """ Generate a list of port description """ @@ -1422,15 +1426,15 @@ def initialise_standard_metadata(): # we declare what are the node model ad hoc data we require: AbstractNode.extend_ad_hoc_slots("position", list, [0, 0], "posx", "posy") Node.extend_ad_hoc_slots("userColor", list, None, "user_color") - Node.extend_ad_hoc_slots("useUserColor", bool, True, "use_user_color",) + Node.extend_ad_hoc_slots("useUserColor", bool, True, "use_user_color", ) Annotation.extend_ad_hoc_slots("text", str, "", "txt") -# Annotation.extend_ad_hoc_slots("htmlText", str, None) + # Annotation.extend_ad_hoc_slots("htmlText", str, None) Annotation.extend_ad_hoc_slots("textColor", list, None) Annotation.extend_ad_hoc_slots("rectP2", tuple, (-1, -1)) Annotation.extend_ad_hoc_slots("color", list, None) Annotation.extend_ad_hoc_slots("visualStyle", int, None) # we declare what are the node model ad hoc data we require: - AbstractPort.extend_ad_hoc_slots("hide" , bool, False) + AbstractPort.extend_ad_hoc_slots("hide", bool, False) AbstractPort.extend_ad_hoc_slots("connectorPosition", list, [0, 0]) diff --git a/src/openalea/core/system/__wralea__.py b/src/openalea/core/system/__wralea__.py index dd3c02ff..bda144ec 100644 --- a/src/openalea/core/system/__wralea__.py +++ b/src/openalea/core/system/__wralea__.py @@ -15,12 +15,12 @@ # ############################################################################### """Wralea for System nodes""" -__revision__ = " $Id$ " - -from openalea.core.external import * #IGNORE:W0614 +from openalea.core import Factory as Fa +from openalea.core import IBool, IFunction, IInt, ISequence, IStr from openalea.core.pkgdict import protected +__revision__ = " $Id$ " __name__ = "openalea.flow control" __alias__ = ["system"] @@ -33,268 +33,247 @@ __all__ = [] -annotation = Factory(name="annotation", - description="Annotation", - category="flow control", - nodemodule="openalea.core.system.systemnodes", - nodeclass="AnnotationNode", - ) +annotation = Fa(uid="3b4eb8dc4e7d11e6bff6d4bed973e64a", + name="annotation", + description="Annotation", + category="flow control", + nodemodule="openalea.core.system.systemnodes", + nodeclass="AnnotationNode", + ) __all__.append('annotation') -iter = Factory(name="iter", - description="Iteration", - category="System", - nodemodule="openalea.core.system.systemnodes", - nodeclass="IterNode", - inputs = (dict(name="generator", interface=None, value=None), - ), - outputs = ( dict(name="value", interface=None), ), - - ) -__all__.append('iter') - -iter_with_delay = Factory(name="iter with delay", - description="Iteration ", - category="flow control", - nodemodule="openalea.core.system.systemnodes", - nodeclass="IterWithDelayNode", - inputs = (dict(name="generator", interface=None, value=None), - dict(name="delay", interface=IInt, value=1), - ), - outputs = ( dict(name="value", interface=None), ), - - ) +iter_ = Fa(uid="3b4eb8dd4e7d11e6bff6d4bed973e64a", + name="iter", + description="Iteration", + category="System", + nodemodule="openalea.core.system.systemnodes", + nodeclass="IterNode", + inputs=(dict(name="generator", interface=None, value=None), + ), + outputs=(dict(name="value", interface=None),), + + ) +__all__.append('iter_') + +iter_with_delay = Fa(uid="3b4eb8de4e7d11e6bff6d4bed973e64a", + name="iter with delay", + description="Iteration ", + category="flow control", + nodemodule="openalea.core.system.systemnodes", + nodeclass="IterWithDelayNode", + inputs=(dict(name="generator", interface=None, value=None), + dict(name="delay", interface=IInt, value=1), + ), + outputs=(dict(name="value", interface=None),), + + ) __all__.append('iter_with_delay') -counter = Factory(name="counter", - description="Count from start to stop, step by step ", - category="flow control", - nodemodule="openalea.core.system.systemnodes", - nodeclass="Counter", - inputs = (dict(name="start", interface=IInt, value=0), - dict(name="stop", interface=IInt, value=10), - dict(name="step", interface=IInt, value=1), - dict(name="dummy", interface=None), - ), - outputs = ( dict(name="value", interface=IInt), ), - delay = 1, - ) +counter = Fa(uid="3b4eb8df4e7d11e6bff6d4bed973e64a", + name="counter", + description="Count from start to stop, step by step ", + category="flow control", + nodemodule="openalea.core.system.systemnodes", + nodeclass="Counter", + inputs=(dict(name="start", interface=IInt, value=0), + dict(name="stop", interface=IInt, value=10), + dict(name="step", interface=IInt, value=1), + dict(name="dummy", interface=None), + ), + outputs=(dict(name="value", interface=IInt),), + delay=1, + ) __all__.append('counter') -stop_simulation = Factory(name="stop simulation", - description="Iteration ", - category="flow control", - nodemodule="openalea.core.system.systemnodes", - nodeclass="StopSimulation", - inputs = (dict(name="any object"), - dict(name="max nb cycles", interface=IInt, value=10), - ), - outputs = ( dict(name="any"), ), - - ) +stop_simulation = Fa(uid="3b4eb8e04e7d11e6bff6d4bed973e64a", + name="stop simulation", + description="Iteration ", + category="flow control", + nodemodule="openalea.core.system.systemnodes", + nodeclass="StopSimulation", + inputs=(dict(name="any object"), + dict(name="max nb cycles", interface=IInt, + value=10), + ), + outputs=(dict(name="any"),), + + ) __all__.append('stop_simulation') -rdv = Factory(name="rendez vous", - description="Synchronize 2 inputs", - category="flow control", - nodemodule="openalea.core.system.systemnodes", - nodeclass="RDVNode", - inputs = (dict(name="value", interface=None, value=None), - dict(name="control_flow", interface=None, value=None), - ), - outputs = ( dict(name="value", interface=None), - dict(name="flow result", interface=None),), +rdv = Fa(uid="3b4eb8e14e7d11e6bff6d4bed973e64a", + name="rendez vous", + description="Synchronize 2 inputs", + category="flow control", + nodemodule="openalea.core.system.systemnodes", + nodeclass="RDVNode", + inputs=(dict(name="value", interface=None, value=None), + dict(name="control_flow", interface=None, value=None), + ), + outputs=(dict(name="value", interface=None), + dict(name="flow result", interface=None),), - ) + ) __all__.append('rdv') -poolreader = Factory( name="pool reader", - description="Read data from the data pool.", - category="flow control", - nodemodule="openalea.core.system.systemnodes", - nodeclass="PoolReader", - inputs = (dict(name='Key', interface=IStr),), - outputs = (dict(name='Obj', interface=None),), - lazy = False, - - ) - +poolreader = Fa(uid="3b4eb8e24e7d11e6bff6d4bed973e64a", + name="pool reader", + description="Read data from the data pool.", + category="flow control", + nodemodule="openalea.core.system.systemnodes", + nodeclass="PoolReader", + inputs=(dict(name='Key', interface=IStr),), + outputs=(dict(name='Obj', interface=None),), + lazy=False, + + ) + __all__.append('poolreader') -poolwriter = Factory(name="pool writer", - description="Write data to the data pool.", - category="flow control", - nodemodule="openalea.core.system.systemnodes", - nodeclass="PoolWriter", - inputs = (dict(name='Key', interface=IStr), - dict(name='Obj', interface=None),), - outputs = (dict(name='Obj', interface=None),), - lazy = False, - ) +poolwriter = Fa(uid="3b4eb8e34e7d11e6bff6d4bed973e64a", + name="pool writer", + description="Write data to the data pool.", + category="flow control", + nodemodule="openalea.core.system.systemnodes", + nodeclass="PoolWriter", + inputs=(dict(name='Key', interface=IStr), + dict(name='Obj', interface=None),), + outputs=(dict(name='Obj', interface=None),), + lazy=False, + ) __all__.append('poolwriter') -pool_rw = Factory(name="pool setdefault", +pool_rw = Fa(uid="3b4eb8e44e7d11e6bff6d4bed973e64a", + name="pool setdefault", description="pool.setdefault(key,value).", category="flow control", nodemodule="openalea.core.system.systemnodes", nodeclass="PoolDefault", - inputs = (dict(name='Key', interface=IStr), - dict(name='Value', interface=None),), - outputs = (dict(name='Obj', interface=None),), - lazy = False, + inputs=(dict(name='Key', interface=IStr), + dict(name='Value', interface=None),), + outputs=(dict(name='Obj', interface=None),), + lazy=False, ) __all__.append('pool_rw') +init = Fa(uid="3b4eb8e54e7d11e6bff6d4bed973e64a", + name="init", + description="Value selector for graph initialisation", + category="flow control", + nodemodule="openalea.core.system.systemnodes", + nodeclass="InitNode", + inputs=(dict(name="val_init", interface=None, value=0.), + dict(name="value", interface=None, value=None), + dict(name="state", interface=IBool, value=True), + ), + outputs=(dict(name="value", interface=None),), -# nf = Factory(name="list accumulator", -# description="List accumulator", -# category="System", -# nodemodule="openalea.core.system.systemnodes", -# nodeclass="AccuList", -# inputs = (dict(name="value", interface=None, value=None), -# dict(name="varname", interface=IStr, value=None), -# ), -# outputs = ( dict(name="list", interface=ISequence), ), - -# ) - -# package.add_factory(nf) - - -# nf = Factory(name="float accumulator", -# description="Float accumulator", -# category="System", -# nodemodule="openalea.core.system.systemnodes", -# nodeclass="AccuFloat", -# inputs = (dict(name="value", interface=IFloat, value=0.), -# dict(name="varname", interface=IStr, value=None), -# ), -# outputs = ( dict(name="float", interface=IFloat), ), - -# ) - -# package.add_factory(nf) - - -init = Factory(name="init", - description="Value selector for graph initialisation", - category="flow control", - nodemodule="openalea.core.system.systemnodes", - nodeclass="InitNode", - inputs = (dict(name="val_init", interface=None, value=0.), - dict(name="value", interface=None, value=None), - dict(name="state", interface=IBool, value=True), - ), - outputs = ( dict(name="value", interface=None), ), - - ) + ) __all__.append('init') - - -X = Factory(name="X", - description="Function variable", - category="flow control", - nodemodule="openalea.core.system.systemnodes", - nodeclass="LambdaVar", - inputs = (dict(name="name", interface=IStr, value='x'), ), - outputs = ( dict(name="lambda", interface=None), ), - ) + +X = Fa(uid="3b4eb8e64e7d11e6bff6d4bed973e64a", + name="X", + description="Function variable", + category="flow control", + nodemodule="openalea.core.system.systemnodes", + nodeclass="LambdaVar", + inputs=(dict(name="name", interface=IStr, value='x'),), + outputs=(dict(name="lambda", interface=None),), + ) __all__.append('X') - -whileuni = Factory(name="while univariate", - description="While Loop (Univariate)", - category="flow control", - nodemodule="openalea.core.system.systemnodes", - nodeclass="WhileUniVar", - inputs = (dict(name="InitValue", interface=None, value=None), - dict(name="Test", interface=IFunction, value=None), - dict(name="Function", interface=IFunction, value=None), - ), - outputs = ( dict(name="Result", interface=None), ), - ) +whileuni = Fa(uid="3b4eb8e74e7d11e6bff6d4bed973e64a", + name="while univariate", + description="While Loop (Univariate)", + category="flow control", + nodemodule="openalea.core.system.systemnodes", + nodeclass="WhileUniVar", + inputs=(dict(name="InitValue", interface=None, value=None), + dict(name="Test", interface=IFunction, value=None), + dict(name="Function", interface=IFunction, value=None), + ), + outputs=(dict(name="Result", interface=None),), + ) __all__.append('whileuni') - -whilemulti = Factory(name="while multivariate", - description="While Loop (Multivariate)", - category="flow control", - nodemodule="openalea.core.system.systemnodes", - nodeclass="WhileMultiVar", - inputs = (dict(name="InitValues", interface=ISequence, value=[]), - dict(name="Test", interface=IFunction, value=None), - dict(name="Functions", interface=IFunction, value=None), - ), - outputs = ( dict(name="Results", interface=ISequence), ), - ) - +whilemulti = Fa(uid="3b4eb8e84e7d11e6bff6d4bed973e64a", + name="while multivariate", + description="While Loop (Multivariate)", + category="flow control", + nodemodule="openalea.core.system.systemnodes", + nodeclass="WhileMultiVar", + inputs=(dict(name="InitValues", interface=ISequence, value=[]), + dict(name="Test", interface=IFunction, value=None), + dict(name="Functions", interface=IFunction, value=None), + ), + outputs=(dict(name="Results", interface=ISequence),), + ) __all__.append('whilemulti') -whilemulti2 = Factory(name="while multivariate2", - description="While Loop (Multivariate)", - category="flow control", - nodemodule="openalea.core.system.systemnodes", - nodeclass="while_multi2", - inputs = (dict(name="InitValues", interface=ISequence, value=[]), - dict(name="Test", interface=IFunction, value=None), - dict(name="Functions", interface=IFunction, value=None), - ), - outputs = ( dict(name="Results", interface=ISequence), ), - ) - +whilemulti2 = Fa(uid="3b4eb8e94e7d11e6bff6d4bed973e64a", + name="while multivariate2", + description="While Loop (Multivariate)", + category="flow control", + nodemodule="openalea.core.system.systemnodes", + nodeclass="while_multi2", + inputs=(dict(name="InitValues", interface=ISequence, value=[]), + dict(name="Test", interface=IFunction, value=None), + dict(name="Functions", interface=IFunction, + value=None), + ), + outputs=(dict(name="Results", interface=ISequence),), + ) __all__.append('whilemulti2') -cmd = Factory(name=protected("command"), - description="Call a system command", - category="System", - nodemodule="openalea.core.system.systemnodes", - nodeclass="system_cmd", - inputs = (dict(name="commands", interface=ISequence, value=[], - desc='List of command strings'), - ), - outputs = ( dict(name="stdout", interface=None, desc='result'), - dict(name="stderr", interface=None, desc='result'), ), - ) - - +cmd = Fa(uid="3b4eb8ea4e7d11e6bff6d4bed973e64a", + name=protected("command"), + description="Call a system command", + category="System", + nodemodule="openalea.core.system.systemnodes", + nodeclass="system_cmd", + inputs=(dict(name="commands", interface=ISequence, value=[], + desc='List of command strings'), + ), + outputs=(dict(name="stdout", interface=None, desc='result'), + dict(name="stderr", interface=None, desc='result'),), + ) __all__.append('cmd') -_delay = Factory(name="delay", - description="Delay return the previous or an init value.", - category="flow control", - nodemodule="openalea.core.system.systemnodes", - nodeclass="Delay", - inputs = (dict(name="init", interface=None), - dict(name="x", interface=None), - dict(name="reset", interface=IBool )), - outputs = ( dict(name="previous", interface=None), ), - lazy = False, - ) +_delay = Fa(uid="3b4eb8eb4e7d11e6bff6d4bed973e64a", + name="delay", + description="Delay return the previous or an init value.", + category="flow control", + nodemodule="openalea.core.system.systemnodes", + nodeclass="Delay", + inputs=(dict(name="init", interface=None), + dict(name="x", interface=None), + dict(name="reset", interface=IBool)), + outputs=(dict(name="previous", interface=None),), + lazy=False, + ) __all__.append('_delay') - -_for = Factory(name="for", - description="for Loop (Univariate)", - category="flow control", - nodemodule="openalea.core.system.systemnodes", - nodeclass="For", - inputs = (dict(name="InitValue", interface=None, value=None), - dict(name="list", interface=ISequence, value=None), - dict(name="Function", interface=IFunction, value=None), - ), - outputs = ( dict(name="Result", interface=None), ), - ) +_for = Fa(uid="3c43d57e4e7d11e6bff6d4bed973e64a", + name="for", + description="for Loop (Univariate)", + category="flow control", + nodemodule="openalea.core.system.systemnodes", + nodeclass="For", + inputs=(dict(name="InitValue", interface=None, value=None), + dict(name="list", interface=ISequence, value=None), + dict(name="Function", interface=IFunction, value=None), + ), + outputs=(dict(name="Result", interface=None),), + ) __all__.append('_for') - diff --git a/src/openalea/core/version.py b/src/openalea/core/version.py index 4d36235b..651ba1b4 100644 --- a/src/openalea/core/version.py +++ b/src/openalea/core/version.py @@ -1,18 +1,9 @@ -""" -Maintain version for this package. -Do not edit this file, use 'version' section of config. -""" # {# pkglts, version # -*- coding: utf-8 -*- -MAJOR = 2 -"""(int) Version major component.""" +major = 1 +minor = 4 +post = 0 -MINOR = 0 -"""(int) Version minor component.""" - -POST = 2 -"""(int) Version post or bugfix component.""" - -__version__ = ".".join([str(s) for s in (MAJOR, MINOR, POST)]) +__version__ = ".".join([str(s) for s in (major, minor, post)]) # #} diff --git a/test/clonepkg/__wralea__.py b/test/clonepkg/__wralea__.py index f4992b43..4a547aeb 100644 --- a/test/clonepkg/__wralea__.py +++ b/test/clonepkg/__wralea__.py @@ -1,10 +1,10 @@ -# This file has been generated at Wed Jan 31 21:28:23 2018 +# This file has been generated at Mon Jul 27 15:31:48 2015 from openalea.core import * -__name__ = 'ClonePkg' +__name__ = 'DummyPkg' __editable__ = True __description__ = 'Base library.' diff --git a/test/test_interpreter.py b/test/test_interpreter.py index a4f4fd7f..0e2ef60f 100644 --- a/test/test_interpreter.py +++ b/test/test_interpreter.py @@ -25,67 +25,35 @@ # dic = interp.get(['c']) # assert dic == {'c': 3} # -############################################################################### - -import unittest - -from openalea.core.service.ipython import interpreter -from openalea.core.interpreter.python import Interpreter as PythonInterpreter -try: - from openalea.core.interpreter.ipython import Interpreter as IPythonInterpreter -except ImportError: - IPythonInterpreter = None - -class TestCase(unittest.TestCase): - - def setUp(self): - self.ip = interpreter() - self.ipy = IPythonInterpreter() if IPythonInterpreter else None - self.py = PythonInterpreter() - - def tearDown(self): - pass - - def _run_all_interpreter(self, test): - test(self.ip) - if self.ipy: - test(self.ipy) - test(self.py) - - def _run_cell(self, interp): - interp.run_cell('a=1\nb=2\nc=a+b') - assert 'c' in interp.user_ns - dic = interp.get(['c']) - assert dic == {'c': 3} - - def _delete(self, interp): - interp.run_cell('a=1') - assert 'a' in interp.user_ns - interp.delete(['a']) - assert 'a' not in interp.user_ns - - def _api(self, interp): - assert interp is not None - assert hasattr(interp, "user_ns") - assert hasattr(interp, "run_cell") - assert hasattr(interp, "run_code") - assert hasattr(interp, "runcode") - assert hasattr(interp, "loadcode") - - def test_api(self): - self._api(self.ip) - - def test_run_cell(self): - self._run_all_interpreter(self._run_cell) - - def test_delete(self): - self._run_all_interpreter(self._delete) - - def test_get_interpreter_twice(self): - interp = interpreter() - interp2 = interpreter() - assert interp is not None - assert interp is interp2 - -if __name__ == '__main__': - unittest.main() +# def _delete(self, interp): +# interp.run_cell('a=1') +# assert 'a' in interp.user_ns +# interp.delete(['a']) +# assert 'a' not in interp.user_ns +# +# def _api(self, interp): +# assert interp is not None +# assert hasattr(interp, "user_ns") +# assert hasattr(interp, "run_cell") +# assert hasattr(interp, "run_code") +# assert hasattr(interp, "runcode") +# assert hasattr(interp, "loadcode") +# +# def test_api(self): +# self._api(self.ip) +# +# def test_run_cell(self): +# self._run_all_interpreter(self._run_cell) +# +# def test_delete(self): +# self._run_all_interpreter(self._delete) +# +# def test_get_interpreter_twice(self): +# interp = interpreter() +# interp2 = interpreter() +# assert interp is not None +# assert interp is interp2 +# +# +# if __name__ == '__main__': +# unittest.main() diff --git a/test/test_model.py b/test/test_model.py index a9867e62..a909bcf1 100644 --- a/test/test_model.py +++ b/test/test_model.py @@ -1,297 +1,300 @@ -import copy - -from openalea.core.model import Model -from openalea.core.service.control import clear_controls, control_namespace -from openalea.core.service.project import create_project - -from openalea.core.model_inout import InputObj, OutputObj - - -project = create_project('unittest', '/tmp/notwritable') - - -def register_model(model): - project.add('model', model) - - -def test_copy(): - m = Model(name='m1') - m.set_step_code('c=a+b') - m.inputs_info = [InputObj('a'), InputObj('b')] - m.outputs_info = [OutputObj('c')] - - m2 = copy.copy(m) - assert m is not m2 - assert m.step_code == m2.step_code - assert [inp.name for inp in m2.inputs_info] == ['a', 'b'] - assert [out.name for out in m2.outputs_info] == ['c'] - - -def test_output(): - step_code = 'c=a+b' - - m = Model('sum') - m.inputs_info = [InputObj('a'), InputObj('b')] - m.outputs_info = [OutputObj('c')] - m.step_code = step_code - - assert m.run(a=1, b=2) == 3 - - -def test_steps(): - step_code = 'a += 10' - - m = Model('sum') - m.inputs_info = [InputObj('a=0')] - m.outputs_info = [OutputObj('a')] - m.step_code = step_code - - assert m.run() == 10 - assert m.run(nstep=10) == 100 - - -def test_global_and_control(): - m1 = Model('IOFullyDefined') - m1.inputs_info = [InputObj('a=0'), InputObj('b=0')] - m1.outputs_info = [OutputObj('c')] - m1.set_step_code('c=a+b') - - m2 = Model('IOWithoutDefault') - m2.inputs_info = [InputObj('a'), InputObj('b')] - m2.outputs_info = [OutputObj('c')] - m2.set_step_code('c=a+b') - - m3 = Model('NoIO') - m3.outputs_info = [OutputObj('c')] - m3.set_step_code('c=a+b') - - ms1 = Model('CheckModelCall') - ms1.outputs_info = [OutputObj('c')] - - register_model(m1) - register_model(m2) - register_model(m3) - - from openalea.core.service.control import new_control - new_control('a', 'IInt', 10) - a = 123456789 - ns = {} - ns.update(control_namespace()) - ns['Model'] = project.get_runnable_model - - ms1.set_step_code('m = Model("IOFullyDefined")\nc = m.run()') - assert ms1.run(namespace=ns) == 0 - ms1.set_step_code('m = Model("IOFullyDefined")\na=44\nc = m.run(b=22)') - assert ms1.run(namespace=ns) == 22 - ms1.set_step_code('m = Model("IOFullyDefined")\na=44\nc = m.run(a=11, b=22)') - assert ms1.run(namespace=ns) == 33 - ms1.set_step_code('m = Model("IOFullyDefined")\na=44\nc = m.run(a)') - assert ms1.run(namespace=ns) == 44 - - assert m1.run() == 0 # 0 - assert m2.run(1, 2) == 3 # 3 - assert m2.run(2, b=3) == 5 # 5 - assert m2.run(a=3, b=4) == 7 # 7 - - # a, b not defined, use default a=0, b=0. Do not use global a because a is not a free variable - assert m1.run(namespace=ns) == 0 - # m2.run(namespace=ns) # FAIL, b not defined - # m3.run(namespace=ns) # FAIL, b not defined - - assert m1.run(1, 2, namespace=ns) == 3 # 3 - assert m2.run(2, b=3, namespace=ns) == 5 # 5 - assert m2.run(a=3, b=4, namespace=ns) == 7 # 7 - assert m3.run(a=3, b=4, namespace=ns) == 7 # 7 - - b = 20 - ns = locals() - ns.update(control_namespace()) - ns['Model'] = project.get_runnable_model - - ms1.set_step_code('m = Model("IOWithoutDefault")\nc = m.run()') - assert ms1.run(namespace=ns) == 30 # 30 free variables a, b. use global var a, b - ms1.set_step_code('m = Model("IOWithoutDefault")\na=456\nc = m.run(a=11, b=22)') - assert ms1.run(namespace=ns) == 33 # 33 - ms1.set_step_code('m = Model("IOWithoutDefault")\nc = m.run(b=22)') - assert ms1.run(namespace=ns) == 32 # 32 a use control value, b set explicitly - ms1.set_step_code('m = Model("IOWithoutDefault")\na=33\nc = m.run(b=22)') - assert ms1.run(namespace=ns) == 55 # 55 a use parent model value, b set explicitly - ms1.set_step_code('m = Model("IOWithoutDefault")\na=44\nc = m.run(a)') - assert ms1.run(namespace=ns) == 64 # 64 - - assert m1.run() == 0 # 0 - assert m2.run(1, 2) == 3 # 3 - assert m2.run(2, b=3) == 5 # 5 - assert m2.run(a=3, b=4) == 7 # 7 - - assert m1.run(namespace=ns) == 0 # Do not use global a or b because a and b are not free variables - assert m2.run(namespace=ns) == 30 # 30 - assert m3.run(namespace=ns) == 30 # 30 - - assert m1.run(1, 2, namespace=ns) == 3 # 3 - assert m2.run(2, b=3, namespace=ns) == 5 # 5 - assert m2.run(a=3, b=4, namespace=ns) == 7 # 7 - assert m3.run(a=3, b=4, namespace=ns) == 7 # 7 - - clear_controls() - - -def test_global_and_control_it(): - from openalea.core.service.control import new_control - - m1 = Model('IterateIOFullyDefined') - m1.inputs_info = [InputObj('a=0')] - m1.outputs_info = [OutputObj('a')] - m1.set_step_code('a+=1') - - m2 = Model('IterateIOWithoutDefault') - m2.inputs_info = [InputObj('a')] - m2.outputs_info = [OutputObj('a')] - m2.set_step_code('a+=1') - - m3 = Model('IterateNoIO') - m3.outputs_info = [OutputObj('a')] - m3.set_step_code('a+=1') - - m4 = Model('IterateNoIO') - m4.set_step_code('a+=1') - - ms1 = Model('IterateCheckModelCall') - ms1.outputs_info = [OutputObj('c')] - - register_model(m1) - register_model(m2) - register_model(m3) - - new_control('a', 'IInt', 10) - a = 123456789 - ns = {} - ns.update(control_namespace()) - ns['Model'] = project.get_runnable_model - - nstep = 3 - - ms1.set_step_code('m = Model("IterateIOFullyDefined")\nc = m.run()') - assert ms1.run(namespace=ns, nstep=nstep) == 1 - ms1.set_step_code('m = Model("IterateIOFullyDefined")\nc = m.run(nstep=3)') - assert ms1.run(namespace=ns, nstep=nstep) == (0 + 3) - - assert m1.run(nstep=nstep) == (0 + nstep) - assert m2.run(1, nstep=nstep) == (1 + nstep) - assert m2.run(a=2, nstep=nstep) == (2 + nstep) - - # a, b not defined, use default a=0, b=0. Do not use global a because a is not a free variable - assert m1.run(namespace=ns, nstep=nstep) == (0 + nstep) - # m2.run(namespace=ns) # FAIL, b not defined - # m3.run(namespace=ns) # FAIL, b not defined - - assert m1.run(1, namespace=ns, nstep=nstep) == (1 + nstep) - assert m2.run(2, namespace=ns, nstep=nstep) == (2 + nstep) - assert m2.run(a=10, namespace=ns, nstep=nstep) == (10 + nstep) - assert m3.run(a=10, namespace=ns, nstep=nstep) == (10 + nstep) - - clear_controls() - -code = """ -model1 = Model("m1") -model2 = Model("m1") - -model1.run() # m1=1 -model2.run() # m2=1 - -model1.step() # m1=2, m2=1 -model1.step() # m1=3, m2=1 -m1 = model1.step() # m1=4, m2=1 -m2 = model2.step() # m1=4, m2=2 -""" - - -def test_multiple_call(): - """ - Check each model instance has is own namespace - """ - model1 = Model('m1') - model1.inputs_info = [InputObj('a=0')] - model1.outputs_info = [OutputObj('a')] - model1.set_step_code('a+=1') - - model2 = Model('m2') - model2.outputs_info = [OutputObj('m1'), OutputObj('m2')] - model2.set_step_code(code) - - project.add("model", model1) - ns = {} - ns['Model'] = project.get_runnable_model - - m1, m2 = project.run_model(model2, namespace=ns) - assert m1 == 4 - assert m2 == 2 - - -def test_parse_code(): - code = ''' -""" -input = a,b -output = c -""" -c = a+b -''' - from openalea.core.model import PythonModel - - m1 = PythonModel(name='m1', code=code) - m2 = PythonModel(name='m2') - - m2.code = code - - m3 = PythonModel(name='m3') - m3.set_code(code) - - assert m1.run(a=1, b=2) == 3 - assert m2.run(a=2, b=3) == 5 - assert m3.run(a=3, b=4) == 7 - - m1 = PythonModel(name='ModelWithoutDoc') - assert m1.get_documentation() == '' - - m1 = PythonModel(name='ModelWithDoc', code=code) - assert m1.get_documentation() - - -def test_clean_ns(): - from openalea.core.model import PythonModel - from openalea.core.service.ipython import interpreter - interp = interpreter() - interp.user_ns = {} - - interp.user_ns['ipython_ns'] = 1 - - model = PythonModel(name='clean') - model.outputs_info = [OutputObj('ns')] - model.inputs_info = [InputObj('a=0'), InputObj('b')] - model.set_step_code('local_to_model=1\nns = dir()') - - initial_ns = dict(initial_ns=1) - model_local_ns = model(namespace=initial_ns, b=1) - model_varname = ['a', 'b', 'local_to_model'] - for varname in model_varname: - assert varname in model_local_ns - for varname in model_varname: - assert varname not in interp.user_ns, str(varname)+', '+str(interp.user_ns) - assert 'ipython_ns' in interp.user_ns - - -def test_in_function(): - code = """ -''' -output=out -''' -a=1 -def f(): - return(a) - -out = f() -""" - from openalea.core.model import PythonModel - model = PythonModel(name='func') - model.set_code(code) - assert model.init() == 1 +# import copy +# +# from openalea.core.model import Model +# from openalea.core.service.control import clear_controls, control_namespace +# from openalea.core.service.project import create_project +# from openalea.oalab.model.parse import InputObj, OutputObj +# +# project = create_project('unittest', '/tmp/notwritable') +# +# +# def register_model(model): +# project.add('model', model) +# +# +# def test_copy(): +# m = Model(name='m1') +# m.set_step_code('c=a+b') +# m.inputs_info = [InputObj('a'), InputObj('b')] +# m.outputs_info = [OutputObj('c')] +# +# m2 = copy.copy(m) +# assert m is not m2 +# assert m.step_code == m2.step_code +# assert [inp.name for inp in m2.inputs_info] == ['a', 'b'] +# assert [out.name for out in m2.outputs_info] == ['c'] +# +# +# def test_output(): +# step_code = 'c=a+b' +# +# m = Model('sum') +# m.inputs_info = [InputObj('a'), InputObj('b')] +# m.outputs_info = [OutputObj('c')] +# m.step_code = step_code +# +# assert m.run(a=1, b=2) == 3 +# +# +# def test_steps(): +# step_code = 'a += 10' +# +# m = Model('sum') +# m.inputs_info = [InputObj('a=0')] +# m.outputs_info = [OutputObj('a')] +# m.step_code = step_code +# +# assert m.run() == 10 +# assert m.run(nstep=10) == 100 +# +# +# def test_global_and_control(): +# m1 = Model('IOFullyDefined') +# m1.inputs_info = [InputObj('a=0'), InputObj('b=0')] +# m1.outputs_info = [OutputObj('c')] +# m1.set_step_code('c=a+b') +# +# m2 = Model('IOWithoutDefault') +# m2.inputs_info = [InputObj('a'), InputObj('b')] +# m2.outputs_info = [OutputObj('c')] +# m2.set_step_code('c=a+b') +# +# m3 = Model('NoIO') +# m3.outputs_info = [OutputObj('c')] +# m3.set_step_code('c=a+b') +# +# ms1 = Model('CheckModelCall') +# ms1.outputs_info = [OutputObj('c')] +# +# register_model(m1) +# register_model(m2) +# register_model(m3) +# +# from openalea.core.service.control import new_control +# new_control('a', 'IInt', 10) +# a = 123456789 +# ns = {} +# ns.update(control_namespace()) +# ns['Model'] = project.get_runnable_model +# +# ms1.set_step_code('m = Model("IOFullyDefined")\nc = m.run()') +# assert ms1.run(namespace=ns) == 0 +# ms1.set_step_code('m = Model("IOFullyDefined")\na=44\nc = m.run(b=22)') +# assert ms1.run(namespace=ns) == 22 +# ms1.set_step_code( +# 'm = Model("IOFullyDefined")\na=44\nc = m.run(a=11, b=22)') +# assert ms1.run(namespace=ns) == 33 +# ms1.set_step_code('m = Model("IOFullyDefined")\na=44\nc = m.run(a)') +# assert ms1.run(namespace=ns) == 44 +# +# assert m1.run() == 0 # 0 +# assert m2.run(1, 2) == 3 # 3 +# assert m2.run(2, b=3) == 5 # 5 +# assert m2.run(a=3, b=4) == 7 # 7 +# +# # a, b not defined, use default a=0, b=0. Do not use global a because a is not a free variable +# assert m1.run(namespace=ns) == 0 +# # m2.run(namespace=ns) # FAIL, b not defined +# # m3.run(namespace=ns) # FAIL, b not defined +# +# assert m1.run(1, 2, namespace=ns) == 3 # 3 +# assert m2.run(2, b=3, namespace=ns) == 5 # 5 +# assert m2.run(a=3, b=4, namespace=ns) == 7 # 7 +# assert m3.run(a=3, b=4, namespace=ns) == 7 # 7 +# +# b = 20 +# ns = locals() +# ns.update(control_namespace()) +# ns['Model'] = project.get_runnable_model +# +# ms1.set_step_code('m = Model("IOWithoutDefault")\nc = m.run()') +# assert ms1.run( +# namespace=ns) == 30 # 30 free variables a, b. use global var a, b +# ms1.set_step_code( +# 'm = Model("IOWithoutDefault")\na=456\nc = m.run(a=11, b=22)') +# assert ms1.run(namespace=ns) == 33 # 33 +# ms1.set_step_code('m = Model("IOWithoutDefault")\nc = m.run(b=22)') +# assert ms1.run( +# namespace=ns) == 32 # 32 a use control value, b set explicitly +# ms1.set_step_code('m = Model("IOWithoutDefault")\na=33\nc = m.run(b=22)') +# assert ms1.run( +# namespace=ns) == 55 # 55 a use parent model value, b set explicitly +# ms1.set_step_code('m = Model("IOWithoutDefault")\na=44\nc = m.run(a)') +# assert ms1.run(namespace=ns) == 64 # 64 +# +# assert m1.run() == 0 # 0 +# assert m2.run(1, 2) == 3 # 3 +# assert m2.run(2, b=3) == 5 # 5 +# assert m2.run(a=3, b=4) == 7 # 7 +# +# assert m1.run( +# namespace=ns) == 0 # Do not use global a or b because a and b are not free variables +# assert m2.run(namespace=ns) == 30 # 30 +# assert m3.run(namespace=ns) == 30 # 30 +# +# assert m1.run(1, 2, namespace=ns) == 3 # 3 +# assert m2.run(2, b=3, namespace=ns) == 5 # 5 +# assert m2.run(a=3, b=4, namespace=ns) == 7 # 7 +# assert m3.run(a=3, b=4, namespace=ns) == 7 # 7 +# +# clear_controls() +# +# +# def test_global_and_control_it(): +# from openalea.core.service.control import new_control +# +# m1 = Model('IterateIOFullyDefined') +# m1.inputs_info = [InputObj('a=0')] +# m1.outputs_info = [OutputObj('a')] +# m1.set_step_code('a+=1') +# +# m2 = Model('IterateIOWithoutDefault') +# m2.inputs_info = [InputObj('a')] +# m2.outputs_info = [OutputObj('a')] +# m2.set_step_code('a+=1') +# +# m3 = Model('IterateNoIO') +# m3.outputs_info = [OutputObj('a')] +# m3.set_step_code('a+=1') +# +# m4 = Model('IterateNoIO') +# m4.set_step_code('a+=1') +# +# ms1 = Model('IterateCheckModelCall') +# ms1.outputs_info = [OutputObj('c')] +# +# register_model(m1) +# register_model(m2) +# register_model(m3) +# +# new_control('a', 'IInt', 10) +# a = 123456789 +# ns = {} +# ns.update(control_namespace()) +# ns['Model'] = project.get_runnable_model +# +# nstep = 3 +# +# ms1.set_step_code('m = Model("IterateIOFullyDefined")\nc = m.run()') +# assert ms1.run(namespace=ns, nstep=nstep) == 1 +# ms1.set_step_code('m = Model("IterateIOFullyDefined")\nc = m.run(nstep=3)') +# assert ms1.run(namespace=ns, nstep=nstep) == (0 + 3) +# +# assert m1.run(nstep=nstep) == (0 + nstep) +# assert m2.run(1, nstep=nstep) == (1 + nstep) +# assert m2.run(a=2, nstep=nstep) == (2 + nstep) +# +# # a, b not defined, use default a=0, b=0. Do not use global a because a is not a free variable +# assert m1.run(namespace=ns, nstep=nstep) == (0 + nstep) +# # m2.run(namespace=ns) # FAIL, b not defined +# # m3.run(namespace=ns) # FAIL, b not defined +# +# assert m1.run(1, namespace=ns, nstep=nstep) == (1 + nstep) +# assert m2.run(2, namespace=ns, nstep=nstep) == (2 + nstep) +# assert m2.run(a=10, namespace=ns, nstep=nstep) == (10 + nstep) +# assert m3.run(a=10, namespace=ns, nstep=nstep) == (10 + nstep) +# +# clear_controls() +# +# +# code = """ +# model1 = Model("m1") +# model2 = Model("m1") +# +# model1.run() # m1=1 +# model2.run() # m2=1 +# +# model1.step() # m1=2, m2=1 +# model1.step() # m1=3, m2=1 +# m1 = model1.step() # m1=4, m2=1 +# m2 = model2.step() # m1=4, m2=2 +# """ +# +# +# def test_multiple_call(): +# """ +# Check each model instance has is own namespace +# """ +# model1 = Model('m1') +# model1.inputs_info = [InputObj('a=0')] +# model1.outputs_info = [OutputObj('a')] +# model1.set_step_code('a+=1') +# +# model2 = Model('m2') +# model2.outputs_info = [OutputObj('m1'), OutputObj('m2')] +# model2.set_step_code(code) +# +# project.add("model", model1) +# ns = {} +# ns['Model'] = project.get_runnable_model +# +# m1, m2 = project.run_model(model2, namespace=ns) +# assert m1 == 4 +# assert m2 == 2 +# +# +# def test_parse_code(): +# code = ''' +# """ +# input = a,b +# output = c +# """ +# c = a+b +# ''' +# from openalea.core.model import PythonModel +# +# m1 = PythonModel(name='m1', code=code) +# m2 = PythonModel(name='m2') +# +# m2.code = code +# +# m3 = PythonModel(name='m3') +# m3.set_code(code) +# +# assert m1.run(a=1, b=2) == 3 +# assert m2.run(a=2, b=3) == 5 +# assert m3.run(a=3, b=4) == 7 +# +# m1 = PythonModel(name='ModelWithoutDoc') +# assert m1.get_documentation() == '' +# +# m1 = PythonModel(name='ModelWithDoc', code=code) +# assert m1.get_documentation() +# +# +# def test_clean_ns(): +# from openalea.core.model import PythonModel +# from openalea.core.service.ipython import interpreter +# interp = interpreter() +# interp.user_ns['ipython_ns'] = 1 +# +# model = PythonModel(name='clean') +# model.outputs_info = [OutputObj('ns')] +# model.inputs_info = [InputObj('a=0'), InputObj('b')] +# model.set_step_code('local_to_model=1\nns = dir()') +# +# initial_ns = dict(initial_ns=1) +# model_local_ns = model(namespace=initial_ns, b=1) +# model_varname = ['a', 'b', 'local_to_model'] +# for varname in model_varname: +# assert varname in model_local_ns +# for varname in model_varname: +# assert varname not in interp.user_ns +# assert 'ipython_ns' in interp.user_ns +# +# +# def test_in_function(): +# code = """ +# ''' +# output=out +# ''' +# a=1 +# def f(): +# return(a) +# +# out = f() +# """ +# from openalea.core.model import PythonModel +# model = PythonModel(name='func') +# model.set_code(code) +# assert model.init() == 1 diff --git a/test/test_project.py b/test/test_project.py index 6965b6aa..14639557 100644 --- a/test/test_project.py +++ b/test/test_project.py @@ -330,5 +330,4 @@ def test_normpath(self): os.chdir(old_path) self.assertNotEqual(p, link_abs) - self.assertEqual(_normpath(p), _normpath(link_abs)) - + self.assertEqual(p, _normpath(link_abs)) diff --git a/test/test_project_manager.py b/test/test_project_manager.py index f372979c..7e14a982 100644 --- a/test/test_project_manager.py +++ b/test/test_project_manager.py @@ -1,185 +1,183 @@ -import shutil -import sys - -from openalea.core.path import path as Path -from openalea.core.path import tempdir -from openalea.core.project.manager import ProjectManager -from openalea.core.project.project import Project, _normpath - -from openalea.core.unittest_tools import TestCase, EventTracker - - - -# used to test symlinks only on unix systems -win = 'win' in sys.platform - -pm = ProjectManager() -pm.count = 0 -ev = EventTracker() -# ev.debug = True -pm.register_listener(ev) - - -def get_data(filename): - return Path(__file__).parent.abspath() / 'data' / filename - - -class TestProjectManager(TestCase): - def setUp(self): - self.tmpdir = tempdir() - self.tmpdir2 = tempdir() - - pm.clear() - # Force to ignore default repositories - pm.repositories = [self.tmpdir] - ev.events # clear events - - def tearDown(self): - pm.close() - self.tmpdir.rmtree() - self.tmpdir2.rmtree() - - def create_projects(self): - """ - tmp1 - - p1 - - p2 - - link -> p2 - tmp2 - - p1 - - p3 - - link -> p4 - """ - project = Project(self.tmpdir / 'p1', alias="p1") - project.save() - - project = Project(self.tmpdir / 'p2') - project.save() - - if not win: - project.path.symlink(self.tmpdir / 'link') - - project = Project(self.tmpdir2 / 'p1', alias="Project 1") - project.save() - - project = Project(self.tmpdir2 / 'p3') - project.save() - - if not win: - project.path.symlink(self.tmpdir2 / 'link') - - def test_create_project_from_manager(self): - proj = pm.create('new_temp_project', projectdir=self.tmpdir) - assert _normpath(proj.projectdir) == _normpath(self.tmpdir), str(proj.projectdir)+' vs '+self.tmpdir - assert proj.path.name == 'new_temp_project' - - def test_discover(self): - pm.discover() - self.assertEqual(pm.items(), []) - - self.create_projects() - pm.discover() - - directories = sorted([str(path.name) for path in self.tmpdir.listdir()]) - project_dirs = sorted([str(project.path.name) for project in pm.items()]) - - # Check that we discover projects in the right repository (tmpdir) - for project in pm.items(): - self.assertEqual(_normpath(project.projectdir), _normpath(self.tmpdir)) - - # Check all directory have been created - if win: - self.assertListEqual(directories, ['p1', 'p2']) - else: - self.assertListEqual(directories, ['link', 'p1', 'p2']) - - # Check symlink has been replaced by right path - # And check projects are not appended twice - self.assertListEqual(project_dirs, ['p1', 'p2']) - - def test_repository_added_twice(self): - # Check if a repository is added twice, projects are discovered only one time - self.create_projects() - assert len(pm.repositories) == 1 - pm.repositories.append(self.tmpdir) - pm.discover() - - assert len(pm.items()) == 2 - - def test_search(self): - self.create_projects() - pm.repositories.append(self.tmpdir2) - pm.discover() - - p1a = self.tmpdir / 'p1' - p2a = self.tmpdir / 'p2' - p1b = self.tmpdir2 / 'p1' - p3b = self.tmpdir2 / 'p3' - - assert len(pm.items()) == 4 - proj_p1 = pm.item('p1') - #name_p1 = sorted([str(project.path) for project in proj_p1]) - - #assert len(proj_p1) == 2 - #assert len(pm.item('p2')) == 1 - #self.assertListEqual(name_p1, sorted([str(p1a), str(p1b)])) - - - def test_load(self): - self.create_projects() - pm.repositories.append(self.tmpdir2) - pm.discover() - project = pm.load('p2') - assert _normpath(project.path) == _normpath(self.tmpdir / 'p2') - project = pm.load('p1') - assert project.name == 'p1' - assert isinstance(project, Project) - - def test_default(self): - project = pm.default() - assert str(project.name) == "temp" - - def test_project_updated_event(self): - project = pm.create('TestProject', self.tmpdir) - project.save() - pm.cproject = project - ev.events # clear - project.add('model', filename='newfile.py') - self.check_events(ev.events, ['project_updated']) - - def test_namespace(self): - proj = pm.create('new_temp_project', projectdir=self.tmpdir) - pm.cproject = proj - user_ns = pm.shell.user_ns - - assert 'project' in user_ns - assert user_ns['project'].path == proj.path - assert user_ns['data'] == proj.path / 'data' - - pm.cproject = None - assert 'project' not in user_ns - assert 'data' not in user_ns - - def test_world_namespace(self): - - from openalea.core.model import Model - - proj = pm.create('new_temp_project', projectdir=self.tmpdir) - user_ns = pm.shell.user_ns - - assert 'world' in user_ns - w = user_ns['world'] - assert w.keys() == [] - - code = "world.add(1, name='i');a=1" - model = Model() - proj.add('model', model) - model.set_code(code) - model.run(**proj.ns) - - assert w.keys() == ['i'] - assert w['i'].obj == 1 - - pm.cproject = None - assert 'world' not in user_ns - assert w.keys() == [] +# import shutil +# import sys +# +# from openalea.core.path import path as Path +# from openalea.core.path import tempdir +# from openalea.core.project.manager import ProjectManager +# from openalea.core.project.project import Project +# +# from openalea.core.unittest_tools import TestCase, EventTracker +# +# # used to test symlinks only on unix systems +# win = 'win' in sys.platform +# +# pm = ProjectManager() +# pm.count = 0 +# ev = EventTracker() +# # ev.debug = True +# pm.register_listener(ev) +# +# +# def get_data(filename): +# return Path(__file__).parent.abspath() / 'data' / filename +# +# +# class TestProjectManager(TestCase): +# def setUp(self): +# self.tmpdir = tempdir() +# self.tmpdir2 = tempdir() +# +# pm.clear() +# # Force to ignore default repositories +# pm.repositories = [self.tmpdir] +# ev.events # clear events +# +# def tearDown(self): +# pm.close() +# self.tmpdir.rmtree() +# self.tmpdir2.rmtree() +# +# def create_projects(self): +# """ +# tmp1 +# - p1 +# - p2 +# - link -> p2 +# tmp2 +# - p1 +# - p3 +# - link -> p4 +# """ +# project = Project(self.tmpdir / 'p1', alias="p1") +# project.save() +# +# project = Project(self.tmpdir / 'p2') +# project.save() +# +# if not win: +# project.path.symlink(self.tmpdir / 'link') +# +# project = Project(self.tmpdir2 / 'p1', alias="Project 1") +# project.save() +# +# project = Project(self.tmpdir2 / 'p3') +# project.save() +# +# if not win: +# project.path.symlink(self.tmpdir2 / 'link') +# +# def test_create_project_from_manager(self): +# proj = pm.create('new_temp_project', projectdir=self.tmpdir) +# assert proj.projectdir == self.tmpdir +# assert proj.path.name == 'new_temp_project' +# +# def test_discover(self): +# pm.discover() +# self.assertEqual(pm.items(), []) +# +# self.create_projects() +# pm.discover() +# +# directories = sorted([str(path.name) for path in self.tmpdir.listdir()]) +# project_dirs = sorted( +# [str(project.path.name) for project in pm.items()]) +# +# # Check that we discover projects in the right repository (tmpdir) +# for project in pm.items(): +# self.assertEqual(project.projectdir, self.tmpdir) +# +# # Check all directory have been created +# if win: +# self.assertListEqual(directories, ['p1', 'p2']) +# else: +# self.assertListEqual(directories, ['link', 'p1', 'p2']) +# +# # Check symlink has been replaced by right path +# # And check projects are not appended twice +# self.assertListEqual(project_dirs, ['p1', 'p2']) +# +# def test_repository_added_twice(self): +# # Check if a repository is added twice, projects are discovered only one time +# self.create_projects() +# assert len(pm.repositories) == 1 +# pm.repositories.append(self.tmpdir) +# pm.discover() +# +# assert len(pm.items()) == 2 +# +# def test_search(self): +# self.create_projects() +# pm.repositories.append(self.tmpdir2) +# pm.discover() +# +# p1a = self.tmpdir / 'p1' +# p2a = self.tmpdir / 'p2' +# p1b = self.tmpdir2 / 'p1' +# p3b = self.tmpdir2 / 'p3' +# +# assert len(pm.items()) == 4 +# proj_p1 = pm.item('p1') +# # name_p1 = sorted([str(project.path) for project in proj_p1]) +# +# # assert len(proj_p1) == 2 +# # assert len(pm.item('p2')) == 1 +# # self.assertListEqual(name_p1, sorted([str(p1a), str(p1b)])) +# +# def test_load(self): +# self.create_projects() +# pm.repositories.append(self.tmpdir2) +# pm.discover() +# project = pm.load('p2') +# assert project.path == self.tmpdir / 'p2' +# project = pm.load('p1') +# assert project.name == 'p1' +# assert isinstance(project, Project) +# +# def test_default(self): +# project = pm.default() +# assert str(project.name) == "temp" +# +# def test_project_updated_event(self): +# project = pm.create('TestProject', self.tmpdir) +# project.save() +# pm.cproject = project +# ev.events # clear +# project.add('model', filename='newfile.py') +# self.check_events(ev.events, ['project_updated']) +# +# def test_namespace(self): +# proj = pm.create('new_temp_project', projectdir=self.tmpdir) +# pm.cproject = proj +# user_ns = pm.shell.user_ns +# +# assert 'project' in user_ns +# assert user_ns['project'].path == proj.path +# assert user_ns['data'] == proj.path / 'data' +# +# pm.cproject = None +# assert 'project' not in user_ns +# assert 'data' not in user_ns +# +# def test_world_namespace(self): +# +# from openalea.core.model import Model +# +# proj = pm.create('new_temp_project', projectdir=self.tmpdir) +# user_ns = pm.shell.user_ns +# +# assert 'world' in user_ns +# w = user_ns['world'] +# assert w.keys() == [] +# +# code = "world.add(1, name='i');a=1" +# model = Model() +# proj.add('model', model) +# model.set_code(code) +# model.run(**proj.ns) +# +# assert w.keys() == ['i'] +# assert w['i'].obj == 1 +# +# pm.cproject = None +# assert 'world' not in user_ns +# assert w.keys() == [] diff --git a/test/test_service_data.py b/test/test_service_data.py index 9c18a965..06e02d12 100644 --- a/test/test_service_data.py +++ b/test/test_service_data.py @@ -97,7 +97,7 @@ def test_arrange_data_args(self): # arrange_data_args is not a public function, please do not use it outside service.data from openalea.core.service.data import arrange_data_args path1 = get_data('model.py') - path2 = 'test.ext2' + path2 = 'test.ext' self.assertEqual(arrange_data_args(path1, None, None), (path1, 'text/x-python')) self.assertEqual(arrange_data_args(path2, None, None), (path2, None)) diff --git a/test/test_session.py b/test/test_session.py index 0deac30b..56a60eb1 100644 --- a/test/test_session.py +++ b/test/test_session.py @@ -62,17 +62,10 @@ def test_save_datapool(): asession.load(os.path.join(tmp_dir, 'test.pic')) assert asession.datapool['i'] == [1, 2, 3] - try: - os.remove('test.pic') - except: - try: - os.remove('test.pic.db') - except: - pass - -# Remove this test: TODO investigate + + @with_setup(setup, teardown) -def no_save_workspace(): +def test_save_workspace(): pm = PackageManager() pm.init() @@ -97,14 +90,7 @@ def no_save_workspace(): asession.save(os.path.join(tmp_dir, 'test.pic')) asession.workspaces = [] - asession.load('test.pic') - try: - os.remove('test.pic') - except: - try: - os.remove('test.pic.db') - except: - pass + asession.load(os.path.join(tmp_dir, 'test.pic')) i = asession.workspaces[0] assert type(i) == type(instance) diff --git a/travis.yml b/travis.yml deleted file mode 100644 index 52069d98..00000000 --- a/travis.yml +++ /dev/null @@ -1,45 +0,0 @@ -language: cpp - -os: - - linux - - osx - -sudo: required - -services: - - docker - -env: - - CONDA_RECIPE=conda - CONDA_VERSION=2 - -install: - - git clone https://github.com/OpenAlea/travis-ci.git - - cd travis-ci - - source install.sh - -before_script: - - source before_script.sh - -script: - - source script.sh - -after_success: - - source after_success.sh - -after_failure: - - source after_failure.sh - -before_deploy: - - source before_deploy.sh - -deploy: - skip_cleanup: true - provider: script - script: bash deploy_script.sh - -after_deploy: - - source after_deploy.sh - -after_script: - - source after_script.sh