diff --git a/inex/inex.py b/inex/inex.py index 9a1f3ef..12adf9d 100644 --- a/inex/inex.py +++ b/inex/inex.py @@ -1,17 +1,28 @@ +import os import re import sys import logging import argparse +from pydoc import locate from pathlib import Path from omegaconf import OmegaConf +from typing import Optional, Any, List from datetime import datetime, timedelta from inex.utils.configure import configure_logging, load_config, bind_plugins from inex.version import __version__ from inex.engine import execute -def fetch(path, value=None): - path = Path(path) +def evaluate(expression: str, initialize: List[str] = None, **kwargs): + if initialize is not None: + for sentence in initialize: + exec(sentence) + expression = expression.format(**kwargs) + return eval(expression) + + +def fetch(path: str, value: str = None): + path = Path(path).absolute() assert path.is_file(), f'File {path} does not exist' config = OmegaConf.load(path) if value is not None: @@ -19,10 +30,85 @@ def fetch(path, value=None): return config +def getenv(name: str, cast: Optional[str] = None) -> Any: + value = os.getenv(name) + assert value is not None, f'Environment variable with name {name} does not exist' + if (cast is None) or (cast == 'str'): + return value + else: + cast = locate(cast) + return cast(value) + + +def setenv(name: str, value: Any) -> Any: + os.environ[name] = str(value) + return value + + +def read_text(path: str, cast: Optional[str] = None) -> Any: + path = Path(path).absolute() + assert path.is_file(), f'File {path} does not exist' + value = path.read_text(encoding='utf-8').strip() + if (cast is None) or (cast == 'str'): + return value + else: + cast = locate(cast) + return cast(value) + + +def num_lines(path: Any) -> int: + if isinstance(path, str): + path = Path(path).absolute() + assert path.is_file(), f'File {path} does not exist' + return len(path.read_text().strip().splitlines()) + else: + return len(path) + + +def path_parent(path: str) -> str: + return str(Path(path).absolute().parent) + + +def path_name(path: str) -> str: + return Path(path).name + + +def path_stem(path: str) -> str: + return Path(path).stem + + +def path_suffix(path: str) -> str: + return Path(path).suffix + + +def path_is_file(path: str) -> bool: + return Path(path).is_file() + + +def path_is_dir(path: str) -> bool: + return Path(path).is_dir() + + +def path_exists(path: str) -> bool: + return Path(path).exists() + + def start(log_level, log_path, sys_path, merge, update, config_path, stop_after=None, final_path=None): begin_time = datetime.now() + OmegaConf.register_new_resolver('__evaluate__', evaluate, replace=True) OmegaConf.register_new_resolver('__fetch__', fetch, replace=True) + OmegaConf.register_new_resolver('__getenv__', getenv, replace=True) + OmegaConf.register_new_resolver('__setenv__', setenv, replace=True) + OmegaConf.register_new_resolver('__read_text__', read_text, replace=True) + OmegaConf.register_new_resolver('__num_lines__', num_lines, replace=True) + OmegaConf.register_new_resolver('__path_parent__', path_parent, replace=True) + OmegaConf.register_new_resolver('__path_name__', path_name, replace=True) + OmegaConf.register_new_resolver('__path_stem__', path_stem, replace=True) + OmegaConf.register_new_resolver('__path_suffix__', path_suffix, replace=True) + OmegaConf.register_new_resolver('__path_is_file__', path_is_file, replace=True) + OmegaConf.register_new_resolver('__path_is_dir__', path_is_dir, replace=True) + OmegaConf.register_new_resolver('__path_exists__', path_exists, replace=True) config = load_config(config_path) if merge is not None: @@ -62,7 +148,7 @@ def start(log_level, log_path, sys_path, merge, update, config_path, stop_after= if final_path is not None: logging.debug(f'Writing final config to {final_path}') - final_path = Path(final_path) + final_path = Path(final_path).absolute() parent = final_path.parent if not parent.exists(): logging.debug(f'Creating directory {parent}') diff --git a/inex/utils/configure.py b/inex/utils/configure.py index 8e47e1a..8ae4a7c 100644 --- a/inex/utils/configure.py +++ b/inex/utils/configure.py @@ -161,6 +161,9 @@ def resolve_option(option, state): def create_plugin(name, config, state): assert name in config, f'Failed find module {name} in config\n{config}' params = config[name] + title = params.get('title', None) + if title is not None: + print(title, flush=True) if name == 'execute': assert 'method' in params, f'Failed to find "method" in execute section\n{params}' modname = params['method'] diff --git a/inex/version.txt b/inex/version.txt index a0cd9f0..50e47c8 100644 --- a/inex/version.txt +++ b/inex/version.txt @@ -1 +1 @@ -3.1.0 \ No newline at end of file +3.1.1 \ No newline at end of file diff --git a/tests/test_fetch.py b/tests/test_fetch.py index 035bfee..fc76cb0 100644 --- a/tests/test_fetch.py +++ b/tests/test_fetch.py @@ -1,5 +1,4 @@ import unittest -import numpy as np from pathlib import Path from typing import Dict, Any from tests.utils import call_engine diff --git a/tests/test_num_lines.py b/tests/test_num_lines.py new file mode 100644 index 0000000..d7b0165 --- /dev/null +++ b/tests/test_num_lines.py @@ -0,0 +1,20 @@ +import unittest +from pathlib import Path +from typing import Dict, Any +from tests.utils import call_engine + + +class TestModule(unittest.TestCase): + @staticmethod + def test_config_file(): + config = Path(__file__).with_suffix('.yaml') + assert config.is_file() + values: Dict[str, Any] = call_engine(config) + assert values['num1'] == 27 + assert values['num2'] == 20 + assert values['num3'] == 3 + assert values['num4'] == 2 + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_num_lines.yaml b/tests/test_num_lines.yaml new file mode 100644 index 0000000..40d5e16 --- /dev/null +++ b/tests/test_num_lines.yaml @@ -0,0 +1,27 @@ +#!/bin/env inex + +path_conf: tests/test_num_lines.yaml +path_test: tests/test_num_lines.py + +lines: + - 1 + - 2 + - 3 + +options: + key1: value1 + key2: value2 + +num1: ${__num_lines__:${path_conf}} +num2: ${__num_lines__:${path_test}} +num3: ${__num_lines__:${lines}} +num4: ${__num_lines__:${options}} + +execute: + method: inex.helpers/assign + options: + value: + num1: ${num1} + num2: ${num2} + num3: ${num3} + num4: ${num4} diff --git a/tests/test_read_text.py b/tests/test_read_text.py new file mode 100644 index 0000000..d21b65a --- /dev/null +++ b/tests/test_read_text.py @@ -0,0 +1,20 @@ +import unittest +from pathlib import Path +from typing import Dict, Any +from tests.utils import call_engine + + +class TestModule(unittest.TestCase): + @staticmethod + def test_config_file(): + config = Path(__file__).with_suffix('.yaml') + assert config.is_file() + values: Dict[str, Any] = call_engine(config) + assert values['value_str'] == '17' + assert values['value_int'] == 17 + assert values['value_float'] == 17.0 + assert values['value_bool'] + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_read_text.txt b/tests/test_read_text.txt new file mode 100644 index 0000000..5ae8c48 --- /dev/null +++ b/tests/test_read_text.txt @@ -0,0 +1,2 @@ + + 17 diff --git a/tests/test_read_text.yaml b/tests/test_read_text.yaml new file mode 100644 index 0000000..d6bf3ad --- /dev/null +++ b/tests/test_read_text.yaml @@ -0,0 +1,17 @@ +#!/bin/env inex + +path_txt: tests/test_read_text.txt + +value_str: ${__read_text__:${path_txt}, str} +value_int: ${__read_text__:${path_txt}, int} +value_float: ${__read_text__:${path_txt}, float} +value_bool: ${__read_text__:${path_txt}, 'bool'} + +execute: + method: inex.helpers/assign + options: + value: + value_str: ${value_str} + value_int: ${value_int} + value_float: ${value_float} + value_bool: ${value_bool} diff --git a/tests/test_resolvers.py b/tests/test_resolvers.py new file mode 100644 index 0000000..fc9ed19 --- /dev/null +++ b/tests/test_resolvers.py @@ -0,0 +1,29 @@ +import os +import unittest +from pathlib import Path +from typing import Dict, Any +from tests.utils import call_engine + + +class TestModule(unittest.TestCase): + @staticmethod + def test_config_file(): + config = Path(__file__).with_suffix('.yaml') + assert config.is_file() + values: Dict[str, Any] = call_engine(config) + assert values['evaluate1'] == 7 + assert values['evaluate2'] == 6 + assert os.getenv('Seven') == '7' + assert values['seven_set'] == 7 + assert values['seven_get'] == 7 + assert values['path_parent'].endswith('tests') + assert values['path_name'] == 'test_resolvers.yaml' + assert values['path_stem'] == 'test_resolvers' + assert values['path_suffix'] == '.yaml' + assert values['path_is_file'] + assert not values['path_is_dir'] + assert values['path_exists'] + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_resolvers.yaml b/tests/test_resolvers.yaml new file mode 100644 index 0000000..8b7e032 --- /dev/null +++ b/tests/test_resolvers.yaml @@ -0,0 +1,36 @@ +#!/bin/env inex + +a: 2 +b: 3 +evaluate1: ${__evaluate__:'${a}^2 + ${b}'} +evaluate2: ${__evaluate__:'int(np.sum(np.array([1, 2, 3])))', [import numpy as np]} + +seven_set: ${__setenv__:Seven, 7} +seven_get: ${__getenv__:Seven, int} + +path_yaml: tests/test_resolvers.yaml + +path_parent: ${__path_parent__:${path_yaml}} +path_name: ${__path_name__:${path_yaml}} +path_stem: ${__path_stem__:${path_yaml}} +path_suffix: ${__path_suffix__:${path_yaml}} +path_is_file: ${__path_is_file__:${path_yaml}} +path_is_dir: ${__path_is_dir__:${path_yaml}} +path_exists: ${__path_exists__:${path_yaml}} + +execute: + title: ' Testing resolvers' + method: inex.helpers/assign + options: + value: + evaluate1: ${evaluate1} + evaluate2: ${evaluate2} + seven_set: ${seven_set} + seven_get: ${seven_get} + path_parent: ${path_parent} + path_name: ${path_name} + path_stem: ${path_stem} + path_suffix: ${path_suffix} + path_is_file: ${path_is_file} + path_is_dir: ${path_is_dir} + path_exists: ${path_exists}