From 1f40ea2875d9ad1424645aa8810f3cce035a103a Mon Sep 17 00:00:00 2001 From: cpelley Date: Thu, 30 Jul 2026 16:03:19 +0100 Subject: [PATCH] testing --- .github/workflows/docs.yml | 70 +++ README.md | 6 +- docs/dagrunner.config.md | 139 ----- docs/dagrunner.events.md | 63 --- docs/dagrunner.execute_graph.md | 169 ------ docs/dagrunner.md | 20 - docs/dagrunner.plugin_framework.md | 539 ------------------ docs/dagrunner.runner.md | 6 - docs/dagrunner.runner.schedulers.asyncmp.md | 96 ---- docs/dagrunner.runner.schedulers.base.md | 4 - docs/dagrunner.runner.schedulers.dask.md | 254 --------- docs/dagrunner.runner.schedulers.md | 21 - docs/dagrunner.utils.logger.md | 320 ----------- docs/dagrunner.utils.md | 573 -------------------- docs/dagrunner.utils.networkx.md | 112 ---- docs/dagrunner.utils.visualisation.md | 234 -------- docs/dagrunner_index.md | 73 --- 17 files changed, 73 insertions(+), 2626 deletions(-) create mode 100644 .github/workflows/docs.yml delete mode 100644 docs/dagrunner.config.md delete mode 100644 docs/dagrunner.events.md delete mode 100644 docs/dagrunner.execute_graph.md delete mode 100644 docs/dagrunner.md delete mode 100644 docs/dagrunner.plugin_framework.md delete mode 100644 docs/dagrunner.runner.md delete mode 100644 docs/dagrunner.runner.schedulers.asyncmp.md delete mode 100644 docs/dagrunner.runner.schedulers.base.md delete mode 100644 docs/dagrunner.runner.schedulers.dask.md delete mode 100644 docs/dagrunner.runner.schedulers.md delete mode 100644 docs/dagrunner.utils.logger.md delete mode 100644 docs/dagrunner.utils.md delete mode 100644 docs/dagrunner.utils.networkx.md delete mode 100644 docs/dagrunner.utils.visualisation.md delete mode 100644 docs/dagrunner_index.md diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..8eb62f4 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,70 @@ +name: Pull Request Tests + +on: + push: + tags: + - '*' + branches: + - main + - release/* + - feature/* + +permissions: + contents: write + +jobs: + docs-build: + if: "!contains(github.event.head_commit.message, '[skip ci]')" + runs-on: ubuntu-latest + + steps: + # SETUP + ############################ + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + ref: ${{ github.head_ref }} # Ensure branch is checked out, not detached state (so we can push a commit later) + token: ${{ secrets.GITHUB_TOKEN }} + persist-credentials: true # Ensure that the token is available for pushing changes + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 + with: + python-version: 3.x + + - name: Add checkout directory to PYTHONPATH + run: echo "PYTHONPATH=$(pwd):$PYTHONPATH" >> $GITHUB_ENV + + - name: Install dependencies + id: install-dependencies + run: | + pip install .[tests,dev] + pip uninstall dagrunner -y + + # DOCUMENTATION + ############################ + - name: Build documentation + run: | + rm -rf ./docs/_build + mkdir -p ./docs/_build + ./docs/gen_docs dagrunner ./docs/_build + + - name: Check if documentation has changed + id: check-docs + run: | + echo "changed=$(git diff --quiet --exit-code || echo true)" | tee -a $GITHUB_OUTPUT + + # https://github.com/orgs/community/discussions/26560#discussioncomment-3531273 + # This must be our very final step to ensure that it runs only on condition of + # success of all previous steps. A pushed commit will not trigger the re-running + # of this workflow. + - name: Commit and push documentation changes + if: steps.check-docs.outputs.changed == 'true' + run: | + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git add docs/. + git commit -am "Automated reference documentation update for PR ${{ github.event.number }} [skip ci]" + git push + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/README.md b/README.md index 5d7e83c..3676c40 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ DAGrunner takes advantage of the native markdown rendering support provided by g ## API -See [DAGrunner API](docs/dagrunner_index.md) +See [DAGrunner API](docs/_build/dagrunner_index.md) ## License/copyright @@ -52,7 +52,7 @@ See [docs/demo.ipynb](docs/demo.ipynb) DAGrunner concerns itself with graph execution and does not strictly require processing modules (plugins) to take any particular form. That is, you may or may not choose to use or subclass the plugins provided by DAGrunner. However, for convenience, DAGrunner does define some plugins which fall into two broad categories, some abstract and some for use as they are. -See [here](docs/dagrunner.plugin_framework.md) for more information. +See [here](docs/_build/dagrunner.plugin_framework.md) for more information. ## Schedulers @@ -63,7 +63,7 @@ These range from [dask](https://www.dask.org/), [ray](https://docs.ray.io/en/lat ## Logging and monitoring DAGrunner provides a script `dagrunner-logger` for running a TCP server. This enables logging to function across the network. Additionally, it will write logs to an sqlite database to aid in realtime monitoring from external tools. -See [logger](docs/dagrunner.utils.logger.md) for more information. +See [logger](docs/_build/dagrunner.utils.logger.md) for more information. ## Logo diff --git a/docs/dagrunner.config.md b/docs/dagrunner.config.md deleted file mode 100644 index 266c896..0000000 --- a/docs/dagrunner.config.md +++ /dev/null @@ -1,139 +0,0 @@ -# module: `dagrunner.config` - -[Source](../dagrunner/config.py#L0) - -This module handles the run-time configuration of the dagrunner library. - -Certain hooks are present in the library for providing detailed control over -dagrunner run-time. - -The configuration of dagrunner follows a first-in first-out approach on parsing -configuration options and is handled by :class:`dagrunner.config.GlobalConfiguration`. -This means that any number of configuration files can be parsed. On import, -`dagrunner.cfg` is parsed when present in the dagrunner root folder. Each successive -configuration file parsing will override existing parameter values. - -see [class: dagrunner.utils.Singleton](dagrunner.utils.md#class-singleton) - -## GlobalConfiguration: `CONFIG` - -## class: `GlobalConfiguration` - -[Source](../dagrunner/config.py#L28) - -### Call Signature: - -```python -GlobalConfiguration(*args, **kwargs) -``` - -The global configuration class handles any number of configuration files, -where subsequent configuration entries act to override previous entries -parsed. - -All group names parsed are prefixed with "dagrunner" and all entries are then -parsed strictly. Those groups not with this prefix are silently ignored. - -The following represents a description of the runtime configuration -options:: - - # Graph visualisation - [dagrunner_visualisation] - enabled - title - collapse_properties - backend - output_filepath - group_by - label_by - - [dagrunner_runtime] - # cache disable/enable: None/False/True. - # None implies enabled only if cache_dir is set. - cache_enabled - # if not specified and cache enabled, uses temp directory 'dagrunner_cache' - # in temp folder - cache_dir - - # Logging - [dagrunner_logging] - enabled - host - port - -### function: `__getitem__` - -[Source](../dagrunner/config.py#L172) - -#### Call Signature: - -```python -__getitem__(self, key) -``` - -### function: `__init__` - -[Source](../dagrunner/config.py#L83) - -#### Call Signature: - -```python -__init__(self) -``` - -Initialize self. See help(type(self)) for accurate signature. - -### function: `__repr__` - -[Source](../dagrunner/config.py#L90) - -#### Call Signature: - -```python -__repr__(self) -``` - -Return repr(self). - -### function: `__setitem__` - -[Source](../dagrunner/config.py#L175) - -#### Call Signature: - -```python -__setitem__(self, key, value) -``` - -### function: `__str__` - -[Source](../dagrunner/config.py#L87) - -#### Call Signature: - -```python -__str__(self) -``` - -Return str(self). - -### function: `parse_configuration` - -[Source](../dagrunner/config.py#L146) - -#### Call Signature: - -```python -parse_configuration(self, filename) -``` - -Parses a new configuration file. - -Entries in 'filename' override existing entries in the configuration, -while entries not set remain unchanged from the previous state. - -Parameters ----------- -filename : str - Name of the configuration file to read. - diff --git a/docs/dagrunner.events.md b/docs/dagrunner.events.md deleted file mode 100644 index 0c7ff5c..0000000 --- a/docs/dagrunner.events.md +++ /dev/null @@ -1,63 +0,0 @@ -# module: `dagrunner.events` - -[Source](../dagrunner/events.py#L0) - -## Overview -DAGrunner defines two special singleton events that plugins can return to control the -execution flow of their graph. - -- event.IGNORE - removes a particular input from further processing. -- event.SKIP - aborts the execution of a plugin (and all downstream nodes) when any - input carries this event. - - -### event.IGNORE -When a plugin returns `event.IGNORE`, the immediate descendant node filters out that -input. -The remaining inputs of that plugin are utilised by that node as normal. -If all inputs to a node are `event.IGNORE`, the node's execution is ignored, and a -`event.IGNORE` event is returned instead, ignoring execution through all descendants. - -```mermaid ---- -title: event.IGNORE (filtering some) ---- -graph - cycle1{cycleX} - cycle1 --> Input1 --> filepath --> Load1 --> cube1 --> Proc - cycle1 --> Input2 --> filepath --> Load2 --> event.IGNORE --> Proc - cycle1 --> Input3 --> filepath --> Load3 --> cube3 --> Proc - cycle1 --> Input4 --> filepath --> Load4 --> event.IGNORE --> Proc - Proc --> cube --> Save - Proc["Proc (cube1, cube3)"] -``` -Here, Input2 and Input4 return an IGNORE event, likely due to there being missing data. -Only the non-ignored cubes (`cube1` and `cube3`) reach `Proc`; the ignored inputs are -dropped. - -### `event.IGNORE` -The SKIP event differs from the IGNORE event in that if **any** input to a plugin is a -SKIP event, node execution is skipped and it instead propagates this skip event so that -all dependent nodes and their descendants aren't executed. - -```mermaid ---- -title: event.SKIP ---- -graph - cycle1{cycleX} - cycle1 --> Input1 --> filepath --> Load1 --> cube --> Proc - cycle1 --> Input2 --> filepath --> Load2 --> event.SKIP --> Proc - cycle1 --> Input3 --> filepath --> Load3 --> cube --> Proc - cycle1 --> Input4 --> filepath --> Load4 --> event.IGNORE --> Proc - Proc --> event.SKIP --> Save -``` -Because `Input2` returns a SKIP event, the Proc node and everything that follows aren't -executed and neither is the Save node since the skip is propagated along the execution -graph. - -see [class: dagrunner.utils.Singleton](dagrunner.utils.md#class-singleton) - -## _IgnoreEvent: `IGNORE_EVENT` - -## _SkipEvent: `SKIP_EVENT` \ No newline at end of file diff --git a/docs/dagrunner.execute_graph.md b/docs/dagrunner.execute_graph.md deleted file mode 100644 index 1a03168..0000000 --- a/docs/dagrunner.execute_graph.md +++ /dev/null @@ -1,169 +0,0 @@ -# module: `dagrunner.execute_graph` - -[Source](../dagrunner/execute_graph.py#L0) - -see [GlobalConfiguration: dagrunner.config.CONFIG](dagrunner.config.md#globalconfiguration-config) - -see [class: dagrunner.utils.CaptureProcMemory](dagrunner.utils.md#class-captureprocmemory) - -see [class: dagrunner.plugin_framework.NodeAwarePlugin](dagrunner.plugin_framework.md#class-nodeawareplugin) - -see [class: dagrunner.utils.TimeIt](dagrunner.utils.md#class-timeit) - -see [function: dagrunner.utils.as_iterable](dagrunner.utils.md#function-as_iterable) - -see [module: dagrunner.events](dagrunner.events.md#module-dagrunnerevents) - -see [function: dagrunner.utils.function_to_argparse_parse_args](dagrunner.utils.md#function-function_to_argparse_parse_args) - -see [module: dagrunner.utils.logger](dagrunner.utils.logger.md#module-dagrunnerutilslogger) - -see [function: dagrunner.utils.networkx.visualise_graph](dagrunner.utils.networkx.md#function-visualise_graph) - -## class: `ExecuteGraph` - -[Source](../dagrunner/execute_graph.py#L274) - -### Call Signature: - -```python -ExecuteGraph(networkx_graph: str, networkx_graph_kwargs: dict = None, , scheduler: str = 'multiprocessing', num_workers: int = 1, profiler_filepath: str = None, config_filepath: str = None, dry_run: bool = False, verbose: bool = False, **kwargs) -``` - -### function: `__call__` - -[Source](../dagrunner/execute_graph.py#L399) - -#### Call Signature: - -```python -__call__(self) -``` - -Call self as a function. - -### function: `__init__` - -[Source](../dagrunner/execute_graph.py#L275) - -#### Call Signature: - -```python -__init__(self, networkx_graph: str, networkx_graph_kwargs: dict = None, , scheduler: str = 'multiprocessing', num_workers: int = 1, profiler_filepath: str = None, config_filepath: str = None, dry_run: bool = False, verbose: bool = False, **kwargs) -``` - -Execute a networkx graph using a chosen scheduler. - -Args: -- `networkx_graph` (networkx.DiGraph, callable or str): - Python dot path to a `networkx.DiGraph` or tuple(edges, settings) object, or - callable that returns one. When called via the library, we support passing - the `networkx.DiGraph` or `tuple(edges, settings)` objects directly. Note - that 'settings' represent a mapping (dictionary) between node and the node - attributes. When provided, DAGrunner will attempt to convert this tuple into - a networkx through the following pseudo-code: - 1. Copy node identity properties into the node attributes dictionary - and remove any attributes that are 'None' ('settings' from the tuple - provided). - 2. Construct an empty networkx.DiGraph object. - 3. Add edges to this graph ('edges' from the tuple provided). - 4. Add node to attributes lookup to this graph ('settings' from the tuple - provided). - It is recommended that the user instead provide the networkx graph directly - rather than relying on DAGrunner to decide how to construct it. -- `networkx_graph_kwargs` (dict): - Keyword arguments to pass to the `networkx_graph` when it represents a - callable. Optional. -- `plugin_executor` (callable): - A callable object that executes a plugin function or method with the provided - arguments and keyword arguments. By default, uses the `plugin_executor` - function. Optional. -- `scheduler` (str): - Accepted values include "ray", "multiprocessing" and those recognised - by dask: "threads", "processes" and "single-threaded" (useful for debugging) - and "distributed". See https://docs.dask.org/en/latest/scheduling.html. - Optional. -- `num_workers` (int): - Number of processes or threads to use. Optional. -- `config_filepath` (str): - Path to the configuration file. See [dagrunner.config](dagrunner.config.md). - Optional. -- `dry_run` (bool): - Print executed commands but don't actually run them. Optional. -- `profiler_filepath` (str): - Output html profile filepath if supported by the chosen scheduler. - See https://docs.dask.org/en/latest/diagnostics-local.html - Optional. -- `verbose` (bool): - Print executed commands. Optional. -- `**kwargs`: - Optional global keyword arguments to apply to all applicable plugins. - -### function: `visualise` - -[Source](../dagrunner/execute_graph.py#L396) - -#### Call Signature: - -```python -visualise(self, **kwargs) -``` - -## dict: `SCHEDULERS` - -## function: `main` - -[Source](../dagrunner/execute_graph.py#L410) - -### Call Signature: - -```python -main() -``` - -Entry point of the program. -Parses command line arguments and executes the graph using the ExecuteGraph class. - -## function: `plugin_executor` - -[Source](../dagrunner/execute_graph.py#L47) - -### Call Signature: - -```python -plugin_executor(*args, call=None, verbose=False, dry_run=False, common_kwargs=None, node_id=None, **node_properties) -``` - -Executes a plugin callable with the provided arguments and keyword arguments. - -Plugins can be functions or classes. If a class, it is instantiated with the -keyword arguments provided in the `call` tuple. The plugin callable is then -executed with the positional arguments provided in `args` and the keyword arguments -provided in the `call` tuple. A plugin call is skipped if 1 or more of the `args` -is the `SKIP_EVENT` object. - -Args: -- `*args`: Positional arguments to be passed to the plugin callable. -- `call`: A tuple containing the callable object (plugin) or python dot path to one - and optionally keyword arguments on instantiating and calling to that plugin: - - `(CallableClass, kwargs_init, kwargs_call)` -> `CallableClass(**kwargs_init)(*args, **kwargs_call)` - - `(CallableClass, {}, kwargs_call)` -> `CallableClass()(*args, **kwargs_call)` - - `(CallableClass)` - `CallableClass()(*args)` - - `(callable, kwargs)` -> `callable(*args, **kwargs)` - - `(callable)` -> `callable(*args)` -- `verbose`: A boolean indicating whether to print verbose output. -- `dry_run`: A boolean indicating whether to perform a dry run without executing - the plugin. -- `common_kwargs`: A dictionary of optional keyword arguments to apply to all - applicable plugins. That is, being passed to the plugin initialisation and or - call if such keywords are expected from the plugin. This is a useful alternative - to global or environment variable usage. -- `**node_properties`: Node properties. These will be passed to 'node-aware' - plugins. - -Returns: -- The result of executing the plugin function or method. - -Raises: -- ValueError: If the `call` argument is not provided. - diff --git a/docs/dagrunner.md b/docs/dagrunner.md deleted file mode 100644 index 3078169..0000000 --- a/docs/dagrunner.md +++ /dev/null @@ -1,20 +0,0 @@ -# module: `dagrunner` - -[Source](../dagrunner/__init__.py#L0) - -see [class: plugin_framework.DataPolling](dagrunner.plugin_framework.md#class-datapolling) - -see [class: plugin_framework.Input](dagrunner.plugin_framework.md#class-input) - -see [class: plugin_framework.NodeAwarePlugin](dagrunner.plugin_framework.md#class-nodeawareplugin) - -see [class: plugin_framework.Plugin](dagrunner.plugin_framework.md#class-plugin) - -see [class: plugin_framework.Shell](dagrunner.plugin_framework.md#class-shell) - -see [module: events](dagrunner.events.md#module-dagrunnerevents) - -see [module: plugin_framework](dagrunner.plugin_framework.md#module-dagrunnerplugin_framework) - -see [module: utils](dagrunner.utils.md#module-dagrunnerutils) - diff --git a/docs/dagrunner.plugin_framework.md b/docs/dagrunner.plugin_framework.md deleted file mode 100644 index 94028b5..0000000 --- a/docs/dagrunner.plugin_framework.md +++ /dev/null @@ -1,539 +0,0 @@ -# module: `dagrunner.plugin_framework` - -[Source](../dagrunner/plugin_framework.py#L0) - -see [function: dagrunner.utils.data_polling](dagrunner.utils.md#function-data_polling) - -see [module: dagrunner.events](dagrunner.events.md#module-dagrunnerevents) - -see [function: dagrunner.utils.process_path](dagrunner.utils.md#function-process_path) - -see [function: dagrunner.utils.stage_to_dir](dagrunner.utils.md#function-stage_to_dir) - -## class: `DataPolling` - -[Source](../dagrunner/plugin_framework.py#L176) - -### Call Signature: - -```python -DataPolling(timeout=120, polling=1, file_count=None, verbose=False) -``` - -A trigger plugin that completes only when data is successfully polled. - -Remote file paths using `:` syntax are supported as well as -local and remote glob patterns. - -### function: `__call__` - -[Source](../dagrunner/plugin_framework.py#L201) - -#### Call Signature: - -```python -__call__(self, *args) -``` - -Poll for data until available or timeout is reached. - -Args: -- *args: File paths or glob patterns to poll for. - -Returns: -- None - -### function: `__init__` - -[Source](../dagrunner/plugin_framework.py#L184) - -#### Call Signature: - -```python -__init__(self, timeout=120, polling=1, file_count=None, verbose=False) -``` - -Initialize the DataPolling plugin. - -Args: -- timeout (int): Maximum time to wait for data in seconds. -- polling (int): Polling interval in seconds. -- file_count (int or None): Expected number of files. - If None, any number greater than 1 per input/glob pattern is not considered - missing. -- verbose (bool): Whether to print verbose output. - -## class: `Input` - -[Source](../dagrunner/plugin_framework.py#L222) - -### Call Signature: - -```python -Input() -``` - -A plugin to expand filepaths using keyword arguments and environment variables. - -### function: `__call__` - -[Source](../dagrunner/plugin_framework.py#L227) - -#### Call Signature: - -```python -__call__(self, filepath, node_properties=None, **kwargs) -``` - -Expand a filepath. - -Expand the provided string (typically representing a filepath) using the -keyword arguments and environment variables. Note that this plugin is -'node aware' since it is derived from the -[NodeAwarePlugin](dagrunner.plugin_framework.md#class-nodeawareplugin). - -Args: -- `filepath` (str): The filepath to be expanded. -- `node_properties`: node properties passed by the plugin executor. -- **kwargs: Keyword arguments to be used in the expansion. - -Returns: -- str: The expanded filepath. - -Raises: -- ValueError: If positional arguments are provided. - -## class: `Load` - -[Source](../dagrunner/plugin_framework.py#L68) - -### Call Signature: - -```python -Load(staging_dir=None, on_missing='error', verbose=False) -``` - -Abstract data loader. - -The `load` method must be implemented by the subclass. -This abstract class handles staging of files from remote hosts -and handling missing files according to the `on_missing` parameter as well -as globbing of file paths (local or remote). - -### function: `__call__` - -[Source](../dagrunner/plugin_framework.py#L119) - -#### Call Signature: - -```python -__call__(self, *args, **kwargs) -``` - -Load data from a file or list of files. - -Args: -- *args: List of filepaths to load. `:` syntax supported - for loading files from a remote host. -- **kwargs: Keyword arguments to pass to. - -Returns: -- Any: User overrode 'load' abstractmethod return value, or - `events.IGNORE` or `events.SKIP` if files are missing and - `on_missing` is set to 'ignore' or 'skip' respectively. - -Raises: -- FileNotFoundError: If any of the files do not exist and `on_missing` is set - to 'error'. - -### function: `__init__` - -[Source](../dagrunner/plugin_framework.py#L78) - -#### Call Signature: - -```python -__init__(self, staging_dir=None, on_missing='error', verbose=False) -``` - -Load data from a file. - -Args: -- staging_dir: Local directory to stage files in. - Staging of remote files where filepaths are of `:` syntax. - A staging directory must be specified when loading remote files. -- on_missing: Action to take when files are missing. Accepted values: 'error', - 'ignore' and 'skip'. - 'ignore' and 'skip' will return `events.IGNORE` and `events.SKIP` - respectively, whilst 'error' will raise a `FileNotFoundError`. - See [dagrunner.events](dagrunner.events.md) -- verbose: Print verbose output. - -### function: `load` - -[Source](../dagrunner/plugin_framework.py#L102) - -#### Call Signature: - -```python -load(self, *args, **kwargs) -``` - -Load data from a file. - -Args: -- *args: Positional arguments. -- **kwargs: Keyword arguments. - -Returns: -- Any: The loaded data. - -Raises: -- NotImplementedError: If the method is not implemented. - -## class: `LoadJson` - -[Source](../dagrunner/plugin_framework.py#L261) - -### Call Signature: - -```python -LoadJson(staging_dir=None, on_missing='error', verbose=False) -``` - -json file loader. - -### function: `__call__` - -[Source](../dagrunner/plugin_framework.py#L119) - -#### Call Signature: - -```python -__call__(self, *args, **kwargs) -``` - -Load data from a file or list of files. - -Args: -- *args: List of filepaths to load. `:` syntax supported - for loading files from a remote host. -- **kwargs: Keyword arguments to pass to. - -Returns: -- Any: User overrode 'load' abstractmethod return value, or - `events.IGNORE` or `events.SKIP` if files are missing and - `on_missing` is set to 'ignore' or 'skip' respectively. - -Raises: -- FileNotFoundError: If any of the files do not exist and `on_missing` is set - to 'error'. - -### function: `__init__` - -[Source](../dagrunner/plugin_framework.py#L78) - -#### Call Signature: - -```python -__init__(self, staging_dir=None, on_missing='error', verbose=False) -``` - -Load data from a file. - -Args: -- staging_dir: Local directory to stage files in. - Staging of remote files where filepaths are of `:` syntax. - A staging directory must be specified when loading remote files. -- on_missing: Action to take when files are missing. Accepted values: 'error', - 'ignore' and 'skip'. - 'ignore' and 'skip' will return `events.IGNORE` and `events.SKIP` - respectively, whilst 'error' will raise a `FileNotFoundError`. - See [dagrunner.events](dagrunner.events.md) -- verbose: Print verbose output. - -### function: `load` - -[Source](../dagrunner/plugin_framework.py#L264) - -#### Call Signature: - -```python -load(self, *args) -``` - -Load data from a file. - -Args: -- *args: Positional arguments. -- **kwargs: Keyword arguments. - -Returns: -- Any: The loaded data. - -Raises: -- NotImplementedError: If the method is not implemented. - -## class: `LoadPickle` - -[Source](../dagrunner/plugin_framework.py#L305) - -### Call Signature: - -```python -LoadPickle(staging_dir=None, on_missing='error', verbose=False) -``` - -pickle file loader. - -### function: `__call__` - -[Source](../dagrunner/plugin_framework.py#L119) - -#### Call Signature: - -```python -__call__(self, *args, **kwargs) -``` - -Load data from a file or list of files. - -Args: -- *args: List of filepaths to load. `:` syntax supported - for loading files from a remote host. -- **kwargs: Keyword arguments to pass to. - -Returns: -- Any: User overrode 'load' abstractmethod return value, or - `events.IGNORE` or `events.SKIP` if files are missing and - `on_missing` is set to 'ignore' or 'skip' respectively. - -Raises: -- FileNotFoundError: If any of the files do not exist and `on_missing` is set - to 'error'. - -### function: `__init__` - -[Source](../dagrunner/plugin_framework.py#L78) - -#### Call Signature: - -```python -__init__(self, staging_dir=None, on_missing='error', verbose=False) -``` - -Load data from a file. - -Args: -- staging_dir: Local directory to stage files in. - Staging of remote files where filepaths are of `:` syntax. - A staging directory must be specified when loading remote files. -- on_missing: Action to take when files are missing. Accepted values: 'error', - 'ignore' and 'skip'. - 'ignore' and 'skip' will return `events.IGNORE` and `events.SKIP` - respectively, whilst 'error' will raise a `FileNotFoundError`. - See [dagrunner.events](dagrunner.events.md) -- verbose: Print verbose output. - -### function: `load` - -[Source](../dagrunner/plugin_framework.py#L308) - -#### Call Signature: - -```python -load(self, *args) -``` - -Load data from a file. - -Args: -- *args: Positional arguments. -- **kwargs: Keyword arguments. - -Returns: -- Any: The loaded data. - -Raises: -- NotImplementedError: If the method is not implemented. - -## class: `NodeAwarePlugin` - -[Source](../dagrunner/plugin_framework.py#L42) - -### Call Signature: - -```python -NodeAwarePlugin() -``` - -An abstract base class plugin that is of type that instructs the plugin -executor to pass it node parameters. This enables the definition of plugins -that are 'node aware'. - -### function: `__call__` - -[Source](../dagrunner/plugin_framework.py#L23) - -#### Call Signature: - -```python -__call__(self, *args, **kwargs) -``` - -The main method of the plugin (abstract method). - -Positional arguments represent the plugin's inputs (dependencies), -while keyword arguments represent the plugin's parameters. - -Args: -- *args: Positional arguments. -- **kwargs: Keyword arguments. - -Returns: -- Any: The output of the plugin. - -## class: `Plugin` - -[Source](../dagrunner/plugin_framework.py#L20) - -### Call Signature: - -```python -Plugin() -``` - -Abstract base class to define our plugin UI - -### function: `__call__` - -[Source](../dagrunner/plugin_framework.py#L23) - -#### Call Signature: - -```python -__call__(self, *args, **kwargs) -``` - -The main method of the plugin (abstract method). - -Positional arguments represent the plugin's inputs (dependencies), -while keyword arguments represent the plugin's parameters. - -Args: -- *args: Positional arguments. -- **kwargs: Keyword arguments. - -Returns: -- Any: The output of the plugin. - -## class: `SaveJson` - -[Source](../dagrunner/plugin_framework.py#L274) - -### Call Signature: - -```python -SaveJson() -``` - -Save data to a JSON file. - -### function: `__call__` - -[Source](../dagrunner/plugin_framework.py#L277) - -#### Call Signature: - -```python -__call__(self, *args, filepath, node_properties=None, **kwargs) -``` - -Save data to a JSON file - -Save the provided data to a JSON file at the specified filepath. The filepath -is expanded using the keyword arguments and environment variables. Note that -this plugin is 'node aware' since it is derived from the -[NodeAwarePlugin](dagrunner.plugin_framework.md#class-nodeawareplugin). - -Args: -- `*args`: Positional arguments (data) to be saved. -- `filepath`: The filepath to save the data to. -- `node_properties`: node properties passed by the plugin executor. -- `**kwargs`: Keyword arguments to be used in the expansion. - -Returns: -- None - -## class: `SavePickle` - -[Source](../dagrunner/plugin_framework.py#L316) - -### Call Signature: - -```python -SavePickle() -``` - -Save data to a Pickle file. - -### function: `__call__` - -[Source](../dagrunner/plugin_framework.py#L319) - -#### Call Signature: - -```python -__call__(self, *args, filepath, node_properties=None, **kwargs) -``` - -Save data to a Pickle file - -Save the provided data to a pickle file at the specified filepath. The filepath -is expanded using the keyword arguments and environment variables. Note that -this plugin is 'node aware' since it is derived from the -[NodeAwarePlugin](dagrunner.plugin_framework.md#class-nodeawareplugin). - -Args: -- `*args`: Positional arguments (data) to be saved. -- `filepath`: The filepath to save the data to. -- `node_properties`: node properties passed by the plugin executor. -- `**kwargs`: Keyword arguments to be used in the expansion. - -Returns: -- None - -## class: `Shell` - -[Source](../dagrunner/plugin_framework.py#L50) - -### Call Signature: - -```python -Shell() -``` - -Abstract base class to define our plugin UI - -### function: `__call__` - -[Source](../dagrunner/plugin_framework.py#L51) - -#### Call Signature: - -```python -__call__(self, *args, **kwargs) -``` - -Execute a subprocess command. - -Args: -- *args: The command to be executed. -- **kwargs: Additional keyword arguments to be passed to `subprocess.run` - -Returns: -- CompletedProcess: An object representing the completed process. - -Raises: -- CalledProcessError: If the command returns a non-zero exit status. - diff --git a/docs/dagrunner.runner.md b/docs/dagrunner.runner.md deleted file mode 100644 index 99ff895..0000000 --- a/docs/dagrunner.runner.md +++ /dev/null @@ -1,6 +0,0 @@ -# module: `dagrunner.runner` - -[Source](../dagrunner/runner/__init__.py#L0) - -see [module: schedulers](dagrunner.runner.schedulers.md#module-dagrunnerrunnerschedulers) - diff --git a/docs/dagrunner.runner.schedulers.asyncmp.md b/docs/dagrunner.runner.schedulers.asyncmp.md deleted file mode 100644 index 8853fae..0000000 --- a/docs/dagrunner.runner.schedulers.asyncmp.md +++ /dev/null @@ -1,96 +0,0 @@ -# module: `dagrunner.runner.schedulers.asyncmp` - -[Source](../dagrunner/runner/schedulers/asyncmp.py#L0) - -## class: `AsyncMP` - -[Source](../dagrunner/runner/schedulers/asyncmp.py#L25) - -### Call Signature: - -```python -AsyncMP(nprocesses, *args, fail_fast=True, profiler_filepath=None, **kwargs) -``` - -Basic asynchronous scheduler using python built-in multiprocessing. - -Context manager for creating a pool of workers, submits jobs based on the -condition of their dependencies completing, then finally tidies up after -itself. - -### function: `__enter__` - -[Source](../dagrunner/runner/schedulers/asyncmp.py#L170) - -#### Call Signature: - -```python -__enter__(self) -``` - -Initiate the pool of workers. - -### function: `__exit__` - -[Source](../dagrunner/runner/schedulers/asyncmp.py#L175) - -#### Call Signature: - -```python -__exit__(self, exc_type, exc_value, exc_traceback) -``` - -Prevents any more tasks from being submitted to the pool. -Once all the tasks have been completed the worker processes will -exit. - -### function: `__init__` - -[Source](../dagrunner/runner/schedulers/asyncmp.py#L35) - -#### Call Signature: - -```python -__init__(self, nprocesses, *args, fail_fast=True, profiler_filepath=None, **kwargs) -``` - -Initialise our asynchronous multiprocessing scheduler, ready to be used. - -Args: -- nprocesses (int): - Number of processes to use. -- *args: - Positional arguments to be passed to the multiprocessing pool call. - -Keyword Args: -- fail_fast (bool): - When a job is found to raise an exception, stop submitting new jobs - to the queue. If fail_fast is True, terminate all currently running - jobs. If False, wait for already queued jobs to complete. -- **kwargs: - Keyword arguments to be passed to the multiprocessing pool call. - -### function: `run` - -[Source](../dagrunner/runner/schedulers/asyncmp.py#L64) - -#### Call Signature: - -```python -run(self, graph, verbose=False, poll_frequency=0.5) -``` - -Run the provided graph using multiprocessing. - -Args: -- graph (dict): - Dictionary, mapping targets to an iterable of commands. - -Keyword Args: -- verbose (bool): - Print out statements indicating progress. -- poll_frequency (float): - This is the frequency in seconds which we poll running processes. - Dependent nodes are run if their predecessors are completed, - as verified by this polling of status. - diff --git a/docs/dagrunner.runner.schedulers.base.md b/docs/dagrunner.runner.schedulers.base.md deleted file mode 100644 index 0c71b3e..0000000 --- a/docs/dagrunner.runner.schedulers.base.md +++ /dev/null @@ -1,4 +0,0 @@ -# module: `dagrunner.runner.schedulers.base` - -[Source](../dagrunner/runner/schedulers/base.py#L0) - diff --git a/docs/dagrunner.runner.schedulers.dask.md b/docs/dagrunner.runner.schedulers.dask.md deleted file mode 100644 index 84f9853..0000000 --- a/docs/dagrunner.runner.schedulers.dask.md +++ /dev/null @@ -1,254 +0,0 @@ -# module: `dagrunner.runner.schedulers.dask` - -[Source](../dagrunner/runner/schedulers/dask.py#L0) - -Standardised UI for dask compatible schedulers (including 'dask on ray') - -All have in common that they convert the provided workflow dictionary into a -dask graph by initiating a dask Delayed container on each node and then -executing it (by calling its compute method) using the specified scheduler. -See the following useful background reading: - - - https://docs.dask.org/en/latest/scheduler-overview.html - - https://docs.dask.org/en/latest/scheduling.html - - https://docs.dask.org/en/latest/delayed.html - -## class: `DaskOnRay` - -[Source](../dagrunner/runner/schedulers/dask.py#L217) - -### Call Signature: - -```python -DaskOnRay(num_workers, profiler_filepath=None, **kwargs) -``` - -A class to run dask graphs using the 'dak-on-ray' scheduler. - -### function: `__enter__` - -[Source](../dagrunner/runner/schedulers/dask.py#L225) - -#### Call Signature: - -```python -__enter__(self) -``` - -### function: `__exit__` - -[Source](../dagrunner/runner/schedulers/dask.py#L228) - -#### Call Signature: - -```python -__exit__(self, exc_type, exc_value, exc_traceback) -``` - -### function: `__init__` - -[Source](../dagrunner/runner/schedulers/dask.py#L220) - -#### Call Signature: - -```python -__init__(self, num_workers, profiler_filepath=None, **kwargs) -``` - -Initialize self. See help(type(self)) for accurate signature. - -### function: `run` - -[Source](../dagrunner/runner/schedulers/dask.py#L233) - -#### Call Signature: - -```python -run(self, dask_graph, verbose=False) -``` - -Execute the provided graph. - -Args: -- dask_graph (dict): Dask graph dictionary - -Keyword Args: -- verbose (bool): Print out statements indicating progress. - -Returns: -- Any: The output of the graph execution. - -## class: `Distributed` - -[Source](../dagrunner/runner/schedulers/dask.py#L84) - -### Call Signature: - -```python -Distributed(num_workers, profiler_filepath=None, **kwargs) -``` - -A class to run dask graphs on a distributed cluster. - -### function: `__enter__` - -[Source](../dagrunner/runner/schedulers/dask.py#L97) - -#### Call Signature: - -```python -__enter__(self) -``` - -Create a local cluster and connect a client to it. - -### function: `__exit__` - -[Source](../dagrunner/runner/schedulers/dask.py#L111) - -#### Call Signature: - -```python -__exit__(self, exc_type, exc_value, exc_traceback) -``` - -### function: `__init__` - -[Source](../dagrunner/runner/schedulers/dask.py#L87) - -#### Call Signature: - -```python -__init__(self, num_workers, profiler_filepath=None, **kwargs) -``` - -Initialize self. See help(type(self)) for accurate signature. - -### function: `run` - -[Source](../dagrunner/runner/schedulers/dask.py#L115) - -#### Call Signature: - -```python -run(self, dask_graph, verbose=False) -``` - -Execute the provided graph. - -Args: -- dask_graph (dict): Dask graph dictionary - -Keyword Args: -- verbose (bool): Print out statements indicating progress. - -Returns: -- Any: The output of the graph execution. - -## class: `SingleMachine` - -[Source](../dagrunner/runner/schedulers/dask.py#L140) - -### Call Signature: - -```python -SingleMachine(num_workers, scheduler='processes', profiler_filepath=None, **kwargs) -``` - -A class to run dask graphs on a single machine. - -### function: `__enter__` - -[Source](../dagrunner/runner/schedulers/dask.py#L152) - -#### Call Signature: - -```python -__enter__(self) -``` - -### function: `__exit__` - -[Source](../dagrunner/runner/schedulers/dask.py#L213) - -#### Call Signature: - -```python -__exit__(self, exc_type, exc_value, exc_traceback) -``` - -### function: `__init__` - -[Source](../dagrunner/runner/schedulers/dask.py#L143) - -#### Call Signature: - -```python -__init__(self, num_workers, scheduler='processes', profiler_filepath=None, **kwargs) -``` - -Initialize self. See help(type(self)) for accurate signature. - -### function: `run` - -[Source](../dagrunner/runner/schedulers/dask.py#L155) - -#### Call Signature: - -```python -run(self, dask_graph, verbose=False) -``` - -Execute the provided graph. - -Args: -- dask_graph (dict): Dask graph dictionary - -Keyword Args: -- verbose (bool): Print out statements indicating progress. - -Returns: -- Any: The output of the graph execution. - -## function: `add_dummy_tasks` - -[Source](../dagrunner/runner/schedulers/dask.py#L38) - -### Call Signature: - -```python -add_dummy_tasks(dask_graph) -``` - -Add a terminating dummy task to the graph as well as to each of our -disconnected branches. - -A terminating dummy node is added to our graph to allow us to run the -complete graph in one call as well as discard the return. -Dummy nodes (as denoted by 'waiter' prefix in their name) are also added -to the termination of each independent branch, as a single terminal task -on the graph would gather all data (as its input) on the single worker and -potentially blow past its limits. That is, one dummy task per output -effectively ensures that no data leaves the worker. - -Args: -- dask_graph (dict): Dask graph dict - -Returns: -- dict: Dask graph - -TODO: -- Potentially skip intermediate dummy for tasks with no return value. - -## function: `no_op` - -[Source](../dagrunner/runner/schedulers/dask.py#L30) - -### Call Signature: - -```python -no_op(*args, **kwargs) -``` - -Dummy operation for our dask graph See [add_dummy_tasks](#function-add_dummy_tasks) - diff --git a/docs/dagrunner.runner.schedulers.md b/docs/dagrunner.runner.schedulers.md deleted file mode 100644 index 7aa178d..0000000 --- a/docs/dagrunner.runner.schedulers.md +++ /dev/null @@ -1,21 +0,0 @@ -# module: `dagrunner.runner.schedulers` - -[Source](../dagrunner/runner/schedulers/__init__.py#L0) - -Supackage which provides access to various schedulers through a common UI. - -A SCHEDULERS dictionary provides access to the available schedulers through a -name-scheduler lookup: -- distributed -- threads -- processes -- single-threaded -- multiprocessing -- ray - -see [module: asyncmp](dagrunner.runner.schedulers.asyncmp.md#module-dagrunnerrunnerschedulersasyncmp) - -see [module: dask](dagrunner.runner.schedulers.dask.md#module-dagrunnerrunnerschedulersdask) - -## dict: `SCHEDULERS` - diff --git a/docs/dagrunner.utils.logger.md b/docs/dagrunner.utils.logger.md deleted file mode 100644 index 65e042d..0000000 --- a/docs/dagrunner.utils.logger.md +++ /dev/null @@ -1,320 +0,0 @@ -# module: `dagrunner.utils.logger` - -[Source](../dagrunner/utils/logger.py#L0) - -This module takes much from the Python logging cookbook: -https://docs.python.org/3/howto/logging-cookbook.html#sending-and-receiving-logging-events-across-a-network - -## Overview - -- `client_attach_socket_handler`, a function that attaches a socket handler - `logging.handlers.SocketHandler` to the root logger with the specified host name and - port number. -- `start_logging_server`, a function to start the TCP server - `LogRecordSocketReceiver` on its own thread, ready to receive log records. - - `SQLiteHandler`, a custom logging handler to write log messages to an SQLite - database. - - `LogRecordSocketReceiver(socketserver.ThreadingTCPServer)`, the TCP server running - on a specified host and port, managed by the server context that receives log - records and utilises the `LogRecordStreamHandler` handler. - - `LogRecordStreamHandler`, a specialisation of the - `socketserver.StreamRequestHandler`, responsible for 'getting' log records. - -see [function: dagrunner.utils.function_to_argparse_parse_args](dagrunner.utils.md#function-function_to_argparse_parse_args) - -## class: `CustomFormatter` - -[Source](../dagrunner/utils/logger.py#L217) - -### Call Signature: - -```python -CustomFormatter(fmt=None, datefmt=None) -``` - -Formatter instances are used to convert a LogRecord to text. - -Formatters need to know how a LogRecord is constructed. They are -responsible for converting a LogRecord to (usually) a string which can -be interpreted by either a human or an external system. The base Formatter -allows a formatting string to be specified. If none is supplied, the -style-dependent default value, "%(message)s", "{message}", or -"${message}", is used. - -The Formatter can be initialized with a format string which makes use of -knowledge of the LogRecord attributes - e.g. the default value mentioned -above makes use of the fact that the user's message and arguments are pre- -formatted into a LogRecord's message attribute. Currently, the useful -attributes in a LogRecord are described by: - -%(name)s Name of the logger (logging channel) -%(levelno)s Numeric logging level for the message (DEBUG, INFO, - WARNING, ERROR, CRITICAL) -%(levelname)s Text logging level for the message ("DEBUG", "INFO", - "WARNING", "ERROR", "CRITICAL") -%(pathname)s Full pathname of the source file where the logging - call was issued (if available) -%(filename)s Filename portion of pathname -%(module)s Module (name portion of filename) -%(lineno)d Source line number where the logging call was issued - (if available) -%(funcName)s Function name -%(created)f Time when the LogRecord was created (time.time_ns() / 1e9 - return value) -%(asctime)s Textual time when the LogRecord was created -%(msecs)d Millisecond portion of the creation time -%(relativeCreated)d Time in milliseconds when the LogRecord was created, - relative to the time the logging module was loaded - (typically at application startup time) -%(thread)d Thread ID (if available) -%(threadName)s Thread name (if available) -%(taskName)s Task name (if available) -%(process)d Process ID (if available) -%(processName)s Process name (if available) -%(message)s The result of record.getMessage(), computed just as - the record is emitted - -### function: `__init__` - -[Source](../dagrunner/utils/logger.py#L218) - -#### Call Signature: - -```python -__init__(self, fmt=None, datefmt=None) -``` - -Initialize the formatter with specified format strings. - -Initialize the formatter either with the specified format string, or a -default as described above. Allow for specialized date formatting with -the optional datefmt argument. If datefmt is omitted, you get an -ISO8601-like (or RFC 3339-like) format. - -Use a style parameter of '%', '{' or '$' to specify that you want to -use one of %-formatting, :meth:`str.format` (``{}``) formatting or -:class:`string.Template` formatting in your format string. - -.. versionchanged:: 3.2 - Added the ``style`` parameter. - -### function: `format` - -[Source](../dagrunner/utils/logger.py#L221) - -#### Call Signature: - -```python -format(self, record) -``` - -Format the specified record as text. - -The record's attribute dictionary is used as the operand to a -string formatting operation which yields the returned string. -Before formatting the dictionary, a couple of preparatory steps -are carried out. The message attribute of the record is computed -using LogRecord.getMessage(). If the formatting string uses the -time (as determined by a call to usesTime(), formatTime() is -called to format the event time. If there is exception information, -it is formatted using formatException() and appended to the message. - -## str: `DATEFMT` - -## class: `LogRecordSocketReceiver` - -[Source](../dagrunner/utils/logger.py#L121) - -### Call Signature: - -```python -LogRecordSocketReceiver(host='localhost', port=9020, ) -``` - -Simple TCP socket-based logging receiver. - -Specialisation of the `socketserver.ThreadingTCPServer` class to handle -log records. - -### function: `__init__` - -[Source](../dagrunner/utils/logger.py#L131) - -#### Call Signature: - -```python -__init__(self, host='localhost', port=9020, ) -``` - -Constructor. May be extended, do not override. - -### function: `serve_until_stopped` - -[Source](../dagrunner/utils/logger.py#L142) - -#### Call Signature: - -```python -serve_until_stopped(self) -``` - -## class: `LogRecordStreamHandler` - -[Source](../dagrunner/utils/logger.py#L76) - -### Call Signature: - -```python -LogRecordStreamHandler(request, client_address, server) -``` - -Handler for a streaming logging request. - -Specialisation of the `socketserver.StreamRequestHandler` class to handle log -records and customise logging events. - -### function: `handle` - -[Source](../dagrunner/utils/logger.py#L84) - -#### Call Signature: - -```python -handle(self) -``` - -Handle multiple requests - each expected to be a 4-byte length, -followed by the LogRecord in pickle format. Logs the record -according to whatever policy is configured locally. - -### function: `handle_log_record` - -[Source](../dagrunner/utils/logger.py#L106) - -#### Call Signature: - -```python -handle_log_record(self, record) -``` - -### function: `unpickle` - -[Source](../dagrunner/utils/logger.py#L103) - -#### Call Signature: - -```python -unpickle(self, data) -``` - -## class: `SQLiteHandler` - -[Source](../dagrunner/utils/logger.py#L153) - -### Call Signature: - -```python -SQLiteHandler(sqfile='logs.sqlite') -``` - -Custom logging handler to write log messages to an SQLite database. - -### function: `__init__` - -[Source](../dagrunner/utils/logger.py#L158) - -#### Call Signature: - -```python -__init__(self, sqfile='logs.sqlite') -``` - -Initializes the instance - basically setting the formatter to None -and the filter list to empty. - -### function: `close` - -[Source](../dagrunner/utils/logger.py#L212) - -#### Call Signature: - -```python -close(self) -``` - -Ensure the database connection is closed cleanly. - -### function: `emit` - -[Source](../dagrunner/utils/logger.py#L185) - -#### Call Signature: - -```python -emit(self, record) -``` - -Emit a log record, and insert it into the database. - -## function: `client_attach_socket_handler` - -[Source](../dagrunner/utils/logger.py#L43) - -### Call Signature: - -```python -client_attach_socket_handler(host: str = 'localhost', port: int = 9020) -``` - -Attach a SocketHandler instance to the root logger at the sending end. - -Now, we can log to the root logger, or any other logger. First the root... - logging.info('Jackdaws love my big sphinx of quartz.') - -Now, define a couple of other loggers which might represent areas in your -application: - - logger1 = logging.getLogger('myapp.area1') - logger2 = logging.getLogger('myapp.area2') - - logger1.debug('Quick zephyrs blow, vexing daft Jim.') - logger1.info('How quickly daft jumping zebras vex.') - logger2.warning('Jail zesty vixen who grabbed pay from quack.') - logger2.error('The five boxing wizards jump quickly.') - -Args: -- `host`: The host name of the server. Optional. -- `port`: The port number the server is listening on. Optional. - -## function: `main` - -[Source](../dagrunner/utils/logger.py#L265) - -### Call Signature: - -```python -main() -``` - -Entry point of the program. - -Parses command line arguments and executes the logging server - -## function: `start_logging_server` - -[Source](../dagrunner/utils/logger.py#L231) - -### Call Signature: - -```python -start_logging_server(sqlite_filepath: str = None, host: str = 'localhost', port: int = 9020, verbose: bool = False) -``` - -Start the logging server. - -Args: -- `sqlite_filepath`: The file path to the SQLite database. Optional. -- `host`: The host name of the server. Optional. -- `port`: The port number the server is listening on. Optional. -- `verbose`: Whether to print verbose output. Optional. - diff --git a/docs/dagrunner.utils.md b/docs/dagrunner.utils.md deleted file mode 100644 index 32da463..0000000 --- a/docs/dagrunner.utils.md +++ /dev/null @@ -1,573 +0,0 @@ -# module: `dagrunner.utils` - -[Source](../dagrunner/utils/__init__.py#L0) - -see [module: _doc_styles](dagrunner.utils._doc_styles.md#module-dagrunnerutils_doc_styles) - -see [module: logger](dagrunner.utils.logger.md#module-dagrunnerutilslogger) - -see [module: networkx](dagrunner.utils.networkx.md#module-dagrunnerutilsnetworkx) - -see [module: visualisation](dagrunner.utils.visualisation.md#module-dagrunnerutilsvisualisation) - -## class: `CaptureProcMemory` - -[Source](../dagrunner/utils/__init__.py#L216) - -### Call Signature: - -```python -CaptureProcMemory(interval=1.0, pid=None) -``` - -Capture maximum process memory statistics. - -See `get_proc_mem_stat` for more information. - -### function: `__enter__` - -[Source](../dagrunner/utils/__init__.py#L197) - -#### Call Signature: - -```python -__enter__(self) -``` - -### function: `__exit__` - -[Source](../dagrunner/utils/__init__.py#L202) - -#### Call Signature: - -```python -__exit__(self, exc_type, exc_value, traceback) -``` - -### function: `__init__` - -[Source](../dagrunner/utils/__init__.py#L227) - -#### Call Signature: - -```python -__init__(self, interval=1.0, pid=None) -``` - -Initialize the memory capture. - -Args: -- `interval`: Time interval in seconds to capture memory statistics. - Note that memory statistics are captured by reading /proc files. It is - advised not to reduce the interval too much, otherwise we increase the - overhead of reading the files. -- `pid`: Process id. Optional. Default is the current process. - -### function: `max` - -[Source](../dagrunner/utils/__init__.py#L206) - -#### Call Signature: - -```python -max(self) -``` - -Return maximum memory statistics. - -Returns: -- Dictionary with memory statistics in MB. - -## class: `CaptureSysMemory` - -[Source](../dagrunner/utils/__init__.py#L268) - -### Call Signature: - -```python -CaptureSysMemory(interval=1.0, **kwargs) -``` - -Capture maximum system memory statistics. - -See `get_sys_mem_stat` for more information. - -### function: `__enter__` - -[Source](../dagrunner/utils/__init__.py#L197) - -#### Call Signature: - -```python -__enter__(self) -``` - -### function: `__exit__` - -[Source](../dagrunner/utils/__init__.py#L202) - -#### Call Signature: - -```python -__exit__(self, exc_type, exc_value, traceback) -``` - -### function: `__init__` - -[Source](../dagrunner/utils/__init__.py#L165) - -#### Call Signature: - -```python -__init__(self, interval=1.0, **kwargs) -``` - -Initialize the memory capture. - -Args: -- `interval`: Time interval in seconds to capture memory statistics. - Note that memory statistics are captured by reading `/proc` files. It is - advised not to reduce the interval too much, otherwise we increase the - overhead of reading the files. - -### function: `max` - -[Source](../dagrunner/utils/__init__.py#L206) - -#### Call Signature: - -```python -max(self) -``` - -Return maximum memory statistics. - -Returns: -- Dictionary with memory statistics in MB. - -## class: `KeyValueAction` - -[Source](../dagrunner/utils/__init__.py#L386) - -### Call Signature: - -```python -KeyValueAction(option_strings, dest, nargs=None, const=None, default=None, type=None, choices=None, required=False, help=None, metavar=None, deprecated=False) -``` - -Information about how to convert command line strings to Python objects. - -Action objects are used by an ArgumentParser to represent the information -needed to parse a single argument from one or more strings from the -command line. The keyword arguments to the Action constructor are also -all attributes of Action instances. - -Keyword Arguments: - - - option_strings -- A list of command-line option strings which - should be associated with this action. - - - dest -- The name of the attribute to hold the created object(s) - - - nargs -- The number of command-line arguments that should be - consumed. By default, one argument will be consumed and a single - value will be produced. Other values include: - - N (an integer) consumes N arguments (and produces a list) - - '?' consumes zero or one arguments - - '*' consumes zero or more arguments (and produces a list) - - '+' consumes one or more arguments (and produces a list) - Note that the difference between the default and nargs=1 is that - with the default, a single value will be produced, while with - nargs=1, a list containing a single value will be produced. - - - const -- The value to be produced if the option is specified and the - option uses an action that takes no values. - - - default -- The value to be produced if the option is not specified. - - - type -- A callable that accepts a single string argument, and - returns the converted value. The standard Python types str, int, - float, and complex are useful examples of such callables. If None, - str is used. - - - choices -- A container of values that should be allowed. If not None, - after a command-line argument has been converted to the appropriate - type, an exception will be raised if it is not a member of this - collection. - - - required -- True if the action must always be specified at the - command line. This is only meaningful for optional command-line - arguments. - - - help -- The help string describing the argument. - - - metavar -- The name to be used for the option's argument with the - help string. If None, the 'dest' value will be used as the name. - -### function: `__call__` - -[Source](../dagrunner/utils/__init__.py#L387) - -#### Call Signature: - -```python -__call__(self, parser, namespace, values, option_string=None) -``` - -Call self as a function. - -## class: `ObjectAsStr` - -[Source](../dagrunner/utils/__init__.py#L280) - -### Call Signature: - -```python -ObjectAsStr(obj, name=None) -``` - -Hide object under a string. - -### function: `__hash__` - -[Source](../dagrunner/utils/__init__.py#L294) - -#### Call Signature: - -```python -__hash__(self) -``` - -Return hash(self). - -### function: `__new__` - -[Source](../dagrunner/utils/__init__.py#L285) - -#### Call Signature: - -```python -__new__(cls, obj, name=None) -``` - -Create and return a new object. See help(type) for accurate signature. - -### function: `obj_to_name` - -[Source](../dagrunner/utils/__init__.py#L298) - -#### Call Signature: - -```python -obj_to_name(obj, cls) -``` - -## class: `Singleton` - -[Source](../dagrunner/utils/__init__.py#L101) - -Singleton metaclass. - -### function: `__call__` - -[Source](../dagrunner/utils/__init__.py#L108) - -#### Call Signature: - -```python -__call__(cls, *args, **kwargs) -``` - -Call self as a function. - -## class: `TimeIt` - -[Source](../dagrunner/utils/__init__.py#L306) - -### Call Signature: - -```python -TimeIt(verbose=False) -``` - -Timer context manager which can also be used as a standalone timer. - -We can query our timer for the elapsed time in seconds even before . - -Example as a context manager: - - >>> with TimeIt() as timer: - >>> sleep(0.05) - >>> print(timer) - "Elapsed time: 0.05s" - -Example as a standalone timer: - - >>> timer = TimeIt() - >>> timer.start_timer() - >>> sleep(0.05) - >>> print(timer) - "Elapsed time: 0.05s" - -### function: `__enter__` - -[Source](../dagrunner/utils/__init__.py#L335) - -#### Call Signature: - -```python -__enter__(self) -``` - -### function: `__exit__` - -[Source](../dagrunner/utils/__init__.py#L339) - -#### Call Signature: - -```python -__exit__(self, *args) -``` - -### function: `__init__` - -[Source](../dagrunner/utils/__init__.py#L329) - -#### Call Signature: - -```python -__init__(self, verbose=False) -``` - -Initialize self. See help(type(self)) for accurate signature. - -### function: `__str__` - -[Source](../dagrunner/utils/__init__.py#L364) - -#### Call Signature: - -```python -__str__(self) -``` - -Print elapsed time in seconds. - -### function: `start` - -[Source](../dagrunner/utils/__init__.py#L345) - -#### Call Signature: - -```python -start(self) -``` - -### function: `stop` - -[Source](../dagrunner/utils/__init__.py#L349) - -#### Call Signature: - -```python -stop(self) -``` - -## function: `as_iterable` - -[Source](../dagrunner/utils/__init__.py#L93) - -### Call Signature: - -```python -as_iterable(obj) -``` - -## function: `data_polling` - -[Source](../dagrunner/utils/__init__.py#L487) - -### Call Signature: - -```python -data_polling(*args, timeout=120, polling=1, file_count=None, fail_fast=True, verbose=False) -``` - -Poll for the availability of files - -Poll for data and return when all data is available or otherwise raise an -exception if the timeout is reached. -This function will not respect input-output ordering. If that is important, -please call this function on each path individually. - -Args: -- *args: Variable length argument list of file patterns to be checked. - `:` syntax supported for files on a remote host. - -Args: -- timeout (int): Timeout in seconds (default is 120 seconds). -- polling (int): Time interval in seconds between each poll (default is 1 - second). -- file_count (int): Expected number of files to be found for globular - expansion (default is >= 1 files per pattern). -- fail_fast (bool): Stop when a file is not found (default is True). -- verbose (bool): Print verbose output. - -Returns: -- fpaths_found (set): Set of file paths that were found. - -## function: `docstring_parse` - -[Source](../dagrunner/utils/__init__.py#L369) - -### Call Signature: - -```python -docstring_parse(obj) -``` - -## function: `function_to_argparse` - -[Source](../dagrunner/utils/__init__.py#L398) - -### Call Signature: - -```python -function_to_argparse(func, parser=None, exclude=None) -``` - -Generate an argparse from a function signature - -## function: `function_to_argparse_parse_args` - -[Source](../dagrunner/utils/__init__.py#L475) - -### Call Signature: - -```python -function_to_argparse_parse_args(*args, **kwargs) -``` - -## function: `get_proc_mem_stat` - -[Source](../dagrunner/utils/__init__.py#L134) - -### Call Signature: - -```python -get_proc_mem_stat(pid=None) -``` - -Get process memory statistics from /proc//status. - -More information can be found at -https://github.com/torvalds/linux/blob/master/Documentation/filesystems/proc.txt - -Args: -- `pid`: Process id. Optional. Default is the current process. - -Returns: -- Dictionary with memory statistics in MB. Fields are VmSize, VmRSS, VmPeak and - VmHWM. - -## function: `get_sys_mem_stat` - -[Source](../dagrunner/utils/__init__.py#L242) - -### Call Signature: - -```python -get_sys_mem_stat() -``` - -Get system memory statistics from /proc/meminfo. - -More information can be found at -https://github.com/torvalds/linux/blob/master/Documentation/filesystems/proc.txt - -Returns: -- Dictionary with memory statistics in MB. Fields are Committed_AS, MemFree, - Buffers, Cached and MemTotal. - -## function: `in_notebook` - -[Source](../dagrunner/utils/__init__.py#L81) - -### Call Signature: - -```python -in_notebook() -``` - -Determine whether we are in a Jupyter notebook. - -## function: `pairwise` - -[Source](../dagrunner/utils/__init__.py#L59) - -### Call Signature: - -```python -pairwise(iterable) -``` - -Return successive overlapping pairs taken from the input iterable. - -The number of 2-tuples in the output iterator will be one fewer than the -number of inputs. It will be empty if the input iterable has fewer than -two values. - -pairwise('ABCDEFG') → AB BC CD DE EF FG - -## function: `process_path` - -[Source](../dagrunner/utils/__init__.py#L114) - -### Call Signature: - -```python -process_path(fpath: str) -``` - -Process path. - -Args: -- `fpath`: Remote path in the format :. If host corresponds to - the local host, then the host element will be removed. - -Returns: -- Processed path - -## function: `stage_to_dir` - -[Source](../dagrunner/utils/__init__.py#L682) - -### Call Signature: - -```python -stage_to_dir(*args, staging_dir, verbose=False) -``` - -Copy input filepaths to a staging area and update paths. - -Hard link copies are preferred (same host) and physical copies are made otherwise. -File name, size and modification time are used to evaluate if the destination file -exists already (matching criteria of rsync). If exists already, skip the copy. -Staged files are named: `__` to avoid -collision with identically names files. - -## function: `subset_equality` - -[Source](../dagrunner/utils/__init__.py#L25) - -### Call Signature: - -```python -subset_equality(obj_a, obj_b) -``` - -Return whether obj_a is a subset of obj_b. - -Supporting namedtuple and dataclasses, otherwise fallback to equality. Note that -a 'None' value in obj_a is considered a wildcard. - diff --git a/docs/dagrunner.utils.networkx.md b/docs/dagrunner.utils.networkx.md deleted file mode 100644 index 6d8d40f..0000000 --- a/docs/dagrunner.utils.networkx.md +++ /dev/null @@ -1,112 +0,0 @@ -# module: `dagrunner.utils.networkx` - -[Source](../dagrunner/utils/networkx.py#L0) - -see [function: dagrunner.utils.as_iterable](dagrunner.utils.md#function-as_iterable) - -see [function: dagrunner.utils.subset_equality](dagrunner.utils.md#function-subset_equality) - -see [module: dagrunner.utils.visualisation](dagrunner.utils.visualisation.md#module-dagrunnerutilsvisualisation) - -## function: `collapse_graph` - -[Source](../dagrunner/utils/networkx.py#L112) - -### Call Signature: - -```python -collapse_graph(graph: networkx.classes.digraph.DiGraph, collapse_properties: str | Iterable[str], collapsed_data_summary: bool = False) -``` - -Collapses a directed graph by grouping nodes based on specified properties. - -This function modifies the input graph by collapsing nodes that share the same -values for the specified properties. It also generates mappings to track the -relationship between the collapsed nodes and the original nodes. - -Args: -- `graph`: The directed graph to be collapsed. The nodes of the graph - must be dataclass instances. -- `collapse_properties`: A single property or an iterable - of properties to collapse the graph along. These properties must exist in the - dataclass definition of the graph nodes. -- `collapsed_data_summary` - Where False, the collapsed node data lookup returned is a set of all - contributing node data dictionaries (excluding any collapse properties). - If True, this set of dictionaries is merged into a single dictionary of value - sets. Useful for representing large amounts of data in a more compact form, - at the cost of loosing associative relationship between data keys and values. - - -Returns: - Tuple[nx.DiGraph, Dict[Any, Set[str]], Dict[Any, Dict[str, List[Any]]]]: - - The collapsed graph as a new `nx.DiGraph` object. - - A dictionary, mapping each collapsed node to a set representing the data - across the uncollapsed node set. See 'collapsed_data_summary' where this - may change. - - A dictionary mapping each collapsed node to a dictionary of the - collapsed properties and their corresponding sorted values from - the original nodes. - -Raises: - TypeError: If the nodes of the graph are not dataclass instances. - -Notes: -- The function uses `dataclasses.replace` to create new nodes with updated - properties for collapsing. -- The `subset_equality` function is used to determine if a node in the original - graph matches a collapsed node. - -## function: `get_subset_with_dependencies` - -[Source](../dagrunner/utils/networkx.py#L50) - -### Call Signature: - -```python -get_subset_with_dependencies(graph: networkx.classes.digraph.DiGraph, filter_list: dict | Iterable[dict]) -``` - -Helper function to easily filter networkx graphs. - -Each item in our filter list determines whether its node should be included or -excluded. - -Args: -- `graph`: The graph to filter. -- `filter_list`: The list of filters to apply. - Each item in this list should take the form: - {node: , exclude: , descendants: , ancestors: } - -## function: `visualise_graph` - -[Source](../dagrunner/utils/networkx.py#L224) - -### Call Signature: - -```python -visualise_graph(graph: networkx.classes.digraph.DiGraph, backend: str = 'mermaid', collapse_properties: str | Iterable[str] = None, title: str = None, output_filepath: str = None, **kwargs) -``` - -Visualise a networkx graph. - -Plots each disconnected branch of nodes with their connected edges of its own -distinct color. Intended for plotting small graphs, whether filtered (e.g. a graph -filtered for a specific leadtime or diagnostic) and or collapsed along specified -dimensions (e.g. collapsing along the 'leadtime' property). - -Args: -- `graph`: The graph to visualise. -- `backend`: The backend to use for visualisation. Supported values include - 'mermaid' (javascript, default) and 'matplotlib' (experimental and unsupported). - See [visualise_graph_mermaid](#function-visualise_graph_mermaid). -- `collapse_properties`: One or more properties to collapse nodes on. Only - supported for nodes represented by dataclasses right now. -- `title`: The title of the visualisation. -- `output_filepath`: The output filepath to save the visualisation to. -- `**kwargs`: Additional keyword arguments to pass to the visualisation backend. - The default and only supported backend right now (mermaid) supports the following - keyword arguments: - - `group_by`: One or more property to group nodes by (i.e. subgraphing) - - `label_by`: One or more property to label visualisation nodes by. - diff --git a/docs/dagrunner.utils.visualisation.md b/docs/dagrunner.utils.visualisation.md deleted file mode 100644 index 0d5a8a3..0000000 --- a/docs/dagrunner.utils.visualisation.md +++ /dev/null @@ -1,234 +0,0 @@ -# module: `dagrunner.utils.visualisation` - -[Source](../dagrunner/utils/visualisation.py#L0) - -Module responsible for scheduler independent graph visualisation - -see [function: dagrunner.utils.as_iterable](dagrunner.utils.md#function-as_iterable) - -see [function: dagrunner.utils.in_notebook](dagrunner.utils.md#function-in_notebook) - -## class: `HTMLTable` - -[Source](../dagrunner/utils/visualisation.py#L44) - -### Call Signature: - -```python -HTMLTable(column_names) -``` - -### function: `__init__` - -[Source](../dagrunner/utils/visualisation.py#L53) - -#### Call Signature: - -```python -__init__(self, column_names) -``` - -Initialize self. See help(type(self)) for accurate signature. - -### function: `__str__` - -[Source](../dagrunner/utils/visualisation.py#L72) - -#### Call Signature: - -```python -__str__(self) -``` - -Return str(self). - -### function: `add_row` - -[Source](../dagrunner/utils/visualisation.py#L62) - -#### Call Signature: - -```python -add_row(self, *args, id=None) -``` - -## list: `MERMAID_SUBGRAPH_COLORS` - -## str: `MERMAID_SUBGRAPH_COLOR_HIGHLIGHT` - -## class: `MermaidGraph` - -[Source](../dagrunner/utils/visualisation.py#L78) - -### Call Signature: - -```python -MermaidGraph(title=None) -``` - -### function: `__init__` - -[Source](../dagrunner/utils/visualisation.py#L85) - -#### Call Signature: - -```python -__init__(self, title=None) -``` - -Initialize self. See help(type(self)) for accurate signature. - -### function: `__str__` - -[Source](../dagrunner/utils/visualisation.py#L109) - -#### Call Signature: - -```python -__str__(self) -``` - -Return str(self). - -### function: `add_connection` - -[Source](../dagrunner/utils/visualisation.py#L106) - -#### Call Signature: - -```python -add_connection(self, id1, id2) -``` - -### function: `add_node` - -[Source](../dagrunner/utils/visualisation.py#L92) - -#### Call Signature: - -```python -add_node(self, nodeid, label=None, tooltip=None, url=None) -``` - -### function: `add_raw` - -[Source](../dagrunner/utils/visualisation.py#L89) - -#### Call Signature: - -```python -add_raw(self, raw) -``` - -### function: `base64` - -[Source](../dagrunner/utils/visualisation.py#L112) - -#### Call Signature: - -```python -base64(self) -``` - -### function: `display` - -[Source](../dagrunner/utils/visualisation.py#L130) - -#### Call Signature: - -```python -display(self, output_filepath: str = None) -``` - -## class: `MermaidHTML` - -[Source](../dagrunner/utils/visualisation.py#L156) - -### Call Signature: - -```python -MermaidHTML(mermaid, table=None) -``` - -### function: `__init__` - -[Source](../dagrunner/utils/visualisation.py#L179) - -#### Call Signature: - -```python -__init__(self, mermaid, table=None) -``` - -Initialize self. See help(type(self)) for accurate signature. - -### function: `__str__` - -[Source](../dagrunner/utils/visualisation.py#L182) - -#### Call Signature: - -```python -__str__(self) -``` - -Return str(self). - -### function: `save` - -[Source](../dagrunner/utils/visualisation.py#L189) - -#### Call Signature: - -```python -save(self, output_filepath) -``` - -## str: `WEBCOMPONENT_PATH` - -## function: `visualise_graph_matplotlib` - -[Source](../dagrunner/utils/visualisation.py#L197) - -### Call Signature: - -```python -visualise_graph_matplotlib(graph: networkx.classes.digraph.DiGraph, node_info_lookup: dict = None, title: str = None, output_filepath: str = None) -``` - -Visualise a networkx graph using matplotlib. - -Note that this backend is provided as-is and not intended for production use. -'mermaid' graph is the recommended approach to graph visualisation. - -Args: -- `graph`: The graph to visualise. -- `node_info_lookup`: A dictionary mapping nodes to their information. -- `title`: The title of the visualisation. -- `output_filepath`: The output filepath to save the visualisation to. - -## function: `visualise_graph_mermaid` - -[Source](../dagrunner/utils/visualisation.py#L341) - -### Call Signature: - -```python -visualise_graph_mermaid(graph: networkx.classes.digraph.DiGraph, node_data_lookup: dict = None, node_tooltip_lookup: dict = None, title: str = None, output_filepath: str = None, group_by: str | Iterable[str] = None, label_by: str | Iterable[str] = None) -``` - -Visualise a networkx graph using mermaid. - -Args: -- `graph`: The graph to visualise. -- `node_info_lookup`: A dictionary mapping nodes to their information. -- `title`: The title of the visualisation. -- `output_filepath`: The output filepath to save the visualisation to. Where not - provided, write a html file to a temporary location and open it with your default - browser. Otherwise, supported extensions include ".html", ".png", ".jpg", - ".jpeg", ".svg" and ".md". Note that not all formats support the full - set of visualisation features, so html is recommended. -- `group_by`: One or more property to group nodes by (i.e. - [subgraph](https://mermaid-js.github.io/mermaid/#/subgraph)). -- `label_by`: One or more property to label visualisation nodes by. - diff --git a/docs/dagrunner_index.md b/docs/dagrunner_index.md deleted file mode 100644 index 34296f0..0000000 --- a/docs/dagrunner_index.md +++ /dev/null @@ -1,73 +0,0 @@ -# Index for 'dagrunner' reference documentation -version: {module.__version__} - - - [module: dagrunner](dagrunner.md#module-dagrunner) - - [module: config](dagrunner.config.md#module-dagrunnerconfig) - - [GlobalConfiguration: CONFIG](dagrunner.config.md#globalconfiguration-config) - - [class: GlobalConfiguration](dagrunner.config.md#class-globalconfiguration) - - [module: events](dagrunner.events.md#module-dagrunnerevents) - - [_IgnoreEvent: IGNORE_EVENT](dagrunner.events.md#_ignoreevent-ignore_event) - - [_SkipEvent: SKIP_EVENT](dagrunner.events.md#_skipevent-skip_event) - - [module: execute_graph](dagrunner.execute_graph.md#module-dagrunnerexecute_graph) - - [class: ExecuteGraph](dagrunner.execute_graph.md#class-executegraph) - - [function: main](dagrunner.execute_graph.md#function-main) - - [function: plugin_executor](dagrunner.execute_graph.md#function-plugin_executor) - - [module: plugin_framework](dagrunner.plugin_framework.md#module-dagrunnerplugin_framework) - - [class: DataPolling](dagrunner.plugin_framework.md#class-datapolling) - - [class: Input](dagrunner.plugin_framework.md#class-input) - - [class: Load](dagrunner.plugin_framework.md#class-load) - - [class: LoadJson](dagrunner.plugin_framework.md#class-loadjson) - - [class: LoadPickle](dagrunner.plugin_framework.md#class-loadpickle) - - [class: NodeAwarePlugin](dagrunner.plugin_framework.md#class-nodeawareplugin) - - [class: Plugin](dagrunner.plugin_framework.md#class-plugin) - - [class: SaveJson](dagrunner.plugin_framework.md#class-savejson) - - [class: SavePickle](dagrunner.plugin_framework.md#class-savepickle) - - [class: Shell](dagrunner.plugin_framework.md#class-shell) - - [module: runner](dagrunner.runner.md#module-dagrunnerrunner) - - [module: schedulers](dagrunner.runner.schedulers.md#module-dagrunnerrunnerschedulers) - - [module: asyncmp](dagrunner.runner.schedulers.asyncmp.md#module-dagrunnerrunnerschedulersasyncmp) - - [class: AsyncMP](dagrunner.runner.schedulers.asyncmp.md#class-asyncmp) - - [module: base](dagrunner.runner.schedulers.base.md#module-dagrunnerrunnerschedulersbase) - - [module: dask](dagrunner.runner.schedulers.dask.md#module-dagrunnerrunnerschedulersdask) - - [class: DaskOnRay](dagrunner.runner.schedulers.dask.md#class-daskonray) - - [class: Distributed](dagrunner.runner.schedulers.dask.md#class-distributed) - - [class: SingleMachine](dagrunner.runner.schedulers.dask.md#class-singlemachine) - - [function: add_dummy_tasks](dagrunner.runner.schedulers.dask.md#function-add_dummy_tasks) - - [function: no_op](dagrunner.runner.schedulers.dask.md#function-no_op) - - [module: utils](dagrunner.utils.md#module-dagrunnerutils) - - [class: CaptureProcMemory](dagrunner.utils.md#class-captureprocmemory) - - [class: CaptureSysMemory](dagrunner.utils.md#class-capturesysmemory) - - [class: KeyValueAction](dagrunner.utils.md#class-keyvalueaction) - - [class: ObjectAsStr](dagrunner.utils.md#class-objectasstr) - - [class: Singleton](dagrunner.utils.md#class-singleton) - - [class: TimeIt](dagrunner.utils.md#class-timeit) - - [function: as_iterable](dagrunner.utils.md#function-as_iterable) - - [function: data_polling](dagrunner.utils.md#function-data_polling) - - [function: docstring_parse](dagrunner.utils.md#function-docstring_parse) - - [function: function_to_argparse](dagrunner.utils.md#function-function_to_argparse) - - [function: function_to_argparse_parse_args](dagrunner.utils.md#function-function_to_argparse_parse_args) - - [function: get_proc_mem_stat](dagrunner.utils.md#function-get_proc_mem_stat) - - [function: get_sys_mem_stat](dagrunner.utils.md#function-get_sys_mem_stat) - - [function: in_notebook](dagrunner.utils.md#function-in_notebook) - - [module: logger](dagrunner.utils.logger.md#module-dagrunnerutilslogger) - - [class: CustomFormatter](dagrunner.utils.logger.md#class-customformatter) - - [class: LogRecordSocketReceiver](dagrunner.utils.logger.md#class-logrecordsocketreceiver) - - [class: LogRecordStreamHandler](dagrunner.utils.logger.md#class-logrecordstreamhandler) - - [class: SQLiteHandler](dagrunner.utils.logger.md#class-sqlitehandler) - - [function: client_attach_socket_handler](dagrunner.utils.logger.md#function-client_attach_socket_handler) - - [function: main](dagrunner.utils.logger.md#function-main) - - [function: start_logging_server](dagrunner.utils.logger.md#function-start_logging_server) - - [module: networkx](dagrunner.utils.networkx.md#module-dagrunnerutilsnetworkx) - - [function: collapse_graph](dagrunner.utils.networkx.md#function-collapse_graph) - - [function: get_subset_with_dependencies](dagrunner.utils.networkx.md#function-get_subset_with_dependencies) - - [function: visualise_graph](dagrunner.utils.networkx.md#function-visualise_graph) - - [function: pairwise](dagrunner.utils.md#function-pairwise) - - [function: process_path](dagrunner.utils.md#function-process_path) - - [function: stage_to_dir](dagrunner.utils.md#function-stage_to_dir) - - [function: subset_equality](dagrunner.utils.md#function-subset_equality) - - [module: visualisation](dagrunner.utils.visualisation.md#module-dagrunnerutilsvisualisation) - - [class: HTMLTable](dagrunner.utils.visualisation.md#class-htmltable) - - [class: MermaidGraph](dagrunner.utils.visualisation.md#class-mermaidgraph) - - [class: MermaidHTML](dagrunner.utils.visualisation.md#class-mermaidhtml) - - [function: visualise_graph_matplotlib](dagrunner.utils.visualisation.md#function-visualise_graph_matplotlib) - - [function: visualise_graph_mermaid](dagrunner.utils.visualisation.md#function-visualise_graph_mermaid) \ No newline at end of file