diff --git a/docs/changes/newsfragments/498.feature b/docs/changes/newsfragments/498.feature new file mode 100644 index 000000000..c7a14e19e --- /dev/null +++ b/docs/changes/newsfragments/498.feature @@ -0,0 +1 @@ +Introduce :func:`.generate_yaml` to generate feature YAML from metadata by `Synchon Mandal`_ diff --git a/docs/links.inc b/docs/links.inc index dfc113e5b..fcaf3ebad 100644 --- a/docs/links.inc +++ b/docs/links.inc @@ -13,6 +13,7 @@ .. _`INM-7`: https://www.fz-juelich.de/inm/inm-7/EN/Home/home_node.html .. _`julearn`: https://juaml.github.io/julearn .. _`junifer-data`: https://github.com/juaml/junifer-data-client +.. _`julio`: https://github.com/juaml/julio .. _`pandas`: https://pandas.pydata.org .. _`pandas.DataFrame` : https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.html diff --git a/docs/using/generate_yaml.rst b/docs/using/generate_yaml.rst new file mode 100644 index 000000000..ddbe68c38 --- /dev/null +++ b/docs/using/generate_yaml.rst @@ -0,0 +1,14 @@ +.. include:: ../links.inc + +.. _generate_yaml: + +Generating YAML from metadata +============================= + +``junifer`` stores the pipeline metadata for a run along with the extracted feature data. +So, the metadata for all the "elements" processed with a pipeline is unique. The metadata +contains all the necessary information to recreate the configuration used for the processing. + +If one wants to generate the processing YAML, :func:`.generate_yaml` can be used for that. +The only requirement is providing the metadata which can be extracted by following the initial steps of +:ref:`analysing results `. diff --git a/docs/using/index.rst b/docs/using/index.rst index 338d9e72c..a1fac87fe 100644 --- a/docs/using/index.rst +++ b/docs/using/index.rst @@ -20,6 +20,7 @@ to interact with HPC and HTC systems. queueing configuring dumping + generate_yaml .. _using_components: diff --git a/junifer/api/__init__.pyi b/junifer/api/__init__.pyi index 943720d80..33c512d81 100644 --- a/junifer/api/__init__.pyi +++ b/junifer/api/__init__.pyi @@ -6,6 +6,7 @@ __all__ = [ "reset", "list_elements", "parse_yaml", + "generate_yaml", ] from . import decorators @@ -13,6 +14,7 @@ from .functions import ( collect, list_elements, parse_yaml, + generate_yaml, reset, run, queue, diff --git a/junifer/api/functions.py b/junifer/api/functions.py index 04392e64a..b7f3314d2 100644 --- a/junifer/api/functions.py +++ b/junifer/api/functions.py @@ -6,14 +6,18 @@ # License: AGPL import atexit +import datetime as dt import importlib import importlib.util +import io import os import shutil import sys from pathlib import Path +from typing import TYPE_CHECKING, Any import structlog +from pydantic import ValidationError from ..api.queue_context import GnuParallelLocalAdapter, HTCondorAdapter from ..datagrabber import BaseDataGrabber @@ -35,8 +39,13 @@ from ..utils import raise_error, warn_with_log, yaml +if TYPE_CHECKING: + from ruamel.yaml.comments import CommentedMap + + __all__ = [ "collect", + "generate_yaml", "list_elements", "parse_yaml", "queue", @@ -600,3 +609,175 @@ def parse_yaml(filepath: str | Path) -> dict: # noqa: C901 ) return contents + + +def generate_yaml(meta: dict) -> "CommentedMap": # noqa: C901 + """Generate the feature YAML from metadata. + + Parameters + ---------- + meta : dict + Feature metadata as dictionary. + + Returns + ------- + ruamel.yaml.comments.CommentedMap + Feature YAML. + + """ + y: dict[str, Any] = {} + y["workdir"] = "" + # Add "with" section if present + if "with" in meta: + y["with"] = meta["with"].copy() + # Init var for post comment and issues + post = "\nIssues:\n" + issue_ext = ( + " - `{0}` is not a built-in component and thus could not be properly " + "regenerated. Some of these entries in the YAML section might be " + "redundant and not needed. Please check the " + "documentation/implementation of this specific component and remove " + "the unnecessary entries.\n" + ) + issue_inv = ( + " - `{0}` failed to initialise and thus could not be properly " + "regenerated. Some of these entries in the YAML section might be " + "redundant and not needed. Please check the " + "documentation/implementation of this specific component and remove " + "the unnecessary entries.\n" + ) + var = "" + # Set datagrabber + meta_dg = meta["datagrabber"].copy() + a = meta_dg.pop("class") + if a not in PipelineComponentRegistry()._components["datagrabber"]: + y["datagrabber"] = {"kind": a, **meta_dg} + post += f"- datagrabber:\n{issue_ext.format(a)}" + else: + dg = PipelineComponentRegistry().get_class(step="datagrabber", name=a) + try: + dg_model = dg.model_validate(meta_dg) + except ValidationError: + y["datagrabber"] = {"kind": a, **meta_dg} + post += f"- datagrabber:\n{issue_inv.format(a)}" + else: + y["datagrabber"] = { + "kind": a, + **dg_model.model_dump( + mode="json", + include=set(dg_model.dump_fields()), + exclude_defaults=True, + exclude_none=True, + ), + } + if meta_dg.get("datalad_dirty"): + var = ( + "- The dataset was 'dirty', there is no guarantee that the " + "results will be reproducible.\n" + ) + # Set preprocessor(s) + if "preprocess" in meta: + y["preprocess"] = [] + meta_p = meta["preprocess"].copy() + if not isinstance(meta_p, list): + meta_p = [meta_p] + for mp in meta_p: + b = mp.pop("class") + if ( + b + not in PipelineComponentRegistry()._components["preprocessing"] + ): + y["preprocess"].append({"kind": b, **mp}) + if "- preprocess:\n" in post: + post += f"{issue_ext.format(b)}" + else: + post += f"- preprocess:\n{issue_ext.format(b)}" + else: + p = PipelineComponentRegistry().get_class( + step="preprocessing", name=b + ) + try: + p_model = p.model_validate(mp) + except ValidationError: + y["preprocess"].append({"kind": b, **mp}) + if "- preprocess:\n" in post: + post += f"{issue_inv.format(b)}" + else: + post += f"- preprocess:\n{issue_inv.format(b)}" + else: + y["preprocess"].append( + { + "kind": b, + **p_model.model_dump( + mode="json", + exclude={"required_data_types"}, + exclude_defaults=True, + exclude_none=True, + ), + } + ) + # Set marker + meta_m = meta["marker"].copy() + c = meta_m.pop("class") + y["markers"] = [] + if c not in PipelineComponentRegistry()._components["marker"]: + y["markers"].append({"kind": c, **meta_m}) + post += f"- markers:\n{issue_ext.format(c)}" + else: + m = PipelineComponentRegistry().get_class(step="marker", name=c) + try: + m_model = m.model_validate(meta_m) + except ValidationError: + y["markers"].append({"kind": c, **meta_m}) + post += f"- markers:\n{issue_inv.format(c)}" + else: + y["markers"].append( + { + "kind": c, + **m_model.model_dump( + mode="json", + exclude_defaults=True, + exclude_none=True, + ), + } + ) + # Set storage + y["storage"] = { + "kind": "HDF5FeatureStorage", + "uri": "", + } + # Set queue + if "queue" in meta: + y["queue"] = meta["queue"].copy() + else: + y["queue"] = { + "jobname": meta["name"], + "kind": "", + } + # Dump and load yaml to format + f = io.StringIO() + yaml.dump(y, stream=f) + f.seek(0) + d = yaml.load(f) + # Write comments + pre = ( + "Auto-generated by junifer on " + f"{dt.datetime.now(tz=dt.timezone.utc).strftime('%Y-%m-%d %H:%M:%S')} " + "UTC\n\n" + ) + if "dependencies" in meta: + for k, v in meta["dependencies"].items(): + pre += f"{k}=={v}\n" + const = ( + "\nNotes:\n" + "- Check the components for possible changes in the API.\n" + "- `datadir` is ignored and not reproduced. " + "If `datadir` used was not a temporary directory, you will have to " + "manually edit this YAML.\n" + ) + post = post if post != "\nIssues:\n" else "" + d.yaml_set_start_comment(pre + const + var + post) + # Add newline between sections + for s in d.keys(): + d.yaml_set_comment_before_after_key(s, before="\n") + return d diff --git a/junifer/api/tests/test_functions.py b/junifer/api/tests/test_functions.py index d6cc8c67c..d9b66cc56 100644 --- a/junifer/api/tests/test_functions.py +++ b/junifer/api/tests/test_functions.py @@ -5,6 +5,7 @@ # Synchon Mandal # License: AGPL +import io import logging import sys from contextlib import AbstractContextManager, nullcontext @@ -16,7 +17,15 @@ from ruamel.yaml import YAML import junifer.testing.registry # noqa: F401 -from junifer.api import collect, list_elements, parse_yaml, queue, reset, run +from junifer.api import ( + collect, + generate_yaml, + list_elements, + parse_yaml, + queue, + reset, + run, +) from junifer.datagrabber.base import BaseDataGrabber from junifer.pipeline import PipelineComponentRegistry from junifer.typing import Elements @@ -1025,3 +1034,351 @@ def test_parse_yaml_queue_venv_relative(tmp_path: Path) -> None: fname = tmp_path / "test_parse_yaml_queue_venv_relative.yaml" fname.write_text("queue:\n env:\n kind: venv\n name: .venv\n") _ = parse_yaml(fname) + + +@pytest.mark.parametrize( + "m, exp", + [ + ( + { + "datagrabber": { + "class": "PartlyCloudyTestingDataGrabber", + "types": ["BOLD"], + "datadir": ( + "/var/folders/dv/2lbr8f8j0q12zrx3mz3ll5m40000gp/T/tmpjeqj9nou" + ), + "reduce_confounds": False, + "age_group": "both", + }, + "dependencies": {"scikit-learn": "1.4.2", "nilearn": "0.10.4"}, + "datareader": {"class": "DefaultDataReader"}, + "type": "BOLD", + "marker": { + "class": "FunctionalConnectivityParcels", + "on": ["BOLD"], + "name": "fc_mean-shen_2015_268_functional_connectivity", + "agg_method": "mean", + "agg_method_params": None, + "conn_method": "correlation", + "conn_method_params": {"empirical": True}, + "masks": None, + "parcellation": ["Shen_2015_268"], + }, + "_element_keys": ["subject"], + "name": "BOLD_fc_mean-shen_2015_268_functional_connectivity", + }, + [ + "Auto-generated by junifer on", + "Check the components for possible changes in the API", + ], + ), + ( + { + "datagrabber": { + "class": "PartlyCloudyTestingDataGrabber", + "types": ["BOLD"], + "datadir": ( + "/var/folders/dv/2lbr8f8j0q12zrx3mz3ll5m40000gp/T/tmpjeqj9nou" + ), + "reduce_confound": True, + "age": "both", + }, + "dependencies": {"scikit-learn": "1.4.2", "nilearn": "0.10.4"}, + "datareader": {"class": "DefaultDataReader"}, + "type": "BOLD", + "marker": { + "class": "FunctionalConnectivityParcels", + "on": ["BOLD"], + "name": "fc_mean-shen_2015_268_functional_connectivity", + "ag_method": "mean", + "ag_method_params": None, + "con_method": "correlation", + "con_method_params": {"empirical": True}, + "masks": None, + "parcellation": ["Shen_2015_268"], + }, + "_element_keys": ["subject"], + "name": "BOLD_fc_mean-shen_2015_268_functional_connectivity", + }, + [ + "Auto-generated by junifer on", + "Check the components for possible changes in the API", + "`PartlyCloudyTestingDataGrabber` failed to initialise and " + "thus could not be properly", + ], + ), + ( + { + "datagrabber": { + "class": "DMCC13Benchmark", + "types": ["BOLD"], + "patterns": { + "BOLD": { + "pattern": ( + "derivatives/fmriprep-1.3.2/{subject}/{session}/func/{subject}_{session}_task-{task}_acq-mb4{phase_encoding}_run-{run}_space-MNI152NLin2009cAsym_desc-preproc_bold.nii.gz" + ), + "space": "MNI152NLin2009cAsym", + "mask": { + "pattern": ( + "derivatives/fmriprep-1.3.2/{subject}/{session}/func/{subject}_{session}_task-{task}_acq-mb4{phase_encoding}_run-{run}_space-MNI152NLin2009cAsym_desc-brain_mask.nii.gz" + ), + "space": "MNI152NLin2009cAsym", + }, + "confounds": { + "pattern": ( + "derivatives/fmriprep-1.3.2/{subject}/{session}/func/{subject}_{session}_task-{task}_acq-mb4{phase_encoding}_run-{run}_desc-confounds_regressors.tsv" + ), + "format": "fmriprep", + }, + }, + "T1w": { + "pattern": ( + "derivatives/fmriprep-1.3.2/{subject}/anat/{subject}_space-MNI152NLin2009cAsym_desc-preproc_T1w.nii.gz" + ), + "space": "MNI152NLin2009cAsym", + "mask": { + "pattern": ( + "derivatives/fmriprep-1.3.2/{subject}/anat/{subject}_space-MNI152NLin2009cAsym_desc-brain_mask.nii.gz" + ), + "space": "MNI152NLin2009cAsym", + }, + }, + }, + "replacements": [ + "subject", + "session", + "task", + "phase_encoding", + "run", + ], + "confounds_format": "fmriprep", + "partial_pattern_ok": False, + "uri": "https://github.com/OpenNeuroDatasets/ds003452.git", + "rootdir": ".", + "datalad_dirty": True, + "datalad_commit_id": ( + "8484f21551af53fcf2bc53878f18ce93dc2d29da" + ), + "datalad_id": "ade00fb6-636f-46fb-b2e6-60958b1b112d", + "sessions": ["ses-wave1bas"], + "tasks": ["Rest"], + "phase_encodings": ["AP"], + "runs": ["1"], + "native_t1w": False, + }, + "dependencies": { + "scikit-learn": "1.4.2", + "nilearn": "0.10.4", + "numpy": "1.26.4", + }, + "datareader": {"class": "DefaultDataReader"}, + "preprocess": { + "class": "fMRIPrepConfoundRemover", + "on": ["BOLD"], + "required_data_types": ["BOLD"], + "strategy": { + "motion": "full", + "wm_csf": "full", + "global_signal": "full", + }, + "spike": None, + "scrub": None, + "fd_threshold": None, + "std_dvars_threshold": None, + "detrend": True, + "standardize": True, + "low_pass": 0.08, + "high_pass": 0.01, + "t_r": None, + "masks": ["compute_epi_mask"], + }, + "type": "BOLD", + "marker": { + "class": "FunctionalConnectivitySpheres", + "on": ["BOLD"], + "name": "fc_spheres_functional_connectivity", + "agg_method": "mean", + "agg_method_params": None, + "conn_method": "correlation", + "conn_method_params": {"empirical": True}, + "masks": None, + "coords": "DMNBuckner", + "radius": 5.0, + "allow_overlap": False, + }, + "_element_keys": [ + "subject", + "session", + "task", + "phase_encoding", + "run", + ], + "name": "BOLD_fc_spheres_functional_connectivity", + }, + [ + "Auto-generated by junifer on", + "Check the components for possible changes in the API", + "The dataset was 'dirty'", + ], + ), + ( + { + "datagrabber": { + "class": "DMCC13Benchmark", + "types": ["BOLD"], + "patterns": { + "BOLD": { + "pattern": ( + "derivatives/fmriprep-1.3.2/{subject}/{session}/func/{subject}_{session}_task-{task}_acq-mb4{phase_encoding}_run-{run}_space-MNI152NLin2009cAsym_desc-preproc_bold.nii.gz" + ), + "space": "MNI152NLin2009cAsym", + "mask": { + "pattern": ( + "derivatives/fmriprep-1.3.2/{subject}/{session}/func/{subject}_{session}_task-{task}_acq-mb4{phase_encoding}_run-{run}_space-MNI152NLin2009cAsym_desc-brain_mask.nii.gz" + ), + "space": "MNI152NLin2009cAsym", + }, + "confounds": { + "pattern": ( + "derivatives/fmriprep-1.3.2/{subject}/{session}/func/{subject}_{session}_task-{task}_acq-mb4{phase_encoding}_run-{run}_desc-confounds_regressors.tsv" + ), + "format": "fmriprep", + }, + }, + }, + "replacements": [ + "subject", + "session", + "task", + "phase_encoding", + "run", + ], + "confounds_format": "fmriprep", + "partial_pattern_ok": False, + "uri": "https://github.com/OpenNeuroDatasets/ds003452.git", + "rootdir": ".", + "datalad_dirty": False, + "datalad_commit_id": ( + "8484f21551af53fcf2bc53878f18ce93dc2d29da" + ), + "datalad_id": "ade00fb6-636f-46fb-b2e6-60958b1b112d", + "sessions": ["ses-wave1bas"], + "tasks": ["Rest"], + "phase_encodings": ["AP"], + "runs": ["1"], + "native_t1w": False, + }, + "dependencies": { + "scikit-learn": "1.4.2", + "nilearn": "0.10.4", + "numpy": "1.26.4", + }, + "datareader": {"class": "DefaultDataReader"}, + "type": "BOLD", + "marker": { + "class": "FunctionalConnectivitySpheres", + "on": ["BOLD"], + "name": "fc_spheres_functional_connectivity", + "agg_method": "mean", + "agg_method_params": None, + "conn_method": "correlation", + "conn_method_params": {"empirical": True}, + "masks": None, + "coords": "DMNBuckner", + "radius": 5.0, + "allow_overlap": False, + }, + "_element_keys": [ + "subject", + "session", + "task", + "phase_encoding", + "run", + ], + "name": "BOLD_fc_spheres_functional_connectivity", + }, + [ + "Auto-generated by junifer on", + "Check the components for possible changes in the API", + ], + ), + ( + { + "datagrabber": { + "class": "ExternalDataGrabber", + "types": ["BOLD"], + "patterns": { + "BOLD": { + "pattern": ( + "derivatives/fmriprep-1.3.2/{subject}/func/{subject}_task-{task}_space-MNI152NLin2009cAsym_desc-preproc_bold.nii.gz" + ), + "space": "MNI152NLin2009cAsym", + }, + }, + "replacements": [ + "subject", + "task", + ], + "confounds_format": "fmriprep", + "partial_pattern_ok": True, + "uri": "https://github.com/datasets/ds11.git", + "rootdir": ".", + "datalad_dirty": False, + "datalad_commit_id": ( + "8484f21551af53fcf2bc53878f18ce93dc2d29da" + ), + "datalad_id": "ade00fb6-636f-46fb-b2e6-60958b1b112d", + "sessions": ["ses-wave1bas"], + "tasks": ["Rest"], + }, + "dependencies": {"scikit-learn": "1.4.2", "nilearn": "0.10.4"}, + "datareader": {"class": "DefaultDataReader"}, + "preprocess": [ + { + "class": "ExternalPreprocessor1", + "on": ["BOLD"], + "required_data_types": ["BOLD"], + }, + { + "class": "ExternalPreprocessor2", + "on": ["BOLD"], + "required_data_types": ["BOLD"], + }, + ], + "type": "BOLD", + "marker": { + "class": "ExternalMarker", + "on": ["BOLD"], + "name": "external", + }, + "_element_keys": ["subject", "task"], + "name": "BOLD_external", + }, + [ + "Auto-generated by junifer on", + "Check the components for possible changes in the API", + "`ExternalDataGrabber` is not a built-in component", + "`ExternalPreprocessor1` is not a built-in component", + "`ExternalMarker` is not a built-in component", + ], + ), + ], +) +def test_generate_yaml(m: dict, exp: list[str]) -> None: + """Test YAML generation from feature metadata. + + Parameters + ---------- + m : dict + The parametrized feature metadata. + exp : list + The parametrized expected comments. + + """ + c = generate_yaml(m) + buf = io.StringIO() + yaml.dump(c, stream=buf) + buf.seek(0) + y = buf.read() + for e in exp: + assert e in y diff --git a/junifer/configs/juseless/datagrabbers/aomic_id1000_vbm.py b/junifer/configs/juseless/datagrabbers/aomic_id1000_vbm.py index 3555fc05d..2bdc89fb1 100644 --- a/junifer/configs/juseless/datagrabbers/aomic_id1000_vbm.py +++ b/junifer/configs/juseless/datagrabbers/aomic_id1000_vbm.py @@ -40,3 +40,8 @@ class JuselessDataladAOMICID1000VBM(PatternDataladDataGrabber): }, } replacements: list[str] = ["subject"] # noqa: RUF012 + + @classmethod + def dump_fields(cls) -> list[str]: + """Fields to include when dumping model.""" + return [] diff --git a/junifer/configs/juseless/datagrabbers/camcan_vbm.py b/junifer/configs/juseless/datagrabbers/camcan_vbm.py index 85fccec6f..ce609af9d 100644 --- a/junifer/configs/juseless/datagrabbers/camcan_vbm.py +++ b/junifer/configs/juseless/datagrabbers/camcan_vbm.py @@ -43,3 +43,8 @@ class JuselessDataladCamCANVBM(PatternDataladDataGrabber): }, } replacements: list[str] = ["subject"] # noqa: RUF012 + + @classmethod + def dump_fields(cls) -> list[str]: + """Fields to include when dumping model.""" + return [] diff --git a/junifer/configs/juseless/datagrabbers/ixi_vbm.py b/junifer/configs/juseless/datagrabbers/ixi_vbm.py index ee1955e39..898494d16 100644 --- a/junifer/configs/juseless/datagrabbers/ixi_vbm.py +++ b/junifer/configs/juseless/datagrabbers/ixi_vbm.py @@ -66,3 +66,8 @@ class JuselessDataladIXIVBM(PatternDataladDataGrabber): }, } replacements: list[str] = ["site", "subject"] # noqa: RUF012 + + @classmethod + def dump_fields(cls) -> list[str]: + """Fields to include when dumping model.""" + return [] diff --git a/junifer/configs/juseless/datagrabbers/ucla.py b/junifer/configs/juseless/datagrabbers/ucla.py index 1e46b1f95..d9ddc1eba 100644 --- a/junifer/configs/juseless/datagrabbers/ucla.py +++ b/junifer/configs/juseless/datagrabbers/ucla.py @@ -143,6 +143,11 @@ class JuselessUCLA(PatternDataGrabber): replacements: list[str] = ["subject", "task"] # noqa: RUF012 confounds_format: ConfoundsFormat = ConfoundsFormat.FMRIPrep + @classmethod + def dump_fields(cls) -> list[str]: + """Fields to include when dumping model.""" + return ["types", "tasks"] + def get_elements(self) -> list: """Implement fetching list of elements in the dataset. diff --git a/junifer/configs/juseless/datagrabbers/ukb_vbm.py b/junifer/configs/juseless/datagrabbers/ukb_vbm.py index 1f05a8bb8..5852cc5c4 100644 --- a/junifer/configs/juseless/datagrabbers/ukb_vbm.py +++ b/junifer/configs/juseless/datagrabbers/ukb_vbm.py @@ -43,3 +43,8 @@ class JuselessDataladUKBVBM(PatternDataladDataGrabber): }, } replacements: list[str] = ["subject", "session"] # noqa: RUF012 + + @classmethod + def dump_fields(cls) -> list[str]: + """Fields to include when dumping model.""" + return [] diff --git a/junifer/datagrabber/aomic/id1000.py b/junifer/datagrabber/aomic/id1000.py index 2a1fd2349..135b363e5 100644 --- a/junifer/datagrabber/aomic/id1000.py +++ b/junifer/datagrabber/aomic/id1000.py @@ -243,3 +243,8 @@ def validate_datagrabber_params(self) -> None: else: self.patterns["BOLD"]["prewarp_space"] = "native" super().validate_datagrabber_params() + + @classmethod + def dump_fields(cls) -> list[str]: + """Fields to include when dumping model.""" + return ["types", "space"] diff --git a/junifer/datagrabber/aomic/piop1.py b/junifer/datagrabber/aomic/piop1.py index cac65e01f..b5935ea77 100644 --- a/junifer/datagrabber/aomic/piop1.py +++ b/junifer/datagrabber/aomic/piop1.py @@ -266,6 +266,11 @@ def validate_datagrabber_params(self) -> None: self.patterns["BOLD"]["prewarp_space"] = "native" super().validate_datagrabber_params() + @classmethod + def dump_fields(cls) -> list[str]: + """Fields to include when dumping model.""" + return ["types", "tasks", "space"] + def get_item(self, subject: str, task: str) -> dict: """Get the specified item from the dataset. diff --git a/junifer/datagrabber/aomic/piop2.py b/junifer/datagrabber/aomic/piop2.py index 1fab2a0a8..bf52f7cb2 100644 --- a/junifer/datagrabber/aomic/piop2.py +++ b/junifer/datagrabber/aomic/piop2.py @@ -262,6 +262,11 @@ def validate_datagrabber_params(self) -> None: self.patterns["BOLD"]["prewarp_space"] = "native" super().validate_datagrabber_params() + @classmethod + def dump_fields(cls) -> list[str]: + """Fields to include when dumping model.""" + return ["types", "tasks", "space"] + def get_elements(self) -> list: """Implement fetching list of elements in the dataset. diff --git a/junifer/datagrabber/base.py b/junifer/datagrabber/base.py index 4fb3796a6..41252c550 100644 --- a/junifer/datagrabber/base.py +++ b/junifer/datagrabber/base.py @@ -85,6 +85,11 @@ def validate_datagrabber_params(self) -> None: """ pass + @classmethod + def dump_fields(cls) -> list[str]: + """Fields to include when dumping model.""" + return ["types", "datadir"] + def __iter__(self) -> Iterator[Elements]: """Enable iterable support. diff --git a/junifer/datagrabber/datalad_base.py b/junifer/datagrabber/datalad_base.py index 9c28fa4a7..b6b8e5eff 100644 --- a/junifer/datagrabber/datalad_base.py +++ b/junifer/datagrabber/datalad_base.py @@ -176,6 +176,11 @@ def __del__(self) -> None: ) and self.datadir.stem.endswith("juniferauto"): _remove_datadir(self.datadir) + @classmethod + def dump_fields(cls) -> list[str]: + """Fields to include when dumping model.""" + return ["types", "uri", "rootdir"] + @property def fulldir(self) -> Path: """Get complete data directory path. diff --git a/junifer/datagrabber/dmcc13_benchmark.py b/junifer/datagrabber/dmcc13_benchmark.py index e726b97c7..e2cbd6a7d 100644 --- a/junifer/datagrabber/dmcc13_benchmark.py +++ b/junifer/datagrabber/dmcc13_benchmark.py @@ -276,6 +276,18 @@ def validate_datagrabber_params(self) -> None: self.types.append(DataType.Warp) super().validate_datagrabber_params() + @classmethod + def dump_fields(cls) -> list[str]: + """Fields to include when dumping model.""" + return [ + "types", + "sessions", + "tasks", + "phase_encodings", + "runs", + "native_t1w", + ] + def get_item( self, subject: str, diff --git a/junifer/datagrabber/hcp1200/datalad_hcp1200.py b/junifer/datagrabber/hcp1200/datalad_hcp1200.py index 8a6abee6a..716314c17 100644 --- a/junifer/datagrabber/hcp1200/datalad_hcp1200.py +++ b/junifer/datagrabber/hcp1200/datalad_hcp1200.py @@ -61,6 +61,11 @@ class DataladHCP1200(DataladDataGrabber, HCP1200): ] rootdir: Path = Path("HCP1200") + @classmethod + def dump_fields(cls) -> list[str]: + """Fields to include when dumping model.""" + return ["types", "tasks", "phase_encodings", "ica_fix"] + # Needed here as HCP1200's subjects are sub-datasets, so will not be # found when elements are checked. @property diff --git a/junifer/datagrabber/hcp1200/hcp1200.py b/junifer/datagrabber/hcp1200/hcp1200.py index 5ea9c5ba8..bb85a781f 100644 --- a/junifer/datagrabber/hcp1200/hcp1200.py +++ b/junifer/datagrabber/hcp1200/hcp1200.py @@ -159,6 +159,11 @@ def validate_datagrabber_params(self) -> None: ].replace("{suffix}", suffix) super().validate_datagrabber_params() + @classmethod + def dump_fields(cls) -> list[str]: + """Fields to include when dumping model.""" + return ["types", "datadir", "tasks", "phase_encodings", "ica_fix"] + def get_item(self, subject: str, task: str, phase_encoding: str) -> dict: """Get the specified item from the dataset. diff --git a/junifer/datagrabber/multiple.py b/junifer/datagrabber/multiple.py index e2bb4eebd..76d7235b3 100644 --- a/junifer/datagrabber/multiple.py +++ b/junifer/datagrabber/multiple.py @@ -94,6 +94,11 @@ def validate_datagrabber_params(self) -> None: klass=RuntimeError, ) + @classmethod + def dump_fields(cls) -> list[str]: + """Fields to include when dumping model.""" + return [*super().dump_fields(), "datagrabbers"] + def __getitem__(self, element: Element) -> dict: """Implement indexing. diff --git a/junifer/datagrabber/pattern.py b/junifer/datagrabber/pattern.py index 3996faa3c..392ab4741 100644 --- a/junifer/datagrabber/pattern.py +++ b/junifer/datagrabber/pattern.py @@ -101,6 +101,17 @@ def validate_datagrabber_params(self) -> None: logger.debug(f"\treplacements = {self.replacements}") logger.debug(f"\tconfounds_format = {self.confounds_format}") + @classmethod + def dump_fields(cls) -> list[str]: + """Fields to include when dumping model.""" + return [ + *super().dump_fields(), + "patterns", + "replacements", + "confounds_format", + "partial_pattern_ok", + ] + @property def skip_file_check(self) -> bool: """Skip file check existence.""" diff --git a/junifer/datagrabber/pattern_datalad.py b/junifer/datagrabber/pattern_datalad.py index 9864591af..3492cff82 100644 --- a/junifer/datagrabber/pattern_datalad.py +++ b/junifer/datagrabber/pattern_datalad.py @@ -61,3 +61,16 @@ def validate_datagrabber_params(self) -> None: logger.debug("Initializing PatternDataladDataGrabber") for key, val in self.__pydantic_extra__.items(): logger.debug(f"\t{key} = {val}") + + @classmethod + def dump_fields(cls) -> list[str]: + """Fields to include when dumping model.""" + return [ + "types", + "patterns", + "replacements", + "confounds_format", + "partial_pattern_ok", + "uri", + "rootdir", + ] diff --git a/junifer/markers/utils.py b/junifer/markers/utils.py index c3ab4ef33..969b32cdf 100644 --- a/junifer/markers/utils.py +++ b/junifer/markers/utils.py @@ -19,7 +19,7 @@ def _ets( bold_ts: np.ndarray, - roi_names: None | list[str] = None, + roi_names: list[str] | None = None, ) -> tuple[np.ndarray, list[str] | None]: """Compute the edge-wise time series based on BOLD time series. diff --git a/junifer/testing/datagrabbers.py b/junifer/testing/datagrabbers.py index 61ada8413..c84976f06 100644 --- a/junifer/testing/datagrabbers.py +++ b/junifer/testing/datagrabbers.py @@ -34,6 +34,11 @@ class OasisVBMTestingDataGrabber(BaseDataGrabber): datadir: Path = Path(tempfile.mkdtemp()) _dataset: Any = None + @classmethod + def dump_fields(cls) -> list[str]: + """Fields to include when dumping model.""" + return ["types"] + def get_element_keys(self) -> list[str]: """Get element keys. @@ -101,6 +106,11 @@ class SPMAuditoryTestingDataGrabber(BaseDataGrabber): types: list[DataType] = [DataType.BOLD, DataType.T1w] # noqa: RUF012 datadir: Path = Path(tempfile.mkdtemp()) + @classmethod + def dump_fields(cls) -> list[str]: + """Fields to include when dumping model.""" + return ["types"] + def get_element_keys(self) -> list[str]: """Get element keys. @@ -189,6 +199,11 @@ class PartlyCloudyTestingDataGrabber(BaseDataGrabber): reduce_confounds: bool = True age_group: PartlyCloudyAgeGroup = PartlyCloudyAgeGroup.Both + @classmethod + def dump_fields(cls) -> list[str]: + """Fields to include when dumping model.""" + return ["types", "reduce_confounds", "age_group"] + def __enter__(self) -> "PartlyCloudyTestingDataGrabber": """Implement context entry.