From 0f9ed1a947efc845d9b06341096b42c9eea932a6 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 29 May 2026 16:46:27 +0200 Subject: [PATCH 01/21] feat: add junifer.api.generate_yaml --- junifer/api/__init__.pyi | 2 + junifer/api/functions.py | 115 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 117 insertions(+) diff --git a/junifer/api/__init__.pyi b/junifer/api/__init__.pyi index 943720d80..33c512d81 100644 --- a/junifer/api/__init__.pyi +++ b/junifer/api/__init__.pyi @@ -6,6 +6,7 @@ __all__ = [ "reset", "list_elements", "parse_yaml", + "generate_yaml", ] from . import decorators @@ -13,6 +14,7 @@ from .functions import ( collect, list_elements, parse_yaml, + generate_yaml, reset, run, queue, diff --git a/junifer/api/functions.py b/junifer/api/functions.py index 04392e64a..e26e25771 100644 --- a/junifer/api/functions.py +++ b/junifer/api/functions.py @@ -6,12 +6,15 @@ # License: AGPL import atexit +import datetime as dt import importlib import importlib.util +import io import os import shutil import sys from pathlib import Path +from typing import TYPE_CHECKING, Any import structlog @@ -35,8 +38,13 @@ from ..utils import raise_error, warn_with_log, yaml +if TYPE_CHECKING: + from ruamel.yaml.comments import CommentedMap + + __all__ = [ "collect", + "generate_yaml", "list_elements", "parse_yaml", "queue", @@ -600,3 +608,110 @@ def parse_yaml(filepath: str | Path) -> dict: # noqa: C901 ) return contents + + +def generate_yaml(meta: dict) -> "CommentedMap": + """Generate the feature YAML from metadata. + + Parameters + ---------- + meta : dict + Feature metadata as dictionary. + + Returns + ------- + ruamel.yaml.comments.CommentedMap + Feature YAML. + + """ + y: dict[str, Any] = {} + y["workdir"] = "" + # Add "with" section if present + if "with" in meta: + y["with"] = meta["with"].copy() + # Set datagrabber + meta_dg = meta["datagrabber"].copy() + a = meta_dg.pop("class") + dg = PipelineComponentRegistry().get_class(step="datagrabber", name=a) + dg_model = dg.model_construct(**meta_dg) + y["datagrabber"] = { + "kind": a, + **dg_model.model_dump( + mode="json", + exclude=dg_model._dump_exclude + if hasattr(dg_model, "_dump_exclude") + else {}, + exclude_defaults=True, + exclude_none=True, + ), + } + # Set preprocessor(s) + if "preprocess" in meta: + y["preprocess"] = [] + meta_p = meta["preprocess"].copy() + if not isinstance(meta_p, list): + meta_p = [meta_p] + for mp in meta_p: + b = mp.pop("class") + p = PipelineComponentRegistry().get_class( + step="preprocessing", name=b + ) + p_model = p.model_construct(**mp) + y["preprocess"].append( + { + "kind": b, + **p_model.model_dump( + mode="json", + exclude={"required_data_types"}, + exclude_defaults=True, + exclude_none=True, + ), + } + ) + # Set marker + meta_m = meta["marker"].copy() + c = meta_m.pop("class") + m = PipelineComponentRegistry().get_class(step="marker", name=c) + m_model = m.model_construct(**meta_m) + y["markers"] = [] + y["markers"].append( + { + "kind": c, + **m_model.model_dump( + mode="json", + exclude_defaults=True, + exclude_none=True, + ), + } + ) + # Set storage + y["storage"] = { + "kind": "HDF5FeatureStorage", + "uri": "", + } + # Set queue + if "queue" in meta: + y["queue"] = meta["queue"].copy() + else: + y["queue"] = { + "jobname": meta["name"], + "kind": "", + } + # Dump and load yaml to format + f = io.StringIO() + yaml.dump(y, stream=f) + f.seek(0) + d = yaml.load(f) + # Add preamble + pre = ( + "Auto-generated by junifer on " + f"{dt.datetime.now(tz=dt.UTC).strftime('%Y-%m-%d %H:%M:%S')} UTC\n\n" + ) + if "dependencies" in meta: + for k, v in meta["dependencies"].items(): + pre += f"{k}=={v}\n" + d.yaml_set_start_comment(pre) + # Add newline between sections + for s in d.keys(): + d.yaml_set_comment_before_after_key(s, before="\n") + return d From 845bd49b623f6499d26f9b2fed49a487d632a11d Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 29 May 2026 18:01:05 +0200 Subject: [PATCH 02/21] feat: add _dump_exclude class variables to datagrabbers --- .../juseless/datagrabbers/aomic_id1000_vbm.py | 15 ++++++++++++++- .../configs/juseless/datagrabbers/camcan_vbm.py | 15 ++++++++++++++- junifer/configs/juseless/datagrabbers/ixi_vbm.py | 13 +++++++++++++ junifer/configs/juseless/datagrabbers/ukb_vbm.py | 15 ++++++++++++++- junifer/datagrabber/aomic/id1000.py | 15 ++++++++++++++- junifer/datagrabber/aomic/piop1.py | 15 ++++++++++++++- junifer/datagrabber/aomic/piop2.py | 15 ++++++++++++++- junifer/datagrabber/dmcc13_benchmark.py | 15 ++++++++++++++- junifer/datagrabber/hcp1200/datalad_hcp1200.py | 15 ++++++++++++++- junifer/datagrabber/pattern.py | 8 ++++++++ junifer/datagrabber/pattern_datalad.py | 10 ++++++++++ 11 files changed, 143 insertions(+), 8 deletions(-) diff --git a/junifer/configs/juseless/datagrabbers/aomic_id1000_vbm.py b/junifer/configs/juseless/datagrabbers/aomic_id1000_vbm.py index 3555fc05d..6e40f2dc0 100644 --- a/junifer/configs/juseless/datagrabbers/aomic_id1000_vbm.py +++ b/junifer/configs/juseless/datagrabbers/aomic_id1000_vbm.py @@ -4,7 +4,7 @@ # Synchon Mandal # License: AGPL -from typing import Literal +from typing import ClassVar, Literal from pydantic import AnyUrl @@ -31,6 +31,19 @@ class JuselessDataladAOMICID1000VBM(PatternDataladDataGrabber): """ + _dump_exclude: ClassVar[set[str]] = { + "patterns", + "replacements", + "confounds_format", + "partial_pattern_ok", + "uri", + "rootdir", + "datadir", + "datalad_id", + "datalad_dirty", + "datalad_commit_id", + } + uri: AnyUrl = AnyUrl("https://gin.g-node.org/felixh/ds003097_ReproVBM") types: list[Literal[DataType.VBM_GM]] = [DataType.VBM_GM] # noqa: RUF012 patterns: DataGrabberPatterns = { # noqa: RUF012 diff --git a/junifer/configs/juseless/datagrabbers/camcan_vbm.py b/junifer/configs/juseless/datagrabbers/camcan_vbm.py index 85fccec6f..9342796f0 100644 --- a/junifer/configs/juseless/datagrabbers/camcan_vbm.py +++ b/junifer/configs/juseless/datagrabbers/camcan_vbm.py @@ -5,7 +5,7 @@ # Synchon Mandal # License: AGPL -from typing import Literal +from typing import ClassVar, Literal from pydantic import AnyUrl @@ -32,6 +32,19 @@ class JuselessDataladCamCANVBM(PatternDataladDataGrabber): """ + _dump_exclude: ClassVar[set[str]] = { + "patterns", + "replacements", + "confounds_format", + "partial_pattern_ok", + "uri", + "rootdir", + "datadir", + "datalad_id", + "datalad_dirty", + "datalad_commit_id", + } + uri: AnyUrl = AnyUrl( "ria+http://cat_12.5.ds.inm7.de#a139b26a-8406-11ea-8f94-a0369f287950" ) diff --git a/junifer/configs/juseless/datagrabbers/ixi_vbm.py b/junifer/configs/juseless/datagrabbers/ixi_vbm.py index ee1955e39..0f58308e3 100644 --- a/junifer/configs/juseless/datagrabbers/ixi_vbm.py +++ b/junifer/configs/juseless/datagrabbers/ixi_vbm.py @@ -48,6 +48,19 @@ class JuselessDataladIXIVBM(PatternDataladDataGrabber): """ + _dump_exclude: ClassVar[set[str]] = { + "patterns", + "replacements", + "confounds_format", + "partial_pattern_ok", + "uri", + "rootdir", + "datadir", + "datalad_id", + "datalad_dirty", + "datalad_commit_id", + } + uri: AnyUrl = AnyUrl( "ria+http://cat_12.5.ds.inm7.de#b7107c52-8408-11ea-89c6-a0369f287950" ) diff --git a/junifer/configs/juseless/datagrabbers/ukb_vbm.py b/junifer/configs/juseless/datagrabbers/ukb_vbm.py index 1f05a8bb8..369b94f1c 100644 --- a/junifer/configs/juseless/datagrabbers/ukb_vbm.py +++ b/junifer/configs/juseless/datagrabbers/ukb_vbm.py @@ -6,7 +6,7 @@ # License: AGPL from pathlib import Path -from typing import Literal +from typing import ClassVar, Literal from pydantic import AnyUrl @@ -33,6 +33,19 @@ class JuselessDataladUKBVBM(PatternDataladDataGrabber): """ + _dump_exclude: ClassVar[set[str]] = { + "patterns", + "replacements", + "confounds_format", + "partial_pattern_ok", + "uri", + "rootdir", + "datadir", + "datalad_id", + "datalad_dirty", + "datalad_commit_id", + } + uri: AnyUrl = AnyUrl("ria+http://ukb.ds.inm7.de#~cat_m0wp1") rootdir: Path = Path("m0wp1") types: list[Literal[DataType.VBM_GM]] = [DataType.VBM_GM] # noqa: RUF012 diff --git a/junifer/datagrabber/aomic/id1000.py b/junifer/datagrabber/aomic/id1000.py index 2a1fd2349..e1ae2e6e1 100644 --- a/junifer/datagrabber/aomic/id1000.py +++ b/junifer/datagrabber/aomic/id1000.py @@ -7,7 +7,7 @@ # Synchon Mandal # License: AGPL -from typing import Annotated, Literal +from typing import Annotated, ClassVar, Literal from pydantic import AnyUrl, BeforeValidator @@ -52,6 +52,19 @@ class DataladAOMICID1000(PatternDataladDataGrabber): """ + _dump_exclude: ClassVar[set[str]] = { + "patterns", + "replacements", + "confounds_format", + "partial_pattern_ok", + "uri", + "rootdir", + "datadir", + "datalad_id", + "datalad_dirty", + "datalad_commit_id", + } + uri: AnyUrl = AnyUrl("https://github.com/OpenNeuroDatasets/ds003097.git") types: Annotated[_types | list[_types], BeforeValidator(ensure_list)] = [ # noqa: RUF012 DataType.BOLD, diff --git a/junifer/datagrabber/aomic/piop1.py b/junifer/datagrabber/aomic/piop1.py index cac65e01f..34a2d7f59 100644 --- a/junifer/datagrabber/aomic/piop1.py +++ b/junifer/datagrabber/aomic/piop1.py @@ -8,7 +8,7 @@ # License: AGPL from itertools import product -from typing import Annotated, Literal +from typing import Annotated, ClassVar, Literal from pydantic import AnyUrl, BeforeValidator @@ -66,6 +66,19 @@ class DataladAOMICPIOP1(PatternDataladDataGrabber): """ + _dump_exclude: ClassVar[set[str]] = { + "patterns", + "replacements", + "confounds_format", + "partial_pattern_ok", + "uri", + "rootdir", + "datadir", + "datalad_id", + "datalad_dirty", + "datalad_commit_id", + } + uri: AnyUrl = AnyUrl("https://github.com/OpenNeuroDatasets/ds002785") types: Annotated[_types | list[_types], BeforeValidator(ensure_list)] = [ # noqa: RUF012 DataType.BOLD, diff --git a/junifer/datagrabber/aomic/piop2.py b/junifer/datagrabber/aomic/piop2.py index 1fab2a0a8..56d8b94ef 100644 --- a/junifer/datagrabber/aomic/piop2.py +++ b/junifer/datagrabber/aomic/piop2.py @@ -8,7 +8,7 @@ # License: AGPL from itertools import product -from typing import Annotated, Literal +from typing import Annotated, ClassVar, Literal from pydantic import AnyUrl, BeforeValidator @@ -64,6 +64,19 @@ class DataladAOMICPIOP2(PatternDataladDataGrabber): """ + _dump_exclude: ClassVar[set[str]] = { + "patterns", + "replacements", + "confounds_format", + "partial_pattern_ok", + "uri", + "rootdir", + "datadir", + "datalad_id", + "datalad_dirty", + "datalad_commit_id", + } + uri: AnyUrl = AnyUrl("https://github.com/OpenNeuroDatasets/ds002790") types: Annotated[_types | list[_types], BeforeValidator(ensure_list)] = [ # noqa: RUF012 DataType.BOLD, diff --git a/junifer/datagrabber/dmcc13_benchmark.py b/junifer/datagrabber/dmcc13_benchmark.py index e726b97c7..0e7f54ba5 100644 --- a/junifer/datagrabber/dmcc13_benchmark.py +++ b/junifer/datagrabber/dmcc13_benchmark.py @@ -5,7 +5,7 @@ from enum import Enum from itertools import product -from typing import Annotated, Literal +from typing import Annotated, ClassVar, Literal from pydantic import AnyUrl, BeforeValidator @@ -124,6 +124,19 @@ class DMCC13Benchmark(PatternDataladDataGrabber): """ + _dump_exclude: ClassVar[set[str]] = { + "patterns", + "replacements", + "confounds_format", + "partial_pattern_ok", + "uri", + "rootdir", + "datadir", + "datalad_id", + "datalad_dirty", + "datalad_commit_id", + } + uri: AnyUrl = AnyUrl("https://github.com/OpenNeuroDatasets/ds003452.git") types: Annotated[_types | list[_types], BeforeValidator(ensure_list)] = [ # noqa: RUF012 DataType.BOLD, diff --git a/junifer/datagrabber/hcp1200/datalad_hcp1200.py b/junifer/datagrabber/hcp1200/datalad_hcp1200.py index 8a6abee6a..5ae2a1396 100644 --- a/junifer/datagrabber/hcp1200/datalad_hcp1200.py +++ b/junifer/datagrabber/hcp1200/datalad_hcp1200.py @@ -6,7 +6,7 @@ # License: AGPL from pathlib import Path -from typing import Annotated, Literal +from typing import Annotated, ClassVar, Literal from pydantic import AnyUrl, BeforeValidator @@ -50,6 +50,19 @@ class DataladHCP1200(DataladDataGrabber, HCP1200): """ + _dump_exclude: ClassVar[set[str]] = { + "patterns", + "replacements", + "confounds_format", + "partial_pattern_ok", + "uri", + "rootdir", + "datadir", + "datalad_id", + "datalad_dirty", + "datalad_commit_id", + } + uri: AnyUrl = AnyUrl( "https://github.com/datalad-datasets/" "human-connectome-project-openaccess.git" diff --git a/junifer/datagrabber/pattern.py b/junifer/datagrabber/pattern.py index 3996faa3c..d398aa948 100644 --- a/junifer/datagrabber/pattern.py +++ b/junifer/datagrabber/pattern.py @@ -8,6 +8,7 @@ import re from copy import deepcopy from pathlib import Path +from typing import ClassVar import numpy as np from aenum import Enum as AEnum @@ -82,6 +83,13 @@ class PatternDataGrabber(BaseDataGrabber, PatternValidationMixin): """ + _dump_exclude: ClassVar[set[str]] = { + "patterns", + "replacements", + "confounds_format", + "partial_pattern_ok", + } + patterns: DataGrabberPatterns = Field(frozen=True) replacements: list[str] = Field(frozen=True) confounds_format: ConfoundsFormat | None = Field(None, frozen=True) diff --git a/junifer/datagrabber/pattern_datalad.py b/junifer/datagrabber/pattern_datalad.py index 9864591af..8e0861c0e 100644 --- a/junifer/datagrabber/pattern_datalad.py +++ b/junifer/datagrabber/pattern_datalad.py @@ -5,6 +5,8 @@ # Synchon Mandal # License: AGPL +from typing import ClassVar + from pydantic import ConfigDict from ..api.decorators import register_datagrabber @@ -53,6 +55,14 @@ class PatternDataladDataGrabber(DataladDataGrabber, PatternDataGrabber): """ + _dump_exclude: ClassVar[set[str]] = { + "uri", + "datadir", + "datalad_dirty", + "datalad_commit_id", + "datalad_id", + } + model_config = ConfigDict(extra="allow") def validate_datagrabber_params(self) -> None: From 5087add4bf87c45c2736278605f34c6fdf8e3ebd Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 29 May 2026 18:02:13 +0200 Subject: [PATCH 03/21] docs: add generate_yaml documentation --- docs/links.inc | 1 + docs/using/generate_yaml.rst | 56 ++++++++++++++++++++++++++++++++++++ docs/using/index.rst | 1 + 3 files changed, 58 insertions(+) create mode 100644 docs/using/generate_yaml.rst diff --git a/docs/links.inc b/docs/links.inc index dfc113e5b..fcaf3ebad 100644 --- a/docs/links.inc +++ b/docs/links.inc @@ -13,6 +13,7 @@ .. _`INM-7`: https://www.fz-juelich.de/inm/inm-7/EN/Home/home_node.html .. _`julearn`: https://juaml.github.io/julearn .. _`junifer-data`: https://github.com/juaml/junifer-data-client +.. _`julio`: https://github.com/juaml/julio .. _`pandas`: https://pandas.pydata.org .. _`pandas.DataFrame` : https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.html diff --git a/docs/using/generate_yaml.rst b/docs/using/generate_yaml.rst new file mode 100644 index 000000000..ba9dcb1f4 --- /dev/null +++ b/docs/using/generate_yaml.rst @@ -0,0 +1,56 @@ +.. include:: ../links.inc + +.. _generate_yaml: + +Generating YAML from metadata +============================= + +``junifer`` stores the pipeline metadata for a run along with the extracted feature data. +So, the metadata for all the "elements" processed with a pipeline is unique. The metadata +contains all the necessary information to recreate the configuration used for the processing. + +If one wants to generate the processing YAML, :func:`.generate_yaml` can be used for that. +The only requirement is providing the metadata which can be extracted by following the initial steps of +:ref:`analysing results `. + + +Configuration for ``julio`` +--------------------------- + +When generating a registry with `julio`_, we can configure the YAML generation process. For now, only +DataGrabbers can be configured through the use of ``_dump_exclude`` class variable, like so: + + .. code-block:: python + + from typing import ClassVar + + from junifer.api.decorators import register_datagrabber + from junifer.datagrabber import PatternDataladDataGrabber + + + @register_datagrabber + class MyDataGrabber(PatternDataladDataGrabber): + + _dump_exclude: ClassVar[set[str]] = { + "patterns", + "replacements", + "confounds_format", + "partial_pattern_ok", + "uri", + "rootdir", + "datadir", + "datalad_id", + "datalad_dirty", + "datalad_commit_id", + } + + +The above can be considered a standard setup for a custom DataGrabber inheriting from :class:`.PatternDataladDataGrabber`. + + +.. admonition:: Tip + + - For DataGrabbers inheriting from :class:`.BaseDataGrabber` custom setup is possible but not required. + - For DataGrabbers inheriting from :class:`.PatternDataGrabber` no extra setup should be required. + - For :class:`.PatternDataladDataGrabber`\s specified via the YAML, it is not possible + to customise and is usually not required. If such a need arises, creating a custom DataGrabber is the only way. diff --git a/docs/using/index.rst b/docs/using/index.rst index 338d9e72c..a1fac87fe 100644 --- a/docs/using/index.rst +++ b/docs/using/index.rst @@ -20,6 +20,7 @@ to interact with HPC and HTC systems. queueing configuring dumping + generate_yaml .. _using_components: From 1d2e2f6b98b9c8254b9d4b20d48a4304adfdafd0 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 29 May 2026 18:05:54 +0200 Subject: [PATCH 04/21] chore: add changelog 498.feature --- docs/changes/newsfragments/498.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 docs/changes/newsfragments/498.feature diff --git a/docs/changes/newsfragments/498.feature b/docs/changes/newsfragments/498.feature new file mode 100644 index 000000000..c7a14e19e --- /dev/null +++ b/docs/changes/newsfragments/498.feature @@ -0,0 +1 @@ +Introduce :func:`.generate_yaml` to generate feature YAML from metadata by `Synchon Mandal`_ From 3daca38c467e08e30e15465a2866c036167337b7 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 8 Jun 2026 14:48:51 +0200 Subject: [PATCH 05/21] Revert "feat: add _dump_exclude class variables to datagrabbers" This reverts commit 99aab929f9250fdea607643c9424a8d43f72af3c. --- .../juseless/datagrabbers/aomic_id1000_vbm.py | 15 +-------------- .../configs/juseless/datagrabbers/camcan_vbm.py | 15 +-------------- junifer/configs/juseless/datagrabbers/ixi_vbm.py | 13 ------------- junifer/configs/juseless/datagrabbers/ukb_vbm.py | 15 +-------------- junifer/datagrabber/aomic/id1000.py | 15 +-------------- junifer/datagrabber/aomic/piop1.py | 15 +-------------- junifer/datagrabber/aomic/piop2.py | 15 +-------------- junifer/datagrabber/dmcc13_benchmark.py | 15 +-------------- junifer/datagrabber/hcp1200/datalad_hcp1200.py | 15 +-------------- junifer/datagrabber/pattern.py | 8 -------- junifer/datagrabber/pattern_datalad.py | 10 ---------- 11 files changed, 8 insertions(+), 143 deletions(-) diff --git a/junifer/configs/juseless/datagrabbers/aomic_id1000_vbm.py b/junifer/configs/juseless/datagrabbers/aomic_id1000_vbm.py index 6e40f2dc0..3555fc05d 100644 --- a/junifer/configs/juseless/datagrabbers/aomic_id1000_vbm.py +++ b/junifer/configs/juseless/datagrabbers/aomic_id1000_vbm.py @@ -4,7 +4,7 @@ # Synchon Mandal # License: AGPL -from typing import ClassVar, Literal +from typing import Literal from pydantic import AnyUrl @@ -31,19 +31,6 @@ class JuselessDataladAOMICID1000VBM(PatternDataladDataGrabber): """ - _dump_exclude: ClassVar[set[str]] = { - "patterns", - "replacements", - "confounds_format", - "partial_pattern_ok", - "uri", - "rootdir", - "datadir", - "datalad_id", - "datalad_dirty", - "datalad_commit_id", - } - uri: AnyUrl = AnyUrl("https://gin.g-node.org/felixh/ds003097_ReproVBM") types: list[Literal[DataType.VBM_GM]] = [DataType.VBM_GM] # noqa: RUF012 patterns: DataGrabberPatterns = { # noqa: RUF012 diff --git a/junifer/configs/juseless/datagrabbers/camcan_vbm.py b/junifer/configs/juseless/datagrabbers/camcan_vbm.py index 9342796f0..85fccec6f 100644 --- a/junifer/configs/juseless/datagrabbers/camcan_vbm.py +++ b/junifer/configs/juseless/datagrabbers/camcan_vbm.py @@ -5,7 +5,7 @@ # Synchon Mandal # License: AGPL -from typing import ClassVar, Literal +from typing import Literal from pydantic import AnyUrl @@ -32,19 +32,6 @@ class JuselessDataladCamCANVBM(PatternDataladDataGrabber): """ - _dump_exclude: ClassVar[set[str]] = { - "patterns", - "replacements", - "confounds_format", - "partial_pattern_ok", - "uri", - "rootdir", - "datadir", - "datalad_id", - "datalad_dirty", - "datalad_commit_id", - } - uri: AnyUrl = AnyUrl( "ria+http://cat_12.5.ds.inm7.de#a139b26a-8406-11ea-8f94-a0369f287950" ) diff --git a/junifer/configs/juseless/datagrabbers/ixi_vbm.py b/junifer/configs/juseless/datagrabbers/ixi_vbm.py index 0f58308e3..ee1955e39 100644 --- a/junifer/configs/juseless/datagrabbers/ixi_vbm.py +++ b/junifer/configs/juseless/datagrabbers/ixi_vbm.py @@ -48,19 +48,6 @@ class JuselessDataladIXIVBM(PatternDataladDataGrabber): """ - _dump_exclude: ClassVar[set[str]] = { - "patterns", - "replacements", - "confounds_format", - "partial_pattern_ok", - "uri", - "rootdir", - "datadir", - "datalad_id", - "datalad_dirty", - "datalad_commit_id", - } - uri: AnyUrl = AnyUrl( "ria+http://cat_12.5.ds.inm7.de#b7107c52-8408-11ea-89c6-a0369f287950" ) diff --git a/junifer/configs/juseless/datagrabbers/ukb_vbm.py b/junifer/configs/juseless/datagrabbers/ukb_vbm.py index 369b94f1c..1f05a8bb8 100644 --- a/junifer/configs/juseless/datagrabbers/ukb_vbm.py +++ b/junifer/configs/juseless/datagrabbers/ukb_vbm.py @@ -6,7 +6,7 @@ # License: AGPL from pathlib import Path -from typing import ClassVar, Literal +from typing import Literal from pydantic import AnyUrl @@ -33,19 +33,6 @@ class JuselessDataladUKBVBM(PatternDataladDataGrabber): """ - _dump_exclude: ClassVar[set[str]] = { - "patterns", - "replacements", - "confounds_format", - "partial_pattern_ok", - "uri", - "rootdir", - "datadir", - "datalad_id", - "datalad_dirty", - "datalad_commit_id", - } - uri: AnyUrl = AnyUrl("ria+http://ukb.ds.inm7.de#~cat_m0wp1") rootdir: Path = Path("m0wp1") types: list[Literal[DataType.VBM_GM]] = [DataType.VBM_GM] # noqa: RUF012 diff --git a/junifer/datagrabber/aomic/id1000.py b/junifer/datagrabber/aomic/id1000.py index e1ae2e6e1..2a1fd2349 100644 --- a/junifer/datagrabber/aomic/id1000.py +++ b/junifer/datagrabber/aomic/id1000.py @@ -7,7 +7,7 @@ # Synchon Mandal # License: AGPL -from typing import Annotated, ClassVar, Literal +from typing import Annotated, Literal from pydantic import AnyUrl, BeforeValidator @@ -52,19 +52,6 @@ class DataladAOMICID1000(PatternDataladDataGrabber): """ - _dump_exclude: ClassVar[set[str]] = { - "patterns", - "replacements", - "confounds_format", - "partial_pattern_ok", - "uri", - "rootdir", - "datadir", - "datalad_id", - "datalad_dirty", - "datalad_commit_id", - } - uri: AnyUrl = AnyUrl("https://github.com/OpenNeuroDatasets/ds003097.git") types: Annotated[_types | list[_types], BeforeValidator(ensure_list)] = [ # noqa: RUF012 DataType.BOLD, diff --git a/junifer/datagrabber/aomic/piop1.py b/junifer/datagrabber/aomic/piop1.py index 34a2d7f59..cac65e01f 100644 --- a/junifer/datagrabber/aomic/piop1.py +++ b/junifer/datagrabber/aomic/piop1.py @@ -8,7 +8,7 @@ # License: AGPL from itertools import product -from typing import Annotated, ClassVar, Literal +from typing import Annotated, Literal from pydantic import AnyUrl, BeforeValidator @@ -66,19 +66,6 @@ class DataladAOMICPIOP1(PatternDataladDataGrabber): """ - _dump_exclude: ClassVar[set[str]] = { - "patterns", - "replacements", - "confounds_format", - "partial_pattern_ok", - "uri", - "rootdir", - "datadir", - "datalad_id", - "datalad_dirty", - "datalad_commit_id", - } - uri: AnyUrl = AnyUrl("https://github.com/OpenNeuroDatasets/ds002785") types: Annotated[_types | list[_types], BeforeValidator(ensure_list)] = [ # noqa: RUF012 DataType.BOLD, diff --git a/junifer/datagrabber/aomic/piop2.py b/junifer/datagrabber/aomic/piop2.py index 56d8b94ef..1fab2a0a8 100644 --- a/junifer/datagrabber/aomic/piop2.py +++ b/junifer/datagrabber/aomic/piop2.py @@ -8,7 +8,7 @@ # License: AGPL from itertools import product -from typing import Annotated, ClassVar, Literal +from typing import Annotated, Literal from pydantic import AnyUrl, BeforeValidator @@ -64,19 +64,6 @@ class DataladAOMICPIOP2(PatternDataladDataGrabber): """ - _dump_exclude: ClassVar[set[str]] = { - "patterns", - "replacements", - "confounds_format", - "partial_pattern_ok", - "uri", - "rootdir", - "datadir", - "datalad_id", - "datalad_dirty", - "datalad_commit_id", - } - uri: AnyUrl = AnyUrl("https://github.com/OpenNeuroDatasets/ds002790") types: Annotated[_types | list[_types], BeforeValidator(ensure_list)] = [ # noqa: RUF012 DataType.BOLD, diff --git a/junifer/datagrabber/dmcc13_benchmark.py b/junifer/datagrabber/dmcc13_benchmark.py index 0e7f54ba5..e726b97c7 100644 --- a/junifer/datagrabber/dmcc13_benchmark.py +++ b/junifer/datagrabber/dmcc13_benchmark.py @@ -5,7 +5,7 @@ from enum import Enum from itertools import product -from typing import Annotated, ClassVar, Literal +from typing import Annotated, Literal from pydantic import AnyUrl, BeforeValidator @@ -124,19 +124,6 @@ class DMCC13Benchmark(PatternDataladDataGrabber): """ - _dump_exclude: ClassVar[set[str]] = { - "patterns", - "replacements", - "confounds_format", - "partial_pattern_ok", - "uri", - "rootdir", - "datadir", - "datalad_id", - "datalad_dirty", - "datalad_commit_id", - } - uri: AnyUrl = AnyUrl("https://github.com/OpenNeuroDatasets/ds003452.git") types: Annotated[_types | list[_types], BeforeValidator(ensure_list)] = [ # noqa: RUF012 DataType.BOLD, diff --git a/junifer/datagrabber/hcp1200/datalad_hcp1200.py b/junifer/datagrabber/hcp1200/datalad_hcp1200.py index 5ae2a1396..8a6abee6a 100644 --- a/junifer/datagrabber/hcp1200/datalad_hcp1200.py +++ b/junifer/datagrabber/hcp1200/datalad_hcp1200.py @@ -6,7 +6,7 @@ # License: AGPL from pathlib import Path -from typing import Annotated, ClassVar, Literal +from typing import Annotated, Literal from pydantic import AnyUrl, BeforeValidator @@ -50,19 +50,6 @@ class DataladHCP1200(DataladDataGrabber, HCP1200): """ - _dump_exclude: ClassVar[set[str]] = { - "patterns", - "replacements", - "confounds_format", - "partial_pattern_ok", - "uri", - "rootdir", - "datadir", - "datalad_id", - "datalad_dirty", - "datalad_commit_id", - } - uri: AnyUrl = AnyUrl( "https://github.com/datalad-datasets/" "human-connectome-project-openaccess.git" diff --git a/junifer/datagrabber/pattern.py b/junifer/datagrabber/pattern.py index d398aa948..3996faa3c 100644 --- a/junifer/datagrabber/pattern.py +++ b/junifer/datagrabber/pattern.py @@ -8,7 +8,6 @@ import re from copy import deepcopy from pathlib import Path -from typing import ClassVar import numpy as np from aenum import Enum as AEnum @@ -83,13 +82,6 @@ class PatternDataGrabber(BaseDataGrabber, PatternValidationMixin): """ - _dump_exclude: ClassVar[set[str]] = { - "patterns", - "replacements", - "confounds_format", - "partial_pattern_ok", - } - patterns: DataGrabberPatterns = Field(frozen=True) replacements: list[str] = Field(frozen=True) confounds_format: ConfoundsFormat | None = Field(None, frozen=True) diff --git a/junifer/datagrabber/pattern_datalad.py b/junifer/datagrabber/pattern_datalad.py index 8e0861c0e..9864591af 100644 --- a/junifer/datagrabber/pattern_datalad.py +++ b/junifer/datagrabber/pattern_datalad.py @@ -5,8 +5,6 @@ # Synchon Mandal # License: AGPL -from typing import ClassVar - from pydantic import ConfigDict from ..api.decorators import register_datagrabber @@ -55,14 +53,6 @@ class PatternDataladDataGrabber(DataladDataGrabber, PatternDataGrabber): """ - _dump_exclude: ClassVar[set[str]] = { - "uri", - "datadir", - "datalad_dirty", - "datalad_commit_id", - "datalad_id", - } - model_config = ConfigDict(extra="allow") def validate_datagrabber_params(self) -> None: From f94ce523d5913bed313bd8011a4f1994a395d2fd Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 8 Jun 2026 16:34:06 +0200 Subject: [PATCH 06/21] update: improve generate_yaml datagrabber module dump exclusion --- junifer/api/functions.py | 43 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/junifer/api/functions.py b/junifer/api/functions.py index e26e25771..2d7aaa946 100644 --- a/junifer/api/functions.py +++ b/junifer/api/functions.py @@ -610,6 +610,45 @@ def parse_yaml(filepath: str | Path) -> dict: # noqa: C901 return contents +def _dg_dump_exclude(dg: str) -> set[str]: + """Generate datagrabber model dump exclusion set. + + Parameters + ---------- + dg : str + The datagrabber kind. + + Returns + ------- + set of str + + """ + if dg == "PatternDataGrabber": + return set() + elif dg == "PatternDataladDataGrabber": + return { + "datadir", + "datalad_dirty", + "datalad_commit_id", + "datalad_id", + } + else: + return { + # from PatternDataGrabber + "patterns", + "replacements", + "confounds_format", + "partial_pattern_ok", + # from DataladDataGrabber + "uri", + "rootdir", + "datadir", + "datalad_dirty", + "datalad_commit_id", + "datalad_id", + } + + def generate_yaml(meta: dict) -> "CommentedMap": """Generate the feature YAML from metadata. @@ -638,9 +677,7 @@ def generate_yaml(meta: dict) -> "CommentedMap": "kind": a, **dg_model.model_dump( mode="json", - exclude=dg_model._dump_exclude - if hasattr(dg_model, "_dump_exclude") - else {}, + exclude=_dg_dump_exclude(a), exclude_defaults=True, exclude_none=True, ), From 13862783a4986f5ea6223507aa24f7d97294fa8d Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Mon, 8 Jun 2026 16:40:15 +0200 Subject: [PATCH 07/21] docs: update generate_yaml.rst --- docs/using/generate_yaml.rst | 42 ------------------------------------ 1 file changed, 42 deletions(-) diff --git a/docs/using/generate_yaml.rst b/docs/using/generate_yaml.rst index ba9dcb1f4..ddbe68c38 100644 --- a/docs/using/generate_yaml.rst +++ b/docs/using/generate_yaml.rst @@ -12,45 +12,3 @@ contains all the necessary information to recreate the configuration used for th If one wants to generate the processing YAML, :func:`.generate_yaml` can be used for that. The only requirement is providing the metadata which can be extracted by following the initial steps of :ref:`analysing results `. - - -Configuration for ``julio`` ---------------------------- - -When generating a registry with `julio`_, we can configure the YAML generation process. For now, only -DataGrabbers can be configured through the use of ``_dump_exclude`` class variable, like so: - - .. code-block:: python - - from typing import ClassVar - - from junifer.api.decorators import register_datagrabber - from junifer.datagrabber import PatternDataladDataGrabber - - - @register_datagrabber - class MyDataGrabber(PatternDataladDataGrabber): - - _dump_exclude: ClassVar[set[str]] = { - "patterns", - "replacements", - "confounds_format", - "partial_pattern_ok", - "uri", - "rootdir", - "datadir", - "datalad_id", - "datalad_dirty", - "datalad_commit_id", - } - - -The above can be considered a standard setup for a custom DataGrabber inheriting from :class:`.PatternDataladDataGrabber`. - - -.. admonition:: Tip - - - For DataGrabbers inheriting from :class:`.BaseDataGrabber` custom setup is possible but not required. - - For DataGrabbers inheriting from :class:`.PatternDataGrabber` no extra setup should be required. - - For :class:`.PatternDataladDataGrabber`\s specified via the YAML, it is not possible - to customise and is usually not required. If such a need arises, creating a custom DataGrabber is the only way. From 07ba047844c9d46f238b244d4d5374846155a772 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 21 Jul 2026 14:48:03 +0200 Subject: [PATCH 08/21] update: add dump_fields class method for datagrabber fields dumping --- junifer/api/functions.py | 41 +------------------ .../juseless/datagrabbers/aomic_id1000_vbm.py | 5 +++ .../juseless/datagrabbers/camcan_vbm.py | 5 +++ .../configs/juseless/datagrabbers/ixi_vbm.py | 5 +++ junifer/configs/juseless/datagrabbers/ucla.py | 5 +++ .../configs/juseless/datagrabbers/ukb_vbm.py | 5 +++ junifer/datagrabber/aomic/id1000.py | 5 +++ junifer/datagrabber/aomic/piop1.py | 5 +++ junifer/datagrabber/aomic/piop2.py | 5 +++ junifer/datagrabber/base.py | 5 +++ junifer/datagrabber/datalad_base.py | 5 +++ junifer/datagrabber/dmcc13_benchmark.py | 12 ++++++ .../datagrabber/hcp1200/datalad_hcp1200.py | 5 +++ junifer/datagrabber/hcp1200/hcp1200.py | 5 +++ junifer/datagrabber/multiple.py | 5 +++ junifer/datagrabber/pattern.py | 11 +++++ junifer/datagrabber/pattern_datalad.py | 13 ++++++ junifer/testing/datagrabbers.py | 15 +++++++ 18 files changed, 117 insertions(+), 40 deletions(-) diff --git a/junifer/api/functions.py b/junifer/api/functions.py index 2d7aaa946..f095b9d09 100644 --- a/junifer/api/functions.py +++ b/junifer/api/functions.py @@ -610,45 +610,6 @@ def parse_yaml(filepath: str | Path) -> dict: # noqa: C901 return contents -def _dg_dump_exclude(dg: str) -> set[str]: - """Generate datagrabber model dump exclusion set. - - Parameters - ---------- - dg : str - The datagrabber kind. - - Returns - ------- - set of str - - """ - if dg == "PatternDataGrabber": - return set() - elif dg == "PatternDataladDataGrabber": - return { - "datadir", - "datalad_dirty", - "datalad_commit_id", - "datalad_id", - } - else: - return { - # from PatternDataGrabber - "patterns", - "replacements", - "confounds_format", - "partial_pattern_ok", - # from DataladDataGrabber - "uri", - "rootdir", - "datadir", - "datalad_dirty", - "datalad_commit_id", - "datalad_id", - } - - def generate_yaml(meta: dict) -> "CommentedMap": """Generate the feature YAML from metadata. @@ -677,7 +638,7 @@ def generate_yaml(meta: dict) -> "CommentedMap": "kind": a, **dg_model.model_dump( mode="json", - exclude=_dg_dump_exclude(a), + include=set(dg_model.dump_fields()), exclude_defaults=True, exclude_none=True, ), diff --git a/junifer/configs/juseless/datagrabbers/aomic_id1000_vbm.py b/junifer/configs/juseless/datagrabbers/aomic_id1000_vbm.py index 3555fc05d..2bdc89fb1 100644 --- a/junifer/configs/juseless/datagrabbers/aomic_id1000_vbm.py +++ b/junifer/configs/juseless/datagrabbers/aomic_id1000_vbm.py @@ -40,3 +40,8 @@ class JuselessDataladAOMICID1000VBM(PatternDataladDataGrabber): }, } replacements: list[str] = ["subject"] # noqa: RUF012 + + @classmethod + def dump_fields(cls) -> list[str]: + """Fields to include when dumping model.""" + return [] diff --git a/junifer/configs/juseless/datagrabbers/camcan_vbm.py b/junifer/configs/juseless/datagrabbers/camcan_vbm.py index 85fccec6f..ce609af9d 100644 --- a/junifer/configs/juseless/datagrabbers/camcan_vbm.py +++ b/junifer/configs/juseless/datagrabbers/camcan_vbm.py @@ -43,3 +43,8 @@ class JuselessDataladCamCANVBM(PatternDataladDataGrabber): }, } replacements: list[str] = ["subject"] # noqa: RUF012 + + @classmethod + def dump_fields(cls) -> list[str]: + """Fields to include when dumping model.""" + return [] diff --git a/junifer/configs/juseless/datagrabbers/ixi_vbm.py b/junifer/configs/juseless/datagrabbers/ixi_vbm.py index ee1955e39..898494d16 100644 --- a/junifer/configs/juseless/datagrabbers/ixi_vbm.py +++ b/junifer/configs/juseless/datagrabbers/ixi_vbm.py @@ -66,3 +66,8 @@ class JuselessDataladIXIVBM(PatternDataladDataGrabber): }, } replacements: list[str] = ["site", "subject"] # noqa: RUF012 + + @classmethod + def dump_fields(cls) -> list[str]: + """Fields to include when dumping model.""" + return [] diff --git a/junifer/configs/juseless/datagrabbers/ucla.py b/junifer/configs/juseless/datagrabbers/ucla.py index 1e46b1f95..7d90b3df6 100644 --- a/junifer/configs/juseless/datagrabbers/ucla.py +++ b/junifer/configs/juseless/datagrabbers/ucla.py @@ -143,6 +143,11 @@ class JuselessUCLA(PatternDataGrabber): replacements: list[str] = ["subject", "task"] # noqa: RUF012 confounds_format: ConfoundsFormat = ConfoundsFormat.FMRIPrep + @classmethod + def dump_fields(cls) -> list[str]: + """Fields to include when dumping model.""" + return [*super().dump_fields(), "tasks"] + def get_elements(self) -> list: """Implement fetching list of elements in the dataset. diff --git a/junifer/configs/juseless/datagrabbers/ukb_vbm.py b/junifer/configs/juseless/datagrabbers/ukb_vbm.py index 1f05a8bb8..5852cc5c4 100644 --- a/junifer/configs/juseless/datagrabbers/ukb_vbm.py +++ b/junifer/configs/juseless/datagrabbers/ukb_vbm.py @@ -43,3 +43,8 @@ class JuselessDataladUKBVBM(PatternDataladDataGrabber): }, } replacements: list[str] = ["subject", "session"] # noqa: RUF012 + + @classmethod + def dump_fields(cls) -> list[str]: + """Fields to include when dumping model.""" + return [] diff --git a/junifer/datagrabber/aomic/id1000.py b/junifer/datagrabber/aomic/id1000.py index 2a1fd2349..135b363e5 100644 --- a/junifer/datagrabber/aomic/id1000.py +++ b/junifer/datagrabber/aomic/id1000.py @@ -243,3 +243,8 @@ def validate_datagrabber_params(self) -> None: else: self.patterns["BOLD"]["prewarp_space"] = "native" super().validate_datagrabber_params() + + @classmethod + def dump_fields(cls) -> list[str]: + """Fields to include when dumping model.""" + return ["types", "space"] diff --git a/junifer/datagrabber/aomic/piop1.py b/junifer/datagrabber/aomic/piop1.py index cac65e01f..b5935ea77 100644 --- a/junifer/datagrabber/aomic/piop1.py +++ b/junifer/datagrabber/aomic/piop1.py @@ -266,6 +266,11 @@ def validate_datagrabber_params(self) -> None: self.patterns["BOLD"]["prewarp_space"] = "native" super().validate_datagrabber_params() + @classmethod + def dump_fields(cls) -> list[str]: + """Fields to include when dumping model.""" + return ["types", "tasks", "space"] + def get_item(self, subject: str, task: str) -> dict: """Get the specified item from the dataset. diff --git a/junifer/datagrabber/aomic/piop2.py b/junifer/datagrabber/aomic/piop2.py index 1fab2a0a8..bf52f7cb2 100644 --- a/junifer/datagrabber/aomic/piop2.py +++ b/junifer/datagrabber/aomic/piop2.py @@ -262,6 +262,11 @@ def validate_datagrabber_params(self) -> None: self.patterns["BOLD"]["prewarp_space"] = "native" super().validate_datagrabber_params() + @classmethod + def dump_fields(cls) -> list[str]: + """Fields to include when dumping model.""" + return ["types", "tasks", "space"] + def get_elements(self) -> list: """Implement fetching list of elements in the dataset. diff --git a/junifer/datagrabber/base.py b/junifer/datagrabber/base.py index 4fb3796a6..41252c550 100644 --- a/junifer/datagrabber/base.py +++ b/junifer/datagrabber/base.py @@ -85,6 +85,11 @@ def validate_datagrabber_params(self) -> None: """ pass + @classmethod + def dump_fields(cls) -> list[str]: + """Fields to include when dumping model.""" + return ["types", "datadir"] + def __iter__(self) -> Iterator[Elements]: """Enable iterable support. diff --git a/junifer/datagrabber/datalad_base.py b/junifer/datagrabber/datalad_base.py index 9c28fa4a7..b6b8e5eff 100644 --- a/junifer/datagrabber/datalad_base.py +++ b/junifer/datagrabber/datalad_base.py @@ -176,6 +176,11 @@ def __del__(self) -> None: ) and self.datadir.stem.endswith("juniferauto"): _remove_datadir(self.datadir) + @classmethod + def dump_fields(cls) -> list[str]: + """Fields to include when dumping model.""" + return ["types", "uri", "rootdir"] + @property def fulldir(self) -> Path: """Get complete data directory path. diff --git a/junifer/datagrabber/dmcc13_benchmark.py b/junifer/datagrabber/dmcc13_benchmark.py index e726b97c7..e2cbd6a7d 100644 --- a/junifer/datagrabber/dmcc13_benchmark.py +++ b/junifer/datagrabber/dmcc13_benchmark.py @@ -276,6 +276,18 @@ def validate_datagrabber_params(self) -> None: self.types.append(DataType.Warp) super().validate_datagrabber_params() + @classmethod + def dump_fields(cls) -> list[str]: + """Fields to include when dumping model.""" + return [ + "types", + "sessions", + "tasks", + "phase_encodings", + "runs", + "native_t1w", + ] + def get_item( self, subject: str, diff --git a/junifer/datagrabber/hcp1200/datalad_hcp1200.py b/junifer/datagrabber/hcp1200/datalad_hcp1200.py index 8a6abee6a..716314c17 100644 --- a/junifer/datagrabber/hcp1200/datalad_hcp1200.py +++ b/junifer/datagrabber/hcp1200/datalad_hcp1200.py @@ -61,6 +61,11 @@ class DataladHCP1200(DataladDataGrabber, HCP1200): ] rootdir: Path = Path("HCP1200") + @classmethod + def dump_fields(cls) -> list[str]: + """Fields to include when dumping model.""" + return ["types", "tasks", "phase_encodings", "ica_fix"] + # Needed here as HCP1200's subjects are sub-datasets, so will not be # found when elements are checked. @property diff --git a/junifer/datagrabber/hcp1200/hcp1200.py b/junifer/datagrabber/hcp1200/hcp1200.py index 5ea9c5ba8..bb85a781f 100644 --- a/junifer/datagrabber/hcp1200/hcp1200.py +++ b/junifer/datagrabber/hcp1200/hcp1200.py @@ -159,6 +159,11 @@ def validate_datagrabber_params(self) -> None: ].replace("{suffix}", suffix) super().validate_datagrabber_params() + @classmethod + def dump_fields(cls) -> list[str]: + """Fields to include when dumping model.""" + return ["types", "datadir", "tasks", "phase_encodings", "ica_fix"] + def get_item(self, subject: str, task: str, phase_encoding: str) -> dict: """Get the specified item from the dataset. diff --git a/junifer/datagrabber/multiple.py b/junifer/datagrabber/multiple.py index e2bb4eebd..76d7235b3 100644 --- a/junifer/datagrabber/multiple.py +++ b/junifer/datagrabber/multiple.py @@ -94,6 +94,11 @@ def validate_datagrabber_params(self) -> None: klass=RuntimeError, ) + @classmethod + def dump_fields(cls) -> list[str]: + """Fields to include when dumping model.""" + return [*super().dump_fields(), "datagrabbers"] + def __getitem__(self, element: Element) -> dict: """Implement indexing. diff --git a/junifer/datagrabber/pattern.py b/junifer/datagrabber/pattern.py index 3996faa3c..392ab4741 100644 --- a/junifer/datagrabber/pattern.py +++ b/junifer/datagrabber/pattern.py @@ -101,6 +101,17 @@ def validate_datagrabber_params(self) -> None: logger.debug(f"\treplacements = {self.replacements}") logger.debug(f"\tconfounds_format = {self.confounds_format}") + @classmethod + def dump_fields(cls) -> list[str]: + """Fields to include when dumping model.""" + return [ + *super().dump_fields(), + "patterns", + "replacements", + "confounds_format", + "partial_pattern_ok", + ] + @property def skip_file_check(self) -> bool: """Skip file check existence.""" diff --git a/junifer/datagrabber/pattern_datalad.py b/junifer/datagrabber/pattern_datalad.py index 9864591af..3492cff82 100644 --- a/junifer/datagrabber/pattern_datalad.py +++ b/junifer/datagrabber/pattern_datalad.py @@ -61,3 +61,16 @@ def validate_datagrabber_params(self) -> None: logger.debug("Initializing PatternDataladDataGrabber") for key, val in self.__pydantic_extra__.items(): logger.debug(f"\t{key} = {val}") + + @classmethod + def dump_fields(cls) -> list[str]: + """Fields to include when dumping model.""" + return [ + "types", + "patterns", + "replacements", + "confounds_format", + "partial_pattern_ok", + "uri", + "rootdir", + ] diff --git a/junifer/testing/datagrabbers.py b/junifer/testing/datagrabbers.py index 61ada8413..c84976f06 100644 --- a/junifer/testing/datagrabbers.py +++ b/junifer/testing/datagrabbers.py @@ -34,6 +34,11 @@ class OasisVBMTestingDataGrabber(BaseDataGrabber): datadir: Path = Path(tempfile.mkdtemp()) _dataset: Any = None + @classmethod + def dump_fields(cls) -> list[str]: + """Fields to include when dumping model.""" + return ["types"] + def get_element_keys(self) -> list[str]: """Get element keys. @@ -101,6 +106,11 @@ class SPMAuditoryTestingDataGrabber(BaseDataGrabber): types: list[DataType] = [DataType.BOLD, DataType.T1w] # noqa: RUF012 datadir: Path = Path(tempfile.mkdtemp()) + @classmethod + def dump_fields(cls) -> list[str]: + """Fields to include when dumping model.""" + return ["types"] + def get_element_keys(self) -> list[str]: """Get element keys. @@ -189,6 +199,11 @@ class PartlyCloudyTestingDataGrabber(BaseDataGrabber): reduce_confounds: bool = True age_group: PartlyCloudyAgeGroup = PartlyCloudyAgeGroup.Both + @classmethod + def dump_fields(cls) -> list[str]: + """Fields to include when dumping model.""" + return ["types", "reduce_confounds", "age_group"] + def __enter__(self) -> "PartlyCloudyTestingDataGrabber": """Implement context entry. From 5fe3936cf22d7534db59680d0ae538c8f3d71991 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Tue, 21 Jul 2026 17:18:27 +0200 Subject: [PATCH 09/21] chore: correct dump_fields for JuselessUCLA --- junifer/configs/juseless/datagrabbers/ucla.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/junifer/configs/juseless/datagrabbers/ucla.py b/junifer/configs/juseless/datagrabbers/ucla.py index 7d90b3df6..d9ddc1eba 100644 --- a/junifer/configs/juseless/datagrabbers/ucla.py +++ b/junifer/configs/juseless/datagrabbers/ucla.py @@ -146,7 +146,7 @@ class JuselessUCLA(PatternDataGrabber): @classmethod def dump_fields(cls) -> list[str]: """Fields to include when dumping model.""" - return [*super().dump_fields(), "tasks"] + return ["types", "tasks"] def get_elements(self) -> list: """Implement fetching list of elements in the dataset. From acad9c2ef35aa7bb1bd30bc0d952d550aa951a9e Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Wed, 22 Jul 2026 11:21:43 +0200 Subject: [PATCH 10/21] update: add comments about datadir to generate_yaml output --- junifer/api/functions.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/junifer/api/functions.py b/junifer/api/functions.py index f095b9d09..f55b0fa6f 100644 --- a/junifer/api/functions.py +++ b/junifer/api/functions.py @@ -708,6 +708,13 @@ def generate_yaml(meta: dict) -> "CommentedMap": if "dependencies" in meta: for k, v in meta["dependencies"].items(): pre += f"{k}=={v}\n" + pre += ( + "\n`datadir` is ignored and not reproduced.\n" + "If `datadir` used was not a temporary directory, you will have to " + "manually edit this YAML.\n" + "In case the dataset was 'dirty', there is no guarantee that the " + "results will be reproducible.\n " + ) d.yaml_set_start_comment(pre) # Add newline between sections for s in d.keys(): From 2555c0a150e78e3deda5aea6640553c7161e5df5 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 23 Jul 2026 11:30:26 +0200 Subject: [PATCH 11/21] refactor: improve external component handling and comments generation in generate_yaml --- junifer/api/functions.py | 115 ++++++++++++++++++++++++--------------- 1 file changed, 70 insertions(+), 45 deletions(-) diff --git a/junifer/api/functions.py b/junifer/api/functions.py index f55b0fa6f..f73bf26db 100644 --- a/junifer/api/functions.py +++ b/junifer/api/functions.py @@ -629,20 +629,31 @@ def generate_yaml(meta: dict) -> "CommentedMap": # Add "with" section if present if "with" in meta: y["with"] = meta["with"].copy() + # Init var for post comment and issues + post = "\nIssues:\n" + issue = ( + " - `{0}` is not a built-in component and thus could not be " + "reproduced. Either replace (or remove) the component or fill it in " + "manually.\n" + ) # Set datagrabber meta_dg = meta["datagrabber"].copy() a = meta_dg.pop("class") - dg = PipelineComponentRegistry().get_class(step="datagrabber", name=a) - dg_model = dg.model_construct(**meta_dg) - y["datagrabber"] = { - "kind": a, - **dg_model.model_dump( - mode="json", - include=set(dg_model.dump_fields()), - exclude_defaults=True, - exclude_none=True, - ), - } + try: + dg = PipelineComponentRegistry().get_class(step="datagrabber", name=a) + dg_model = dg.model_construct(**meta_dg) + y["datagrabber"] = { + "kind": a, + **dg_model.model_dump( + mode="json", + include=set(dg_model.dump_fields()), + exclude_defaults=True, + exclude_none=True, + ), + } + except ValueError: + y["datagrabber"] = {"kind": a} + post += f"- datagrabber:\n{issue.format(a)}" # Set preprocessor(s) if "preprocess" in meta: y["preprocess"] = [] @@ -651,37 +662,48 @@ def generate_yaml(meta: dict) -> "CommentedMap": meta_p = [meta_p] for mp in meta_p: b = mp.pop("class") - p = PipelineComponentRegistry().get_class( - step="preprocessing", name=b - ) - p_model = p.model_construct(**mp) - y["preprocess"].append( - { - "kind": b, - **p_model.model_dump( - mode="json", - exclude={"required_data_types"}, - exclude_defaults=True, - exclude_none=True, - ), - } - ) + try: + p = PipelineComponentRegistry().get_class( + step="preprocessing", name=b + ) + p_model = p.model_construct(**mp) + y["preprocess"].append( + { + "kind": b, + **p_model.model_dump( + mode="json", + exclude={"required_data_types"}, + exclude_defaults=True, + exclude_none=True, + ), + } + ) + except ValueError: + y["preprocess"].append({"kind": b}) + if "- preprocess:\n" in post: + post += f"{issue.format(b)}" + else: + post += f"- preprocess:\n{issue.format(b)}" # Set marker meta_m = meta["marker"].copy() c = meta_m.pop("class") - m = PipelineComponentRegistry().get_class(step="marker", name=c) - m_model = m.model_construct(**meta_m) y["markers"] = [] - y["markers"].append( - { - "kind": c, - **m_model.model_dump( - mode="json", - exclude_defaults=True, - exclude_none=True, - ), - } - ) + try: + m = PipelineComponentRegistry().get_class(step="marker", name=c) + m_model = m.model_construct(**meta_m) + y["markers"].append( + { + "kind": c, + **m_model.model_dump( + mode="json", + exclude_defaults=True, + exclude_none=True, + ), + } + ) + except ValueError: + y["markers"].append({"kind": c}) + post += f"- markers:\n{issue.format(c)}" # Set storage y["storage"] = { "kind": "HDF5FeatureStorage", @@ -700,22 +722,25 @@ def generate_yaml(meta: dict) -> "CommentedMap": yaml.dump(y, stream=f) f.seek(0) d = yaml.load(f) - # Add preamble + # Write comments pre = ( "Auto-generated by junifer on " - f"{dt.datetime.now(tz=dt.UTC).strftime('%Y-%m-%d %H:%M:%S')} UTC\n\n" + f"{dt.datetime.now(tz=dt.timezone.utc).strftime('%Y-%m-%d %H:%M:%S')} " + "UTC\n\n" ) if "dependencies" in meta: for k, v in meta["dependencies"].items(): pre += f"{k}=={v}\n" - pre += ( - "\n`datadir` is ignored and not reproduced.\n" + const = ( + "\nNotes:\n" + "- `datadir` is ignored and not reproduced. " "If `datadir` used was not a temporary directory, you will have to " "manually edit this YAML.\n" - "In case the dataset was 'dirty', there is no guarantee that the " - "results will be reproducible.\n " + "- In case the dataset was 'dirty', there is no guarantee that the " + "results will be reproducible.\n" ) - d.yaml_set_start_comment(pre) + post = post if post != "\nIssues:\n" else "" + d.yaml_set_start_comment(pre + const + post) # Add newline between sections for s in d.keys(): d.yaml_set_comment_before_after_key(s, before="\n") From c4109f6631205b31182ee07d65ce9e4b04aaec1c Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 23 Jul 2026 11:31:19 +0200 Subject: [PATCH 12/21] chore: add test for generate_yaml --- junifer/api/tests/test_functions.py | 230 +++++++++++++++++++++++++++- 1 file changed, 229 insertions(+), 1 deletion(-) diff --git a/junifer/api/tests/test_functions.py b/junifer/api/tests/test_functions.py index d6cc8c67c..ac9405f4e 100644 --- a/junifer/api/tests/test_functions.py +++ b/junifer/api/tests/test_functions.py @@ -16,7 +16,15 @@ from ruamel.yaml import YAML import junifer.testing.registry # noqa: F401 -from junifer.api import collect, list_elements, parse_yaml, queue, reset, run +from junifer.api import ( + collect, + generate_yaml, + list_elements, + parse_yaml, + queue, + reset, + run, +) from junifer.datagrabber.base import BaseDataGrabber from junifer.pipeline import PipelineComponentRegistry from junifer.typing import Elements @@ -1025,3 +1033,223 @@ def test_parse_yaml_queue_venv_relative(tmp_path: Path) -> None: fname = tmp_path / "test_parse_yaml_queue_venv_relative.yaml" fname.write_text("queue:\n env:\n kind: venv\n name: .venv\n") _ = parse_yaml(fname) + + +@pytest.mark.parametrize( + "m", + [ + { + "datagrabber": { + "class": "PartlyCloudyTestingDataGrabber", + "types": ["BOLD"], + "datadir": ( + "/var/folders/dv/2lbr8f8j0q12zrx3mz3ll5m40000gp/T/tmpjeqj9nou" + ), + "reduce_confounds": True, + "age_group": "both", + }, + "dependencies": {"scikit-learn": "1.4.2", "nilearn": "0.10.4"}, + "datareader": {"class": "DefaultDataReader"}, + "type": "BOLD", + "marker": { + "class": "FunctionalConnectivityParcels", + "on": ["BOLD"], + "name": "fc_mean-shen_2015_268_functional_connectivity", + "agg_method": "mean", + "agg_method_params": None, + "conn_method": "correlation", + "conn_method_params": {"empirical": True}, + "masks": None, + "parcellation": ["Shen_2015_268"], + }, + "_element_keys": ["subject"], + "name": "BOLD_fc_mean-shen_2015_268_functional_connectivity", + }, + { + "datagrabber": { + "class": "DMCC13Benchmark", + "types": ["BOLD"], + "patterns": { + "BOLD": { + "pattern": ( + "derivatives/fmriprep-1.3.2/{subject}/{session}/func/{subject}_{session}_task-{task}_acq-mb4{phase_encoding}_run-{run}_space-MNI152NLin2009cAsym_desc-preproc_bold.nii.gz" + ), + "space": "MNI152NLin2009cAsym", + "mask": { + "pattern": ( + "derivatives/fmriprep-1.3.2/{subject}/{session}/func/{subject}_{session}_task-{task}_acq-mb4{phase_encoding}_run-{run}_space-MNI152NLin2009cAsym_desc-brain_mask.nii.gz" + ), + "space": "MNI152NLin2009cAsym", + }, + "confounds": { + "pattern": ( + "derivatives/fmriprep-1.3.2/{subject}/{session}/func/{subject}_{session}_task-{task}_acq-mb4{phase_encoding}_run-{run}_desc-confounds_regressors.tsv" + ), + "format": "fmriprep", + }, + }, + "T1w": { + "pattern": ( + "derivatives/fmriprep-1.3.2/{subject}/anat/{subject}_space-MNI152NLin2009cAsym_desc-preproc_T1w.nii.gz" + ), + "space": "MNI152NLin2009cAsym", + "mask": { + "pattern": ( + "derivatives/fmriprep-1.3.2/{subject}/anat/{subject}_space-MNI152NLin2009cAsym_desc-brain_mask.nii.gz" + ), + "space": "MNI152NLin2009cAsym", + }, + }, + "VBM_CSF": { + "pattern": ( + "derivatives/fmriprep-1.3.2/{subject}/anat/{subject}_space-MNI152NLin2009cAsym_label-CSF_probseg.nii.gz" + ), + "space": "MNI152NLin2009cAsym", + }, + "VBM_GM": { + "pattern": ( + "derivatives/fmriprep-1.3.2/{subject}/anat/{subject}_space-MNI152NLin2009cAsym_label-GM_probseg.nii.gz" + ), + "space": "MNI152NLin2009cAsym", + }, + "VBM_WM": { + "pattern": ( + "derivatives/fmriprep-1.3.2/{subject}/anat/{subject}_space-MNI152NLin2009cAsym_label-WM_probseg.nii.gz" + ), + "space": "MNI152NLin2009cAsym", + }, + }, + "replacements": [ + "subject", + "session", + "task", + "phase_encoding", + "run", + ], + "confounds_format": "fmriprep", + "partial_pattern_ok": False, + "uri": "https://github.com/OpenNeuroDatasets/ds003452.git", + "rootdir": ".", + "datalad_dirty": True, + "datalad_commit_id": ( + "8484f21551af53fcf2bc53878f18ce93dc2d29da" + ), + "datalad_id": "ade00fb6-636f-46fb-b2e6-60958b1b112d", + "sessions": ["ses-wave1bas"], + "tasks": ["Rest"], + "phase_encodings": ["AP"], + "runs": ["1"], + "native_t1w": False, + }, + "dependencies": { + "scikit-learn": "1.4.2", + "nilearn": "0.10.4", + "numpy": "1.26.4", + }, + "datareader": {"class": "DefaultDataReader"}, + "preprocess": { + "class": "fMRIPrepConfoundRemover", + "on": ["BOLD"], + "required_data_types": ["BOLD"], + "strategy": { + "motion": "full", + "wm_csf": "full", + "global_signal": "full", + }, + "spike": None, + "scrub": None, + "fd_threshold": None, + "std_dvars_threshold": None, + "detrend": True, + "standardize": True, + "low_pass": 0.08, + "high_pass": 0.01, + "t_r": None, + "masks": ["compute_epi_mask"], + }, + "type": "BOLD", + "marker": { + "class": "FunctionalConnectivitySpheres", + "on": ["BOLD"], + "name": "fc_spheres_functional_connectivity", + "agg_method": "mean", + "agg_method_params": None, + "conn_method": "correlation", + "conn_method_params": {"empirical": True}, + "masks": None, + "coords": "DMNBuckner", + "radius": 5.0, + "allow_overlap": False, + }, + "_element_keys": [ + "subject", + "session", + "task", + "phase_encoding", + "run", + ], + "name": "BOLD_fc_spheres_functional_connectivity", + }, + { + "datagrabber": { + "class": "ExternalDataGrabber", + "types": ["BOLD"], + "patterns": { + "BOLD": { + "pattern": ( + "derivatives/fmriprep-1.3.2/{subject}/func/{subject}_task-{task}_space-MNI152NLin2009cAsym_desc-preproc_bold.nii.gz" + ), + "space": "MNI152NLin2009cAsym", + }, + }, + "replacements": [ + "subject", + "task", + ], + "confounds_format": "fmriprep", + "partial_pattern_ok": True, + "uri": "https://github.com/datasets/ds11.git", + "rootdir": ".", + "datalad_dirty": False, + "datalad_commit_id": ( + "8484f21551af53fcf2bc53878f18ce93dc2d29da" + ), + "datalad_id": "ade00fb6-636f-46fb-b2e6-60958b1b112d", + "sessions": ["ses-wave1bas"], + "tasks": ["Rest"], + }, + "dependencies": {"scikit-learn": "1.4.2", "nilearn": "0.10.4"}, + "datareader": {"class": "DefaultDataReader"}, + "preprocess": [ + { + "class": "ExternalPreprocessor1", + "on": ["BOLD"], + "required_data_types": ["BOLD"], + }, + { + "class": "ExternalPreprocessor2", + "on": ["BOLD"], + "required_data_types": ["BOLD"], + }, + ], + "type": "BOLD", + "marker": { + "class": "ExternalMarker", + "on": ["BOLD"], + "name": "external", + }, + "_element_keys": ["subject", "task"], + "name": "BOLD_external", + }, + ], +) +def test_generate_yaml(m: dict) -> None: + """Test YAML generation from feature metadata. + + Parameters + ---------- + m : dict + The parametrized feature metadata. + + """ + _ = generate_yaml(m) From b3a010480cf2f55499cefbef26876c3f2942616a Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 23 Jul 2026 11:43:50 +0200 Subject: [PATCH 13/21] update: make external component check exception more targeted in generate_yaml --- junifer/api/functions.py | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/junifer/api/functions.py b/junifer/api/functions.py index f73bf26db..12db37c78 100644 --- a/junifer/api/functions.py +++ b/junifer/api/functions.py @@ -641,6 +641,10 @@ def generate_yaml(meta: dict) -> "CommentedMap": a = meta_dg.pop("class") try: dg = PipelineComponentRegistry().get_class(step="datagrabber", name=a) + except ValueError: + y["datagrabber"] = {"kind": a} + post += f"- datagrabber:\n{issue.format(a)}" + else: dg_model = dg.model_construct(**meta_dg) y["datagrabber"] = { "kind": a, @@ -651,9 +655,6 @@ def generate_yaml(meta: dict) -> "CommentedMap": exclude_none=True, ), } - except ValueError: - y["datagrabber"] = {"kind": a} - post += f"- datagrabber:\n{issue.format(a)}" # Set preprocessor(s) if "preprocess" in meta: y["preprocess"] = [] @@ -666,6 +667,13 @@ def generate_yaml(meta: dict) -> "CommentedMap": p = PipelineComponentRegistry().get_class( step="preprocessing", name=b ) + except ValueError: + y["preprocess"].append({"kind": b}) + if "- preprocess:\n" in post: + post += f"{issue.format(b)}" + else: + post += f"- preprocess:\n{issue.format(b)}" + else: p_model = p.model_construct(**mp) y["preprocess"].append( { @@ -678,18 +686,16 @@ def generate_yaml(meta: dict) -> "CommentedMap": ), } ) - except ValueError: - y["preprocess"].append({"kind": b}) - if "- preprocess:\n" in post: - post += f"{issue.format(b)}" - else: - post += f"- preprocess:\n{issue.format(b)}" # Set marker meta_m = meta["marker"].copy() c = meta_m.pop("class") y["markers"] = [] try: m = PipelineComponentRegistry().get_class(step="marker", name=c) + except ValueError: + y["markers"].append({"kind": c}) + post += f"- markers:\n{issue.format(c)}" + else: m_model = m.model_construct(**meta_m) y["markers"].append( { @@ -701,9 +707,6 @@ def generate_yaml(meta: dict) -> "CommentedMap": ), } ) - except ValueError: - y["markers"].append({"kind": c}) - post += f"- markers:\n{issue.format(c)}" # Set storage y["storage"] = { "kind": "HDF5FeatureStorage", From a59f96e155acb6cba9c003284b278ef0b76853f6 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 23 Jul 2026 12:20:36 +0200 Subject: [PATCH 14/21] update: conditional comment addition in generate_yaml --- junifer/api/functions.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/junifer/api/functions.py b/junifer/api/functions.py index 12db37c78..162d1455e 100644 --- a/junifer/api/functions.py +++ b/junifer/api/functions.py @@ -636,6 +636,7 @@ def generate_yaml(meta: dict) -> "CommentedMap": "reproduced. Either replace (or remove) the component or fill it in " "manually.\n" ) + var = "" # Set datagrabber meta_dg = meta["datagrabber"].copy() a = meta_dg.pop("class") @@ -655,6 +656,11 @@ def generate_yaml(meta: dict) -> "CommentedMap": exclude_none=True, ), } + if meta_dg.get("datalad_dirty"): + var = ( + "- The dataset was 'dirty', there is no guarantee that the " + "results will be reproducible.\n" + ) # Set preprocessor(s) if "preprocess" in meta: y["preprocess"] = [] @@ -739,11 +745,9 @@ def generate_yaml(meta: dict) -> "CommentedMap": "- `datadir` is ignored and not reproduced. " "If `datadir` used was not a temporary directory, you will have to " "manually edit this YAML.\n" - "- In case the dataset was 'dirty', there is no guarantee that the " - "results will be reproducible.\n" ) post = post if post != "\nIssues:\n" else "" - d.yaml_set_start_comment(pre + const + post) + d.yaml_set_start_comment(pre + const + var + post) # Add newline between sections for s in d.keys(): d.yaml_set_comment_before_after_key(s, before="\n") From 73effdf2a7d2a88ec9ebda791102cda73cb96e5b Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 23 Jul 2026 12:51:00 +0200 Subject: [PATCH 15/21] update: add parameters for external components in generate_yaml --- junifer/api/functions.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/junifer/api/functions.py b/junifer/api/functions.py index 162d1455e..d94e7c49c 100644 --- a/junifer/api/functions.py +++ b/junifer/api/functions.py @@ -643,7 +643,7 @@ def generate_yaml(meta: dict) -> "CommentedMap": try: dg = PipelineComponentRegistry().get_class(step="datagrabber", name=a) except ValueError: - y["datagrabber"] = {"kind": a} + y["datagrabber"] = {"kind": a, **meta_dg} post += f"- datagrabber:\n{issue.format(a)}" else: dg_model = dg.model_construct(**meta_dg) @@ -674,7 +674,7 @@ def generate_yaml(meta: dict) -> "CommentedMap": step="preprocessing", name=b ) except ValueError: - y["preprocess"].append({"kind": b}) + y["preprocess"].append({"kind": b, **mp}) if "- preprocess:\n" in post: post += f"{issue.format(b)}" else: @@ -699,7 +699,7 @@ def generate_yaml(meta: dict) -> "CommentedMap": try: m = PipelineComponentRegistry().get_class(step="marker", name=c) except ValueError: - y["markers"].append({"kind": c}) + y["markers"].append({"kind": c, **meta_m}) post += f"- markers:\n{issue.format(c)}" else: m_model = m.model_construct(**meta_m) From cdd7d8b648704c2860a92ccca86f3d4d79c039c0 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 23 Jul 2026 12:52:10 +0200 Subject: [PATCH 16/21] update: improve external component comment in generate_yaml --- junifer/api/functions.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/junifer/api/functions.py b/junifer/api/functions.py index d94e7c49c..1a34dad7f 100644 --- a/junifer/api/functions.py +++ b/junifer/api/functions.py @@ -632,9 +632,11 @@ def generate_yaml(meta: dict) -> "CommentedMap": # Init var for post comment and issues post = "\nIssues:\n" issue = ( - " - `{0}` is not a built-in component and thus could not be " - "reproduced. Either replace (or remove) the component or fill it in " - "manually.\n" + " - `{0}` is not a built-in component and thus could not be properly " + "regenerated. Some of these entries in the YAML section might be " + "redundant and not needed. Please check the " + "documentation/implementation of this specific datagrabber and remove " + "the unnecessary entries.\n" ) var = "" # Set datagrabber From 64d052e13beaafdff21e51ec18f9d9573ae48c03 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 23 Jul 2026 12:57:50 +0200 Subject: [PATCH 17/21] update: make external component check non-idiomatic in generate_yaml --- junifer/api/functions.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/junifer/api/functions.py b/junifer/api/functions.py index 1a34dad7f..a9d74878a 100644 --- a/junifer/api/functions.py +++ b/junifer/api/functions.py @@ -642,12 +642,11 @@ def generate_yaml(meta: dict) -> "CommentedMap": # Set datagrabber meta_dg = meta["datagrabber"].copy() a = meta_dg.pop("class") - try: - dg = PipelineComponentRegistry().get_class(step="datagrabber", name=a) - except ValueError: + if a not in PipelineComponentRegistry()._components["datagrabber"]: y["datagrabber"] = {"kind": a, **meta_dg} post += f"- datagrabber:\n{issue.format(a)}" else: + dg = PipelineComponentRegistry().get_class(step="datagrabber", name=a) dg_model = dg.model_construct(**meta_dg) y["datagrabber"] = { "kind": a, @@ -671,17 +670,19 @@ def generate_yaml(meta: dict) -> "CommentedMap": meta_p = [meta_p] for mp in meta_p: b = mp.pop("class") - try: - p = PipelineComponentRegistry().get_class( - step="preprocessing", name=b - ) - except ValueError: + if ( + b + not in PipelineComponentRegistry()._components["preprocessing"] + ): y["preprocess"].append({"kind": b, **mp}) if "- preprocess:\n" in post: post += f"{issue.format(b)}" else: post += f"- preprocess:\n{issue.format(b)}" else: + p = PipelineComponentRegistry().get_class( + step="preprocessing", name=b + ) p_model = p.model_construct(**mp) y["preprocess"].append( { @@ -698,12 +699,11 @@ def generate_yaml(meta: dict) -> "CommentedMap": meta_m = meta["marker"].copy() c = meta_m.pop("class") y["markers"] = [] - try: - m = PipelineComponentRegistry().get_class(step="marker", name=c) - except ValueError: + if c not in PipelineComponentRegistry()._components["marker"]: y["markers"].append({"kind": c, **meta_m}) post += f"- markers:\n{issue.format(c)}" else: + m = PipelineComponentRegistry().get_class(step="marker", name=c) m_model = m.model_construct(**meta_m) y["markers"].append( { From 12332b0d3ad970486eff943d0eae3b589174f1fb Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 23 Jul 2026 15:56:45 +0200 Subject: [PATCH 18/21] refactor: replace model_construct with model_validate and add comments and checks in generate_yaml --- junifer/api/functions.py | 107 +++++++++++++++++----------- junifer/api/tests/test_functions.py | 27 +++++++ 2 files changed, 94 insertions(+), 40 deletions(-) diff --git a/junifer/api/functions.py b/junifer/api/functions.py index a9d74878a..5858afd5d 100644 --- a/junifer/api/functions.py +++ b/junifer/api/functions.py @@ -17,6 +17,7 @@ from typing import TYPE_CHECKING, Any import structlog +from pydantic import ValidationError from ..api.queue_context import GnuParallelLocalAdapter, HTCondorAdapter from ..datagrabber import BaseDataGrabber @@ -610,7 +611,7 @@ def parse_yaml(filepath: str | Path) -> dict: # noqa: C901 return contents -def generate_yaml(meta: dict) -> "CommentedMap": +def generate_yaml(meta: dict) -> "CommentedMap": # noqa: C901 """Generate the feature YAML from metadata. Parameters @@ -631,11 +632,18 @@ def generate_yaml(meta: dict) -> "CommentedMap": y["with"] = meta["with"].copy() # Init var for post comment and issues post = "\nIssues:\n" - issue = ( + issue_ext = ( " - `{0}` is not a built-in component and thus could not be properly " "regenerated. Some of these entries in the YAML section might be " "redundant and not needed. Please check the " - "documentation/implementation of this specific datagrabber and remove " + "documentation/implementation of this specific component and remove " + "the unnecessary entries.\n" + ) + issue_inv = ( + " - `{0}` failed to initialise and thus could not be properly " + "regenerated. Some of these entries in the YAML section might be " + "redundant and not needed. Please check the " + "documentation/implementation of this specific component and remove " "the unnecessary entries.\n" ) var = "" @@ -644,19 +652,24 @@ def generate_yaml(meta: dict) -> "CommentedMap": a = meta_dg.pop("class") if a not in PipelineComponentRegistry()._components["datagrabber"]: y["datagrabber"] = {"kind": a, **meta_dg} - post += f"- datagrabber:\n{issue.format(a)}" + post += f"- datagrabber:\n{issue_ext.format(a)}" else: dg = PipelineComponentRegistry().get_class(step="datagrabber", name=a) - dg_model = dg.model_construct(**meta_dg) - y["datagrabber"] = { - "kind": a, - **dg_model.model_dump( - mode="json", - include=set(dg_model.dump_fields()), - exclude_defaults=True, - exclude_none=True, - ), - } + try: + dg_model = dg.model_validate(**meta_dg) + except (ValidationError, TypeError): + y["datagrabber"] = {"kind": a, **meta_dg} + post += f"- datagrabber:\n{issue_inv.format(a)}" + else: + y["datagrabber"] = { + "kind": a, + **dg_model.model_dump( + mode="json", + include=set(dg_model.dump_fields()), + exclude_defaults=True, + exclude_none=True, + ), + } if meta_dg.get("datalad_dirty"): var = ( "- The dataset was 'dirty', there is no guarantee that the " @@ -676,45 +689,58 @@ def generate_yaml(meta: dict) -> "CommentedMap": ): y["preprocess"].append({"kind": b, **mp}) if "- preprocess:\n" in post: - post += f"{issue.format(b)}" + post += f"{issue_ext.format(b)}" else: - post += f"- preprocess:\n{issue.format(b)}" + post += f"- preprocess:\n{issue_ext.format(b)}" else: p = PipelineComponentRegistry().get_class( step="preprocessing", name=b ) - p_model = p.model_construct(**mp) - y["preprocess"].append( - { - "kind": b, - **p_model.model_dump( - mode="json", - exclude={"required_data_types"}, - exclude_defaults=True, - exclude_none=True, - ), - } - ) + try: + p_model = p.model_validate(**mp) + except (ValidationError, TypeError): + y["preprocess"].append({"kind": b, **mp}) + if "- preprocess:\n" in post: + post += f"{issue_inv.format(b)}" + else: + post += f"- preprocess:\n{issue_inv.format(b)}" + else: + y["preprocess"].append( + { + "kind": b, + **p_model.model_dump( + mode="json", + exclude={"required_data_types"}, + exclude_defaults=True, + exclude_none=True, + ), + } + ) # Set marker meta_m = meta["marker"].copy() c = meta_m.pop("class") y["markers"] = [] if c not in PipelineComponentRegistry()._components["marker"]: y["markers"].append({"kind": c, **meta_m}) - post += f"- markers:\n{issue.format(c)}" + post += f"- markers:\n{issue_ext.format(c)}" else: m = PipelineComponentRegistry().get_class(step="marker", name=c) - m_model = m.model_construct(**meta_m) - y["markers"].append( - { - "kind": c, - **m_model.model_dump( - mode="json", - exclude_defaults=True, - exclude_none=True, - ), - } - ) + try: + m_model = m.model_validate(**meta_m) + except (ValidationError, TypeError): + y["markers"].append({"kind": c, **meta_m}) + post += f"- markers:\n{issue_inv.format(c)}" + else: + y["markers"].append( + { + "kind": c, + **m_model.model_dump( + mode="json", + exclude_defaults=True, + exclude_none=True, + ), + } + ) # Set storage y["storage"] = { "kind": "HDF5FeatureStorage", @@ -744,6 +770,7 @@ def generate_yaml(meta: dict) -> "CommentedMap": pre += f"{k}=={v}\n" const = ( "\nNotes:\n" + "- Check the components for possible changes in the API.\n" "- `datadir` is ignored and not reproduced. " "If `datadir` used was not a temporary directory, you will have to " "manually edit this YAML.\n" diff --git a/junifer/api/tests/test_functions.py b/junifer/api/tests/test_functions.py index ac9405f4e..59d503935 100644 --- a/junifer/api/tests/test_functions.py +++ b/junifer/api/tests/test_functions.py @@ -1065,6 +1065,33 @@ def test_parse_yaml_queue_venv_relative(tmp_path: Path) -> None: "_element_keys": ["subject"], "name": "BOLD_fc_mean-shen_2015_268_functional_connectivity", }, + { + "datagrabber": { + "class": "PartlyCloudyTestingDataGrabber", + "types": ["BOLD"], + "datadir": ( + "/var/folders/dv/2lbr8f8j0q12zrx3mz3ll5m40000gp/T/tmpjeqj9nou" + ), + "reduce_confound": True, + "age": "both", + }, + "dependencies": {"scikit-learn": "1.4.2", "nilearn": "0.10.4"}, + "datareader": {"class": "DefaultDataReader"}, + "type": "BOLD", + "marker": { + "class": "FunctionalConnectivityParcels", + "on": ["BOLD"], + "name": "fc_mean-shen_2015_268_functional_connectivity", + "ag_method": "mean", + "ag_method_params": None, + "con_method": "correlation", + "con_method_params": {"empirical": True}, + "masks": None, + "parcellation": ["Shen_2015_268"], + }, + "_element_keys": ["subject"], + "name": "BOLD_fc_mean-shen_2015_268_functional_connectivity", + }, { "datagrabber": { "class": "DMCC13Benchmark", From cb88686882ada405741f8516b3dd4d72b5cf7041 Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Thu, 23 Jul 2026 16:47:07 +0200 Subject: [PATCH 19/21] fix: correct usage of model_validate in generate_yaml --- junifer/api/functions.py | 12 ++++++------ junifer/api/tests/test_functions.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/junifer/api/functions.py b/junifer/api/functions.py index 5858afd5d..b7f3314d2 100644 --- a/junifer/api/functions.py +++ b/junifer/api/functions.py @@ -656,8 +656,8 @@ def generate_yaml(meta: dict) -> "CommentedMap": # noqa: C901 else: dg = PipelineComponentRegistry().get_class(step="datagrabber", name=a) try: - dg_model = dg.model_validate(**meta_dg) - except (ValidationError, TypeError): + dg_model = dg.model_validate(meta_dg) + except ValidationError: y["datagrabber"] = {"kind": a, **meta_dg} post += f"- datagrabber:\n{issue_inv.format(a)}" else: @@ -697,8 +697,8 @@ def generate_yaml(meta: dict) -> "CommentedMap": # noqa: C901 step="preprocessing", name=b ) try: - p_model = p.model_validate(**mp) - except (ValidationError, TypeError): + p_model = p.model_validate(mp) + except ValidationError: y["preprocess"].append({"kind": b, **mp}) if "- preprocess:\n" in post: post += f"{issue_inv.format(b)}" @@ -726,8 +726,8 @@ def generate_yaml(meta: dict) -> "CommentedMap": # noqa: C901 else: m = PipelineComponentRegistry().get_class(step="marker", name=c) try: - m_model = m.model_validate(**meta_m) - except (ValidationError, TypeError): + m_model = m.model_validate(meta_m) + except ValidationError: y["markers"].append({"kind": c, **meta_m}) post += f"- markers:\n{issue_inv.format(c)}" else: diff --git a/junifer/api/tests/test_functions.py b/junifer/api/tests/test_functions.py index 59d503935..fc472c4a3 100644 --- a/junifer/api/tests/test_functions.py +++ b/junifer/api/tests/test_functions.py @@ -1045,7 +1045,7 @@ def test_parse_yaml_queue_venv_relative(tmp_path: Path) -> None: "datadir": ( "/var/folders/dv/2lbr8f8j0q12zrx3mz3ll5m40000gp/T/tmpjeqj9nou" ), - "reduce_confounds": True, + "reduce_confounds": False, "age_group": "both", }, "dependencies": {"scikit-learn": "1.4.2", "nilearn": "0.10.4"}, From ec63d265bcf70d2a28b60001be3a26f0554c68cf Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 24 Jul 2026 14:28:56 +0200 Subject: [PATCH 20/21] chore: update test for generate_yaml --- junifer/api/tests/test_functions.py | 506 +++++++++++++++++----------- 1 file changed, 304 insertions(+), 202 deletions(-) diff --git a/junifer/api/tests/test_functions.py b/junifer/api/tests/test_functions.py index fc472c4a3..d9b66cc56 100644 --- a/junifer/api/tests/test_functions.py +++ b/junifer/api/tests/test_functions.py @@ -5,6 +5,7 @@ # Synchon Mandal # License: AGPL +import io import logging import sys from contextlib import AbstractContextManager, nullcontext @@ -1036,247 +1037,348 @@ def test_parse_yaml_queue_venv_relative(tmp_path: Path) -> None: @pytest.mark.parametrize( - "m", + "m, exp", [ - { - "datagrabber": { - "class": "PartlyCloudyTestingDataGrabber", - "types": ["BOLD"], - "datadir": ( - "/var/folders/dv/2lbr8f8j0q12zrx3mz3ll5m40000gp/T/tmpjeqj9nou" - ), - "reduce_confounds": False, - "age_group": "both", - }, - "dependencies": {"scikit-learn": "1.4.2", "nilearn": "0.10.4"}, - "datareader": {"class": "DefaultDataReader"}, - "type": "BOLD", - "marker": { - "class": "FunctionalConnectivityParcels", - "on": ["BOLD"], - "name": "fc_mean-shen_2015_268_functional_connectivity", - "agg_method": "mean", - "agg_method_params": None, - "conn_method": "correlation", - "conn_method_params": {"empirical": True}, - "masks": None, - "parcellation": ["Shen_2015_268"], - }, - "_element_keys": ["subject"], - "name": "BOLD_fc_mean-shen_2015_268_functional_connectivity", - }, - { - "datagrabber": { - "class": "PartlyCloudyTestingDataGrabber", - "types": ["BOLD"], - "datadir": ( - "/var/folders/dv/2lbr8f8j0q12zrx3mz3ll5m40000gp/T/tmpjeqj9nou" - ), - "reduce_confound": True, - "age": "both", + ( + { + "datagrabber": { + "class": "PartlyCloudyTestingDataGrabber", + "types": ["BOLD"], + "datadir": ( + "/var/folders/dv/2lbr8f8j0q12zrx3mz3ll5m40000gp/T/tmpjeqj9nou" + ), + "reduce_confounds": False, + "age_group": "both", + }, + "dependencies": {"scikit-learn": "1.4.2", "nilearn": "0.10.4"}, + "datareader": {"class": "DefaultDataReader"}, + "type": "BOLD", + "marker": { + "class": "FunctionalConnectivityParcels", + "on": ["BOLD"], + "name": "fc_mean-shen_2015_268_functional_connectivity", + "agg_method": "mean", + "agg_method_params": None, + "conn_method": "correlation", + "conn_method_params": {"empirical": True}, + "masks": None, + "parcellation": ["Shen_2015_268"], + }, + "_element_keys": ["subject"], + "name": "BOLD_fc_mean-shen_2015_268_functional_connectivity", }, - "dependencies": {"scikit-learn": "1.4.2", "nilearn": "0.10.4"}, - "datareader": {"class": "DefaultDataReader"}, - "type": "BOLD", - "marker": { - "class": "FunctionalConnectivityParcels", - "on": ["BOLD"], - "name": "fc_mean-shen_2015_268_functional_connectivity", - "ag_method": "mean", - "ag_method_params": None, - "con_method": "correlation", - "con_method_params": {"empirical": True}, - "masks": None, - "parcellation": ["Shen_2015_268"], + [ + "Auto-generated by junifer on", + "Check the components for possible changes in the API", + ], + ), + ( + { + "datagrabber": { + "class": "PartlyCloudyTestingDataGrabber", + "types": ["BOLD"], + "datadir": ( + "/var/folders/dv/2lbr8f8j0q12zrx3mz3ll5m40000gp/T/tmpjeqj9nou" + ), + "reduce_confound": True, + "age": "both", + }, + "dependencies": {"scikit-learn": "1.4.2", "nilearn": "0.10.4"}, + "datareader": {"class": "DefaultDataReader"}, + "type": "BOLD", + "marker": { + "class": "FunctionalConnectivityParcels", + "on": ["BOLD"], + "name": "fc_mean-shen_2015_268_functional_connectivity", + "ag_method": "mean", + "ag_method_params": None, + "con_method": "correlation", + "con_method_params": {"empirical": True}, + "masks": None, + "parcellation": ["Shen_2015_268"], + }, + "_element_keys": ["subject"], + "name": "BOLD_fc_mean-shen_2015_268_functional_connectivity", }, - "_element_keys": ["subject"], - "name": "BOLD_fc_mean-shen_2015_268_functional_connectivity", - }, - { - "datagrabber": { - "class": "DMCC13Benchmark", - "types": ["BOLD"], - "patterns": { - "BOLD": { - "pattern": ( - "derivatives/fmriprep-1.3.2/{subject}/{session}/func/{subject}_{session}_task-{task}_acq-mb4{phase_encoding}_run-{run}_space-MNI152NLin2009cAsym_desc-preproc_bold.nii.gz" - ), - "space": "MNI152NLin2009cAsym", - "mask": { + [ + "Auto-generated by junifer on", + "Check the components for possible changes in the API", + "`PartlyCloudyTestingDataGrabber` failed to initialise and " + "thus could not be properly", + ], + ), + ( + { + "datagrabber": { + "class": "DMCC13Benchmark", + "types": ["BOLD"], + "patterns": { + "BOLD": { "pattern": ( - "derivatives/fmriprep-1.3.2/{subject}/{session}/func/{subject}_{session}_task-{task}_acq-mb4{phase_encoding}_run-{run}_space-MNI152NLin2009cAsym_desc-brain_mask.nii.gz" + "derivatives/fmriprep-1.3.2/{subject}/{session}/func/{subject}_{session}_task-{task}_acq-mb4{phase_encoding}_run-{run}_space-MNI152NLin2009cAsym_desc-preproc_bold.nii.gz" ), "space": "MNI152NLin2009cAsym", + "mask": { + "pattern": ( + "derivatives/fmriprep-1.3.2/{subject}/{session}/func/{subject}_{session}_task-{task}_acq-mb4{phase_encoding}_run-{run}_space-MNI152NLin2009cAsym_desc-brain_mask.nii.gz" + ), + "space": "MNI152NLin2009cAsym", + }, + "confounds": { + "pattern": ( + "derivatives/fmriprep-1.3.2/{subject}/{session}/func/{subject}_{session}_task-{task}_acq-mb4{phase_encoding}_run-{run}_desc-confounds_regressors.tsv" + ), + "format": "fmriprep", + }, }, - "confounds": { + "T1w": { "pattern": ( - "derivatives/fmriprep-1.3.2/{subject}/{session}/func/{subject}_{session}_task-{task}_acq-mb4{phase_encoding}_run-{run}_desc-confounds_regressors.tsv" - ), - "format": "fmriprep", - }, - }, - "T1w": { - "pattern": ( - "derivatives/fmriprep-1.3.2/{subject}/anat/{subject}_space-MNI152NLin2009cAsym_desc-preproc_T1w.nii.gz" - ), - "space": "MNI152NLin2009cAsym", - "mask": { - "pattern": ( - "derivatives/fmriprep-1.3.2/{subject}/anat/{subject}_space-MNI152NLin2009cAsym_desc-brain_mask.nii.gz" + "derivatives/fmriprep-1.3.2/{subject}/anat/{subject}_space-MNI152NLin2009cAsym_desc-preproc_T1w.nii.gz" ), "space": "MNI152NLin2009cAsym", + "mask": { + "pattern": ( + "derivatives/fmriprep-1.3.2/{subject}/anat/{subject}_space-MNI152NLin2009cAsym_desc-brain_mask.nii.gz" + ), + "space": "MNI152NLin2009cAsym", + }, }, }, - "VBM_CSF": { - "pattern": ( - "derivatives/fmriprep-1.3.2/{subject}/anat/{subject}_space-MNI152NLin2009cAsym_label-CSF_probseg.nii.gz" - ), - "space": "MNI152NLin2009cAsym", - }, - "VBM_GM": { - "pattern": ( - "derivatives/fmriprep-1.3.2/{subject}/anat/{subject}_space-MNI152NLin2009cAsym_label-GM_probseg.nii.gz" - ), - "space": "MNI152NLin2009cAsym", - }, - "VBM_WM": { - "pattern": ( - "derivatives/fmriprep-1.3.2/{subject}/anat/{subject}_space-MNI152NLin2009cAsym_label-WM_probseg.nii.gz" - ), - "space": "MNI152NLin2009cAsym", + "replacements": [ + "subject", + "session", + "task", + "phase_encoding", + "run", + ], + "confounds_format": "fmriprep", + "partial_pattern_ok": False, + "uri": "https://github.com/OpenNeuroDatasets/ds003452.git", + "rootdir": ".", + "datalad_dirty": True, + "datalad_commit_id": ( + "8484f21551af53fcf2bc53878f18ce93dc2d29da" + ), + "datalad_id": "ade00fb6-636f-46fb-b2e6-60958b1b112d", + "sessions": ["ses-wave1bas"], + "tasks": ["Rest"], + "phase_encodings": ["AP"], + "runs": ["1"], + "native_t1w": False, + }, + "dependencies": { + "scikit-learn": "1.4.2", + "nilearn": "0.10.4", + "numpy": "1.26.4", + }, + "datareader": {"class": "DefaultDataReader"}, + "preprocess": { + "class": "fMRIPrepConfoundRemover", + "on": ["BOLD"], + "required_data_types": ["BOLD"], + "strategy": { + "motion": "full", + "wm_csf": "full", + "global_signal": "full", }, + "spike": None, + "scrub": None, + "fd_threshold": None, + "std_dvars_threshold": None, + "detrend": True, + "standardize": True, + "low_pass": 0.08, + "high_pass": 0.01, + "t_r": None, + "masks": ["compute_epi_mask"], }, - "replacements": [ + "type": "BOLD", + "marker": { + "class": "FunctionalConnectivitySpheres", + "on": ["BOLD"], + "name": "fc_spheres_functional_connectivity", + "agg_method": "mean", + "agg_method_params": None, + "conn_method": "correlation", + "conn_method_params": {"empirical": True}, + "masks": None, + "coords": "DMNBuckner", + "radius": 5.0, + "allow_overlap": False, + }, + "_element_keys": [ "subject", "session", "task", "phase_encoding", "run", ], - "confounds_format": "fmriprep", - "partial_pattern_ok": False, - "uri": "https://github.com/OpenNeuroDatasets/ds003452.git", - "rootdir": ".", - "datalad_dirty": True, - "datalad_commit_id": ( - "8484f21551af53fcf2bc53878f18ce93dc2d29da" - ), - "datalad_id": "ade00fb6-636f-46fb-b2e6-60958b1b112d", - "sessions": ["ses-wave1bas"], - "tasks": ["Rest"], - "phase_encodings": ["AP"], - "runs": ["1"], - "native_t1w": False, + "name": "BOLD_fc_spheres_functional_connectivity", }, - "dependencies": { - "scikit-learn": "1.4.2", - "nilearn": "0.10.4", - "numpy": "1.26.4", - }, - "datareader": {"class": "DefaultDataReader"}, - "preprocess": { - "class": "fMRIPrepConfoundRemover", - "on": ["BOLD"], - "required_data_types": ["BOLD"], - "strategy": { - "motion": "full", - "wm_csf": "full", - "global_signal": "full", - }, - "spike": None, - "scrub": None, - "fd_threshold": None, - "std_dvars_threshold": None, - "detrend": True, - "standardize": True, - "low_pass": 0.08, - "high_pass": 0.01, - "t_r": None, - "masks": ["compute_epi_mask"], - }, - "type": "BOLD", - "marker": { - "class": "FunctionalConnectivitySpheres", - "on": ["BOLD"], - "name": "fc_spheres_functional_connectivity", - "agg_method": "mean", - "agg_method_params": None, - "conn_method": "correlation", - "conn_method_params": {"empirical": True}, - "masks": None, - "coords": "DMNBuckner", - "radius": 5.0, - "allow_overlap": False, - }, - "_element_keys": [ - "subject", - "session", - "task", - "phase_encoding", - "run", + [ + "Auto-generated by junifer on", + "Check the components for possible changes in the API", + "The dataset was 'dirty'", ], - "name": "BOLD_fc_spheres_functional_connectivity", - }, - { - "datagrabber": { - "class": "ExternalDataGrabber", - "types": ["BOLD"], - "patterns": { - "BOLD": { - "pattern": ( - "derivatives/fmriprep-1.3.2/{subject}/func/{subject}_task-{task}_space-MNI152NLin2009cAsym_desc-preproc_bold.nii.gz" - ), - "space": "MNI152NLin2009cAsym", + ), + ( + { + "datagrabber": { + "class": "DMCC13Benchmark", + "types": ["BOLD"], + "patterns": { + "BOLD": { + "pattern": ( + "derivatives/fmriprep-1.3.2/{subject}/{session}/func/{subject}_{session}_task-{task}_acq-mb4{phase_encoding}_run-{run}_space-MNI152NLin2009cAsym_desc-preproc_bold.nii.gz" + ), + "space": "MNI152NLin2009cAsym", + "mask": { + "pattern": ( + "derivatives/fmriprep-1.3.2/{subject}/{session}/func/{subject}_{session}_task-{task}_acq-mb4{phase_encoding}_run-{run}_space-MNI152NLin2009cAsym_desc-brain_mask.nii.gz" + ), + "space": "MNI152NLin2009cAsym", + }, + "confounds": { + "pattern": ( + "derivatives/fmriprep-1.3.2/{subject}/{session}/func/{subject}_{session}_task-{task}_acq-mb4{phase_encoding}_run-{run}_desc-confounds_regressors.tsv" + ), + "format": "fmriprep", + }, + }, }, + "replacements": [ + "subject", + "session", + "task", + "phase_encoding", + "run", + ], + "confounds_format": "fmriprep", + "partial_pattern_ok": False, + "uri": "https://github.com/OpenNeuroDatasets/ds003452.git", + "rootdir": ".", + "datalad_dirty": False, + "datalad_commit_id": ( + "8484f21551af53fcf2bc53878f18ce93dc2d29da" + ), + "datalad_id": "ade00fb6-636f-46fb-b2e6-60958b1b112d", + "sessions": ["ses-wave1bas"], + "tasks": ["Rest"], + "phase_encodings": ["AP"], + "runs": ["1"], + "native_t1w": False, + }, + "dependencies": { + "scikit-learn": "1.4.2", + "nilearn": "0.10.4", + "numpy": "1.26.4", + }, + "datareader": {"class": "DefaultDataReader"}, + "type": "BOLD", + "marker": { + "class": "FunctionalConnectivitySpheres", + "on": ["BOLD"], + "name": "fc_spheres_functional_connectivity", + "agg_method": "mean", + "agg_method_params": None, + "conn_method": "correlation", + "conn_method_params": {"empirical": True}, + "masks": None, + "coords": "DMNBuckner", + "radius": 5.0, + "allow_overlap": False, }, - "replacements": [ + "_element_keys": [ "subject", + "session", "task", + "phase_encoding", + "run", ], - "confounds_format": "fmriprep", - "partial_pattern_ok": True, - "uri": "https://github.com/datasets/ds11.git", - "rootdir": ".", - "datalad_dirty": False, - "datalad_commit_id": ( - "8484f21551af53fcf2bc53878f18ce93dc2d29da" - ), - "datalad_id": "ade00fb6-636f-46fb-b2e6-60958b1b112d", - "sessions": ["ses-wave1bas"], - "tasks": ["Rest"], + "name": "BOLD_fc_spheres_functional_connectivity", }, - "dependencies": {"scikit-learn": "1.4.2", "nilearn": "0.10.4"}, - "datareader": {"class": "DefaultDataReader"}, - "preprocess": [ - { - "class": "ExternalPreprocessor1", - "on": ["BOLD"], - "required_data_types": ["BOLD"], + [ + "Auto-generated by junifer on", + "Check the components for possible changes in the API", + ], + ), + ( + { + "datagrabber": { + "class": "ExternalDataGrabber", + "types": ["BOLD"], + "patterns": { + "BOLD": { + "pattern": ( + "derivatives/fmriprep-1.3.2/{subject}/func/{subject}_task-{task}_space-MNI152NLin2009cAsym_desc-preproc_bold.nii.gz" + ), + "space": "MNI152NLin2009cAsym", + }, + }, + "replacements": [ + "subject", + "task", + ], + "confounds_format": "fmriprep", + "partial_pattern_ok": True, + "uri": "https://github.com/datasets/ds11.git", + "rootdir": ".", + "datalad_dirty": False, + "datalad_commit_id": ( + "8484f21551af53fcf2bc53878f18ce93dc2d29da" + ), + "datalad_id": "ade00fb6-636f-46fb-b2e6-60958b1b112d", + "sessions": ["ses-wave1bas"], + "tasks": ["Rest"], }, - { - "class": "ExternalPreprocessor2", + "dependencies": {"scikit-learn": "1.4.2", "nilearn": "0.10.4"}, + "datareader": {"class": "DefaultDataReader"}, + "preprocess": [ + { + "class": "ExternalPreprocessor1", + "on": ["BOLD"], + "required_data_types": ["BOLD"], + }, + { + "class": "ExternalPreprocessor2", + "on": ["BOLD"], + "required_data_types": ["BOLD"], + }, + ], + "type": "BOLD", + "marker": { + "class": "ExternalMarker", "on": ["BOLD"], - "required_data_types": ["BOLD"], + "name": "external", }, - ], - "type": "BOLD", - "marker": { - "class": "ExternalMarker", - "on": ["BOLD"], - "name": "external", + "_element_keys": ["subject", "task"], + "name": "BOLD_external", }, - "_element_keys": ["subject", "task"], - "name": "BOLD_external", - }, + [ + "Auto-generated by junifer on", + "Check the components for possible changes in the API", + "`ExternalDataGrabber` is not a built-in component", + "`ExternalPreprocessor1` is not a built-in component", + "`ExternalMarker` is not a built-in component", + ], + ), ], ) -def test_generate_yaml(m: dict) -> None: +def test_generate_yaml(m: dict, exp: list[str]) -> None: """Test YAML generation from feature metadata. Parameters ---------- m : dict The parametrized feature metadata. + exp : list + The parametrized expected comments. """ - _ = generate_yaml(m) + c = generate_yaml(m) + buf = io.StringIO() + yaml.dump(c, stream=buf) + buf.seek(0) + y = buf.read() + for e in exp: + assert e in y From ca6cc8a71347e045067768418c240502a8c4a70b Mon Sep 17 00:00:00 2001 From: Synchon Mandal Date: Fri, 24 Jul 2026 14:30:35 +0200 Subject: [PATCH 21/21] chore: lint --- junifer/markers/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/junifer/markers/utils.py b/junifer/markers/utils.py index c3ab4ef33..969b32cdf 100644 --- a/junifer/markers/utils.py +++ b/junifer/markers/utils.py @@ -19,7 +19,7 @@ def _ets( bold_ts: np.ndarray, - roi_names: None | list[str] = None, + roi_names: list[str] | None = None, ) -> tuple[np.ndarray, list[str] | None]: """Compute the edge-wise time series based on BOLD time series.