From a7ee05e860875b6817f5e5ebfe8e5a94219cb000 Mon Sep 17 00:00:00 2001 From: thomasATbayer <105632614+thomasATbayer@users.noreply.github.com> Date: Thu, 2 Jul 2026 12:13:20 +0000 Subject: [PATCH 1/8] enable hold out tuning and make gp sample the standard sampler --- src/mother/optimization/core.py | 43 +++++++++++++++++++++++++-------- uv.lock | 2 +- 2 files changed, 34 insertions(+), 11 deletions(-) diff --git a/src/mother/optimization/core.py b/src/mother/optimization/core.py index 2eb8f3d..e7bfa4f 100644 --- a/src/mother/optimization/core.py +++ b/src/mother/optimization/core.py @@ -136,21 +136,35 @@ def __init__( self.early_stopping_optuna: bool = early_stopping_optuna self.tuning_direction: typing.Union[StudyDirection, str] = tuning_direction self.scorer: typing.Callable = skl_metrics.get_scorer(scorer) + if sampler is None: - module_logger.debug("Setting up default sampler TPE") - self.sampler = optuna.samplers.TPESampler( - multivariate=kwargs.get("multivariate", True), - group=True, - constant_liar=True, - seed=seed, - n_startup_trials=n_startup_trials, - ) + if torch_available: + module_logger.debug("torch available — using GPSampler as default") + self.sampler = optuna.samplers.GPSampler( + seed=seed, + n_startup_trials=n_startup_trials, + deterministic_objective=False, + # constant liar is aalways True for GPSampler, + # so we don't need to set it explicitly + ) + else: + module_logger.debug("torch not available — falling back to TPESampler") + self.sampler = optuna.samplers.TPESampler( + multivariate=True, + group=True, + constant_liar=True, + seed=seed, + n_startup_trials=n_startup_trials, + ) else: self.sampler = sampler self.study: typing.Optional[Study] = None - def get_callbacks(self): + def get_callbacks( + self, + cross_validation: typing.Optional[skl_model_sel.BaseCrossValidator] = None, + ): """ Prepares and returns a list of callbacks for early stopping in Optuna optimization. @@ -164,6 +178,15 @@ def get_callbacks(self): """ callbacks: typing.Optional[typing.List[TerminatorCallback]] = None if self.early_stopping_optuna: + if cross_validation is not None: + n_splits = cross_validation.get_n_splits() + if n_splits < 2: + module_logger.warning( + "Optuna early stopping requires at least 2 CV folds. " + "Skipping early stopping (hold-out detected, n_splits=%s)", + n_splits, + ) + return None if not torch_available: module_logger.warning("Torch not installed, early optuna termination will not be available") module_logger.warning( @@ -277,7 +300,7 @@ def objective(trial: optuna.trial.Trial) -> float: objective, n_trials=self.n_trials_optuna, gc_after_trial=True, - callbacks=self.get_callbacks(), + callbacks=self.get_callbacks(cross_validation=cross_validation), ) if default_parameters != {}: diff --git a/uv.lock b/uv.lock index ba00735..dceb8f0 100644 --- a/uv.lock +++ b/uv.lock @@ -2810,7 +2810,7 @@ wheels = [ [[package]] name = "mother-ml" -version = "1.0.0" +version = "1.0.1" source = { editable = "." } dependencies = [ { name = "boruta" }, From 2f1d862a8d6a6b8f75d2192caaab2c242fbeffe5 Mon Sep 17 00:00:00 2001 From: thomasATbayer <105632614+thomasATbayer@users.noreply.github.com> Date: Thu, 2 Jul 2026 12:48:37 +0000 Subject: [PATCH 2/8] update optuna --- pyproject.toml | 3 +-- uv.lock | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 98af21d..5ae4aa3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ dependencies = [ "catboost>=1.2.6,<=1.2.8", "feature-engine>=1.8.0,<2", "numpy>=2.2,<2.3", - "optuna>=4.2.0,<5", + "optuna>=4.9.0,<5", "pandas>=2.2,<3.0", "pydantic-settings>=2.4.0,<3", "rdkit>=2024.9.1", @@ -358,4 +358,3 @@ module-name = "mother" [build-system] requires = ["uv_build>=0.11.1,<0.12.0","setuptools>=76.0.0"] build-backend = "uv_build" - diff --git a/uv.lock b/uv.lock index dceb8f0..48222fe 100644 --- a/uv.lock +++ b/uv.lock @@ -2905,7 +2905,7 @@ requires-dist = [ { name = "leidenalg", marker = "extra == 'report'", specifier = ">=0.10.2,<0.11" }, { name = "matplotlib", marker = "extra == 'report'", specifier = ">3.7.0" }, { name = "numpy", specifier = ">=2.2,<2.3" }, - { name = "optuna", specifier = ">=4.2.0,<5" }, + { name = "optuna", specifier = ">=4.9.0,<5" }, { name = "pandas", specifier = ">=2.2,<3.0" }, { name = "plotly", marker = "extra == 'report'", specifier = ">=6.0.1,<7" }, { name = "pydantic-settings", specifier = ">=2.4.0,<3" }, From 96e860a47fb91108655c9594e6433760c5fe54e0 Mon Sep 17 00:00:00 2001 From: thomasATbayer <105632614+thomasATbayer@users.noreply.github.com> Date: Thu, 2 Jul 2026 14:30:51 +0000 Subject: [PATCH 3/8] add test for hold out set usage --- test/unit/test_model_tuner.py | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/test/unit/test_model_tuner.py b/test/unit/test_model_tuner.py index bb71e29..0352d2c 100644 --- a/test/unit/test_model_tuner.py +++ b/test/unit/test_model_tuner.py @@ -7,7 +7,7 @@ from sklearn.datasets import make_blobs, make_classification from sklearn.linear_model import Lasso from sklearn.metrics import accuracy_score, make_scorer, mean_squared_error -from sklearn.model_selection import GroupKFold +from sklearn.model_selection import GroupKFold, KFold, PredefinedSplit from sklearn.pipeline import Pipeline from sklearn.preprocessing import MinMaxScaler @@ -208,6 +208,39 @@ def test_tune_mother_model_classification_with_catboost( assert probabilities.shape[1] == n_classes, f"Multiclass probabilities should have {n_classes} columns." +class TestGetCallbacks: + def test_hold_out_returns_none_with_warning(self, caplog): + """get_callbacks must return None and warn when n_splits < 2 (hold-out).""" + tuner = MotherTuner( + scorer=make_scorer(mean_squared_error, greater_is_better=False), + early_stopping_optuna=True, + ) + hold_out_cv = PredefinedSplit(test_fold=[-1, -1, -1, -1, -1, 0, 0, 0, 0, 0]) + with caplog.at_level("WARNING"): + callbacks = tuner.get_callbacks(cross_validation=hold_out_cv) + assert callbacks is None + assert "hold-out" in caplog.text.lower() or "n_splits" in caplog.text + + def test_multi_fold_cv_does_not_skip(self): + """get_callbacks must not skip early stopping for a proper k-fold CV.""" + tuner = MotherTuner( + scorer=make_scorer(mean_squared_error, greater_is_better=False), + early_stopping_optuna=False, + ) + kfold_cv = KFold(n_splits=5) + callbacks = tuner.get_callbacks(cross_validation=kfold_cv) + # early_stopping_optuna=False → always None regardless of CV + assert callbacks is None + + def test_no_cv_argument_does_not_raise(self): + """get_callbacks without cross_validation argument must not raise.""" + tuner = MotherTuner( + scorer=make_scorer(mean_squared_error, greater_is_better=False), + early_stopping_optuna=False, + ) + assert tuner.get_callbacks() is None + + @pytest.mark.slow @pytest.mark.serial class TestTuneMotherModels: From ec60175cb67106805be470b796f404012785c8f5 Mon Sep 17 00:00:00 2001 From: thomasATbayer Date: Tue, 21 Jul 2026 08:57:44 +0000 Subject: [PATCH 4/8] fix: properly test the usage of the GPSampler --- test/unit/test_model_tuner.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/test/unit/test_model_tuner.py b/test/unit/test_model_tuner.py index 0352d2c..caf946f 100644 --- a/test/unit/test_model_tuner.py +++ b/test/unit/test_model_tuner.py @@ -1,4 +1,5 @@ import numpy as np +import optuna import pandas as pd import pytest from optuna.samplers import RandomSampler @@ -241,6 +242,33 @@ def test_no_cv_argument_does_not_raise(self): assert tuner.get_callbacks() is None +class TestDefaultSamplerSelection: + def test_gp_sampler_when_torch_available(self, monkeypatch): + """GPSampler should be selected when torch is available.""" + monkeypatch.setattr("mother.optimization.core.torch_available", True) + tuner = MotherTuner( + scorer=make_scorer(mean_squared_error, greater_is_better=False), + ) + assert isinstance(tuner.sampler, optuna.samplers.GPSampler) + + def test_tpe_sampler_when_torch_unavailable(self, monkeypatch): + """TPESampler should be selected when torch is not available.""" + monkeypatch.setattr("mother.optimization.core.torch_available", False) + tuner = MotherTuner( + scorer=make_scorer(mean_squared_error, greater_is_better=False), + ) + assert isinstance(tuner.sampler, optuna.samplers.TPESampler) + + def test_custom_sampler_used_directly(self): + """A user-provided sampler should be used as-is.""" + custom = RandomSampler(seed=0) + tuner = MotherTuner( + scorer=make_scorer(mean_squared_error, greater_is_better=False), + sampler=custom, + ) + assert tuner.sampler is custom + + @pytest.mark.slow @pytest.mark.serial class TestTuneMotherModels: From 43ffc9abe8e11bff619b0023024ebc6181cac4f7 Mon Sep 17 00:00:00 2001 From: thomasATbayer Date: Thu, 23 Jul 2026 10:41:06 +0000 Subject: [PATCH 5/8] fix(optimization): harden default Optuna sampler selection Use a safe fallback to TPESampler when GPSampler initialization fails, even if torch is available. Preserve the multivariate override in the TPESampler path. Refactor sampler construction into dedicated builder methods for cleaner maintainability. Add tests covering GPSampler failure fallback and multivariate override behavior. --- src/mother/optimization/core.py | 54 +++++++++++++++++++++++++-------- test/unit/test_model_tuner.py | 14 +++++++++ 2 files changed, 56 insertions(+), 12 deletions(-) diff --git a/src/mother/optimization/core.py b/src/mother/optimization/core.py index 39678ab..cc9dc33 100644 --- a/src/mother/optimization/core.py +++ b/src/mother/optimization/core.py @@ -125,6 +125,32 @@ class MotherTuner: -> Pipeline: """ + @staticmethod + def _build_tpe_sampler( + seed: int, + n_startup_trials: int, + ) -> optuna.samplers.TPESampler: + return optuna.samplers.TPESampler( + multivariate=True, + group=True, + constant_liar=True, + seed=seed, + n_startup_trials=n_startup_trials, + ) + + @staticmethod + def _build_gp_sampler( + seed: int, + n_startup_trials: int, + ) -> optuna.samplers.GPSampler: + return optuna.samplers.GPSampler( + seed=seed, + n_startup_trials=n_startup_trials, + deterministic_objective=False, + # constant liar is always True for GPSampler, + # so we don't need to set it explicitly + ) + def __init__( self, scorer: typing.Union[typing.Callable, str], @@ -146,20 +172,24 @@ def __init__( if sampler is None: if torch_available: - module_logger.debug("torch available — using GPSampler as default") - self.sampler = optuna.samplers.GPSampler( - seed=seed, - n_startup_trials=n_startup_trials, - deterministic_objective=False, - # constant liar is aalways True for GPSampler, - # so we don't need to set it explicitly - ) + try: + module_logger.debug("torch available — using GPSampler as default") + self.sampler = self._build_gp_sampler( + seed=seed, + n_startup_trials=n_startup_trials, + ) + except Exception as gpsampler_error: + module_logger.warning( + "GPSampler initialization failed (%s). Falling back to TPESampler.", + gpsampler_error, + ) + self.sampler = self._build_tpe_sampler( + seed=seed, + n_startup_trials=n_startup_trials, + ) else: module_logger.debug("torch not available — falling back to TPESampler") - self.sampler = optuna.samplers.TPESampler( - multivariate=True, - group=True, - constant_liar=True, + self.sampler = self._build_tpe_sampler( seed=seed, n_startup_trials=n_startup_trials, ) diff --git a/test/unit/test_model_tuner.py b/test/unit/test_model_tuner.py index caf946f..b8178a8 100644 --- a/test/unit/test_model_tuner.py +++ b/test/unit/test_model_tuner.py @@ -259,6 +259,20 @@ def test_tpe_sampler_when_torch_unavailable(self, monkeypatch): ) assert isinstance(tuner.sampler, optuna.samplers.TPESampler) + def test_tpe_sampler_fallback_when_gp_sampler_init_fails(self, monkeypatch): + """If GPSampler init fails, MotherTuner should fall back to TPESampler.""" + + def _raise_gp_init_error(*args, **kwargs): + raise RuntimeError("gp init failed") + + monkeypatch.setattr("mother.optimization.core.torch_available", True) + monkeypatch.setattr("optuna.samplers.GPSampler", _raise_gp_init_error) + + tuner = MotherTuner( + scorer=make_scorer(mean_squared_error, greater_is_better=False), + ) + assert isinstance(tuner.sampler, optuna.samplers.TPESampler) + def test_custom_sampler_used_directly(self): """A user-provided sampler should be used as-is.""" custom = RandomSampler(seed=0) From e7c262fd98f2f38a3ae53a3e07d5d1bb2adc4bd7 Mon Sep 17 00:00:00 2001 From: thomasATbayer Date: Thu, 23 Jul 2026 11:55:20 +0000 Subject: [PATCH 6/8] fix(optimization): harden default Optuna sampler selection Use GPSampler as the default sampler when torch is available, with safe fallback to TPESampler if GP initialization fails. Configure TPESampler as the independent sampler for GPSampler so conditional parameters fall back to history-informed sampling instead of pure random sampling. Fix sampler assignment in MotherTuner.init and skip TerminatorCallback for hold-out validation where early stopping is invalid. Add test coverage for default sampler selection, GP fallback behavior, and hold-out callback handling. No breaking changes. --- src/mother/optimization/core.py | 4 ++++ test/unit/test_model_tuner.py | 27 +++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/src/mother/optimization/core.py b/src/mother/optimization/core.py index cc9dc33..616ace3 100644 --- a/src/mother/optimization/core.py +++ b/src/mother/optimization/core.py @@ -145,6 +145,10 @@ def _build_gp_sampler( ) -> optuna.samplers.GPSampler: return optuna.samplers.GPSampler( seed=seed, + independent_sampler=optuna.samplers.TPESampler( + seed=seed, + n_startup_trials=n_startup_trials, + ), n_startup_trials=n_startup_trials, deterministic_objective=False, # constant liar is always True for GPSampler, diff --git a/test/unit/test_model_tuner.py b/test/unit/test_model_tuner.py index b8178a8..066cedb 100644 --- a/test/unit/test_model_tuner.py +++ b/test/unit/test_model_tuner.py @@ -250,6 +250,7 @@ def test_gp_sampler_when_torch_available(self, monkeypatch): scorer=make_scorer(mean_squared_error, greater_is_better=False), ) assert isinstance(tuner.sampler, optuna.samplers.GPSampler) + assert isinstance(tuner.sampler._independent_sampler, optuna.samplers.TPESampler) def test_tpe_sampler_when_torch_unavailable(self, monkeypatch): """TPESampler should be selected when torch is not available.""" @@ -273,6 +274,32 @@ def _raise_gp_init_error(*args, **kwargs): ) assert isinstance(tuner.sampler, optuna.samplers.TPESampler) + def test_gp_sampler_warns_for_dynamic_search_space(self, caplog): + """GPSampler should warn that dynamic search spaces fall back to independent sampling.""" + pytest.importorskip("torch") + pytest.importorskip("scipy") + + sampler = optuna.samplers.GPSampler( + seed=0, + n_startup_trials=1, + warn_independent_sampling=True, + ) + study = optuna.create_study(direction="minimize", sampler=sampler) + + def objective(trial): + # Alternate parameter names across trials to force a dynamic search space. + x = trial.suggest_float("x", 0.0, 1.0) + if trial.number % 2 == 0: + trial.suggest_float("even_only", 0.0, 1.0) + else: + trial.suggest_float("odd_only", 0.0, 1.0) + return x + + with caplog.at_level("WARNING", logger="optuna.samplers._gp.sampler"): + study.optimize(objective, n_trials=6) + + assert "dynamic search space is not supported by GPSampler" in caplog.text + def test_custom_sampler_used_directly(self): """A user-provided sampler should be used as-is.""" custom = RandomSampler(seed=0) From bed73906af7e95f659d1508b543817042efde526 Mon Sep 17 00:00:00 2001 From: thomasATbayer Date: Thu, 23 Jul 2026 13:17:37 +0000 Subject: [PATCH 7/8] fix(optimization): default to GPSampler with TPE independent fallback use GPSampler as default sampler when torch is available configure TPESampler as independent_sampler for GPSampler keep robust fallback to TPESampler when GPSampler init fails or torch is unavailable remove hold-out-specific callback behavior from this branch update tests for default sampler selection and callback behavior --- src/mother/optimization/core.py | 16 ++------------ test/unit/test_model_tuner.py | 39 +++++++++++++++++++-------------- 2 files changed, 24 insertions(+), 31 deletions(-) diff --git a/src/mother/optimization/core.py b/src/mother/optimization/core.py index 616ace3..129b771 100644 --- a/src/mother/optimization/core.py +++ b/src/mother/optimization/core.py @@ -202,10 +202,7 @@ def __init__( self.study: typing.Optional[Study] = None - def get_callbacks( - self, - cross_validation: typing.Optional[skl_model_sel.BaseCrossValidator] = None, - ): + def get_callbacks(self): """ Prepares and returns a list of callbacks for early stopping in Optuna optimization. @@ -219,15 +216,6 @@ def get_callbacks( """ callbacks: typing.Optional[typing.List[TerminatorCallback]] = None if self.early_stopping_optuna: - if cross_validation is not None: - n_splits = cross_validation.get_n_splits() - if n_splits < 2: - module_logger.warning( - "Optuna early stopping requires at least 2 CV folds. " - "Skipping early stopping (hold-out detected, n_splits=%s)", - n_splits, - ) - return None if not torch_available: module_logger.warning("Torch not installed, early optuna termination will not be available") module_logger.warning( @@ -341,7 +329,7 @@ def objective(trial: optuna.trial.Trial) -> float: objective, n_trials=self.n_trials_optuna, gc_after_trial=True, - callbacks=self.get_callbacks(cross_validation=cross_validation), + callbacks=self.get_callbacks(), ) if default_parameters != {}: diff --git a/test/unit/test_model_tuner.py b/test/unit/test_model_tuner.py index 066cedb..8707368 100644 --- a/test/unit/test_model_tuner.py +++ b/test/unit/test_model_tuner.py @@ -8,7 +8,7 @@ from sklearn.datasets import make_blobs, make_classification from sklearn.linear_model import Lasso from sklearn.metrics import accuracy_score, make_scorer, mean_squared_error -from sklearn.model_selection import GroupKFold, KFold, PredefinedSplit +from sklearn.model_selection import GroupKFold from sklearn.pipeline import Pipeline from sklearn.preprocessing import MinMaxScaler @@ -210,36 +210,41 @@ def test_tune_mother_model_classification_with_catboost( class TestGetCallbacks: - def test_hold_out_returns_none_with_warning(self, caplog): - """get_callbacks must return None and warn when n_splits < 2 (hold-out).""" + def test_early_stopping_enabled_returns_callback_when_torch_available(self, monkeypatch): + """get_callbacks should return terminator callback when enabled and torch is available.""" + monkeypatch.setattr("mother.optimization.core.torch_available", True) tuner = MotherTuner( scorer=make_scorer(mean_squared_error, greater_is_better=False), early_stopping_optuna=True, ) - hold_out_cv = PredefinedSplit(test_fold=[-1, -1, -1, -1, -1, 0, 0, 0, 0, 0]) - with caplog.at_level("WARNING"): - callbacks = tuner.get_callbacks(cross_validation=hold_out_cv) - assert callbacks is None - assert "hold-out" in caplog.text.lower() or "n_splits" in caplog.text - def test_multi_fold_cv_does_not_skip(self): - """get_callbacks must not skip early stopping for a proper k-fold CV.""" + callbacks = tuner.get_callbacks() + + assert callbacks is not None + assert len(callbacks) == 1 + + def test_early_stopping_disabled_returns_none(self): tuner = MotherTuner( scorer=make_scorer(mean_squared_error, greater_is_better=False), early_stopping_optuna=False, ) - kfold_cv = KFold(n_splits=5) - callbacks = tuner.get_callbacks(cross_validation=kfold_cv) - # early_stopping_optuna=False → always None regardless of CV + + callbacks = tuner.get_callbacks() + assert callbacks is None - def test_no_cv_argument_does_not_raise(self): - """get_callbacks without cross_validation argument must not raise.""" + def test_early_stopping_enabled_torch_missing_returns_none_with_warning(self, monkeypatch, caplog): + monkeypatch.setattr("mother.optimization.core.torch_available", False) tuner = MotherTuner( scorer=make_scorer(mean_squared_error, greater_is_better=False), - early_stopping_optuna=False, + early_stopping_optuna=True, ) - assert tuner.get_callbacks() is None + + with caplog.at_level("WARNING"): + callbacks = tuner.get_callbacks() + + assert callbacks is None + assert "Torch not installed" in caplog.text class TestDefaultSamplerSelection: From 491d206f682283c420b8fc72ca887c4ea191f79e Mon Sep 17 00:00:00 2001 From: thomasATbayer Date: Thu, 23 Jul 2026 14:13:36 +0000 Subject: [PATCH 8/8] test(optimization): make GP sampler selection test hermetic for optional deps gate test_gp_sampler_when_torch_available with pytest.importorskip("torch") gate test_gp_sampler_when_torch_available with pytest.importorskip("scipy") align optional-dependency handling with existing test patterns avoid false failures when optional extras are not installed --- test/unit/test_model_tuner.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/unit/test_model_tuner.py b/test/unit/test_model_tuner.py index 8707368..d0dac43 100644 --- a/test/unit/test_model_tuner.py +++ b/test/unit/test_model_tuner.py @@ -250,6 +250,8 @@ def test_early_stopping_enabled_torch_missing_returns_none_with_warning(self, mo class TestDefaultSamplerSelection: def test_gp_sampler_when_torch_available(self, monkeypatch): """GPSampler should be selected when torch is available.""" + pytest.importorskip("torch") + pytest.importorskip("scipy") monkeypatch.setattr("mother.optimization.core.torch_available", True) tuner = MotherTuner( scorer=make_scorer(mean_squared_error, greater_is_better=False),