diff --git a/CHANGELOG.md b/CHANGELOG.md index 7021059..6953acc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ ## Unreleased +## 1.0.1 + +- Fresh GMM studies now use the held-out-calibrated defaults: twice the raw + warm-up rule before power-of-two rounding, a 12.5% elite fraction, no ongoing + post-warm-up Sobol' cadence, one mixture component, and a five-sample elite + floor. Direct low-level GMM constructors and refit configuration use the same + component cap. Checkpoints that predate these explicit fields continue to + resolve to the historical defaults when loaded. - Multi-group GMM refits now select elites by Pareto rank and crowding distance, and grouped objectives apply priority weights only within their explicit group. @@ -18,6 +26,8 @@ seeds, immutable provenance manifests, failure-visible reporting, fixed metric scales, and strict result-coverage validation. The suite also adds analytic grouped-TLP and sealed-test mixed-space HPO capability studies. +- Updated the documentation dependency lockfile to resolve CVE-2026-61632 in + `pymdown-extensions`. ## 1.0.1-rc8 diff --git a/Cargo.lock b/Cargo.lock index 7fa6477..eea1ccd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -531,7 +531,7 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "hola" -version = "1.0.1-rc8" +version = "1.0.1" dependencies = [ "axum", "constant_time_eq", @@ -553,7 +553,7 @@ dependencies = [ [[package]] name = "hola-cli" -version = "1.0.1-rc8" +version = "1.0.1" dependencies = [ "clap", "command-group", @@ -569,7 +569,7 @@ dependencies = [ [[package]] name = "hola-py" -version = "1.0.1-rc8" +version = "1.0.1" dependencies = [ "hola", "pyo3", @@ -1077,7 +1077,7 @@ checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" [[package]] name = "opt_engine" -version = "1.0.1-rc8" +version = "1.0.1" dependencies = [ "chrono", "nalgebra", diff --git a/docs/cli-guide.md b/docs/cli-guide.md index 9819e02..b0785ad 100644 --- a/docs/cli-guide.md +++ b/docs/cli-guide.md @@ -69,6 +69,8 @@ objectives: priority: 0.5 group: cost +max_trials: 200 # trial cap and S for the automatic GMM warm-up + strategy: type: gmm refit_interval: 20 @@ -81,6 +83,11 @@ strategy: # max_checkpoints: 5 ``` +`max_trials` is optional. When set, it caps dispatched trials and supplies the +total budget `S` used by the automatic GMM warm-up. If neither `max_trials` nor +`strategy.total_budget` is set, the warm-up calculation uses `S=200` without +imposing a trial cap. + ### Space Configuration Each parameter in the `space:` section has a `type` and @@ -174,10 +181,10 @@ strategy: refit_interval: 20 # how often GMM refits (used by "gmm") seed: 42 # optional seed for reproducible runs exploration_budget: 50 # number of Sobol asks before switching to GMM - elite_fraction: 0.25 # fraction of top trials used for GMM fitting (default: 0.25) - ongoing_exploration_period: 5 # every Nth post-warmup ask is Sobol (0 disables) - max_components: 3 # maximum fitted GMM components - min_elite_samples: 1 # minimum feasible elite workset before fitting + elite_fraction: 0.125 # fraction of top trials used for GMM fitting + ongoing_exploration_period: 0 # every Nth post-warmup ask is Sobol (0 disables) + max_components: 1 # maximum fitted GMM components + min_elite_samples: 5 # minimum feasible elite workset before fitting max_refit_samples: 4096 # maximum elite samples passed to one GMM fit max_refit_candidates: 16384 # maximum trials ranked to choose elites ``` @@ -187,11 +194,12 @@ strategy: | `type` | `"gmm"` | Strategy type: `"gmm"`, `"sobol"`, or `"random"` | | `refit_interval` | `20` | How often the GMM refits (only used by `"gmm"`) | | `seed` | none | Seed for reproducible runs. When omitted, HOLA draws one seed once and records it in full checkpoints. | -| `exploration_budget` | none | Number of issued Sobol exploration suggestions before switching to GMM exploitation. Pending asks count against this budget. When omitted, we use a formula based on `total_budget` and the search dimension. | -| `elite_fraction` | `0.25` | Fraction of top trials used for GMM refitting. Must be in (0.0, 1.0]. | -| `ongoing_exploration_period` | `5` | Continue global Sobol' exploration every Nth post-warmup suggestion. Use `0` to disable; explicit periods must be at least 2. | -| `max_components` | `3` | Maximum fitted GMM components. The effective count can be lower for small elite sets. | -| `min_elite_samples` | `1` | Minimum feasible elite workset required before fitting. Must not exceed `max_refit_samples`. | +| `total_budget` | none | Alternative source for `S` and the trial cap when top-level `max_trials` is omitted. | +| `exploration_budget` | none | Number of issued Sobol exploration suggestions before switching to GMM exploitation. Pending asks count against this budget. When omitted, HOLA doubles `min(floor(S/5), 50 + 2n)` and then rounds down to a power of two, for total budget `S` and dimension `n`; `S=200` when neither budget field is set. | +| `elite_fraction` | `0.125` | Fraction of top trials used for GMM refitting. Must be in (0.0, 1.0]. | +| `ongoing_exploration_period` | `0` | Continue global Sobol' exploration every Nth post-warmup suggestion. The default `0` disables it; explicit periods must be at least 2. | +| `max_components` | `1` | Maximum fitted GMM components. The effective count can be lower for small elite sets. | +| `min_elite_samples` | `5` | Minimum feasible elite workset required before fitting. Must not exceed `max_refit_samples`. | | `max_refit_samples` | `4096` | Maximum elite samples passed to one GMM fit. Must be at least 1. | | `max_refit_candidates` | `16384` | Maximum retained trials ranked to choose elites. Must be at least `max_refit_samples`; longer histories use deterministic stratified coverage of the full retained history. | diff --git a/docs/concepts.md b/docs/concepts.md index bd1c659..ebce4d7 100644 --- a/docs/concepts.md +++ b/docs/concepts.md @@ -113,15 +113,19 @@ rank followed by descending crowding distance. The lifecycle follows three phases. 1. **Warmup.** The first trials build up the leaderboard. HOLA continues - Sobol' sampling until the first empirical fit is installed. + Sobol' sampling until the first empirical fit is installed. With no explicit + exploration budget, the implementation doubles + `min(floor(S/5), 50 + 2n)` and rounds down to a power of two, for total + budget `S` and dimension `n`. The warm-up calculation uses `S=200` when no + total trial budget is configured; this fallback does not impose a trial cap. 2. **Refit.** Every `refit_interval` trials (default 20), we refit - the GMM to the top 25% of trials, subject to the configured minimum + the GMM to the top 12.5% of trials, subject to the configured minimum feasible elite workset. If the first scheduled fit lacks that workset, each subsequent completion retries until the first empirical model is installed; later refits return to the configured cadence. 3. **Exploit.** New samples are drawn from the updated GMM, - focusing on promising regions. By default, every fifth post-warmup - suggestion remains a global Sobol' exploration point. + focusing on promising regions. Ongoing post-warmup Sobol' exploration is + disabled by default and can be enabled with an explicit period. GMM exploitation uses seeded Owen-scrambled Gauss–Sobol' points. One Sobol' coordinate selects the mixture component, and inverse-normal @@ -136,9 +140,9 @@ on completed trials in the leaderboard. Pending asks that cross the nominal warmup boundary before the first completed-data fit remain Sobol' suggestions; the unfitted prior is never used as exploitation. -`max_components` bounds mixture complexity (default 3), while +`max_components` bounds mixture complexity (default 1), while `min_elite_samples` can delay fitting until a feasible elite workset reaches a -requested size (default 1). The effective component count may be smaller: the +requested size (default 5). The effective component count may be smaller: the implementation requires enough elite samples to support each component. Two implementation limits keep refitting bounded on unusually long studies. diff --git a/docs/python-guide.md b/docs/python-guide.md index 1540e16..aedd465 100644 --- a/docs/python-guide.md +++ b/docs/python-guide.md @@ -260,7 +260,7 @@ study = Study( | `objectives` | `list` | required | List of `Minimize` / `Maximize` objectives (at least one) | | `strategy` | `str` or strategy class | `"gmm"` | Search strategy. Pass a string (`"gmm"`, `"sobol"`, `"random"`) for defaults, or a configuration class (`Gmm(...)`, `Sobol()`, `Random()`) for fine-grained control. | | `seed` | `int` or `None` | `None` | Random seed for reproducibility. When set, the same seed produces the same candidate sequence. | -| `max_trials` | `int` or `None` | `None` | Maximum number of trials. When set, `ask()` raises after this many trials have been dispatched. | +| `max_trials` | `int` or `None` | `None` | Maximum number of trials. When set, `ask()` raises after this many trials have been dispatched and the value supplies `S` for the automatic GMM warm-up. When omitted, the warm-up calculation uses `S=200` without imposing a trial cap. | ## The Ask/Tell Loop @@ -458,7 +458,7 @@ Study(strategy=Gmm(refit_interval=10, elite_fraction=0.1), ...) Gaussian Mixture Model strategy. Uses Sobol exploration followed by GMM exploitation. Refits a GMM to the top `elite_fraction` -(default 25%) of trials every `refit_interval` (default 20) +(default 12.5%) of trials every `refit_interval` (default 20) completed trials. With multiple objective groups, elites are ordered by non-domination rank and then descending crowding distance. The exploration budget counts issued `ask` suggestions, including pending @@ -488,11 +488,11 @@ Study(strategy=Gmm(refit_interval=10, elite_fraction=0.1), ...) | Parameter | Type | Default | Description | |-----------|------|---------|-------------| | `refit_interval` | `int` or `None` | 20 | How often the GMM is refit, in completed trials | -| `elite_fraction` | `float` or `None` | 0.25 | Fraction of top trials used for refitting. Must be in (0, 1]. | -| `exploration_budget` | `int` or `None` | auto | Number of issued Sobol exploration suggestions before GMM exploitation begins. Pending asks count against this budget. When omitted, computed automatically from the total budget and number of dimensions. | -| `ongoing_exploration_period` | `int` or `None` | 5 | Continue global Sobol' exploration every Nth post-warmup suggestion. Use `0` to disable; explicit periods must be at least 2. | -| `max_components` | `int` or `None` | 3 | Maximum fitted mixture components. The effective count can be lower when the elite set is small. | -| `min_elite_samples` | `int` or `None` | 1 | Minimum feasible elite workset required before fitting. Must not exceed `max_refit_samples`. | +| `elite_fraction` | `float` or `None` | 0.125 | Fraction of top trials used for refitting. Must be in (0, 1]. | +| `exploration_budget` | `int` or `None` | auto | Number of issued Sobol exploration suggestions before GMM exploitation begins. Pending asks count against this budget. When omitted, HOLA doubles `min(floor(S/5), 50 + 2n)` and then rounds down to a power of two, for total budget `S` and dimension `n`; `S=200` when `max_trials` is unset. | +| `ongoing_exploration_period` | `int` or `None` | 0 | Continue global Sobol' exploration every Nth post-warmup suggestion. The default `0` disables it; explicit periods must be at least 2. | +| `max_components` | `int` or `None` | 1 | Maximum fitted mixture components. The effective count can be lower when the elite set is small. | +| `min_elite_samples` | `int` or `None` | 5 | Minimum feasible elite workset required before fitting. Must not exceed `max_refit_samples`. | | `max_refit_samples` | `int` or `None` | 4096 | Maximum elite samples used by one GMM fit. Must be at least 1. | | `max_refit_candidates` | `int` or `None` | 16384 | Maximum retained trials ranked during elite selection. Must be at least `max_refit_samples`; longer histories use deterministic stratified coverage. | diff --git a/hola-cli/Cargo.toml b/hola-cli/Cargo.toml index c0b7629..e4d5802 100644 --- a/hola-cli/Cargo.toml +++ b/hola-cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "hola-cli" -version = "1.0.1-rc8" +version = "1.0.1" edition = "2024" description = "Command-line server and distributed worker for HOLA" documentation = "https://github.com/blackrock/HOLA/blob/main/docs/cli-guide.md" @@ -19,7 +19,7 @@ path = "src/main.rs" doc = false [dependencies] -hola = { path = "../hola", features = ["server"], version = "1.0.1-rc8" } +hola = { path = "../hola", features = ["server"], version = "1.0.1" } clap = { version = "4", features = ["derive"] } # Keep the established module name while using the maintained, API-compatible fork. serde_yaml = { package = "serde_yaml_ng", version = "0.10" } diff --git a/hola-cli/src/main.rs b/hola-cli/src/main.rs index 588d322..691cd9f 100644 --- a/hola-cli/src/main.rs +++ b/hola-cli/src/main.rs @@ -1526,16 +1526,17 @@ async fn main() -> Result<(), Box> { .checkpoint .as_ref() .map(|checkpoint| PathBuf::from(&checkpoint.directory)); - let engine = HolaEngine::from_config(study_config) - .map_err(|e| format!("Failed to create engine: {e}"))?; - - if let Some(path) = load_from { - let checkpoint_kind = engine - .load_checkpoint_with_fallback(&path) - .await - .map_err(|e| format!("Failed to load checkpoint '{path}': {e}"))?; + let engine = if let Some(path) = load_from { + let (engine, checkpoint_kind) = + HolaEngine::load_configured_checkpoint(study_config, &path) + .await + .map_err(|e| format!("Failed to load checkpoint '{path}': {e}"))?; eprintln!("Loaded {} checkpoint from {path}", checkpoint_kind.as_str()); - } + engine + } else { + HolaEngine::from_config(study_config) + .map_err(|e| format!("Failed to create engine: {e}"))? + }; let auth_token = configured_token(auth_token); if !is_local_host(&host) && auth_token.is_none() { diff --git a/hola-py/Cargo.toml b/hola-py/Cargo.toml index 8bb220a..6ebbd7e 100644 --- a/hola-py/Cargo.toml +++ b/hola-py/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "hola-py" -version = "1.0.1-rc8" +version = "1.0.1" edition = "2024" description = "Python bindings for the HOLA hyperparameter optimization engine" documentation = "https://github.com/blackrock/HOLA/blob/main/docs/python-guide.md" @@ -18,7 +18,7 @@ name = "hola_opt" crate-type = ["cdylib"] [dependencies] -hola_engine = { package = "hola", version = "1.0.1-rc8", path = "../hola", features = ["server"] } +hola_engine = { package = "hola", version = "1.0.1", path = "../hola", features = ["server"] } pyo3 = { version = "0.29", features = ["extension-module", "abi3-py310"] } serde = "1" serde_json = "1" diff --git a/hola-py/benchmarks/adapters/base.py b/hola-py/benchmarks/adapters/base.py index 87619d1..fee410e 100644 --- a/hola-py/benchmarks/adapters/base.py +++ b/hola-py/benchmarks/adapters/base.py @@ -13,9 +13,10 @@ from __future__ import annotations -from collections.abc import Callable +import json +from collections.abc import Callable, Mapping from dataclasses import dataclass, field -from typing import Any, Protocol, runtime_checkable +from typing import Any, Protocol, cast, runtime_checkable import numpy as np @@ -70,6 +71,92 @@ def __init__(self, actual: int, expected: int, optimizer: str) -> None: ) +class EmpiricalExploitationError(RuntimeError): + """A GMM benchmark run did not produce authenticated empirical exploitation.""" + + def __init__(self, actual: int, observed_diagnostics: Mapping[str, object]) -> None: + self.actual = actual + self.observed_diagnostics = dict(observed_diagnostics) + observed = json.dumps( + self.observed_diagnostics, + allow_nan=False, + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ) + super().__init__( + "GMM empirical-exploitation gate failed " + "(requires gmm_fit_epoch>=1, gmm_sampling_ready=true, " + "gmm_origin_suggestions>=5, and " + "issued_suggestions==completed_evaluations); " + f"observed_diagnostics={observed}" + ) + + +def empirical_exploitation_gate_configuration() -> dict[str, object]: + """Return the manifest-bound practical-benchmark GMM gate.""" + + return { + "minimum_gmm_fit_epoch": 1, + "minimum_gmm_origin_suggestions": 5, + "on_failure": "preserved_error_outcome", + } + + +def _stable_observed_value(value: object) -> object: + if value is None or type(value) in {bool, int, str}: + return value + if type(value) is float: + return value if np.isfinite(value) else f"" + return f"" + + +def require_empirical_gmm_exploitation(study: object, completed_evaluations: int) -> None: + """Fail closed unless a completed GMM run demonstrably used its fitted sampler.""" + + if type(completed_evaluations) is not int or completed_evaluations < 0: + raise ValueError("completed_evaluations must be a non-negative integer") + required_fields = ( + "gmm_fit_epoch", + "gmm_origin_suggestions", + "gmm_sampling_ready", + "issued_suggestions", + ) + observed: dict[str, object] = {"completed_evaluations": completed_evaluations} + try: + diagnostics_method = cast(Any, study).strategy_diagnostics + diagnostics = diagnostics_method() + except Exception as error: + observed["diagnostics_error"] = f"{type(error).__name__}: {error}" + raise EmpiricalExploitationError(completed_evaluations, observed) from error + if type(diagnostics) is not dict: + observed["diagnostics"] = f"" + raise EmpiricalExploitationError(completed_evaluations, observed) + diagnostics = cast(dict[object, object], diagnostics) + for field_name in required_fields: + observed[field_name] = ( + "" + if field_name not in diagnostics + else _stable_observed_value(diagnostics[field_name]) + ) + + fit_epoch = diagnostics.get("gmm_fit_epoch") + origin_suggestions = diagnostics.get("gmm_origin_suggestions") + sampling_ready = diagnostics.get("gmm_sampling_ready") + issued_suggestions = diagnostics.get("issued_suggestions") + valid = ( + type(fit_epoch) is int + and fit_epoch >= 1 + and type(origin_suggestions) is int + and origin_suggestions >= 5 + and sampling_ready is True + and type(issued_suggestions) is int + and issued_suggestions == completed_evaluations + ) + if not valid: + raise EmpiricalExploitationError(completed_evaluations, observed) + + def assert_exact_evaluations(actual: int, expected: int, optimizer: str) -> None: """Fail a benchmark run whose objective-call count missed its contract.""" if actual != expected: diff --git a/hola-py/benchmarks/adapters/hola_adapter.py b/hola-py/benchmarks/adapters/hola_adapter.py index e3f4f45..06a59fd 100644 --- a/hola-py/benchmarks/adapters/hola_adapter.py +++ b/hola-py/benchmarks/adapters/hola_adapter.py @@ -23,6 +23,8 @@ MultiObjectiveResult, SingleObjectiveResult, assert_exact_evaluations, + empirical_exploitation_gate_configuration, + require_empirical_gmm_exploitation, ) from benchmarks.problems.registry import ( GroupedTlpProblem, @@ -41,13 +43,18 @@ def __init__(self, strategy: str = "gmm") -> None: self.name = f"HOLA ({label})" def configuration(self, budget: int) -> dict[str, object]: - return { + configuration: dict[str, object] = { "adapter": type(self).__name__, "strategy": self.strategy, "max_trials": budget, "n_workers": 1, "objective": "raw minimization", } + if self.strategy == "gmm": + configuration["empirical_exploitation_gate"] = ( + empirical_exploitation_gate_configuration() + ) + return configuration def optimize( self, @@ -77,6 +84,8 @@ def wrapped(params: dict) -> dict: trials = study.trials() n_evaluations = len(trials) assert_exact_evaluations(n_evaluations, budget, self.name) + if self.strategy == "gmm": + require_empirical_gmm_exploitation(study, n_evaluations) for trial in trials: score = trial.score_vector.get("value", float("inf")) if math.isfinite(score): @@ -109,13 +118,18 @@ def __init__(self, strategy: str = "gmm") -> None: self.name = f"HOLA MO ({label})" def configuration(self, budget: int) -> dict[str, object]: - return { + configuration: dict[str, object] = { "adapter": type(self).__name__, "strategy": self.strategy, "max_trials": budget, "n_workers": 1, "objectives": "unrestricted raw minimization, one group per field", } + if self.strategy == "gmm": + configuration["empirical_exploitation_gate"] = ( + empirical_exploitation_gate_configuration() + ) + return configuration @staticmethod def _build_objectives(problem: MultiObjectiveProblem) -> list[Minimize | Maximize]: @@ -158,6 +172,8 @@ def optimize( all_trials = study.trials(sorted_by="index", include_infeasible=True) n_evaluations = len(all_trials) assert_exact_evaluations(n_evaluations, budget, self.name) + if self.strategy == "gmm": + require_empirical_gmm_exploitation(study, n_evaluations) if all_trials: raw_objectives = np.array( [[t.metrics[name] for name in problem.objective_names] for t in all_trials] @@ -192,7 +208,7 @@ def __init__(self, grouped_problem: GroupedTlpProblem, strategy: str = "gmm") -> self.name = f"HOLA grouped TLP ({label})" def configuration(self, budget: int) -> dict[str, object]: - return { + configuration: dict[str, object] = { "adapter": type(self).__name__, "strategy": self.strategy, "problem": self.grouped_problem.name, @@ -211,6 +227,11 @@ def configuration(self, budget: int) -> dict[str, object]: for objective in self.grouped_problem.objectives ], } + if self.strategy == "gmm": + configuration["empirical_exploitation_gate"] = ( + empirical_exploitation_gate_configuration() + ) + return configuration @staticmethod def _build_objectives(problem: GroupedTlpProblem) -> list[Minimize | Maximize]: @@ -259,6 +280,8 @@ def optimize( trials = study.trials(sorted_by="index", include_infeasible=True) n_evaluations = len(trials) assert_exact_evaluations(n_evaluations, budget, self.name) + if self.strategy == "gmm": + require_empirical_gmm_exploitation(study, n_evaluations) if not trials: return MultiObjectiveResult( pareto_front=np.empty((0, grouped.n_groups)), diff --git a/hola-py/benchmarks/adapters/hpo.py b/hola-py/benchmarks/adapters/hpo.py index ea23eb4..b439e58 100644 --- a/hola-py/benchmarks/adapters/hpo.py +++ b/hola-py/benchmarks/adapters/hpo.py @@ -19,7 +19,12 @@ import optuna -from benchmarks.adapters.base import HpoOptimizationResult, assert_exact_evaluations +from benchmarks.adapters.base import ( + HpoOptimizationResult, + assert_exact_evaluations, + empirical_exploitation_gate_configuration, + require_empirical_gmm_exploitation, +) from benchmarks.problems.hpo import ( CategoricalParameter, HpoProblem, @@ -83,13 +88,18 @@ def __init__(self, strategy: str) -> None: self.name = f"HOLA HPO ({label})" def configuration(self, budget: int) -> dict[str, object]: - return { + configuration: dict[str, object] = { "adapter": type(self).__name__, "strategy": self.strategy, "max_trials": budget, "n_workers": 1, "objective": "maximize fixed-split validation R2", } + if self.strategy == "gmm": + configuration["empirical_exploitation_gate"] = ( + empirical_exploitation_gate_configuration() + ) + return configuration def optimize( self, @@ -115,6 +125,8 @@ def objective(params: dict[str, Any]) -> dict[str, float]: trials = study.trials(sorted_by="index", include_infeasible=True) n_evaluations = len(trials) assert_exact_evaluations(n_evaluations, budget, self.name) + if self.strategy == "gmm": + require_empirical_gmm_exploitation(study, n_evaluations) validation_trace = [float(trial.metrics[problem.objective_name]) for trial in trials] best = max(trials, key=lambda trial: float(trial.metrics[problem.objective_name])) return HpoOptimizationResult( diff --git a/hola-py/benchmarks/runner/executor.py b/hola-py/benchmarks/runner/executor.py index 4c09496..f1f0274 100644 --- a/hola-py/benchmarks/runner/executor.py +++ b/hola-py/benchmarks/runner/executor.py @@ -19,6 +19,7 @@ from typing import Any, TypeAlias from benchmarks.adapters.base import ( + EmpiricalExploitationError, EvaluationCountError, MultiObjectiveOptimizer, SingleObjectiveOptimizer, @@ -145,7 +146,7 @@ def _run_single_one( except Exception as error: row["error"] = _format_error(error) row["wall_time_seconds"] = time.perf_counter() - started - if isinstance(error, EvaluationCountError): + if isinstance(error, (EvaluationCountError, EmpiricalExploitationError)): row["n_evaluations"] = error.actual return row @@ -339,7 +340,7 @@ def _run_multi_one( row["error"] = _format_error(error) if row["wall_time_seconds"] is None: row["wall_time_seconds"] = time.perf_counter() - started - if isinstance(error, EvaluationCountError): + if isinstance(error, (EvaluationCountError, EmpiricalExploitationError)): row["n_evaluations"] = error.actual return row diff --git a/hola-py/examples/gmm_explore_exploit.py b/hola-py/examples/gmm_explore_exploit.py index 5c5c444..39ef5d7 100644 --- a/hola-py/examples/gmm_explore_exploit.py +++ b/hola-py/examples/gmm_explore_exploit.py @@ -16,8 +16,8 @@ how the GMM strategy concentrates samples around promising regions. The GMM strategy: - 1. First ~20 trials: Sobol quasi-random exploration (warmup) - 2. After warmup: fits a Gaussian Mixture Model to the top 25% of trials + 1. Initial trials: Sobol quasi-random exploration (warmup) + 2. After warmup: fits a Gaussian Mixture Model to the top 12.5% of trials 3. Every 20 trials: refits the GMM to the latest top performers 4. New samples are drawn from the GMM, focusing on promising regions """ @@ -39,6 +39,8 @@ def run_with_strategy( space=Space(**space_kwargs), objectives=[Minimize("value", target=target, limit=limit)], strategy=strategy, + seed=42, + max_trials=n_trials, ) history = [] @@ -99,8 +101,8 @@ def main(): print(" Sobol (pure exploration) vs GMM (explore → exploit)") print("=" * 60) print() - print(" Phase 1 (trials 1-20): Both strategies explore via Sobol") - print(" Phase 2 (trials 21+): GMM refits to top 25% and exploits") + print(" Phase 1: Both strategies explore via Sobol") + print(" Phase 2: GMM refits to top 12.5% and exploits") print(" Sobol continues uniform exploration") # 2D: Branin — should see clear GMM advantage diff --git a/hola-py/hola_opt/__init__.pyi b/hola-py/hola_opt/__init__.pyi index b5e5931..be333a8 100644 --- a/hola-py/hola_opt/__init__.pyi +++ b/hola-py/hola_opt/__init__.pyi @@ -135,10 +135,13 @@ class Gmm: refit_interval: How often the GMM is refit, in completed trials (default: 20). elite_fraction: Fraction of top trials used for GMM refitting - (default: 0.25). Must be between 0.0 and 1.0. + (default: 0.125). Must be between 0.0 and 1.0. exploration_budget: Number of Sobol exploration trials before GMM - exploitation begins. When omitted, computed automatically from - the total budget and number of dimensions. + exploitation begins. When omitted, doubles + ``min(floor(S/5), 50 + 2n)`` and rounds down to a power of two, + for total budget ``S`` and dimension ``n``. When ``max_trials`` + is also omitted, the warm-up calculation uses ``S=200`` without + imposing a trial cap. max_refit_samples: Maximum elite samples used by one GMM fit (default: 4096). max_refit_candidates: Maximum retained trials ranked during elite @@ -146,12 +149,11 @@ class Gmm: stratified coverage. ongoing_exploration_period: Period between Sobol exploration trials after GMM exploitation begins. ``None`` uses the default (currently - 5), ``0`` disables ongoing exploration, and explicit periods must - be at least 2. + 0, disabled), and explicit periods must be at least 2. max_components: Maximum number of Gaussian mixture components. When - omitted, uses the default (currently 3). Must be at least 1. + omitted, uses the default (currently 1). Must be at least 1. min_elite_samples: Minimum feasible elite samples required before GMM - fitting. When omitted, uses the default (currently 1). Must be at + fitting. When omitted, uses the default (currently 5). Must be at least 1 and must not exceed ``max_refit_samples``. Raises: @@ -355,6 +357,9 @@ class Study: def trial_count(self) -> int: """Number of completed trials.""" ... + def strategy_diagnostics(self) -> dict[str, int | bool | None]: + """Return read-only strategy routing and fitted-model diagnostics.""" + ... def update_objectives(self, objectives: list[Minimize | Maximize]) -> None: """Update objectives mid-run, re-scalarizing all trials.""" ... diff --git a/hola-py/pyproject.toml b/hola-py/pyproject.toml index 02b7426..91e52b4 100644 --- a/hola-py/pyproject.toml +++ b/hola-py/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "maturin" [project] name = "hola-opt" -version = "1.0.1-rc8" +version = "1.0.1" description = "Lightweight asynchronous hyperparameter optimization backed by Rust" readme = "README.md" requires-python = ">=3.10" @@ -13,7 +13,7 @@ license-files = ["LICENSE-APACHE"] authors = [{ name = "BlackRock, Inc." }] keywords = ["optimization", "hyperparameter-optimization", "machine-learning"] classifiers = [ - "Development Status :: 4 - Beta", + "Development Status :: 5 - Production/Stable", "Intended Audience :: Science/Research", "Operating System :: OS Independent", "Programming Language :: Rust", diff --git a/hola-py/src/lib.rs b/hola-py/src/lib.rs index 567010e..10ace8f 100644 --- a/hola-py/src/lib.rs +++ b/hola-py/src/lib.rs @@ -12,8 +12,8 @@ //! Python bindings for the HOLA optimization engine via PyO3. use hola_engine::hola_engine::{ - DEFAULT_MAX_REFIT_CANDIDATES, DEFAULT_MAX_REFIT_SAMPLES, HolaEngine, ObjectiveConfig, - ParamConfig, StrategyConfig, StudyConfig, + DEFAULT_MAX_REFIT_CANDIDATES, DEFAULT_MAX_REFIT_SAMPLES, DEFAULT_MIN_ELITE_SAMPLES, HolaEngine, + ObjectiveConfig, ParamConfig, StrategyConfig, StudyConfig, }; use pyo3::create_exception; use pyo3::exceptions::{PyRuntimeWarning, PyValueError}; @@ -250,21 +250,23 @@ impl Maximize { /// /// Args: /// refit_interval: How often the GMM is refit, in completed trials (default: 20). -/// elite_fraction: Fraction of top trials used for GMM refitting (default: 0.25). +/// elite_fraction: Fraction of top trials used for GMM refitting (default: 0.125). /// Must be between 0.0 and 1.0. /// exploration_budget: Number of Sobol exploration trials before GMM exploitation -/// begins. When omitted, computed automatically from the total budget and -/// number of dimensions. +/// begins. When omitted, doubles ``min(floor(S/5), 50 + 2n)`` and rounds +/// down to a power of two, for total budget ``S`` and dimension ``n``. +/// When ``max_trials`` is also omitted, the warm-up calculation uses +/// ``S=200`` without imposing a trial cap. /// max_refit_samples: Maximum elite samples used by one GMM fit (default: 4096). /// max_refit_candidates: Maximum retained trials ranked during elite selection /// (default: 16384). Longer histories use deterministic stratified coverage. /// ongoing_exploration_period: Period between Sobol exploration trials after -/// GMM exploitation begins. ``None`` uses the default (currently 5), ``0`` -/// disables ongoing exploration, and explicit periods must be at least 2. +/// GMM exploitation begins. ``None`` uses the default (currently 0, disabled), +/// and explicit periods must be at least 2. /// max_components: Maximum number of Gaussian mixture components. When omitted, -/// uses the default (currently 3). Must be at least 1 when specified. +/// uses the default (currently 1). Must be at least 1 when specified. /// min_elite_samples: Minimum feasible elite samples required before GMM fitting. -/// When omitted, uses the default (currently 1). Must be at least 1 when +/// When omitted, uses the default (currently 5). Must be at least 1 when /// specified and must not exceed ``max_refit_samples``. #[pyclass(from_py_object)] #[derive(Clone)] @@ -339,7 +341,8 @@ impl Gmm { "max_refit_samples must be at least 1", )); } - if min_elite_samples.is_some_and(|minimum| minimum > effective_max_refit_samples) { + let effective_min_elite_samples = min_elite_samples.unwrap_or(DEFAULT_MIN_ELITE_SAMPLES); + if effective_min_elite_samples > effective_max_refit_samples { return Err(ConfigurationError::new_err(format!( "min_elite_samples must not exceed max_refit_samples ({effective_max_refit_samples})", ))); @@ -1356,6 +1359,27 @@ impl Study { } } + /// Read-only strategy routing and fitted-model diagnostics. + /// + /// GMM-specific values are ``None`` for Random and Sobol strategies. The + /// cumulative GMM-origin count is also ``None`` when exact route provenance + /// was unavailable in a legacy or leaderboard-only checkpoint. + fn strategy_diagnostics(&self, py: Python<'_>) -> PyResult> { + let StudyInner::Local { engine } = &self.inner else { + return Err(ConfigurationError::new_err( + "strategy_diagnostics() is only available for local studies, not remote connections", + )); + }; + let runtime = shared_runtime()?; + let diagnostics = py.detach(|| runtime.block_on(engine.strategy_diagnostics())); + let result = PyDict::new(py); + result.set_item("gmm_fit_epoch", diagnostics.gmm_fit_epoch)?; + result.set_item("gmm_origin_suggestions", diagnostics.gmm_origin_suggestions)?; + result.set_item("gmm_sampling_ready", diagnostics.gmm_sampling_ready)?; + result.set_item("issued_suggestions", diagnostics.issued_suggestions)?; + Ok(result.unbind()) + } + /// Update objectives mid-run, re-scalarizing all trials. fn update_objectives(&self, py: Python<'_>, objectives: &Bound<'_, PyList>) -> PyResult<()> { // Convert objectives to owned Rust configs before releasing the GIL. diff --git a/hola-py/tests/test_benchmark_empirical_exploitation.py b/hola-py/tests/test_benchmark_empirical_exploitation.py new file mode 100644 index 0000000..2490d48 --- /dev/null +++ b/hola-py/tests/test_benchmark_empirical_exploitation.py @@ -0,0 +1,299 @@ +# Copyright 2026 BlackRock, Inc. +# Licensed under the Apache License, Version 2.0. + +"""Fail-closed empirical-exploitation checks for practical GMM adapters.""" + +from __future__ import annotations + +import copy +from pathlib import Path + +import numpy as np +import pytest + +import benchmarks.adapters.hola_adapter as hola_adapter_module +from benchmarks.adapters.base import ( + EmpiricalExploitationError, + empirical_exploitation_gate_configuration, + require_empirical_gmm_exploitation, +) +from benchmarks.adapters.hola_adapter import ( + HolaGroupedTlpAdapter, + HolaMultiObjectiveAdapter, + HolaSingleObjectiveAdapter, +) +from benchmarks.adapters.hpo import HolaHpoAdapter +from benchmarks.data.manifest import build_campaign_manifest, validate_manifest +from benchmarks.data.persistence import ResultStore +from benchmarks.data.schema import SO_COLUMNS +from benchmarks.problems.grouped_tlp import SYNTHETIC_GROUPED_TLP +from benchmarks.problems.registry import MultiObjectiveProblem, SingleObjectiveProblem +from benchmarks.runner.executor import _run_multi_one, _run_single_one + +pytestmark = pytest.mark.benchmarks + + +class _DiagnosticStudy: + def __init__(self, diagnostics: object) -> None: + self.diagnostics = diagnostics + + def strategy_diagnostics(self) -> object: + return self.diagnostics + + +def _diagnostics(origin_suggestions: int | None) -> dict[str, int | bool | None]: + return { + "gmm_fit_epoch": 1, + "gmm_origin_suggestions": origin_suggestions, + "gmm_sampling_ready": True, + "issued_suggestions": 9, + } + + +def _fail_gate_at_four(study: object, completed_evaluations: int) -> None: + del study + raise EmpiricalExploitationError( + completed_evaluations, + { + "completed_evaluations": completed_evaluations, + "gmm_fit_epoch": 1, + "gmm_origin_suggestions": 4, + "gmm_sampling_ready": True, + "issued_suggestions": completed_evaluations, + }, + ) + + +@pytest.mark.parametrize("origin_suggestions", [0, None, 4]) +def test_origin_suggestion_threshold_fails_closed( + origin_suggestions: int | None, +) -> None: + with pytest.raises(EmpiricalExploitationError) as raised: + require_empirical_gmm_exploitation( + _DiagnosticStudy(_diagnostics(origin_suggestions)), + completed_evaluations=9, + ) + + assert raised.value.actual == 9 + assert raised.value.observed_diagnostics == { + "completed_evaluations": 9, + "gmm_fit_epoch": 1, + "gmm_origin_suggestions": origin_suggestions, + "gmm_sampling_ready": True, + "issued_suggestions": 9, + } + assert "observed_diagnostics=" in str(raised.value) + + +def test_five_empirical_suggestions_pass_the_gate() -> None: + require_empirical_gmm_exploitation( + _DiagnosticStudy(_diagnostics(5)), + completed_evaluations=9, + ) + + +@pytest.mark.parametrize( + "diagnostics", + [ + { + "gmm_fit_epoch": 0, + "gmm_origin_suggestions": 5, + "gmm_sampling_ready": True, + "issued_suggestions": 9, + }, + { + "gmm_fit_epoch": 1, + "gmm_origin_suggestions": 5, + "gmm_sampling_ready": False, + "issued_suggestions": 9, + }, + { + "gmm_fit_epoch": 1, + "gmm_origin_suggestions": 5, + "gmm_sampling_ready": True, + "issued_suggestions": 8, + }, + {"gmm_fit_epoch": 1}, + None, + ], +) +def test_missing_malformed_or_inconsistent_diagnostics_fail_closed( + diagnostics: object, +) -> None: + with pytest.raises(EmpiricalExploitationError): + require_empirical_gmm_exploitation( + _DiagnosticStudy(diagnostics), + completed_evaluations=9, + ) + + +def _provenance() -> dict[str, object]: + return { + "code": {"commit": "test", "dirty": False, "source_hash": "a" * 64}, + "lock_hash": "b" * 64, + "python": {"implementation": "CPython", "version": "test"}, + "platform": {"platform": "test", "machine": "test", "system": "Linux"}, + "dependencies": {"hola-opt": None}, + "native_extension": None, + } + + +def test_only_gmm_configurations_authenticate_the_empirical_gate() -> None: + expected = { + "minimum_gmm_fit_epoch": 1, + "minimum_gmm_origin_suggestions": 5, + "on_failure": "preserved_error_outcome", + } + gmm_adapters = ( + HolaSingleObjectiveAdapter("gmm"), + HolaMultiObjectiveAdapter("gmm"), + HolaGroupedTlpAdapter(SYNTHETIC_GROUPED_TLP, "gmm"), + HolaHpoAdapter("gmm"), + ) + for adapter in gmm_adapters: + assert adapter.configuration(25)["empirical_exploitation_gate"] == expected + for adapter in ( + HolaSingleObjectiveAdapter("random"), + HolaMultiObjectiveAdapter("sobol"), + HolaGroupedTlpAdapter(SYNTHETIC_GROUPED_TLP, "random"), + HolaHpoAdapter("sobol"), + ): + assert "empirical_exploitation_gate" not in adapter.configuration(25) + + configuration = gmm_adapters[0].configuration(25) + manifest = build_campaign_manifest( + run_kind="single_objective", + budgets=[25], + n_runs=1, + problem_names=["problem"], + optimizer_names=[gmm_adapters[0].name], + optimizer_configurations=[ + { + "optimizer": gmm_adapters[0].name, + "by_budget": [{"budget": 25, "configuration": configuration}], + } + ], + provenance=_provenance(), + ) + validate_manifest(manifest) + assert ( + manifest["optimizer_configurations"][0]["by_budget"][0]["configuration"][ + "empirical_exploitation_gate" + ] + == empirical_exploitation_gate_configuration() + ) + tampered = copy.deepcopy(manifest) + tampered["optimizer_configurations"][0]["by_budget"][0]["configuration"][ + "empirical_exploitation_gate" + ]["minimum_gmm_origin_suggestions"] = 4 + with pytest.raises(RuntimeError, match="fingerprint"): + validate_manifest(tampered) + + +def _scalar_problem() -> SingleObjectiveProblem: + return SingleObjectiveProblem( + name="gated_scalar", + func=lambda params: (params["x"] - 0.25) ** 2, + bounds={"x": (0.0, 1.0)}, + known_minimum=0.0, + ) + + +def _multi_problem() -> MultiObjectiveProblem: + return MultiObjectiveProblem( + name="gated_multi", + func=lambda params: { + "left": params["x"] ** 2, + "right": (1.0 - params["x"]) ** 2, + }, + bounds={"x": (0.0, 1.0)}, + objective_names=("left", "right"), + reference_point=(2.0, 2.0), + ideal_point=(0.0, 0.0), + true_pareto_front=np.asarray([[0.0, 1.0], [1.0, 0.0]]), + ) + + +def test_single_and_multi_error_rows_preserve_completed_gmm_count( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + hola_adapter_module, + "require_empirical_gmm_exploitation", + _fail_gate_at_four, + ) + single = _run_single_one( + _scalar_problem(), + HolaSingleObjectiveAdapter("gmm"), + budget=5, + run_id=0, + ) + multi = _run_multi_one( + _multi_problem(), + HolaMultiObjectiveAdapter("gmm"), + budget=5, + run_id=0, + ) + + for row in (single, multi): + assert row["status"] == "error" + assert row["n_evaluations"] == 5 + assert "EmpiricalExploitationError" in row["error"] + assert "observed_diagnostics=" in row["error"] + + +def test_gate_failure_persists_in_the_existing_single_objective_csv_schema( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + hola_adapter_module, + "require_empirical_gmm_exploitation", + _fail_gate_at_four, + ) + optimizer = HolaSingleObjectiveAdapter("gmm") + problem = _scalar_problem() + configuration = optimizer.configuration(5) + manifest = build_campaign_manifest( + run_kind="single_objective", + budgets=[5], + n_runs=1, + problem_names=[problem.name], + optimizer_names=[optimizer.name], + optimizer_configurations=[ + { + "optimizer": optimizer.name, + "by_budget": [{"budget": 5, "configuration": configuration}], + } + ], + provenance=_provenance(), + ) + store = ResultStore(tmp_path) + store.prepare_campaign(manifest, resume=False) + store.append_single(_run_single_one(problem, optimizer, budget=5, run_id=0)) + + rows = store.load_complete_single() + assert rows.columns.tolist() == SO_COLUMNS + assert rows.loc[0, "status"] == "error" + assert rows.loc[0, "n_evaluations"] == 5 + assert "EmpiricalExploitationError" in rows.loc[0, "error"] + + +def test_grouped_gmm_adapter_invokes_gate_after_completed_run( + monkeypatch: pytest.MonkeyPatch, +) -> None: + observed: list[int] = [] + monkeypatch.setattr( + hola_adapter_module, + "require_empirical_gmm_exploitation", + lambda study, completed_evaluations: observed.append(completed_evaluations), + ) + problem = SYNTHETIC_GROUPED_TLP + result = HolaGroupedTlpAdapter(problem, "gmm").optimize( + problem.as_multi_objective_problem(), + budget=1, + seed=123, + ) + + assert result.n_evaluations == 1 + assert observed == [1] diff --git a/hola-py/tests/test_hola.py b/hola-py/tests/test_hola.py index aed2a95..6ba3392 100644 --- a/hola-py/tests/test_hola.py +++ b/hola-py/tests/test_hola.py @@ -211,6 +211,54 @@ def test_study_strategy_gmm(): assert study.trial_count() == 1 +def test_strategy_diagnostics_python_exposure(): + from hola_opt import Gmm, Minimize, Real, Space, Study + + study = Study( + space=Space(x=Real(0.0, 1.0)), + objectives=[Minimize("loss")], + strategy=Gmm( + exploration_budget=1, + refit_interval=1, + ongoing_exploration_period=0, + min_elite_samples=1, + ), + ) + assert study.strategy_diagnostics() == { + "gmm_fit_epoch": 0, + "gmm_origin_suggestions": 0, + "gmm_sampling_ready": False, + "issued_suggestions": 0, + } + + warmup = study.ask() + assert study.strategy_diagnostics()["gmm_origin_suggestions"] == 0 + study.tell(warmup.trial_id, {"loss": 0.5}) + fitted = study.strategy_diagnostics() + assert fitted["gmm_fit_epoch"] == 1 + assert fitted["gmm_sampling_ready"] is True + + study.ask() + assert study.strategy_diagnostics() == { + "gmm_fit_epoch": 1, + "gmm_origin_suggestions": 1, + "gmm_sampling_ready": True, + "issued_suggestions": 2, + } + + random_study = Study( + space=Space(x=Real(0.0, 1.0)), + objectives=[Minimize("loss")], + strategy="random", + ) + assert random_study.strategy_diagnostics() == { + "gmm_fit_epoch": None, + "gmm_origin_suggestions": None, + "gmm_sampling_ready": None, + "issued_suggestions": 0, + } + + def test_gmm_refit_limit_configuration_and_validation(): from hola_opt import ConfigurationError, Gmm, Minimize, Real, Space, Study @@ -281,9 +329,14 @@ def test_gmm_calibration_controls_configuration_and_validation(tmp_path): default_path = tmp_path / "resolved-defaults.json" default_study.save(str(default_path)) resolved_defaults = json.loads(default_path.read_text())["config"]["strategy"] - assert resolved_defaults["ongoing_exploration_period"] == 5 - assert resolved_defaults["max_components"] == 3 - assert resolved_defaults["min_elite_samples"] == 1 + assert resolved_defaults["refit_interval"] == 20 + assert resolved_defaults["exploration_budget"] == 64 + assert resolved_defaults["elite_fraction"] == 0.125 + assert resolved_defaults["ongoing_exploration_period"] == 0 + assert resolved_defaults["max_components"] == 1 + assert resolved_defaults["min_elite_samples"] == 5 + assert resolved_defaults["max_refit_samples"] == 4096 + assert resolved_defaults["max_refit_candidates"] == 16_384 with pytest.raises(ConfigurationError, match=r"0 \(disabled\) or at least 2"): Gmm(ongoing_exploration_period=1) @@ -293,6 +346,8 @@ def test_gmm_calibration_controls_configuration_and_validation(tmp_path): Gmm(min_elite_samples=0) with pytest.raises(ConfigurationError, match="must not exceed max_refit_samples"): Gmm(min_elite_samples=5, max_refit_samples=4) + with pytest.raises(ConfigurationError, match="must not exceed max_refit_samples"): + Gmm(max_refit_samples=4) # ========================================================================== @@ -453,6 +508,57 @@ def test_study_save_load_resume_uses_fresh_trial_id(tmp_path): assert [trial.trial_id for trial in restored.trials()] == [0, 1, 2] +def test_study_load_migrates_missing_gmm_fields_to_legacy_defaults(tmp_path): + from hola_opt import Gmm, Minimize, Real, Space, Study + + source = Study( + space=Space(x=Real(0.0, 1.0)), + objectives=[Minimize("loss")], + strategy=Gmm( + exploration_budget=32, + elite_fraction=0.25, + ongoing_exploration_period=5, + max_components=3, + min_elite_samples=1, + max_refit_samples=4, + ), + seed=7, + ) + first = source.ask() + source.tell(first.trial_id, {"loss": first.params["x"]}) + path = tmp_path / "legacy-gmm-defaults.json" + source.save(str(path)) + + legacy = json.loads(path.read_text()) + saved_strategy = legacy["config"]["strategy"] + for field in ( + "exploration_budget", + "elite_fraction", + "ongoing_exploration_period", + "max_components", + "min_elite_samples", + ): + saved_strategy.pop(field) + legacy["checkpoint"]["strategy_state"]["inner"].pop("ongoing_exploration_period") + path.write_text(json.dumps(legacy)) + + restored = Study.load(str(path)) + source_next = source.ask() + restored_next = restored.ask() + assert restored_next.trial_id == source_next.trial_id + assert restored_next.params == source_next.params + + migrated_path = tmp_path / "migrated-gmm-defaults.json" + restored.save(str(migrated_path)) + migrated = json.loads(migrated_path.read_text())["config"]["strategy"] + assert migrated["exploration_budget"] == 32 + assert migrated["elite_fraction"] == 0.25 + assert migrated["ongoing_exploration_period"] == 5 + assert migrated["max_components"] == 3 + assert migrated["min_elite_samples"] == 1 + assert migrated["max_refit_samples"] == 4 + + @pytest.mark.skipif(os.name == "nt", reason="named-pipe checkpoint test requires POSIX") @pytest.mark.timeout(10) def test_study_load_releases_gil_while_checkpoint_read_blocks(tmp_path): diff --git a/hola-py/tests/test_hpo_benchmark.py b/hola-py/tests/test_hpo_benchmark.py index 697b9c3..cbf85d4 100644 --- a/hola-py/tests/test_hpo_benchmark.py +++ b/hola-py/tests/test_hpo_benchmark.py @@ -15,7 +15,8 @@ import optuna import pytest -from benchmarks.adapters.base import HpoOptimizationResult +import benchmarks.adapters.hpo as hpo_adapter_module +from benchmarks.adapters.base import EmpiricalExploitationError, HpoOptimizationResult from benchmarks.adapters.hpo import ( HolaHpoAdapter, OptunaTpeHpoAdapter, @@ -139,7 +140,6 @@ def test_native_space_translation_preserves_types_scales_and_choices() -> None: [ HolaHpoAdapter("random"), HolaHpoAdapter("sobol"), - HolaHpoAdapter("gmm"), OptunaTpeHpoAdapter(), ], ids=lambda adapter: adapter.name, @@ -161,6 +161,42 @@ def test_native_hpo_adapters_use_exact_validation_budget(adapter: object) -> Non assert result.best_params["loss"] in {"squared_error", "huber"} +def test_gmm_gate_failure_preserves_validation_count_and_never_calls_heldout( + monkeypatch: pytest.MonkeyPatch, +) -> None: + evaluator = RecordingEvaluator(_toy_problem(), validation_budget=5) + problem = _toy_problem(evaluator) + evaluator.problem = problem + + def fail_gate(study: object, completed_evaluations: int) -> None: + del study + raise EmpiricalExploitationError( + completed_evaluations, + { + "completed_evaluations": completed_evaluations, + "gmm_fit_epoch": 1, + "gmm_origin_suggestions": 4, + "gmm_sampling_ready": True, + "issued_suggestions": completed_evaluations, + }, + ) + + monkeypatch.setattr( + hpo_adapter_module, + "require_empirical_gmm_exploitation", + fail_gate, + ) + + row = _run_hpo_one(problem, HolaHpoAdapter("gmm"), budget=5, run_id=0) + + assert row["status"] == "error" + assert row["n_validation_evaluations"] == 5 + assert row["n_heldout_evaluations"] == 0 + assert "EmpiricalExploitationError" in row["error"] + assert "gmm_origin_suggestions" in row["error"] + assert evaluator.events == ["validation"] * 5 + + def test_runner_calls_heldout_once_only_after_exact_validation_budget() -> None: evaluator = RecordingEvaluator(_toy_problem(), validation_budget=3) problem = _toy_problem(evaluator) diff --git a/hola-py/tests/test_stubs.py b/hola-py/tests/test_stubs.py index cb2490d..67acdfc 100644 --- a/hola-py/tests/test_stubs.py +++ b/hola-py/tests/test_stubs.py @@ -35,6 +35,7 @@ "pareto_front", "trials", "trial_count", + "strategy_diagnostics", "update_objectives", "save", "run", diff --git a/hola-py/uv.lock b/hola-py/uv.lock index 99c7a04..0380638 100644 --- a/hola-py/uv.lock +++ b/hola-py/uv.lock @@ -653,7 +653,7 @@ wheels = [ [[package]] name = "hola-opt" -version = "1.0.1rc8" +version = "1.0.1" source = { editable = "." } [package.dev-dependencies] diff --git a/hola/Cargo.toml b/hola/Cargo.toml index 70615e0..ba2774d 100644 --- a/hola/Cargo.toml +++ b/hola/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "hola" -version = "1.0.1-rc8" +version = "1.0.1" edition = "2024" description = "Dynamic optimization engine and optional REST server for HOLA" documentation = "https://docs.rs/hola" @@ -18,7 +18,7 @@ default = [] server = ["axum/http1", "axum/json", "axum/query", "axum/tokio", "tokio/net", "tokio/signal", "tokio-stream", "tower-http", "tracing"] [dependencies] -opt_engine = { version = "1.0.1-rc8", path = "../opt_engine" } +opt_engine = { version = "1.0.1", path = "../opt_engine" } serde = { version = "1", features = ["derive"] } serde_json = "1" # Core engine needs async locks and blocking refit tasks; server adds net support. @@ -32,7 +32,7 @@ tokio-stream = { version = "0.1", default-features = false, features = ["sync"], tower-http = { version = "0.6.8", default-features = false, features = ["cors", "fs", "request-id", "timeout", "trace"], optional = true } [dev-dependencies] -opt_engine = { version = "1.0.1-rc8", path = "../opt_engine" } +opt_engine = { version = "1.0.1", path = "../opt_engine" } serde_yaml = { package = "serde_yaml_ng", version = "0.10" } tempfile = "3" tower = { version = "0.5", features = ["util"] } diff --git a/hola/src/hola_engine.rs b/hola/src/hola_engine.rs index 433b9fe..ee139fb 100644 --- a/hola/src/hola_engine.rs +++ b/hola/src/hola_engine.rs @@ -29,7 +29,9 @@ use opt_engine::persistence::{ }; use opt_engine::scales::{LinearScale, Log10Scale, LogScale, Scale}; use opt_engine::spaces::{CategoricalSpace, ContinuousSpace, DiscreteSpace}; -use opt_engine::strategies::{GmmRefitConfig, GmmStrategy, RandomStrategy, SobolStrategy}; +use opt_engine::strategies::{ + DEFAULT_GMM_COMPONENTS, GmmRefitConfig, GmmStrategy, RandomStrategy, SobolStrategy, +}; use opt_engine::traits::{RefitConfig, SampleSpace, StandardizedSpace, Strategy}; use serde::{Deserialize, Serialize}; use std::collections::{BTreeMap, HashSet, VecDeque}; @@ -70,14 +72,22 @@ const MAX_ASK_IDEMPOTENCY_KEYS: usize = MAX_PENDING_TRIALS; /// the idempotency ledger into unbounded study history. const MAX_COMPLETION_RECEIPTS: usize = 4096; -/// Legacy cadence for low-discrepancy exploration after the initial warm-up. -/// This prevents a fitted GMM from permanently collapsing around sparse early -/// elites. -pub const DEFAULT_ONGOING_EXPLORATION_PERIOD: usize = 5; -/// Legacy upper bound on fitted GMM components. -pub const DEFAULT_MAX_COMPONENTS: usize = 3; -/// Legacy lower bound on the elite workset passed to a GMM refit. -pub const DEFAULT_MIN_ELITE_SAMPLES: usize = 1; +/// Default fraction of the ranked trial set used for GMM refitting. +pub const DEFAULT_ELITE_FRACTION: f64 = 0.125; +/// Default cadence for low-discrepancy exploration after the initial warm-up. +/// Zero disables periodic exploration once an empirical GMM is available. +pub const DEFAULT_ONGOING_EXPLORATION_PERIOD: usize = 0; +/// Default upper bound on fitted GMM components. +pub const DEFAULT_MAX_COMPONENTS: usize = DEFAULT_GMM_COMPONENTS; +/// Default lower bound on the elite workset passed to a GMM refit. +pub const DEFAULT_MIN_ELITE_SAMPLES: usize = 5; +/// Defaults used only to migrate checkpoints written before these controls +/// were recorded explicitly. They must remain fixed to preserve the behavior +/// of historical studies when an old checkpoint omits the corresponding field. +const LEGACY_DEFAULT_ELITE_FRACTION: f64 = 0.25; +const LEGACY_DEFAULT_ONGOING_EXPLORATION_PERIOD: usize = 5; +const LEGACY_DEFAULT_MAX_COMPONENTS: usize = 3; +const LEGACY_DEFAULT_MIN_ELITE_SAMPLES: usize = 1; /// Variance of the neutral GMM placeholder used before the first empirical fit. const AUTO_GMM_PRIOR_VARIANCE: f64 = 0.1; /// Default bound on the number of elite samples passed to full-covariance EM. @@ -89,6 +99,10 @@ pub const DEFAULT_MAX_REFIT_SAMPLES: usize = 4096; /// strata rather than a newest-only window. pub const DEFAULT_MAX_REFIT_CANDIDATES: usize = 16_384; +fn fold_sobol_seed(seed: u64) -> u32 { + (seed ^ (seed >> 32)) as u32 +} + fn unix_time_millis() -> u64 { SystemTime::now() .duration_since(UNIX_EPOCH) @@ -488,8 +502,9 @@ enum DynStrategyInner { /// model once its first empirical fit is available; the model is periodically /// refit to elite trials. /// -/// The default exploration budget follows the formula from the paper: -/// `min(floor(S / 5), 50 + 2n)`, where `S` is the intended number of +/// The calibrated default applies its factor of two before rounding down to a +/// power of two: +/// `2 * min(floor(S / 5), 50 + 2n)`, where `S` is the intended number of /// simulations and `n` is the dimensionality. #[derive(Debug)] pub struct AutoStrategy { @@ -503,19 +518,40 @@ pub struct AutoStrategy { gmm_sampling_ready: bool, trial_count: usize, issued_count: AtomicUsize, + /// Cumulative suggestions routed through the empirical GMM. + /// + /// `None` means the exact historical count is unknowable because strategy + /// state came from an older checkpoint or history was imported without its + /// sampler state. Keeping unknown state explicit avoids treating the GMM + /// sampling cursor's conservative import watermark as real exploitation. + gmm_origin_suggestions: Option, } impl AutoStrategy { - /// Compute the default exploration budget from the paper's formula, - /// rounded down to the nearest power of two to preserve the balanced - /// space-filling properties of the Sobol sequence. + /// Compute the calibrated default exploration budget. Applying the factor + /// of two before rounding preserves the balanced space-filling properties + /// of the Sobol sequence. /// /// `total_budget` is `S`, the intended total number of simulations. /// `dim` is `n`, the dimensionality of the search space. pub fn default_exploration_budget(total_budget: usize, dim: usize) -> usize { + Self::exploration_budget_with_multiplier(total_budget, dim, 2) + } + + /// Reconstruct the pre-calibration warm-up for a checkpoint whose embedded + /// config predates the concrete exploration-budget field. + fn legacy_default_exploration_budget(total_budget: usize, dim: usize) -> usize { + Self::exploration_budget_with_multiplier(total_budget, dim, 1) + } + + fn exploration_budget_with_multiplier( + total_budget: usize, + dim: usize, + multiplier: usize, + ) -> usize { let a = total_budget / 5; - let b = 50 + 2 * dim; - let raw = a.min(b); + let b = 50usize.saturating_add(2usize.saturating_mul(dim)); + let raw = a.min(b).saturating_mul(multiplier); // Round down to the nearest power of two so the Sobol sequence // retains its low-discrepancy guarantee. if raw < 2 { @@ -544,7 +580,7 @@ impl AutoStrategy { // Fold the high 32 bits into the low 32 instead of truncating, so two // u64 seeds that differ only in their high bits yield distinct Sobol // seeds. Deterministic: the same u64 always folds to the same u32. - Some(s) => ((s ^ (s >> 32)) as u32, s), + Some(s) => (fold_sobol_seed(s), s), None => (42, rand::random()), }; Self { @@ -556,6 +592,7 @@ impl AutoStrategy { gmm_sampling_ready: false, trial_count: 0, issued_count: AtomicUsize::new(0), + gmm_origin_suggestions: Some(AtomicU64::new(0)), } } } @@ -570,6 +607,10 @@ impl Clone for AutoStrategy { gmm_sampling_ready: self.gmm_sampling_ready, trial_count: self.trial_count, issued_count: AtomicUsize::new(self.issued_count.load(Ordering::Relaxed)), + gmm_origin_suggestions: self + .gmm_origin_suggestions + .as_ref() + .map(|count| AtomicU64::new(count.load(Ordering::Relaxed))), } } } @@ -581,7 +622,7 @@ impl Serialize for AutoStrategy { { use serde::ser::SerializeStruct; - let mut state = serializer.serialize_struct("AutoStrategy", 7)?; + let mut state = serializer.serialize_struct("AutoStrategy", 8)?; state.serialize_field("sobol", &self.sobol)?; state.serialize_field("gmm", &self.gmm)?; state.serialize_field("exploration_budget", &self.exploration_budget)?; @@ -592,6 +633,11 @@ impl Serialize for AutoStrategy { state.serialize_field("gmm_sampling_ready", &self.gmm_sampling_ready)?; state.serialize_field("trial_count", &self.trial_count)?; state.serialize_field("issued_count", &self.issued_count.load(Ordering::Relaxed))?; + let gmm_origin_suggestions = self + .gmm_origin_suggestions + .as_ref() + .map(|count| count.load(Ordering::Relaxed)); + state.serialize_field("gmm_origin_suggestions", &gmm_origin_suggestions)?; state.end() } } @@ -606,13 +652,15 @@ impl<'de> Deserialize<'de> for AutoStrategy { sobol: SobolStrategy, gmm: GmmStrategy, exploration_budget: usize, - #[serde(default = "default_ongoing_exploration_period")] + #[serde(default = "legacy_default_ongoing_exploration_period")] ongoing_exploration_period: usize, #[serde(default)] gmm_sampling_ready: Option, trial_count: usize, #[serde(default)] issued_count: Option, + #[serde(default)] + gmm_origin_suggestions: Option, } let state = AutoStrategySerde::deserialize(deserializer)?; @@ -644,6 +692,7 @@ impl<'de> Deserialize<'de> for AutoStrategy { gmm_sampling_ready, trial_count: state.trial_count, issued_count: AtomicUsize::new(issued_count), + gmm_origin_suggestions: state.gmm_origin_suggestions.map(AtomicU64::new), }) } } @@ -676,7 +725,11 @@ impl Strategy for DynStrategy { if issued < s.exploration_budget || !s.gmm_sampling_ready || periodic_exploration { s.sobol.suggest(space) } else { - s.gmm.suggest(space) + let suggestion = s.gmm.suggest(space); + if let Some(count) = &s.gmm_origin_suggestions { + count.fetch_add(1, Ordering::Relaxed); + } + suggestion } } } @@ -698,6 +751,43 @@ impl Strategy for DynStrategy { } impl DynStrategy { + fn diagnostics(&self) -> StrategyDiagnostics { + match &self.inner { + DynStrategyInner::Random(strategy) => StrategyDiagnostics { + gmm_fit_epoch: None, + gmm_origin_suggestions: None, + gmm_sampling_ready: None, + issued_suggestions: strategy.counter(), + }, + DynStrategyInner::Sobol(strategy) => StrategyDiagnostics { + gmm_fit_epoch: None, + gmm_origin_suggestions: None, + gmm_sampling_ready: None, + issued_suggestions: u64::from(strategy.index()), + }, + DynStrategyInner::Gmm(strategy) => StrategyDiagnostics { + gmm_fit_epoch: Some(strategy.refit_epoch()), + // This legacy variant predates route provenance. Its sampling + // cursor may also have been conservatively advanced during an + // import, so it cannot truthfully stand in for the dedicated + // empirical-origin counter owned by AutoStrategy. + gmm_origin_suggestions: None, + gmm_sampling_ready: Some(true), + issued_suggestions: strategy.counter(), + }, + DynStrategyInner::Auto(strategy) => StrategyDiagnostics { + gmm_fit_epoch: Some(strategy.gmm.refit_epoch()), + gmm_origin_suggestions: strategy + .gmm_origin_suggestions + .as_ref() + .map(|count| count.load(Ordering::Relaxed)), + gmm_sampling_ready: Some(strategy.gmm_sampling_ready), + issued_suggestions: u64::try_from(strategy.issued_count.load(Ordering::Relaxed)) + .expect("supported targets have at most 64-bit usize"), + }, + } + } + /// Epoch of an `auto` strategy that is still waiting for its first /// empirical GMM fit. Capturing this before queued maintenance and /// comparing it again under `refit_lock` coalesces concurrent retries once @@ -807,13 +897,42 @@ impl DynStrategy { usize::MAX - 1 )); } + if let Some(gmm_origin_suggestions) = strategy + .gmm_origin_suggestions + .as_ref() + .map(|count| count.load(Ordering::Relaxed)) + { + if gmm_origin_suggestions != strategy.gmm.counter() { + return Err(format!( + "checkpoint auto GMM-origin count {gmm_origin_suggestions} does not match GMM sampling cursor {}", + strategy.gmm.counter() + )); + } + if u128::from(gmm_origin_suggestions) > issued_count as u128 { + return Err(format!( + "checkpoint auto GMM-origin count {gmm_origin_suggestions} exceeds issued_count {issued_count}" + )); + } + } if let Some(seed) = config.seed { - let expected_sobol_seed = (seed ^ (seed >> 32)) as u32; + let expected_sobol_seed = fold_sobol_seed(seed); if strategy.gmm.seed() != seed || strategy.sobol.seed() != expected_sobol_seed { return Err(format!( "checkpoint auto strategy seeds do not match configured seed {seed}" )); } + } else { + let sobol_seed = strategy.sobol.seed(); + let gmm_seed = strategy.gmm.seed(); + let current_pair = sobol_seed == fold_sobol_seed(gmm_seed); + let historical_auto_seed = sobol_seed == 42; + let historical_explicit_seed = sobol_seed == gmm_seed as u32; + if !current_pair && !historical_auto_seed && !historical_explicit_seed { + return Err( + "checkpoint auto strategy with seed null has no supported current or historical seed relationship" + .to_string(), + ); + } } let initial_sobol = issued_count.min(strategy.exploration_budget); let post_exploration = issued_count.saturating_sub(strategy.exploration_budget); @@ -895,7 +1014,7 @@ impl DynStrategy { } DynStrategyInner::Sobol(strategy) => { if let Some(seed) = config.seed { - let expected_seed = (seed ^ (seed >> 32)) as u32; + let expected_seed = fold_sobol_seed(seed); if strategy.seed() != expected_seed { return Err(format!( "checkpoint Sobol seed {} does not match configured folded seed {expected_seed}", @@ -918,12 +1037,21 @@ impl DynStrategy { Ok(()) } - fn resolved_seed(&self) -> u64 { + /// Return a public seed only when constructing the strategy from that seed + /// under the current rules reproduces the serialized sampler seed(s). + /// + /// Historical Auto strategies could contain independent Sobol/GMM seeds, + /// or a truncated Sobol seed for an explicit `u64`. Such a pair has no + /// truthful single-seed representation in today's folded scheme. + fn representable_config_seed(&self) -> Option { match &self.inner { - DynStrategyInner::Random(strategy) => strategy.seed(), - DynStrategyInner::Sobol(strategy) => u64::from(strategy.seed()), - DynStrategyInner::Gmm(strategy) => strategy.seed(), - DynStrategyInner::Auto(strategy) => strategy.gmm.seed(), + DynStrategyInner::Random(strategy) => Some(strategy.seed()), + DynStrategyInner::Sobol(strategy) => Some(u64::from(strategy.seed())), + DynStrategyInner::Gmm(strategy) => Some(strategy.seed()), + DynStrategyInner::Auto(strategy) => { + let seed = strategy.gmm.seed(); + (strategy.sobol.seed() == fold_sobol_seed(seed)).then_some(seed) + } } } @@ -971,6 +1099,10 @@ impl DynStrategy { .map_err(|error| error.to_string())?; } DynStrategyInner::Auto(strategy) => { + // Imported history has no exact sampler-route provenance. The + // GMM cursor below is only a conservative watermark, so it must + // never be exposed as a cumulative empirical-origin count. + strategy.gmm_origin_suggestions = None; strategy.sobol.advance_to(sobol_index); strategy.trial_count = completed_count; strategy @@ -1027,6 +1159,10 @@ impl opt_engine::traits::RefittableStrategy for DynStrategy { // an empty no-op clone carries the live value already. s.issued_count .store(l.issued_count.load(Ordering::Relaxed), Ordering::Relaxed); + s.gmm_origin_suggestions = l + .gmm_origin_suggestions + .as_ref() + .map(|count| AtomicU64::new(count.load(Ordering::Relaxed))); } (DynStrategyInner::Gmm(s), DynStrategyInner::Gmm(l)) => s.reconcile_after_refit(l), // Sobol/Random refit is a no-op, so the off-lock snapshot is stale; @@ -1098,31 +1234,31 @@ pub struct StrategyConfig { pub strategy_type: String, #[serde(default = "default_refit_interval")] pub refit_interval: usize, - /// Total simulation budget S (used by "auto" to compute exploration threshold). + /// Total simulation budget S (used by GMM/auto to compute the exploration threshold). #[serde(default)] pub total_budget: Option, /// Override the exploration budget directly instead of using the formula. #[serde(default)] pub exploration_budget: Option, /// Cadence for ongoing Sobol exploration after the initial warm-up. - /// Missing resolves to the legacy cadence of 5; 0 disables it. + /// In a new study, missing resolves to 0 (disabled). #[serde(default)] pub ongoing_exploration_period: Option, /// Optional seed for reproducible runs. When `None`, strategies use their /// default seeding (Sobol=42, others use random seeds). #[serde(default)] pub seed: Option, - /// Fraction of top trials used for GMM refitting (default: 0.25). + /// Fraction of top trials used for GMM refitting (default: 0.125). /// Must be in (0.0, 1.0]. #[serde(default)] pub elite_fraction: Option, /// Maximum number of GMM components considered during fitting. Missing - /// resolves to the legacy cap of 3. + /// resolves to 1 in a new study. #[serde(default)] pub max_components: Option, /// Minimum feasible elite workset size required for refitting. A scheduled /// refit is skipped until selection can supply this many trials. Missing - /// resolves to the legacy floor of 1. + /// resolves to 5 in a new study. #[serde(default)] pub min_elite_samples: Option, /// Maximum elite samples used by one GMM fit. This bounds EM cost but does @@ -1139,8 +1275,8 @@ fn default_refit_interval() -> usize { 20 } -fn default_ongoing_exploration_period() -> usize { - DEFAULT_ONGOING_EXPLORATION_PERIOD +fn legacy_default_ongoing_exploration_period() -> usize { + LEGACY_DEFAULT_ONGOING_EXPLORATION_PERIOD } fn default_max_refit_samples() -> usize { @@ -1152,16 +1288,25 @@ fn default_max_refit_candidates() -> usize { } impl StrategyConfig { - /// Resolve controls added after the original checkpoint schema. Keeping - /// these as optional on input lets old YAML and embedded checkpoint configs - /// retain their historical behavior, while exported configs record the - /// concrete values that actually govern sampling. - fn resolve_calibration_control_defaults(&mut self) { + /// Resolve controls omitted by an embedded historical checkpoint config. + /// + /// This is deliberately not used for ordinary config deserialization: + /// omitted settings in a new study use the current calibrated defaults, + /// while missing checkpoint fields must retain the pre-calibration route. + fn resolve_legacy_checkpoint_defaults(&mut self, total_budget: usize, dim: usize) { self.ongoing_exploration_period - .get_or_insert(DEFAULT_ONGOING_EXPLORATION_PERIOD); - self.max_components.get_or_insert(DEFAULT_MAX_COMPONENTS); + .get_or_insert(LEGACY_DEFAULT_ONGOING_EXPLORATION_PERIOD); + self.max_components + .get_or_insert(LEGACY_DEFAULT_MAX_COMPONENTS); self.min_elite_samples - .get_or_insert(DEFAULT_MIN_ELITE_SAMPLES); + .get_or_insert(LEGACY_DEFAULT_MIN_ELITE_SAMPLES); + if matches!(self.strategy_type.as_str(), "gmm" | "auto") { + self.elite_fraction + .get_or_insert(LEGACY_DEFAULT_ELITE_FRACTION); + self.exploration_budget.get_or_insert_with(|| { + AutoStrategy::legacy_default_exploration_budget(total_budget, dim) + }); + } } } @@ -1188,6 +1333,386 @@ pub struct StudyConfig { pub max_leaderboard_size: Option, } +impl StudyConfig { + /// Materialize only historical checkpoint defaults. Fresh user configs + /// intentionally bypass this migration and resolve against current values + /// in [`HolaEngine::from_config`]. + fn resolve_legacy_checkpoint_defaults( + &mut self, + checkpoint_document: &serde_json::Value, + ) -> Result<(), String> { + let dim = self.space.len(); + if self.strategy.is_none() { + self.strategy = Some(infer_legacy_strategy_config( + checkpoint_document, + self.max_trials, + dim, + )?); + } + if let Some(strategy) = &mut self.strategy { + let total_budget = self.max_trials.or(strategy.total_budget).unwrap_or(200); + strategy.resolve_legacy_checkpoint_defaults(total_budget, dim); + + // Version-one Auto checkpoints used the GMM's full `u64` seed but + // truncated that value for Sobol (and used an independent pair for + // an omitted seed). A high-bit explicit seed can therefore be + // historically valid without being reproducible from one seed + // under the current folding rule. Preserve the exact sampler state + // while refusing to export a misleading public seed. A malformed + // legacy pair is left intact here so ordinary strict validation + // rejects it below. + if checkpoint_embedded_strategy_was_absent_or_null(checkpoint_document) + && checkpoint_format_version(checkpoint_document) == Some(1) + && matches!(strategy.strategy_type.as_str(), "gmm" | "auto") + { + if let Some(pair) = serialized_auto_seed_pair(checkpoint_document)? { + if let Some(seed) = strategy.seed { + if pair.current_config_seed() != Some(seed) + && pair.matches_legacy_explicit_seed(seed) + { + strategy.seed = None; + } + } + } + } + } + Ok(()) + } +} + +fn checkpoint_embedded_strategy_was_absent_or_null( + checkpoint_document: &serde_json::Value, +) -> bool { + checkpoint_document + .get("config") + .and_then(|config| config.get("strategy")) + .is_none_or(serde_json::Value::is_null) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct SerializedAutoSeedPair { + sobol: u32, + gmm: u64, +} + +impl SerializedAutoSeedPair { + fn current_config_seed(self) -> Option { + (self.sobol == fold_sobol_seed(self.gmm)).then_some(self.gmm) + } + + fn matches_legacy_explicit_seed(self, seed: u64) -> bool { + self.gmm == seed && self.sobol == seed as u32 + } +} + +fn checkpoint_format_version(checkpoint_document: &serde_json::Value) -> Option { + checkpoint_document + .get("checkpoint") + .unwrap_or(checkpoint_document) + .get("metadata") + .and_then(|metadata| metadata.get("format_version")) + .and_then(serde_json::Value::as_u64) +} + +fn serialized_auto_seed_pair( + checkpoint_document: &serde_json::Value, +) -> Result, String> { + let inner = checkpoint_document + .get("checkpoint") + .unwrap_or(checkpoint_document) + .get("strategy_state") + .and_then(|state| state.get("inner")) + .and_then(serde_json::Value::as_object) + .ok_or_else(|| "checkpoint strategy_state has no object-valued inner state".to_string())?; + if inner.get("type").and_then(serde_json::Value::as_str) != Some("auto") { + return Ok(None); + } + + let sobol = checkpoint_u64( + inner.get("sobol").and_then(|sobol| sobol.get("seed")), + "checkpoint Sobol seed", + )?; + let sobol = u32::try_from(sobol) + .map_err(|_| "checkpoint Sobol seed exceeds the supported u32 range".to_string())?; + let gmm = checkpoint_u64( + inner.get("gmm").and_then(|gmm| gmm.get("seed")), + "checkpoint GMM seed", + )?; + Ok(Some(SerializedAutoSeedPair { sobol, gmm })) +} + +fn serialized_sobol_seed(checkpoint_document: &serde_json::Value) -> Result, String> { + let inner = checkpoint_document + .get("checkpoint") + .unwrap_or(checkpoint_document) + .get("strategy_state") + .and_then(|state| state.get("inner")) + .and_then(serde_json::Value::as_object) + .ok_or_else(|| "checkpoint strategy_state has no object-valued inner state".to_string())?; + if inner.get("type").and_then(serde_json::Value::as_str) != Some("sobol") { + return Ok(None); + } + let seed = checkpoint_u64(inner.get("seed"), "checkpoint Sobol seed")?; + u32::try_from(seed) + .map(Some) + .map_err(|_| "checkpoint Sobol seed exceeds the supported u32 range".to_string()) +} + +/// Reconstruct the strategy config omitted by the original full-checkpoint +/// schema. Only strategy-state variants that map unambiguously onto a current +/// configured strategy are accepted; legacy direct-GMM state has no supported +/// two-phase config equivalent and therefore fails closed. +fn infer_legacy_strategy_config( + checkpoint_document: &serde_json::Value, + max_trials: Option, + dim: usize, +) -> Result { + let state = checkpoint_document + .get("checkpoint") + .unwrap_or(checkpoint_document) + .get("strategy_state") + .ok_or_else(|| "checkpoint with strategy: null has no strategy_state".to_string())?; + let inner = state + .get("inner") + .and_then(serde_json::Value::as_object) + .ok_or_else(|| "checkpoint strategy_state has no object-valued inner state".to_string())?; + let state_type = inner + .get("type") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| "checkpoint strategy inner state has no string type".to_string())?; + + let mut strategy = StrategyConfig { + strategy_type: match state_type { + "auto" => "gmm", + "random" => "random", + "sobol" => "sobol", + "gmm" => { + return Err( + "legacy direct-GMM strategy state cannot be reconstructed as a configured strategy" + .to_string(), + ); + } + other => { + return Err(format!( + "unsupported legacy checkpoint strategy state type '{other}'" + )); + } + } + .to_string(), + refit_interval: default_refit_interval(), + total_budget: max_trials, + exploration_budget: None, + ongoing_exploration_period: None, + seed: None, + elite_fraction: None, + max_components: None, + min_elite_samples: None, + max_refit_samples: DEFAULT_MAX_REFIT_SAMPLES, + max_refit_candidates: DEFAULT_MAX_REFIT_CANDIDATES, + }; + + if state_type == "auto" { + strategy.exploration_budget = Some(checkpoint_usize( + inner.get("exploration_budget"), + "checkpoint strategy exploration_budget", + )?); + strategy.ongoing_exploration_period = Some(match inner.get("ongoing_exploration_period") { + Some(value) => checkpoint_usize( + Some(value), + "checkpoint strategy ongoing_exploration_period", + )?, + None => LEGACY_DEFAULT_ONGOING_EXPLORATION_PERIOD, + }); + strategy.max_components = Some( + inner + .get("gmm") + .and_then(|gmm| gmm.get("refit_config")) + .and_then(|config| config.get("n_components")) + .map(|value| checkpoint_usize(Some(value), "checkpoint GMM n_components")) + .transpose()? + .unwrap_or(LEGACY_DEFAULT_MAX_COMPONENTS), + ); + strategy.elite_fraction = Some(LEGACY_DEFAULT_ELITE_FRACTION); + strategy.min_elite_samples = Some(LEGACY_DEFAULT_MIN_ELITE_SAMPLES); + strategy.seed = serialized_auto_seed_pair(checkpoint_document)? + .and_then(SerializedAutoSeedPair::current_config_seed); + } else { + strategy.resolve_legacy_checkpoint_defaults(max_trials.unwrap_or(200), dim); + strategy.seed = Some(checkpoint_u64( + inner.get("seed"), + "checkpoint strategy seed", + )?); + } + Ok(strategy) +} + +fn checkpoint_u64(value: Option<&serde_json::Value>, field: &str) -> Result { + value + .and_then(serde_json::Value::as_u64) + .ok_or_else(|| format!("{field} must be a non-negative integer")) +} + +fn apply_v1_caller_controls_not_encoded_in_state( + migrated: &mut StudyConfig, + requested: &StudyConfig, + checkpoint_document: &serde_json::Value, +) -> Result<(), String> { + let Some(requested_strategy) = requested.strategy.as_ref() else { + return Ok(()); + }; + let migrated_strategy = migrated + .strategy + .as_mut() + .ok_or_else(|| "legacy checkpoint strategy could not be reconstructed".to_string())?; + let inner = checkpoint_document + .get("checkpoint") + .unwrap_or(checkpoint_document) + .get("strategy_state") + .and_then(|state| state.get("inner")) + .and_then(serde_json::Value::as_object) + .ok_or_else(|| "checkpoint strategy_state has no object-valued inner state".to_string())?; + + // Both configured names map to the serialized Auto strategy. The v1 file + // cannot distinguish which spelling the caller originally used. + if migrated_strategy.strategy_type == "gmm" + && matches!(requested_strategy.strategy_type.as_str(), "gmm" | "auto") + { + migrated_strategy.strategy_type = requested_strategy.strategy_type.clone(); + } + + // The original embedded config omitted all engine-level refit controls. + // Caller values are therefore the only source for non-default intent. + migrated_strategy.refit_interval = requested_strategy.refit_interval; + if requested_strategy.elite_fraction.is_some() { + migrated_strategy.elite_fraction = requested_strategy.elite_fraction; + } + if requested_strategy.min_elite_samples.is_some() { + migrated_strategy.min_elite_samples = requested_strategy.min_elite_samples; + } + migrated_strategy.max_refit_samples = requested_strategy.max_refit_samples; + migrated_strategy.max_refit_candidates = requested_strategy.max_refit_candidates; + + // Later Auto-state revisions encoded these controls. Only inherit a caller + // value when the legacy state itself lacks the corresponding evidence. + if inner.get("ongoing_exploration_period").is_none() + && requested_strategy.ongoing_exploration_period.is_some() + { + migrated_strategy.ongoing_exploration_period = + requested_strategy.ongoing_exploration_period; + } + let state_has_components = inner + .get("gmm") + .and_then(|gmm| gmm.get("refit_config")) + .and_then(|config| config.get("n_components")) + .is_some(); + if !state_has_components && requested_strategy.max_components.is_some() { + migrated_strategy.max_components = requested_strategy.max_components; + } + Ok(()) +} + +fn checkpoint_usize(value: Option<&serde_json::Value>, field: &str) -> Result { + let value = value + .and_then(serde_json::Value::as_u64) + .ok_or_else(|| format!("{field} must be a non-negative integer"))?; + usize::try_from(value).map_err(|_| format!("{field} exceeds the supported integer range")) +} + +fn validate_configured_checkpoint_request( + requested: &StudyConfig, + saved: &StudyConfig, + serialized_auto_seeds: Option, + legacy_truncated_sobol_seed: Option, +) -> Result<(), String> { + if serde_json::to_value(&requested.space).map_err(|error| error.to_string())? + != serde_json::to_value(&saved.space).map_err(|error| error.to_string())? + { + return Err("checkpoint search space does not match the caller config".to_string()); + } + if serde_json::to_value(&requested.objectives).map_err(|error| error.to_string())? + != serde_json::to_value(&saved.objectives).map_err(|error| error.to_string())? + { + return Err("checkpoint objectives do not match the caller config".to_string()); + } + if requested.max_leaderboard_size != saved.max_leaderboard_size { + return Err("checkpoint max_leaderboard_size does not match the caller config".to_string()); + } + + let requested_max_trials = requested.max_trials.or_else(|| { + requested + .strategy + .as_ref() + .and_then(|strategy| strategy.total_budget) + }); + if requested_max_trials != saved.max_trials { + return Err("checkpoint trial budget does not match the caller config".to_string()); + } + + let saved_strategy = saved + .strategy + .as_ref() + .ok_or_else(|| "checkpoint has no reconstructable strategy config".to_string())?; + if saved_strategy.total_budget != requested_max_trials { + return Err("checkpoint strategy budget does not match the caller config".to_string()); + } + + // An absent caller strategy delegates the entire strategy choice to the + // full checkpoint. When one is supplied, its non-optional fields and each + // explicitly populated optional control are compatibility constraints. + let Some(requested_strategy) = requested.strategy.as_ref() else { + return Ok(()); + }; + if requested_strategy.strategy_type != saved_strategy.strategy_type { + return Err("checkpoint strategy type does not match the caller config".to_string()); + } + if requested_strategy.refit_interval != saved_strategy.refit_interval { + return Err("checkpoint refit_interval does not match the caller config".to_string()); + } + if requested_strategy.max_refit_samples != saved_strategy.max_refit_samples { + return Err("checkpoint max_refit_samples does not match the caller config".to_string()); + } + if requested_strategy.max_refit_candidates != saved_strategy.max_refit_candidates { + return Err("checkpoint max_refit_candidates does not match the caller config".to_string()); + } + + macro_rules! require_explicit_match { + ($field:ident) => { + if requested_strategy.$field.is_some() + && requested_strategy.$field != saved_strategy.$field + { + return Err(format!( + "checkpoint {} does not match the caller config", + stringify!($field) + )); + } + }; + } + require_explicit_match!(exploration_budget); + require_explicit_match!(ongoing_exploration_period); + if let Some(requested_seed) = requested_strategy.seed { + if saved_strategy.seed != Some(requested_seed) { + // A migrated Auto pair that cannot be reconstructed under today's + // folded-seed rule is deliberately represented as `seed: null`. An + // explicitly configured resume can still prove it is the original + // historical seed by matching both serialized sampler seeds exactly. + let matches_serialized_pair = saved_strategy.seed.is_none() + && serialized_auto_seeds.is_some_and(|pair| { + pair.current_config_seed() == Some(requested_seed) + || pair.matches_legacy_explicit_seed(requested_seed) + }); + let matches_legacy_sobol = legacy_truncated_sobol_seed + .is_some_and(|sobol_seed| sobol_seed == requested_seed as u32); + if !matches_serialized_pair && !matches_legacy_sobol { + return Err("checkpoint seed does not match the caller config".to_string()); + } + } + } + require_explicit_match!(elite_fraction); + require_explicit_match!(max_components); + require_explicit_match!(min_elite_samples); + Ok(()) +} + /// Configuration for automatic checkpointing. #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(deny_unknown_fields)] @@ -1388,7 +1913,10 @@ fn validate_strategy_config(strategy: &StrategyConfig) -> Result<(), String> { if strategy.min_elite_samples == Some(0) { return Err("strategy.min_elite_samples must be at least 1".to_string()); } - if let Some(min_elite_samples) = strategy.min_elite_samples { + if matches!(strategy.strategy_type.as_str(), "gmm" | "auto") { + let min_elite_samples = strategy + .min_elite_samples + .unwrap_or(DEFAULT_MIN_ELITE_SAMPLES); if min_elite_samples > strategy.max_refit_samples { return Err(format!( "strategy.min_elite_samples must not exceed max_refit_samples ({}), got {min_elite_samples}", @@ -1417,6 +1945,24 @@ pub struct DynTrial { pub params: serde_json::Value, } +/// Read-only strategy state used to attribute calibration outcomes. +/// +/// The three GMM-specific fields are `None` for non-GMM strategies. An +/// `auto` strategy can also report an unknown GMM-origin count after loading +/// history whose exact sampler routes were not persisted. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct StrategyDiagnostics { + /// Number of successfully installed GMM parameter sets after the prior. + pub gmm_fit_epoch: Option, + /// Exact cumulative GMM-origin suggestions, when route provenance is known. + /// For `auto`, only routes through a ready empirical model are counted. + pub gmm_origin_suggestions: Option, + /// Whether the auto strategy may currently sample its fitted GMM. + pub gmm_sampling_ready: Option, + /// Total suggestions issued by the strategy cursor. + pub issued_suggestions: u64, +} + /// A completed trial with full scoring, ranking, and Pareto front information. /// /// This is the public-facing trial type returned by `tell()`, `top_k()`, @@ -3073,7 +3619,7 @@ impl HolaEngine { inner: DynStrategyInner::Sobol(SobolStrategy::new( // Fold high bits into low instead of truncating so seeds // differing only in bits >= 32 produce distinct sequences. - (seed ^ (seed >> 32)) as u32, + fold_sobol_seed(seed), )), }, None, @@ -3086,12 +3632,14 @@ impl HolaEngine { let total = max_trials.unwrap_or(200); AutoStrategy::default_exploration_budget(total, dim) }); - let elite_fraction = strategy_cfg.and_then(|s| s.elite_fraction).unwrap_or(0.25); + let elite_fraction = strategy_cfg + .and_then(|s| s.elite_fraction) + .unwrap_or(DEFAULT_ELITE_FRACTION); // Anchor the cadence at the first statistically permitted fit. - // With the legacy floor of one this is exactly the historical - // K0 schedule; an explicit larger floor waits until that many - // completed observations exist and then refits at the requested - // interval from that boundary. + // With the historical floor of one this is exactly the old K0 + // schedule; a larger floor waits until that many completed + // observations exist and then refits at the requested interval + // from that boundary. let first_refit_trials = exploration_budget.max(min_elite_samples); effective_exploration_budget = Some(exploration_budget); effective_elite_fraction = Some(elite_fraction); @@ -3809,6 +4357,11 @@ impl HolaEngine { self.state.read().await.leaderboard.len() } + /// Return a read-only snapshot of strategy routing and fit state. + pub async fn strategy_diagnostics(&self) -> StrategyDiagnostics { + self.state.read().await.strategy.diagnostics() + } + /// Number of unattended auto-checkpoint or rotation failures observed by /// this engine process. pub fn checkpoint_failure_count(&self) -> u64 { @@ -4120,6 +4673,150 @@ impl HolaEngine { } } + /// Construct and load a study from a caller configuration plus checkpoint. + /// + /// This is the configured-resume counterpart to [`Self::load_from_checkpoint`]. + /// A full checkpoint owns its exact sampling semantics, so the returned + /// engine is reconstructed from the checkpoint's migrated embedded config + /// rather than from newly resolved defaults. Structural study fields and + /// every strategy control explicitly supplied by the caller must match; + /// calibration controls omitted by the caller inherit the checkpoint's + /// recorded or historical value. The caller's operational checkpoint + /// settings (directory, cadence, retention, and load path) are reapplied. + /// Leaderboard-only checkpoints instead use the caller config unchanged. + /// + /// The existing [`Self::load_checkpoint_with_fallback`] method remains + /// strict because an already-constructed engine cannot replace immutable + /// refit controls safely. + pub async fn load_configured_checkpoint( + requested_config: StudyConfig, + path: impl AsRef, + ) -> std::io::Result<(Self, CheckpointLoadKind)> { + let path = path.as_ref().to_path_buf(); + let mut raw = tokio::task::spawn_blocking(move || read_checkpoint_document(&path)) + .await + .map_err(|error| std::io::Error::other(format!("checkpoint task failed: {error}")))??; + let has_strategy_state = raw + .get("checkpoint") + .unwrap_or(&raw) + .get("strategy_state") + .is_some(); + + if !has_strategy_state { + let engine = Self::from_config(requested_config) + .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidInput, error))?; + engine.load_leaderboard_checkpoint_document(raw).await?; + return Ok((engine, CheckpointLoadKind::Leaderboard)); + } + + let operational_checkpoint = requested_config.checkpoint.clone(); + let mut resume_config = if let Some(config_value) = raw.get("config") { + let strategy_was_null = checkpoint_embedded_strategy_was_absent_or_null(&raw); + let mut saved_config: StudyConfig = serde_json::from_value(config_value.clone()) + .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?; + saved_config + .resolve_legacy_checkpoint_defaults(&raw) + .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?; + if strategy_was_null { + apply_v1_caller_controls_not_encoded_in_state( + &mut saved_config, + &requested_config, + &raw, + ) + .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?; + } + let serialized_auto_seeds = serialized_auto_seed_pair(&raw) + .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?; + let legacy_truncated_sobol_seed = if strategy_was_null + && checkpoint_format_version(&raw) == Some(1) + { + serialized_sobol_seed(&raw) + .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))? + } else { + None + }; + validate_configured_checkpoint_request( + &requested_config, + &saved_config, + serialized_auto_seeds, + legacy_truncated_sobol_seed, + ) + .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidInput, error))?; + saved_config + } else { + // Direct full checkpoints predate embedded study configs. The + // caller supplies the structure; omitted strategy controls are + // migrated from the supported serialized legacy strategy state. + let mut legacy_config = requested_config.clone(); + legacy_config + .resolve_legacy_checkpoint_defaults(&raw) + .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?; + if let Some(pair) = serialized_auto_seed_pair(&raw) + .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))? + { + let strategy = legacy_config.strategy.as_mut().ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "legacy Auto checkpoint requires a configured strategy", + ) + })?; + if let Some(requested_seed) = strategy.seed { + if pair.current_config_seed() != Some(requested_seed) + && !pair.matches_legacy_explicit_seed(requested_seed) + { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "checkpoint seed does not match the caller config", + )); + } + } + strategy.seed = pair.current_config_seed(); + } else if let Some(sobol_seed) = serialized_sobol_seed(&raw) + .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))? + { + let strategy = legacy_config.strategy.as_mut().ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "legacy Sobol checkpoint requires a configured strategy", + ) + })?; + if let Some(requested_seed) = strategy.seed { + if sobol_seed != requested_seed as u32 { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "checkpoint seed does not match the caller config", + )); + } + } + // The serialized `u32` is a truthful current public seed. The + // original high bits cannot be recovered, so do not re-export + // the caller's pre-fold `u64` as if current folding reproduced + // this state. + strategy.seed = Some(u64::from(sobol_seed)); + } + legacy_config + }; + resume_config.checkpoint = operational_checkpoint; + let mut embedded_resume_config = resume_config.clone(); + embedded_resume_config.checkpoint = None; + raw.as_object_mut() + .ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + "full checkpoint document must be a JSON object", + ) + })? + .insert( + "config".to_string(), + serde_json::to_value(embedded_resume_config) + .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?, + ); + let engine = Self::from_config(resume_config) + .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidInput, error))?; + engine.load_full_checkpoint_document(raw).await?; + Ok((engine, CheckpointLoadKind::Full)) + } + // ========================================================================= // Persistence (internal) // ========================================================================= @@ -4385,9 +5082,9 @@ impl HolaEngine { if let Some(config_value) = raw.get("config") { let mut saved_config: StudyConfig = serde_json::from_value(config_value.clone()) .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; - if let Some(strategy) = &mut saved_config.strategy { - strategy.resolve_calibration_control_defaults(); - } + saved_config + .resolve_legacy_checkpoint_defaults(&raw) + .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?; loaded_strategy_template = saved_config.strategy.clone(); let mut current_config = current_study_config.clone(); // A loaded strategy replaces its sampler state wholesale. Its seed @@ -4434,7 +5131,7 @@ impl HolaEngine { ), }; let mut strategy_template = strategy_template; - let resolved_seed = strategy.resolved_seed(); + let representable_seed = strategy.representable_config_seed(); let strategy_config = strategy_template.as_mut().ok_or_else(|| { std::io::Error::new( std::io::ErrorKind::InvalidData, @@ -4442,10 +5139,20 @@ impl HolaEngine { ) })?; // Older direct full checkpoints did not carry config metadata. Use - // the loaded sampler's real seed so subsequent exports never claim - // the discarded target engine's seed. - if !has_embedded_config || strategy_config.seed.is_none() { - strategy_config.seed = Some(resolved_seed); + // a seed only when it truthfully reconstructs the serialized + // sampler state; a historical Auto pair can require `None` because + // its Sobol and GMM seeds were independent or used truncation. + // Embedded Auto `seed: null` is itself legacy provenance and must + // remain null through subsequent save/load cycles. Other sampler + // variants always have a truthful single-seed representation, so + // retain the established behavior of materializing their seed. + let preserve_embedded_auto_none = has_embedded_config + && strategy_config.seed.is_none() + && matches!(&strategy.inner, DynStrategyInner::Auto(_)); + if !preserve_embedded_auto_none + && (!has_embedded_config || strategy_config.seed.is_none()) + { + strategy_config.seed = representable_seed; } let strategy_config = strategy_config.clone(); let mut replacement = HolaEngineState { @@ -4529,8 +5236,9 @@ impl HolaEngine { .to_string() })?; - let config: StudyConfig = serde_json::from_value(config_value.clone()) + let mut config: StudyConfig = serde_json::from_value(config_value.clone()) .map_err(|e| format!("Failed to parse StudyConfig from checkpoint: {e}"))?; + config.resolve_legacy_checkpoint_defaults(&raw)?; let engine = Self::from_config(config)?; engine @@ -5311,6 +6019,18 @@ mod tests { .unwrap_err() .contains("min_elite_samples") ); + + strategy.min_elite_samples = None; + strategy.max_refit_samples = DEFAULT_MIN_ELITE_SAMPLES - 1; + assert!( + validate_strategy_config(&strategy) + .unwrap_err() + .contains("must not exceed max_refit_samples") + ); + + strategy.strategy_type = "random".to_string(); + validate_strategy_config(&strategy) + .expect("unused GMM elite defaults must not constrain random strategies"); } #[test] @@ -5733,6 +6453,65 @@ mod tests { } } + fn downgrade_full_checkpoint_to_historical_v1(document: &mut serde_json::Value) { + document["config"]["strategy"] = serde_json::Value::Null; + document["checkpoint"]["metadata"]["format_version"] = serde_json::json!(1); + document.as_object_mut().unwrap().remove("runtime_state"); + + let inner = document["checkpoint"]["strategy_state"]["inner"] + .as_object_mut() + .unwrap(); + if inner.get("type").and_then(serde_json::Value::as_str) == Some("auto") { + inner.remove("ongoing_exploration_period"); + inner.remove("gmm_sampling_ready"); + inner.remove("issued_count"); + inner.remove("gmm_origin_suggestions"); + let gmm = inner["gmm"].as_object_mut().unwrap(); + if let Some(counter) = gmm["counter"].get("value").cloned() { + gmm.insert("counter".to_string(), counter); + } + gmm.remove("epoch_start"); + gmm.remove("refit_epoch"); + } + } + + fn downgrade_full_checkpoint_to_v1_with_embedded_strategy(document: &mut serde_json::Value) { + let strategy = document["config"]["strategy"].clone(); + downgrade_full_checkpoint_to_historical_v1(document); + document["config"]["strategy"] = strategy; + } + + fn historical_auto_config(seed: Option) -> StudyConfig { + let mut config = single_objective_config("gmm"); + let strategy = config.strategy.as_mut().unwrap(); + strategy.exploration_budget = Some(8); + strategy.ongoing_exploration_period = Some(LEGACY_DEFAULT_ONGOING_EXPLORATION_PERIOD); + strategy.seed = seed; + strategy.elite_fraction = Some(LEGACY_DEFAULT_ELITE_FRACTION); + strategy.max_components = Some(LEGACY_DEFAULT_MAX_COMPONENTS); + strategy.min_elite_samples = Some(LEGACY_DEFAULT_MIN_ELITE_SAMPLES); + config + } + + async fn install_historical_auto_seed_pair( + engine: &HolaEngine, + sobol_seed: u32, + gmm_seed: u64, + exposed_seed: Option, + ) { + let mut state = engine.state.write().await; + let DynStrategyInner::Auto(auto) = &mut state.strategy.inner else { + panic!("expected auto strategy"); + }; + assert_eq!(auto.trial_count, 0); + assert_eq!(auto.issued_count.load(Ordering::Relaxed), 0); + let refit_config = auto.gmm.get_refit_config().clone(); + auto.sobol = SobolStrategy::new(sobol_seed); + auto.gmm = GmmStrategy::uniform_prior(gmm_seed, 1, AUTO_GMM_PRIOR_VARIANCE).unwrap(); + auto.gmm.set_refit_config(refit_config); + state.strategy_template.as_mut().unwrap().seed = exposed_seed; + } + #[test] fn auto_strategy_period_has_no_off_by_one_and_zero_disables_it() { use opt_engine::strategies::GmmParams; @@ -5786,16 +6565,18 @@ mod tests { let restored: AutoStrategy = serde_json::from_value(encoded).unwrap(); assert_eq!( restored.ongoing_exploration_period, - DEFAULT_ONGOING_EXPLORATION_PERIOD + LEGACY_DEFAULT_ONGOING_EXPLORATION_PERIOD ); } #[tokio::test] - async fn omitted_calibration_controls_resolve_to_legacy_values() { + async fn omitted_calibration_controls_resolve_to_calibrated_values() { let config = single_objective_config("gmm"); - let explicit_legacy = { + let explicit_defaults = { let mut config = config.clone(); let strategy = config.strategy.as_mut().unwrap(); + strategy.exploration_budget = Some(AutoStrategy::default_exploration_budget(200, 1)); + strategy.elite_fraction = Some(DEFAULT_ELITE_FRACTION); strategy.ongoing_exploration_period = Some(DEFAULT_ONGOING_EXPLORATION_PERIOD); strategy.max_components = Some(DEFAULT_MAX_COMPONENTS); strategy.min_elite_samples = Some(DEFAULT_MIN_ELITE_SAMPLES); @@ -5803,9 +6584,14 @@ mod tests { }; let implicit = HolaEngine::from_config(config).unwrap(); - let explicit = HolaEngine::from_config(explicit_legacy).unwrap(); + let explicit = HolaEngine::from_config(explicit_defaults).unwrap(); let exported = implicit.study_config().await; let exported = exported.strategy.unwrap(); + assert_eq!( + exported.exploration_budget, + Some(AutoStrategy::default_exploration_budget(200, 1)) + ); + assert_eq!(exported.elite_fraction, Some(DEFAULT_ELITE_FRACTION)); assert_eq!( exported.ongoing_exploration_period, Some(DEFAULT_ONGOING_EXPLORATION_PERIOD) @@ -5813,7 +6599,7 @@ mod tests { assert_eq!(exported.max_components, Some(DEFAULT_MAX_COMPONENTS)); assert_eq!(exported.min_elite_samples, Some(DEFAULT_MIN_ELITE_SAMPLES)); - // The omitted and explicit legacy forms must remain behaviorally + // The omitted and explicit calibrated forms must remain behaviorally // identical, including the first refit and periodic exploration. for _ in 0..40 { let implicit_trial = implicit.ask().await.unwrap(); @@ -5870,48 +6656,240 @@ mod tests { } #[tokio::test] - async fn min_elite_samples_counts_only_feasible_fit_inputs() { + async fn strategy_diagnostics_count_only_empirical_gmm_routes() { let mut config = single_objective_config("gmm"); - config.objectives[0].target = Some(0.0); - config.objectives[0].limit = Some(1.0); let strategy = config.strategy.as_mut().unwrap(); strategy.exploration_budget = Some(1); + strategy.ongoing_exploration_period = Some(4); + strategy.min_elite_samples = Some(1); strategy.refit_interval = 1; - strategy.min_elite_samples = Some(4); let engine = HolaEngine::from_config(config).unwrap(); - for completed in 1..=5 { - let trial = engine.ask().await.unwrap(); - let loss = if completed == 1 { 2.0 } else { 0.5 }; - engine - .tell(trial.trial_id, serde_json::json!({"loss": loss})) - .await - .unwrap(); - let state = engine.state.read().await; - let DynStrategyInner::Auto(auto) = &state.strategy.inner else { - panic!("expected auto strategy"); - }; - assert_eq!( - auto.gmm.refit_epoch(), - u64::from(completed == 5), - "an infeasible observation must not satisfy the elite floor" - ); - } - } + assert_eq!( + engine.strategy_diagnostics().await, + StrategyDiagnostics { + gmm_fit_epoch: Some(0), + gmm_origin_suggestions: Some(0), + gmm_sampling_ready: Some(false), + issued_suggestions: 0, + } + ); - #[tokio::test] - async fn initial_fit_retries_before_the_next_periodic_cadence() { - let mut config = single_objective_config("gmm"); - config.objectives[0].target = Some(0.0); - config.objectives[0].limit = Some(1.0); - let strategy = config.strategy.as_mut().unwrap(); - strategy.exploration_budget = Some(10); - strategy.refit_interval = 20; - strategy.min_elite_samples = Some(10); - let engine = HolaEngine::from_config(config).unwrap(); + // Concurrent asks can cross the nominal warm-up boundary before any + // tell has produced an empirical model. All three stay on Sobol and + // must therefore leave the GMM-origin count at zero. + let warmup = engine.ask().await.unwrap(); + let _prefit_overflow_a = engine.ask().await.unwrap(); + let _prefit_overflow_b = engine.ask().await.unwrap(); + assert_eq!( + engine.strategy_diagnostics().await, + StrategyDiagnostics { + gmm_fit_epoch: Some(0), + gmm_origin_suggestions: Some(0), + gmm_sampling_ready: Some(false), + issued_suggestions: 3, + } + ); - for completed in 1..=10 { - let trial = engine.ask().await.unwrap(); + engine + .tell(warmup.trial_id, serde_json::json!({"loss": 0.5})) + .await + .unwrap(); + assert_eq!( + engine.strategy_diagnostics().await, + StrategyDiagnostics { + gmm_fit_epoch: Some(1), + gmm_origin_suggestions: Some(0), + gmm_sampling_ready: Some(true), + issued_suggestions: 3, + } + ); + + // The third post-warm-up request is empirical GMM; the fourth is the + // configured periodic Sobol request and must not increment the count. + engine.ask().await.unwrap(); + assert_eq!( + engine.strategy_diagnostics().await.gmm_origin_suggestions, + Some(1) + ); + engine.ask().await.unwrap(); + assert_eq!( + engine.strategy_diagnostics().await, + StrategyDiagnostics { + gmm_fit_epoch: Some(1), + gmm_origin_suggestions: Some(1), + gmm_sampling_ready: Some(true), + issued_suggestions: 5, + } + ); + } + + #[tokio::test] + async fn non_gmm_strategy_diagnostics_have_no_gmm_fields() { + for strategy_type in ["random", "sobol"] { + let engine = HolaEngine::from_config(single_objective_config(strategy_type)).unwrap(); + assert_eq!( + engine.strategy_diagnostics().await, + StrategyDiagnostics { + gmm_fit_epoch: None, + gmm_origin_suggestions: None, + gmm_sampling_ready: None, + issued_suggestions: 0, + } + ); + engine.ask().await.unwrap(); + assert_eq!(engine.strategy_diagnostics().await.issued_suggestions, 1); + } + } + + #[test] + fn legacy_direct_gmm_diagnostics_do_not_claim_empirical_origin() { + let space = DynSpace::new().add_real("x", 0.0, 1.0); + let strategy = DynStrategy { + inner: DynStrategyInner::Gmm( + GmmStrategy::uniform_prior(7, 1, AUTO_GMM_PRIOR_VARIANCE).unwrap(), + ), + }; + strategy.suggest(&space); + assert_eq!( + strategy.diagnostics(), + StrategyDiagnostics { + gmm_fit_epoch: Some(0), + gmm_origin_suggestions: None, + gmm_sampling_ready: Some(true), + issued_suggestions: 1, + } + ); + } + + #[tokio::test] + async fn gmm_origin_diagnostics_roundtrip_and_legacy_unknown() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("diagnostics.json"); + let mut config = single_objective_config("gmm"); + let strategy = config.strategy.as_mut().unwrap(); + strategy.exploration_budget = Some(1); + strategy.ongoing_exploration_period = Some(0); + strategy.min_elite_samples = Some(1); + strategy.refit_interval = 1; + let source = HolaEngine::from_config(config).unwrap(); + + let warmup = source.ask().await.unwrap(); + source + .tell(warmup.trial_id, serde_json::json!({"loss": 0.5})) + .await + .unwrap(); + source.ask().await.unwrap(); + let expected = source.strategy_diagnostics().await; + assert_eq!(expected.gmm_origin_suggestions, Some(1)); + source.save_full_checkpoint(&path, None).await.unwrap(); + + let resumed = HolaEngine::load_from_checkpoint(&path).await.unwrap(); + assert_eq!(resumed.strategy_diagnostics().await, expected); + + let mut legacy: serde_json::Value = + serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap(); + legacy["checkpoint"]["strategy_state"]["inner"] + .as_object_mut() + .unwrap() + .remove("gmm_origin_suggestions"); + std::fs::write(&path, serde_json::to_vec_pretty(&legacy).unwrap()).unwrap(); + + let legacy_resumed = HolaEngine::load_from_checkpoint(&path).await.unwrap(); + let legacy_diagnostics = legacy_resumed.strategy_diagnostics().await; + assert_eq!(legacy_diagnostics.gmm_origin_suggestions, None); + assert_eq!(legacy_diagnostics.gmm_fit_epoch, expected.gmm_fit_epoch); + assert_eq!( + legacy_diagnostics.gmm_sampling_ready, + expected.gmm_sampling_ready + ); + assert_eq!( + legacy_diagnostics.issued_suggestions, + expected.issued_suggestions + ); + + // Once cumulative history is unknown, later GMM suggestions cannot + // reconstruct it and must leave the diagnostic unknown. + legacy_resumed.ask().await.unwrap(); + assert_eq!( + legacy_resumed + .strategy_diagnostics() + .await + .gmm_origin_suggestions, + None + ); + } + + #[tokio::test] + async fn leaderboard_import_makes_gmm_origin_count_unknown() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("leaderboard.json"); + let source = HolaEngine::from_config(single_objective_config("random")).unwrap(); + let trial = source.ask().await.unwrap(); + source + .tell(trial.trial_id, serde_json::json!({"loss": 0.5})) + .await + .unwrap(); + source + .save_leaderboard_checkpoint_to(&path, None) + .await + .unwrap(); + + let target = HolaEngine::from_config(single_objective_config("gmm")).unwrap(); + assert_eq!( + target.strategy_diagnostics().await.gmm_origin_suggestions, + Some(0) + ); + target.load_leaderboard_checkpoint(&path).await.unwrap(); + assert_eq!( + target.strategy_diagnostics().await.gmm_origin_suggestions, + None + ); + } + + #[tokio::test] + async fn min_elite_samples_counts_only_feasible_fit_inputs() { + let mut config = single_objective_config("gmm"); + config.objectives[0].target = Some(0.0); + config.objectives[0].limit = Some(1.0); + let strategy = config.strategy.as_mut().unwrap(); + strategy.exploration_budget = Some(1); + strategy.refit_interval = 1; + strategy.min_elite_samples = Some(4); + let engine = HolaEngine::from_config(config).unwrap(); + + for completed in 1..=5 { + let trial = engine.ask().await.unwrap(); + let loss = if completed == 1 { 2.0 } else { 0.5 }; + engine + .tell(trial.trial_id, serde_json::json!({"loss": loss})) + .await + .unwrap(); + let state = engine.state.read().await; + let DynStrategyInner::Auto(auto) = &state.strategy.inner else { + panic!("expected auto strategy"); + }; + assert_eq!( + auto.gmm.refit_epoch(), + u64::from(completed == 5), + "an infeasible observation must not satisfy the elite floor" + ); + } + } + + #[tokio::test] + async fn initial_fit_retries_before_the_next_periodic_cadence() { + let mut config = single_objective_config("gmm"); + config.objectives[0].target = Some(0.0); + config.objectives[0].limit = Some(1.0); + let strategy = config.strategy.as_mut().unwrap(); + strategy.exploration_budget = Some(10); + strategy.refit_interval = 20; + strategy.min_elite_samples = Some(10); + let engine = HolaEngine::from_config(config).unwrap(); + + for completed in 1..=10 { + let trial = engine.ask().await.unwrap(); let loss = if completed == 1 { 2.0 } else { 0.5 }; engine .tell(trial.trial_id, serde_json::json!({"loss": loss})) @@ -6106,7 +7084,15 @@ mod tests { async fn checkpoint_missing_calibration_fields_loads_with_legacy_defaults() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("legacy-controls.json"); - let source = HolaEngine::from_config(single_objective_config("gmm")).unwrap(); + let mut config = single_objective_config("gmm"); + let strategy = config.strategy.as_mut().unwrap(); + strategy.exploration_budget = Some(AutoStrategy::legacy_default_exploration_budget(200, 1)); + strategy.elite_fraction = Some(LEGACY_DEFAULT_ELITE_FRACTION); + strategy.ongoing_exploration_period = Some(LEGACY_DEFAULT_ONGOING_EXPLORATION_PERIOD); + strategy.max_components = Some(LEGACY_DEFAULT_MAX_COMPONENTS); + strategy.min_elite_samples = Some(LEGACY_DEFAULT_MIN_ELITE_SAMPLES); + strategy.max_refit_samples = 4; + let source = HolaEngine::from_config(config).unwrap(); let trial = source.ask().await.unwrap(); source .tell(trial.trial_id, serde_json::json!({"loss": 0.5})) @@ -6117,6 +7103,8 @@ mod tests { let mut legacy: serde_json::Value = serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap(); let strategy_config = legacy["config"]["strategy"].as_object_mut().unwrap(); + strategy_config.remove("exploration_budget"); + strategy_config.remove("elite_fraction"); strategy_config.remove("ongoing_exploration_period"); strategy_config.remove("max_components"); strategy_config.remove("min_elite_samples"); @@ -6128,12 +7116,617 @@ mod tests { let restored = HolaEngine::load_from_checkpoint(&path).await.unwrap(); let strategy = restored.study_config().await.strategy.unwrap(); + assert_eq!( + strategy.exploration_budget, + Some(AutoStrategy::legacy_default_exploration_budget(200, 1)) + ); + assert_eq!(strategy.elite_fraction, Some(LEGACY_DEFAULT_ELITE_FRACTION)); assert_eq!( strategy.ongoing_exploration_period, - Some(DEFAULT_ONGOING_EXPLORATION_PERIOD) + Some(LEGACY_DEFAULT_ONGOING_EXPLORATION_PERIOD) + ); + assert_eq!(strategy.max_components, Some(LEGACY_DEFAULT_MAX_COMPONENTS)); + assert_eq!( + strategy.min_elite_samples, + Some(LEGACY_DEFAULT_MIN_ELITE_SAMPLES) + ); + assert_eq!(strategy.max_refit_samples, 4); + + let source_next = source.ask().await.unwrap(); + let restored_next = restored.ask().await.unwrap(); + assert_eq!(source_next.trial_id, restored_next.trial_id); + assert_eq!(source_next.params, restored_next.params); + } + + #[tokio::test] + async fn configured_checkpoint_load_inherits_only_omitted_calibration_controls() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("pre-adoption.json"); + let checkpoint_dir = dir.path().join("continued-checkpoints"); + let mut legacy_config = single_objective_config("gmm"); + let strategy = legacy_config.strategy.as_mut().unwrap(); + strategy.exploration_budget = Some(AutoStrategy::legacy_default_exploration_budget(200, 1)); + strategy.elite_fraction = Some(LEGACY_DEFAULT_ELITE_FRACTION); + strategy.ongoing_exploration_period = Some(LEGACY_DEFAULT_ONGOING_EXPLORATION_PERIOD); + strategy.max_components = Some(LEGACY_DEFAULT_MAX_COMPONENTS); + strategy.min_elite_samples = Some(LEGACY_DEFAULT_MIN_ELITE_SAMPLES); + let source = HolaEngine::from_config(legacy_config).unwrap(); + let first = source.ask().await.unwrap(); + source + .tell( + first.trial_id, + serde_json::json!({"loss": first.params["x"]}), + ) + .await + .unwrap(); + source.save_full_checkpoint(&path, None).await.unwrap(); + + let mut requested = single_objective_config("gmm"); + requested.checkpoint = Some(CheckpointConfig { + directory: checkpoint_dir.to_string_lossy().into_owned(), + interval: 1, + max_checkpoints: Some(2), + load_from: Some(path.to_string_lossy().into_owned()), + }); + let (restored, kind) = HolaEngine::load_configured_checkpoint(requested.clone(), &path) + .await + .unwrap(); + assert_eq!(kind, CheckpointLoadKind::Full); + let restored_strategy = restored.study_config().await.strategy.unwrap(); + assert_eq!( + restored_strategy.exploration_budget, + Some(AutoStrategy::legacy_default_exploration_budget(200, 1)) + ); + assert_eq!( + restored_strategy.elite_fraction, + Some(LEGACY_DEFAULT_ELITE_FRACTION) + ); + assert_eq!( + restored_strategy.ongoing_exploration_period, + Some(LEGACY_DEFAULT_ONGOING_EXPLORATION_PERIOD) + ); + assert_eq!( + restored_strategy.max_components, + Some(LEGACY_DEFAULT_MAX_COMPONENTS) + ); + assert_eq!( + restored_strategy.min_elite_samples, + Some(LEGACY_DEFAULT_MIN_ELITE_SAMPLES) + ); + + let source_next = source.ask().await.unwrap(); + let restored_next = restored.ask().await.unwrap(); + assert_eq!(source_next.trial_id, restored_next.trial_id); + assert_eq!(source_next.params, restored_next.params); + restored + .tell( + restored_next.trial_id, + serde_json::json!({"loss": restored_next.params["x"]}), + ) + .await + .unwrap(); + assert!( + std::fs::read_dir(&checkpoint_dir) + .unwrap() + .filter_map(Result::ok) + .any(|entry| entry.path().extension().is_some_and(|ext| ext == "json")), + "caller auto-checkpoint settings must remain active after full resume" + ); + + requested.strategy.as_mut().unwrap().elite_fraction = Some(DEFAULT_ELITE_FRACTION); + let checkpoint_before_conflict = std::fs::read(&path).unwrap(); + let error = HolaEngine::load_configured_checkpoint(requested, &path) + .await + .err() + .expect("an explicit conflicting control must reject configured resume"); + assert!(error.to_string().contains("elite_fraction")); + assert_eq!(std::fs::read(&path).unwrap(), checkpoint_before_conflict); + } + + #[tokio::test] + async fn configured_v1_null_strategy_uses_caller_refit_controls_and_continues_exactly() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("v1-null-strategy.json"); + let mut source_config = single_objective_config("gmm"); + let strategy = source_config.strategy.as_mut().unwrap(); + strategy.exploration_budget = Some(4); + strategy.refit_interval = 7; + strategy.elite_fraction = Some(0.4); + strategy.ongoing_exploration_period = Some(LEGACY_DEFAULT_ONGOING_EXPLORATION_PERIOD); + strategy.max_components = Some(2); + strategy.min_elite_samples = Some(2); + strategy.max_refit_samples = 17; + strategy.max_refit_candidates = 53; + let source = HolaEngine::from_config(source_config.clone()).unwrap(); + for _ in 0..3 { + let trial = source.ask().await.unwrap(); + source + .tell( + trial.trial_id, + serde_json::json!({"loss": trial.params["x"]}), + ) + .await + .unwrap(); + } + source.save_full_checkpoint(&path, None).await.unwrap(); + let mut v1: serde_json::Value = + serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap(); + downgrade_full_checkpoint_to_historical_v1(&mut v1); + std::fs::write(&path, serde_json::to_vec_pretty(&v1).unwrap()).unwrap(); + + let (restored, kind) = HolaEngine::load_configured_checkpoint(source_config, &path) + .await + .unwrap(); + assert_eq!(kind, CheckpointLoadKind::Full); + let restored_strategy = restored.study_config().await.strategy.unwrap(); + assert_eq!(restored_strategy.refit_interval, 7); + assert_eq!(restored_strategy.elite_fraction, Some(0.4)); + assert_eq!(restored_strategy.min_elite_samples, Some(2)); + assert_eq!(restored_strategy.max_refit_samples, 17); + assert_eq!(restored_strategy.max_refit_candidates, 53); + assert_eq!(restored_strategy.exploration_budget, Some(4)); + assert_eq!(restored_strategy.max_components, Some(2)); + + let source_next = source.ask().await.unwrap(); + let restored_next = restored.ask().await.unwrap(); + assert_eq!(source_next.trial_id, restored_next.trial_id); + assert_eq!(source_next.params, restored_next.params); + let next_loss = source_next.params["x"].as_f64().unwrap(); + source + .tell(source_next.trial_id, serde_json::json!({"loss": next_loss})) + .await + .unwrap(); + restored + .tell( + restored_next.trial_id, + serde_json::json!({"loss": next_loss}), + ) + .await + .unwrap(); + assert_eq!(source.strategy_diagnostics().await.gmm_fit_epoch, Some(1)); + assert_eq!(restored.strategy_diagnostics().await.gmm_fit_epoch, Some(1)); + let source_after_refit = source.ask().await.unwrap(); + let restored_after_refit = restored.ask().await.unwrap(); + assert_eq!(source_after_refit.params, restored_after_refit.params); + } + + #[tokio::test] + async fn v1_null_strategy_without_caller_uses_historical_defaults() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("v1-defaults.json"); + let mut legacy_config = single_objective_config("gmm"); + let strategy = legacy_config.strategy.as_mut().unwrap(); + strategy.exploration_budget = Some(AutoStrategy::legacy_default_exploration_budget(200, 1)); + strategy.elite_fraction = Some(LEGACY_DEFAULT_ELITE_FRACTION); + strategy.ongoing_exploration_period = Some(LEGACY_DEFAULT_ONGOING_EXPLORATION_PERIOD); + strategy.max_components = Some(LEGACY_DEFAULT_MAX_COMPONENTS); + strategy.min_elite_samples = Some(LEGACY_DEFAULT_MIN_ELITE_SAMPLES); + let source = HolaEngine::from_config(legacy_config).unwrap(); + let trial = source.ask().await.unwrap(); + source + .tell( + trial.trial_id, + serde_json::json!({"loss": trial.params["x"]}), + ) + .await + .unwrap(); + source.save_full_checkpoint(&path, None).await.unwrap(); + let mut v1: serde_json::Value = + serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap(); + downgrade_full_checkpoint_to_historical_v1(&mut v1); + std::fs::write(&path, serde_json::to_vec_pretty(&v1).unwrap()).unwrap(); + + let restored = HolaEngine::load_from_checkpoint(&path).await.unwrap(); + let restored_strategy = restored.study_config().await.strategy.unwrap(); + assert_eq!( + restored_strategy.elite_fraction, + Some(LEGACY_DEFAULT_ELITE_FRACTION) + ); + assert_eq!( + restored_strategy.ongoing_exploration_period, + Some(LEGACY_DEFAULT_ONGOING_EXPLORATION_PERIOD) + ); + assert_eq!( + restored_strategy.max_components, + Some(LEGACY_DEFAULT_MAX_COMPONENTS) + ); + assert_eq!( + restored_strategy.min_elite_samples, + Some(LEGACY_DEFAULT_MIN_ELITE_SAMPLES) + ); + let source_next = source.ask().await.unwrap(); + let restored_next = restored.ask().await.unwrap(); + assert_eq!(source_next.params, restored_next.params); + } + + #[tokio::test] + async fn v1_null_strategy_infers_random_and_sobol_state() { + let dir = tempfile::tempdir().unwrap(); + for strategy_type in ["random", "sobol"] { + let path = dir.path().join(format!("v1-{strategy_type}.json")); + let source = HolaEngine::from_config(single_objective_config(strategy_type)).unwrap(); + let first = source.ask().await.unwrap(); + source + .tell( + first.trial_id, + serde_json::json!({"loss": first.params["x"]}), + ) + .await + .unwrap(); + source.save_full_checkpoint(&path, None).await.unwrap(); + let mut v1: serde_json::Value = + serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap(); + downgrade_full_checkpoint_to_historical_v1(&mut v1); + std::fs::write(&path, serde_json::to_vec_pretty(&v1).unwrap()).unwrap(); + + let restored = HolaEngine::load_from_checkpoint(&path).await.unwrap(); + assert_eq!( + restored + .study_config() + .await + .strategy + .unwrap() + .strategy_type, + strategy_type + ); + let source_next = source.ask().await.unwrap(); + let restored_next = restored.ask().await.unwrap(); + assert_eq!(source_next.trial_id, restored_next.trial_id); + assert_eq!(source_next.params, restored_next.params); + } + } + + #[tokio::test] + async fn legacy_auto_independent_seed_pair_stays_unrepresented_across_loads() { + let dir = tempfile::tempdir().unwrap(); + let legacy_path = dir.path().join("legacy-independent-seeds.json"); + let resaved_path = dir.path().join("resaved-independent-seeds.json"); + let gmm_seed = 0x0123_4567_89ab_cdef; + + // This is the exact old `seed: null` construction: deterministic Sobol + // seed 42 plus an independently generated full-width GMM seed. + let source = HolaEngine::from_config(historical_auto_config(None)).unwrap(); + install_historical_auto_seed_pair(&source, 42, gmm_seed, None).await; + for _ in 0..3 { + let trial = source.ask().await.unwrap(); + source + .tell( + trial.trial_id, + serde_json::json!({"loss": trial.params["x"]}), + ) + .await + .unwrap(); + } + source + .save_full_checkpoint(&legacy_path, None) + .await + .unwrap(); + let mut legacy: serde_json::Value = + serde_json::from_slice(&std::fs::read(&legacy_path).unwrap()).unwrap(); + downgrade_full_checkpoint_to_historical_v1(&mut legacy); + let mut direct = legacy.clone(); + direct.as_object_mut().unwrap().remove("config"); + std::fs::write(&legacy_path, serde_json::to_vec_pretty(&legacy).unwrap()).unwrap(); + + let restored = HolaEngine::load_from_checkpoint(&legacy_path) + .await + .unwrap(); + assert_eq!( + restored.study_config().await.strategy.unwrap().seed, + None, + "an independent historical pair has no truthful current config seed" + ); + + // A direct legacy full load must replace the temporary target's seed + // metadata with the same honest `None`, while retaining exact sampler + // continuation. + let direct_target = HolaEngine::from_config(historical_auto_config(Some(999))).unwrap(); + direct_target + .load_full_checkpoint_document(direct) + .await + .unwrap(); + assert_eq!( + direct_target.study_config().await.strategy.unwrap().seed, + None + ); + + let source_next = source.ask().await.unwrap(); + let restored_next = restored.ask().await.unwrap(); + let direct_next = direct_target.ask().await.unwrap(); + assert_eq!(source_next.params, restored_next.params); + assert_eq!(source_next.params, direct_next.params); + for (engine, trial) in [ + (&source, source_next), + (&restored, restored_next), + (&direct_target, direct_next), + ] { + engine + .tell( + trial.trial_id, + serde_json::json!({"loss": trial.params["x"]}), + ) + .await + .unwrap(); + } + + restored + .save_full_checkpoint(&resaved_path, None) + .await + .unwrap(); + let resaved: serde_json::Value = + serde_json::from_slice(&std::fs::read(&resaved_path).unwrap()).unwrap(); + assert!(resaved["config"]["strategy"]["seed"].is_null()); + let reloaded = HolaEngine::load_from_checkpoint(&resaved_path) + .await + .unwrap(); + assert_eq!(reloaded.study_config().await.strategy.unwrap().seed, None); + assert_eq!( + restored.ask().await.unwrap().params, + reloaded.ask().await.unwrap().params, + "resaving must not reinterpret the independent pair as a folded seed" + ); + } + + #[tokio::test] + async fn configured_v1_auto_resume_accepts_only_the_exact_truncated_high_seed() { + let dir = tempfile::tempdir().unwrap(); + let modern_path = dir.path().join("modern-inconsistent-seeds.json"); + let non_null_v1_path = dir.path().join("v1-non-null-inconsistent-seeds.json"); + let unsupported_null_path = dir.path().join("modern-unsupported-null-seeds.json"); + let legacy_path = dir.path().join("legacy-high-seed.json"); + let resaved_path = dir.path().join("resaved-high-seed.json"); + let high_seed = 0x0000_0001_0000_0007; + assert_ne!(fold_sobol_seed(high_seed), high_seed as u32); + + let source = HolaEngine::from_config(historical_auto_config(Some(high_seed))).unwrap(); + // Old explicit-seed semantics retained the full seed for the GMM and + // truncated it to the low 32 bits for Sobol. + install_historical_auto_seed_pair(&source, high_seed as u32, high_seed, Some(high_seed)) + .await; + for _ in 0..3 { + let trial = source.ask().await.unwrap(); + source + .tell( + trial.trial_id, + serde_json::json!({"loss": trial.params["x"]}), + ) + .await + .unwrap(); + } + source + .save_full_checkpoint(&modern_path, None) + .await + .unwrap(); + + // The same mismatch is not accepted as a modern checkpoint: an + // embedded concrete seed still requires the current folded pair. + let modern_error = HolaEngine::load_from_checkpoint(&modern_path) + .await + .err() + .expect("a modern checkpoint must enforce current seed folding"); + assert!(modern_error.contains("auto strategy seeds")); + + // The fold and the non-null strategy template shipped together while + // checkpoint format version was still one. Such a checkpoint must not + // be reinterpreted as a pre-fold null-strategy artifact. + let mut non_null_v1: serde_json::Value = + serde_json::from_slice(&std::fs::read(&modern_path).unwrap()).unwrap(); + downgrade_full_checkpoint_to_v1_with_embedded_strategy(&mut non_null_v1); + std::fs::write( + &non_null_v1_path, + serde_json::to_vec_pretty(&non_null_v1).unwrap(), + ) + .unwrap(); + let non_null_v1_error = HolaEngine::load_from_checkpoint(&non_null_v1_path) + .await + .err() + .expect("v1 with a concrete strategy must enforce current seed folding"); + assert!(non_null_v1_error.contains("auto strategy seeds")); + + let mut unsupported_null: serde_json::Value = + serde_json::from_slice(&std::fs::read(&modern_path).unwrap()).unwrap(); + unsupported_null["config"]["strategy"]["seed"] = serde_json::Value::Null; + unsupported_null["checkpoint"]["strategy_state"]["inner"]["sobol"]["seed"] = + serde_json::json!(8); + std::fs::write( + &unsupported_null_path, + serde_json::to_vec_pretty(&unsupported_null).unwrap(), + ) + .unwrap(); + let unsupported_error = HolaEngine::load_from_checkpoint(&unsupported_null_path) + .await + .err() + .expect("seed null must only admit a supported historical pair"); + assert!(unsupported_error.contains("seed relationship")); + + let mut legacy: serde_json::Value = + serde_json::from_slice(&std::fs::read(&modern_path).unwrap()).unwrap(); + downgrade_full_checkpoint_to_historical_v1(&mut legacy); + std::fs::write(&legacy_path, serde_json::to_vec_pretty(&legacy).unwrap()).unwrap(); + + let mut wrong_config = historical_auto_config(Some(high_seed ^ (1 << 40))); + wrong_config.max_trials = None; + let wrong = HolaEngine::load_configured_checkpoint(wrong_config, &legacy_path) + .await + .err() + .expect("a different explicit seed must be rejected"); + assert!(wrong.to_string().contains("seed")); + + let (restored, kind) = HolaEngine::load_configured_checkpoint( + historical_auto_config(Some(high_seed)), + &legacy_path, + ) + .await + .unwrap(); + assert_eq!(kind, CheckpointLoadKind::Full); + assert_eq!( + restored.study_config().await.strategy.unwrap().seed, + None, + "the exact old seed authorizes resume but cannot describe the pair under current folding" + ); + assert_eq!( + source.ask().await.unwrap().params, + restored.ask().await.unwrap().params + ); + restored + .save_full_checkpoint(&resaved_path, None) + .await + .unwrap(); + let resaved: serde_json::Value = + serde_json::from_slice(&std::fs::read(&resaved_path).unwrap()).unwrap(); + assert!(resaved["config"]["strategy"]["seed"].is_null()); + let reloaded = HolaEngine::load_from_checkpoint(&resaved_path) + .await + .unwrap(); + assert_eq!(reloaded.study_config().await.strategy.unwrap().seed, None); + assert_eq!( + restored.ask().await.unwrap().params, + reloaded.ask().await.unwrap().params + ); + } + + #[tokio::test] + async fn v1_non_null_auto_with_folded_high_seed_roundtrips() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("v1-folded-high-seed.json"); + let high_seed = 0x0000_0001_0000_0007; + let source = HolaEngine::from_config(historical_auto_config(Some(high_seed))).unwrap(); + for _ in 0..3 { + let trial = source.ask().await.unwrap(); + source + .tell( + trial.trial_id, + serde_json::json!({"loss": trial.params["x"]}), + ) + .await + .unwrap(); + } + source.save_full_checkpoint(&path, None).await.unwrap(); + let mut v1: serde_json::Value = + serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap(); + downgrade_full_checkpoint_to_v1_with_embedded_strategy(&mut v1); + std::fs::write(&path, serde_json::to_vec_pretty(&v1).unwrap()).unwrap(); + + let restored = HolaEngine::load_from_checkpoint(&path).await.unwrap(); + assert_eq!( + restored.study_config().await.strategy.unwrap().seed, + Some(high_seed) + ); + assert_eq!( + source.ask().await.unwrap().params, + restored.ask().await.unwrap().params + ); + } + + #[tokio::test] + async fn v1_non_null_auto_with_independent_omitted_seed_roundtrips() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("v1-independent-omitted-seed.json"); + let gmm_seed = 0x0123_4567_89ab_cdef; + let source = HolaEngine::from_config(historical_auto_config(None)).unwrap(); + install_historical_auto_seed_pair(&source, 42, gmm_seed, None).await; + for _ in 0..3 { + let trial = source.ask().await.unwrap(); + source + .tell( + trial.trial_id, + serde_json::json!({"loss": trial.params["x"]}), + ) + .await + .unwrap(); + } + source.save_full_checkpoint(&path, None).await.unwrap(); + let mut v1: serde_json::Value = + serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap(); + downgrade_full_checkpoint_to_v1_with_embedded_strategy(&mut v1); + std::fs::write(&path, serde_json::to_vec_pretty(&v1).unwrap()).unwrap(); + + let restored = HolaEngine::load_from_checkpoint(&path).await.unwrap(); + assert_eq!(restored.study_config().await.strategy.unwrap().seed, None); + assert_eq!( + source.ask().await.unwrap().params, + restored.ask().await.unwrap().params + ); + } + + #[tokio::test] + async fn configured_v1_sobol_resume_accepts_the_historical_truncated_seed() { + let dir = tempfile::tempdir().unwrap(); + let modern_path = dir.path().join("modern-sobol-seed-mismatch.json"); + let legacy_path = dir.path().join("legacy-sobol-high-seed.json"); + let direct_path = dir.path().join("direct-sobol-high-seed.json"); + let high_seed = 0x0000_0001_0000_0007; + + let mut source_config = single_objective_config("sobol"); + source_config.strategy.as_mut().unwrap().seed = Some(high_seed); + let source = HolaEngine::from_config(source_config.clone()).unwrap(); + { + let mut state = source.state.write().await; + state.strategy.inner = DynStrategyInner::Sobol(SobolStrategy::new(high_seed as u32)); + state.strategy_template.as_mut().unwrap().seed = Some(high_seed); + } + for _ in 0..3 { + let trial = source.ask().await.unwrap(); + source + .tell( + trial.trial_id, + serde_json::json!({"loss": trial.params["x"]}), + ) + .await + .unwrap(); + } + source + .save_full_checkpoint(&modern_path, None) + .await + .unwrap(); + + let modern_error = HolaEngine::load_from_checkpoint(&modern_path) + .await + .err() + .expect("modern Sobol checkpoints must enforce current seed folding"); + assert!(modern_error.contains("folded seed")); + + let mut legacy: serde_json::Value = + serde_json::from_slice(&std::fs::read(&modern_path).unwrap()).unwrap(); + downgrade_full_checkpoint_to_historical_v1(&mut legacy); + let mut direct = legacy.clone(); + direct.as_object_mut().unwrap().remove("config"); + std::fs::write(&legacy_path, serde_json::to_vec_pretty(&legacy).unwrap()).unwrap(); + std::fs::write(&direct_path, serde_json::to_vec_pretty(&direct).unwrap()).unwrap(); + + let mut wrong_config = source_config.clone(); + wrong_config.strategy.as_mut().unwrap().seed = Some(high_seed + 1); + let wrong = HolaEngine::load_configured_checkpoint(wrong_config, &legacy_path) + .await + .err() + .expect("a different truncated Sobol seed must be rejected"); + assert!(wrong.to_string().contains("seed")); + + let (restored, kind) = + HolaEngine::load_configured_checkpoint(source_config.clone(), &legacy_path) + .await + .unwrap(); + assert_eq!(kind, CheckpointLoadKind::Full); + assert_eq!( + restored.study_config().await.strategy.unwrap().seed, + Some(u64::from(high_seed as u32)), + "the serialized u32 is a truthful current seed even though the original high bits are not" + ); + let (direct_restored, direct_kind) = + HolaEngine::load_configured_checkpoint(source_config, &direct_path) + .await + .unwrap(); + assert_eq!(direct_kind, CheckpointLoadKind::Full); + assert_eq!( + direct_restored.study_config().await.strategy.unwrap().seed, + Some(u64::from(high_seed as u32)) + ); + let source_next = source.ask().await.unwrap(); + assert_eq!(source_next.params, restored.ask().await.unwrap().params); + assert_eq!( + source_next.params, + direct_restored.ask().await.unwrap().params ); - assert_eq!(strategy.max_components, Some(DEFAULT_MAX_COMPONENTS)); - assert_eq!(strategy.min_elite_samples, Some(DEFAULT_MIN_ELITE_SAMPLES)); } #[tokio::test] @@ -7196,6 +8789,7 @@ mod tests { let mut config = single_objective_config("gmm"); let strategy = config.strategy.as_mut().unwrap(); strategy.exploration_budget = Some(1); + strategy.min_elite_samples = Some(1); strategy.refit_interval = 20; let engine = HolaEngine::from_config(config).unwrap(); engine.force_refit_failure.store(true, Ordering::SeqCst); diff --git a/hola/src/lib.rs b/hola/src/lib.rs index abb7600..bb0276e 100644 --- a/hola/src/lib.rs +++ b/hola/src/lib.rs @@ -32,5 +32,5 @@ pub mod server; pub use hola_engine::{ AutoStrategy, CompletedTrial, DEFAULT_MAX_REFIT_CANDIDATES, DEFAULT_MAX_REFIT_SAMPLES, DynSpace, DynTrial, HolaEngine, ObjectiveConfig, ParamConfig, ParamInfo, StrategyConfig, - StudyConfig, + StrategyDiagnostics, StudyConfig, }; diff --git a/hola/tests/integration/hola_engine.rs b/hola/tests/integration/hola_engine.rs index 7f4cb12..c12eda0 100644 --- a/hola/tests/integration/hola_engine.rs +++ b/hola/tests/integration/hola_engine.rs @@ -1985,7 +1985,7 @@ async fn test_dyn_engine_full_checkpoint_resume_preserves_vector_trial_ids() { } #[tokio::test] -async fn test_dyn_engine_checkpoint_load_with_fallback_supports_full_and_leaderboard() { +async fn test_configured_checkpoint_load_supports_full_and_leaderboard() { let config = scalar_checkpoint_config(None); let engine = HolaEngine::from_config(config.clone()).unwrap(); let trial = engine.ask().await.unwrap(); @@ -2001,29 +2001,42 @@ async fn test_dyn_engine_checkpoint_load_with_fallback_supports_full_and_leaderb .await .unwrap(); - let restored_full = HolaEngine::from_config(config.clone()).unwrap(); - let kind = restored_full - .load_checkpoint_with_fallback(&full_path) + let (restored_full, kind) = HolaEngine::load_configured_checkpoint(config.clone(), &full_path) .await .unwrap(); assert_eq!(kind, CheckpointLoadKind::Full); assert_eq!(restored_full.trial_count().await, 1); assert_eq!(restored_full.ask().await.unwrap().trial_id, 1); + let strict_full = HolaEngine::from_config(config.clone()).unwrap(); + let strict_kind = strict_full + .load_checkpoint_with_fallback(&full_path) + .await + .unwrap(); + assert_eq!(strict_kind, CheckpointLoadKind::Full); + assert_eq!(strict_full.trial_count().await, 1); + let leaderboard_path = dir.path().join("leaderboard.json"); engine .save_leaderboard_checkpoint_to(&leaderboard_path, Some("leaderboard")) .await .unwrap(); - let restored_leaderboard = HolaEngine::from_config(config).unwrap(); - let kind = restored_leaderboard - .load_checkpoint_with_fallback(&leaderboard_path) - .await - .unwrap(); + let (restored_leaderboard, kind) = + HolaEngine::load_configured_checkpoint(config, &leaderboard_path) + .await + .unwrap(); assert_eq!(kind, CheckpointLoadKind::Leaderboard); assert_eq!(restored_leaderboard.trial_count().await, 1); assert!(restored_leaderboard.ask().await.unwrap().trial_id >= (1_u64 << 62)); + + let strict_leaderboard = HolaEngine::from_config(scalar_checkpoint_config(None)).unwrap(); + let strict_kind = strict_leaderboard + .load_checkpoint_with_fallback(&leaderboard_path) + .await + .unwrap(); + assert_eq!(strict_kind, CheckpointLoadKind::Leaderboard); + assert_eq!(strict_leaderboard.trial_count().await, 1); } #[tokio::test] @@ -2265,21 +2278,21 @@ async fn test_auto_strategy_with_explicit_exploration_budget() { fn test_auto_strategy_default_exploration_budget() { use hola::hola_engine::AutoStrategy; - // min(40, 56) = 40 -> round down to 32 - assert_eq!(AutoStrategy::default_exploration_budget(200, 3), 32); + // 2 * min(40, 56) = 80 -> round down to 64 + assert_eq!(AutoStrategy::default_exploration_budget(200, 3), 64); - // min(20, 52) = 20 -> round down to 16 - assert_eq!(AutoStrategy::default_exploration_budget(100, 1), 16); + // 2 * min(20, 52) = 40 -> round down to 32 + assert_eq!(AutoStrategy::default_exploration_budget(100, 1), 32); - // min(200, 60) = 60 -> round down to 32 - assert_eq!(AutoStrategy::default_exploration_budget(1000, 5), 32); + // 2 * min(200, 60) = 120 -> round down to 64 + assert_eq!(AutoStrategy::default_exploration_budget(1000, 5), 64); - // min(10, 70) = 10 -> round down to 8 - assert_eq!(AutoStrategy::default_exploration_budget(50, 10), 8); + // 2 * min(10, 70) = 20 -> round down to 16 + assert_eq!(AutoStrategy::default_exploration_budget(50, 10), 16); // Edge cases - assert_eq!(AutoStrategy::default_exploration_budget(10, 1), 2); // min(2, 52) = 2 - assert_eq!(AutoStrategy::default_exploration_budget(5, 1), 1); // min(1, 52) = 1 + assert_eq!(AutoStrategy::default_exploration_budget(10, 1), 4); + assert_eq!(AutoStrategy::default_exploration_budget(5, 1), 2); } #[tokio::test] diff --git a/opt_engine/Cargo.toml b/opt_engine/Cargo.toml index dce47cb..cf4dcae 100644 --- a/opt_engine/Cargo.toml +++ b/opt_engine/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "opt_engine" -version = "1.0.1-rc8" +version = "1.0.1" edition = "2024" description = "Type-safe optimization primitives and search strategies for HOLA" documentation = "https://docs.rs/opt_engine" diff --git a/opt_engine/src/lib.rs b/opt_engine/src/lib.rs index 9b0bca9..cebc263 100644 --- a/opt_engine/src/lib.rs +++ b/opt_engine/src/lib.rs @@ -48,7 +48,9 @@ pub mod prelude { BranchingSpace, CategoricalSpace, ContinuousSpace, DiscreteSpace, EitherDomain, ProductSpace, }; - pub use crate::strategies::{GmmRefitConfig, GmmStrategy, RandomStrategy, SobolStrategy}; + pub use crate::strategies::{ + DEFAULT_GMM_COMPONENTS, GmmRefitConfig, GmmStrategy, RandomStrategy, SobolStrategy, + }; pub use crate::traits::{ RefitConfig, RefittableStrategy, SampleSpace, StandardizedSpace, Strategy, }; @@ -63,5 +65,7 @@ pub use scales::{LinearScale, Log10Scale, LogScale, Scale}; pub use spaces::{ BranchingSpace, CategoricalSpace, ContinuousSpace, DiscreteSpace, EitherDomain, ProductSpace, }; -pub use strategies::{GmmRefitConfig, GmmStrategy, RandomStrategy, SobolStrategy}; +pub use strategies::{ + DEFAULT_GMM_COMPONENTS, GmmRefitConfig, GmmStrategy, RandomStrategy, SobolStrategy, +}; pub use traits::{RefitConfig, RefittableStrategy, SampleSpace, StandardizedSpace, Strategy}; diff --git a/opt_engine/src/strategies/gmm.rs b/opt_engine/src/strategies/gmm.rs index 9e776fd..285f931 100644 --- a/opt_engine/src/strategies/gmm.rs +++ b/opt_engine/src/strategies/gmm.rs @@ -1315,7 +1315,7 @@ struct SerializedGmmStrategy { #[serde(default)] refit_epoch: Option, params: GmmParams, - #[serde(default)] + #[serde(default = "legacy_gmm_refit_config")] refit_config: GmmRefitConfig, } @@ -1656,6 +1656,20 @@ where use crate::traits::RefittableStrategy; +/// Default upper bound on the number of components fitted by a new GMM strategy. +pub const DEFAULT_GMM_COMPONENTS: usize = 1; + +/// Preserve refit behavior for strategy states written before this field was +/// serialized. This must remain independent of the default for new strategies. +fn legacy_gmm_refit_config() -> GmmRefitConfig { + GmmRefitConfig { + n_components: 3, + max_iters: 100, + tolerance: 1e-6, + regularization: 1e-4, + } +} + /// Configuration for GMM refitting. #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(try_from = "GmmRefitConfigSerde", into = "GmmRefitConfigSerde")] @@ -1764,7 +1778,7 @@ impl GmmRefitConfig { impl Default for GmmRefitConfig { fn default() -> Self { Self { - n_components: 3, + n_components: DEFAULT_GMM_COMPONENTS, max_iters: 100, tolerance: 1e-6, regularization: 1e-4, @@ -2651,11 +2665,30 @@ mod tests { #[test] fn test_gmm_refit_config_defaults() { + assert_eq!(DEFAULT_GMM_COMPONENTS, 1); let config = GmmRefitConfig::default(); - assert_eq!(config.n_components(), 3); + assert_eq!(config.n_components(), DEFAULT_GMM_COMPONENTS); assert_eq!(config.max_iters(), 100); assert!((config.tolerance() - 1e-6).abs() < 1e-12); assert!((config.regularization() - 1e-4).abs() < 1e-12); + + let strategy = + GmmStrategy::::new(42, GmmParams::uniform_prior(2, 0.25).unwrap()); + assert_eq!( + strategy.get_refit_config().n_components(), + DEFAULT_GMM_COMPONENTS + ); + } + + #[test] + fn test_legacy_checkpoint_without_refit_config_uses_historical_default() { + let strategy = + GmmStrategy::::new(42, GmmParams::uniform_prior(2, 0.25).unwrap()); + let mut legacy = serde_json::to_value(strategy).unwrap(); + legacy.as_object_mut().unwrap().remove("refit_config"); + + let restored: GmmStrategy = serde_json::from_value(legacy).unwrap(); + assert_eq!(restored.get_refit_config().n_components(), 3); } #[test] diff --git a/opt_engine/src/strategies/mod.rs b/opt_engine/src/strategies/mod.rs index 4bdb627..fa9a57c 100644 --- a/opt_engine/src/strategies/mod.rs +++ b/opt_engine/src/strategies/mod.rs @@ -23,6 +23,8 @@ mod gmm; mod random; mod sobol; -pub use gmm::{GaussianComponent, GmmError, GmmParams, GmmRefitConfig, GmmStrategy}; +pub use gmm::{ + DEFAULT_GMM_COMPONENTS, GaussianComponent, GmmError, GmmParams, GmmRefitConfig, GmmStrategy, +}; pub use random::RandomStrategy; pub use sobol::SobolStrategy;