From f1c8330a6e36eed6aba48448e2b1c08bfe2e4568 Mon Sep 17 00:00:00 2001 From: Enrico Antonini <50218270+eantonini@users.noreply.github.com> Date: Tue, 1 Sep 2026 11:45:23 +0200 Subject: [PATCH 1/5] Add multiple seed option --- runner/config/solvers.yaml | 18 ++++++++++++ runner/tests/test_config.py | 11 +++++++ runner/tests/test_execution.py | 22 ++++++++++++++ runner/tests/test_orchestrator.py | 25 ++++++++++++++++ runner/tests/test_solver.py | 23 +++++++++++++++ runner/utils/config.py | 23 +++++++++++++++ runner/utils/execution.py | 7 +++++ runner/utils/orchestrator.py | 19 ++++++++++-- runner/utils/solver.py | 48 ++++++++++++++++++++++++++----- 9 files changed, 186 insertions(+), 10 deletions(-) diff --git a/runner/config/solvers.yaml b/runner/config/solvers.yaml index 1d6384c5..85ba1030 100644 --- a/runner/config/solvers.yaml +++ b/runner/config/solvers.yaml @@ -20,6 +20,13 @@ # needs, forwarded to systemd-run's isolated scope via `--setenv=` (it # doesn't inherit the caller's environment by default -- see # `execution.run_solver`). Only solvers that need one are listed. +# +# `seed_options`: the options key that holds a solver package's random seed, +# in its own option vocabulary (see solver_configurations.yaml) -- lets a +# caller override just the seed while keeping a configuration's other tuned +# options (e.g. for running multiple seeds of the same configuration; see +# `solver.get_solver`'s `seed` parameter). Every solver package listed in +# `packages` above has an entry here. solvers: glpk: "5.0": @@ -134,3 +141,14 @@ license_env_vars: knitro: [ARTELYS_LICENSE] xpress: [XPRESS, LD_LIBRARY_PATH] mosek: [MOSEKLM_LICENSE_FILE] + +seed_options: + glpk: seed + highs: random_seed + scip: randomization/randomseedshift + gurobi: seed + cbc: randomCbcSeed + cplex: randomseed + knitro: ms_seed + xpress: randomseed + mosek: MSK_IPAR_MIO_SEED diff --git a/runner/tests/test_config.py b/runner/tests/test_config.py index 5e990974..dbfeb7a5 100644 --- a/runner/tests/test_config.py +++ b/runner/tests/test_config.py @@ -11,6 +11,7 @@ get_default_configurations, get_license_env_vars, get_package_name, + get_seed_option, get_solver_configuration, get_solver_options, is_solver_eligible, @@ -131,6 +132,16 @@ def test_solver_with_no_entry_returns_empty_list(self): assert get_license_env_vars("highs", config) == [] +class TestGetSeedOption: + def test_looks_up_seed_option(self): + config = {"seed_options": {"highs": "random_seed"}} + assert get_seed_option("highs", config) == "random_seed" + + def test_solver_with_no_entry_returns_none(self): + config = {"seed_options": {}} + assert get_seed_option("highs", config) is None + + class TestConditionMatches: def test_empty_condition_matches_anything(self): assert _condition_matches({"year": "2025"}, {}) is True diff --git a/runner/tests/test_execution.py b/runner/tests/test_execution.py index f33a01f1..5f6c44ac 100644 --- a/runner/tests/test_execution.py +++ b/runner/tests/test_execution.py @@ -117,6 +117,28 @@ def test_command_invokes_solver_module_via_dash_m(self, mocker): ] assert called_cmd[called_cmd.index("runner.utils.solver") - 1] == "-m" + def test_seed_is_appended_to_command(self, mocker): + cp = subprocess.CompletedProcess( + args=[], returncode=124, stdout="", stderr="MaxResidentSetSizeKB=1000" + ) + mocker.patch("runner.utils.execution._systemd_available", return_value=False) + run_mock = mocker.patch( + "runner.utils.execution.subprocess.run", return_value=cp + ) + run_solver("problem.lp", "highs", timeout=3600, solver_version="1.9.0", seed=42) + called_cmd = run_mock.call_args[0][0] + assert called_cmd[-2:] == ["--seed", "42"] + + def test_no_seed_omits_seed_flag(self, mocker): + # Backward compatibility: omitting `seed` must produce the exact + # same command as before this argument existed. + cp = subprocess.CompletedProcess( + args=[], returncode=124, stdout="", stderr="MaxResidentSetSizeKB=1000" + ) + _, run_mock = self._run(mocker, cp) + called_cmd = run_mock.call_args[0][0] + assert "--seed" not in called_cmd + def test_env_name_uses_pixi_run(self, mocker): cp = subprocess.CompletedProcess( args=[], returncode=124, stdout="", stderr="MaxResidentSetSizeKB=1000" diff --git a/runner/tests/test_orchestrator.py b/runner/tests/test_orchestrator.py index eaca447e..b4847f9f 100644 --- a/runner/tests/test_orchestrator.py +++ b/runner/tests/test_orchestrator.py @@ -147,6 +147,31 @@ def test_multiple_iterations_computes_mean_and_stddev( ) assert summary.iloc[0]["Runtime StdDev (s)"] == 0.0 + def test_iterations_greater_than_one_varies_seed(self, problems_yaml, mocker): + run_solver_mock = mocker.patch.object( + orchestrator, "run_solver", return_value=dict(_FAKE_METRICS) + ) + orchestrator.run_benchmark( + problems_yaml, + ["highs-default"], + year="2025", + run_id="test-run", + iterations=3, + ) + seeds = [call.kwargs["seed"] for call in run_solver_mock.call_args_list] + assert seeds == [0, 1, 2] + + def test_single_iteration_passes_no_seed(self, problems_yaml, mocker): + # Backward compatibility: the default `iterations=1` must not + # override the configuration's own fixed seed. + run_solver_mock = mocker.patch.object( + orchestrator, "run_solver", return_value=dict(_FAKE_METRICS) + ) + orchestrator.run_benchmark( + problems_yaml, ["highs-default"], year="2025", run_id="test-run" + ) + assert run_solver_mock.call_args.kwargs["seed"] is None + def test_error_status_stops_further_iterations( self, problems_yaml, tmp_path, mocker ): diff --git a/runner/tests/test_solver.py b/runner/tests/test_solver.py index ca05b7e0..ab57a4c1 100644 --- a/runner/tests/test_solver.py +++ b/runner/tests/test_solver.py @@ -52,6 +52,29 @@ def test_unsupported_solver_name_raises(self): with pytest.raises(ValueError): get_solver("not-a-solver") + def test_seed_overrides_configurations_own_seed(self, monkeypatch): + captured = self._patch_solver_class(monkeypatch, "Highs") + get_solver("highs-default", seed=42) + assert captured["options"]["random_seed"] == 42 + # Other options are untouched + assert captured["options"]["mip_rel_gap"] == pytest.approx(1e-4) + + def test_no_seed_keeps_configurations_own_seed(self, monkeypatch): + captured = self._patch_solver_class(monkeypatch, "Highs") + get_solver("highs-default", seed=None) + assert captured["options"]["random_seed"] == 4 + + def test_seed_ignored_with_warning_when_no_seed_options_entry( + self, monkeypatch, capsys + ): + monkeypatch.setattr( + "runner.utils.solver.config.get_seed_option", lambda *_a, **_k: None + ) + captured = self._patch_solver_class(monkeypatch, "Highs") + get_solver("highs-default", seed=42) + assert captured["options"]["random_seed"] == 4 + assert "no seed_options entry" in capsys.readouterr().err + class TestIsMipProblem: def test_none_model_is_false(self): diff --git a/runner/utils/config.py b/runner/utils/config.py index 921229a9..0a87512f 100644 --- a/runner/utils/config.py +++ b/runner/utils/config.py @@ -293,6 +293,29 @@ def get_license_env_vars( return list(config.get("license_env_vars", {}).get(solver_package, [])) +def get_seed_option( + solver_package: str, config: dict[str, Any] | None = None +) -> str | None: + """Return the options key that holds a solver package's random seed. + + Parameters + ---------- + solver_package : str + The underlying solver package, e.g. ``"highs"``. + config : dict[str, Any], optional + A pre-loaded solver registry. Defaults to :func:`load_solver_registry`. + + Returns + ------- + str | None + The key into a configuration's ``options`` dict that holds its seed + (e.g. ``"random_seed"`` for ``"highs"``), or None if `solver_package` + has no entry in ``solvers.yaml``'s ``seed_options`` map. + """ + config = config if config is not None else load_solver_registry() + return config.get("seed_options", {}).get(solver_package) + + def _resolve_fact(facts: dict[str, Any], path: str) -> Any: """Look up a fact, following a dotted `path` into nested dict facts. diff --git a/runner/utils/execution.py b/runner/utils/execution.py index 107e84bc..724e80a9 100644 --- a/runner/utils/execution.py +++ b/runner/utils/execution.py @@ -68,6 +68,7 @@ def run_solver( timeout: int, solver_version: str, env_name: str | None = None, + seed: int | None = None, ) -> dict[str, Any]: """Run one solver configuration on one problem file, with resource limits. @@ -92,6 +93,10 @@ def run_solver( If given, run inside this env (a pixi manifest at `runner/envs//`) via `pixi run --manifest-path` instead of the current one. + seed : int, optional + If given, overrides the configuration's own fixed seed (see + `solver.get_solver`), e.g. for running the same configuration under + several different seeds. Notes ----- @@ -163,6 +168,8 @@ def run_solver( solver_version, ] ) + if seed is not None: + command.extend(["--seed", str(seed)]) # Prepend (not replace) PYTHONPATH so `runner` resolves as a package -- # see this module's docstring for why PYTHONPATH rather than `cwd`. diff --git a/runner/utils/orchestrator.py b/runner/utils/orchestrator.py index bb794a22..46b8c1d8 100644 --- a/runner/utils/orchestrator.py +++ b/runner/utils/orchestrator.py @@ -98,9 +98,14 @@ def run_benchmark( year : str, optional The solver-version year to run, e.g. `"2025"`. iterations : int, optional - Repetitions per (problem, solver configuration) pair. A timeout or + Repetitions per (problem, solver configuration) pair. When greater + than 1, each iteration `i` overrides the configuration's own fixed + seed with `i` (see `execution.run_solver`'s `seed` parameter), so + repeated runs sample the solver's actual sensitivity to its seed + rather than just re-measuring one deterministic solve. A timeout or error on one iteration skips the rest. Statistics are still recorded - when this is 1 (mean == the single value, stddev == 0). + when this is 1 (mean == the single value, stddev == 0), and the seed + is left unset (the configuration's own fixed seed applies). reference_interval : int, optional Minimum seconds between reference-benchmark runs (see `execution.run_reference_highs_binary`), interleaved between real @@ -192,9 +197,15 @@ def run_benchmark( timestamp = "" for i in range(iterations): + # Vary the seed across iterations so repeated runs sample the + # solver's actual sensitivity to it. + seed = i if iterations > 1 else None + print( f"Running solver {solver_configuration} (version {solver_version}) " - f"on {problem['path']} ({i})...", + f"on {problem['path']} ({i})" + + (f" with seed {seed}" if seed is not None else "") + + "...", flush=True, ) @@ -207,6 +218,7 @@ def run_benchmark( timeout, solver_version, env_name=env_name, + seed=seed, ) # NOTE: results.csv_record expects the kwarg "solver" (its CSV @@ -215,6 +227,7 @@ def run_benchmark( metrics["solver"] = solver_configuration metrics["solver_version"] = solver_version metrics["solver_release_year"] = year + metrics["seed"] = seed runtimes.append(metrics["runtime"]) memory_usages.append(metrics["memory"]) diff --git a/runner/utils/solver.py b/runner/utils/solver.py index e7d3dc38..4c63782a 100644 --- a/runner/utils/solver.py +++ b/runner/utils/solver.py @@ -10,7 +10,7 @@ Keeps the `if __name__ == "__main__"` entrypoint so a single solver run can still be driven directly, e.g. for debugging: - python -m runner.utils.solver + python -m runner.utils.solver [--seed N] """ import json @@ -40,13 +40,20 @@ highspy = None -def get_solver(solver_configuration: str) -> tuple[Any, str]: +def get_solver(solver_configuration: str, seed: int | None = None) -> tuple[Any, str]: """Build a linopy solver instance with this project's tuning options. Parameters ---------- solver_configuration : str The configuration to run, as requested by a caller (e.g. ``"highs-hipo"``). + seed : int, optional + If given, overrides the configuration's own fixed seed (e.g. for + running the same configuration under several different seeds to + gauge a solver's sensitivity to it -- see `solvers.yaml`'s + `seed_options` map for which options key holds a solver package's + seed). Ignored (with a warning) if `solver_package` has no entry in + `seed_options`. Returns ------- @@ -66,6 +73,17 @@ def get_solver(solver_configuration: str) -> tuple[Any, str]: solver_package = solver_configuration.lower() kwargs = {} + if seed is not None: + seed_key = config.get_seed_option(solver_package) + if seed_key is None: + print( + f"WARNING: '{solver_package}' has no seed_options entry in " + "solvers.yaml; --seed ignored", + file=sys.stderr, + ) + else: + kwargs[seed_key] = seed + solver_enum = SolverName(solver_package) solver_class = getattr(solvers, solver_enum.name) return solver_class(options=kwargs), solver_package @@ -264,7 +282,12 @@ def get_reported_runtime(solver_package: str, solver_model: Any) -> float | None return None -def main(solver_configuration: str, input_file: str, solver_version: str) -> None: +def main( + solver_configuration: str, + input_file: str, + solver_version: str, + seed: int | None = None, +) -> None: """Run one solver on one problem file and print the resulting metrics as JSON. Parameters @@ -277,12 +300,15 @@ def main(solver_configuration: str, input_file: str, solver_version: str) -> Non solver_version : str The solver version, included in output filenames and the printed metrics (not otherwise used to select behavior). + seed : int, optional + If given, overrides the configuration's own fixed seed (see + `get_solver`). """ problem_file = Path(input_file) # keep the requested configuration name (e.g. "highs-hipo") for filenames output_name = solver_configuration - solver, solver_package = get_solver(solver_configuration) + solver, solver_package = get_solver(solver_configuration, seed=seed) solution_dir = Path(__file__).resolve().parent.parent / "solutions" solution_dir.mkdir(parents=True, exist_ok=True) @@ -357,10 +383,18 @@ def main(solver_configuration: str, input_file: str, solver_version: str) -> Non if __name__ == "__main__": - if len(sys.argv) != 4: + argv = sys.argv[1:] + cli_seed = None + if "--seed" in argv: + seed_index = argv.index("--seed") + cli_seed = int(argv[seed_index + 1]) + del argv[seed_index : seed_index + 2] + + if len(argv) != 3: print( - "Usage: python -m runner.utils.solver " + "Usage: python -m runner.utils.solver " + " [--seed N]" ) sys.exit(1) - main(sys.argv[1], sys.argv[2], sys.argv[3]) + main(argv[0], argv[1], argv[2], seed=cli_seed) From d2fc1fb7f1def20c61d00059f617c3e18164fc88 Mon Sep 17 00:00:00 2001 From: Enrico Antonini <50218270+eantonini@users.noreply.github.com> Date: Tue, 1 Sep 2026 11:51:33 +0200 Subject: [PATCH 2/5] Add new column into result csv --- runner/tests/test_results.py | 82 ++++++++++++++++++++ runner/utils/orchestrator.py | 9 ++- runner/utils/results.py | 140 ++++++++++++++++++++++++++++++----- 3 files changed, 208 insertions(+), 23 deletions(-) diff --git a/runner/tests/test_results.py b/runner/tests/test_results.py index 6655ff2d..d65b2269 100644 --- a/runner/tests/test_results.py +++ b/runner/tests/test_results.py @@ -5,7 +5,9 @@ import pytest from runner.utils.results import ( + _MEAN_STDDEV_HEADERS, csv_record, + ensure_csv_schema, write_csv_headers, write_csv_row, write_csv_summary_row, @@ -60,6 +62,7 @@ def test_check_true_passes_when_all_fields_present(self): "vm_instance_type": "vm", "vm_zone": "z", "solver_benchmark_version": "abc123", + "seed": 4, } record = csv_record(check=True, **full_kwargs) assert record["Problem"] == "p" @@ -86,6 +89,7 @@ def test_column_order_is_stable(self): "VM Instance Type", "VM Zone", "Solver benchmark version", + "Seed", ] @@ -151,6 +155,7 @@ def test_mean_stddev_headers_have_no_size_column(self, tmp_path): "Objective Value", "Run ID", "Timestamp", + "Seed", ] def test_write_csv_summary_row(self, tmp_path): @@ -178,3 +183,80 @@ def test_write_csv_summary_row(self, tmp_path): rows = list(csv.reader(f)) assert rows[1][0] == "problem-a" assert rows[1][6] == "1.5" # Runtime Mean (s) + + +class TestEnsureCsvSchema: + def test_append_false_overwrites_existing_content(self, tmp_path): + results_csv = tmp_path / "results.csv" + mean_stddev_csv = tmp_path / "mean_stddev.csv" + results_csv.write_text("stale header\nstale,row\n") + mean_stddev_csv.write_text("stale header\nstale,row\n") + + ensure_csv_schema(results_csv, mean_stddev_csv, append=False) + + assert results_csv.read_text().splitlines()[0] == ",".join( + csv_record(check=False).keys() + ) + + def test_append_true_missing_file_creates_headers(self, tmp_path): + results_csv = tmp_path / "results.csv" + mean_stddev_csv = tmp_path / "mean_stddev.csv" + + ensure_csv_schema(results_csv, mean_stddev_csv, append=True) + + assert results_csv.exists() + assert mean_stddev_csv.exists() + + def test_append_true_matching_schema_is_untouched(self, tmp_path): + results_csv = tmp_path / "results.csv" + mean_stddev_csv = tmp_path / "mean_stddev.csv" + write_csv_headers(results_csv, mean_stddev_csv) + write_csv_row( + results_csv, + problem_id="problem-a", + metrics={"solver": "highs", "status": "ok"}, + run_id="run-1", + timestamp="t", + vm_instance_type="vm", + vm_zone="z", + hostname="h", + solver_benchmark_version="abc123", + ) + before = results_csv.read_text() + + ensure_csv_schema(results_csv, mean_stddev_csv, append=True) + + assert results_csv.read_text() == before + + def test_append_true_widens_a_csv_missing_a_newer_column(self, tmp_path): + results_csv = tmp_path / "results.csv" + mean_stddev_csv = tmp_path / "mean_stddev.csv" + # An "old" file predating the `Seed` column, with one real data row. + old_headers = [h for h in csv_record(check=False).keys() if h != "Seed"] + with open(results_csv, "w", newline="") as f: + writer = csv.writer(f) + writer.writerow(old_headers) + writer.writerow(["problem-a", "highs"] + [""] * (len(old_headers) - 2)) + with open(mean_stddev_csv, "w", newline="") as f: + csv.writer(f).writerow(_MEAN_STDDEV_HEADERS) + + ensure_csv_schema(results_csv, mean_stddev_csv, append=True) + + with open(results_csv, newline="") as f: + rows = list(csv.reader(f)) + assert rows[0] == list(csv_record(check=False).keys()) + assert rows[0][-1] == "Seed" + assert rows[1][0] == "problem-a" # old data preserved + assert rows[1][1] == "highs" + assert rows[1][-1] == "" # new column, blank for the old row + + def test_append_true_raises_on_unrecognized_column(self, tmp_path): + results_csv = tmp_path / "results.csv" + mean_stddev_csv = tmp_path / "mean_stddev.csv" + with open(results_csv, "w", newline="") as f: + csv.writer(f).writerow(["Problem", "Some Removed Column"]) + with open(mean_stddev_csv, "w", newline="") as f: + csv.writer(f).writerow(_MEAN_STDDEV_HEADERS) + + with pytest.raises(ValueError, match="Some Removed Column"): + ensure_csv_schema(results_csv, mean_stddev_csv, append=True) diff --git a/runner/utils/orchestrator.py b/runner/utils/orchestrator.py index 46b8c1d8..22e9798d 100644 --- a/runner/utils/orchestrator.py +++ b/runner/utils/orchestrator.py @@ -19,7 +19,7 @@ from . import config, env from .execution import get_highs_binary_version, run_reference_highs_binary, run_solver from .metadata import load_problems -from .results import write_csv_headers, write_csv_row, write_csv_summary_row +from .results import ensure_csv_schema, write_csv_row, write_csv_summary_row _REPO_ROOT = Path(__file__).resolve().parent.parent.parent _PROBLEMS_FOLDER = Path(__file__).resolve().parent.parent / "benchmarks" @@ -143,9 +143,10 @@ def run_benchmark( results_csv = results_folder / "benchmark_results.csv" mean_stddev_csv = results_folder / "benchmark_results_mean_stddev.csv" - # Write headers if overriding or file doesn't exist - if not append or not results_csv.exists() or not mean_stddev_csv.exists(): - write_csv_headers(results_csv, mean_stddev_csv) + # Write headers if overriding or a file doesn't exist yet; otherwise + # widen an existing file to the current schema in place if it predates a + # column added since (see `ensure_csv_schema`'s own docstring). + ensure_csv_schema(results_csv, mean_stddev_csv, append) os.makedirs(_PROBLEMS_FOLDER, exist_ok=True) registered_solver_versions = env.get_registered_solver_versions( diff --git a/runner/utils/results.py b/runner/utils/results.py index e64418de..25d5539e 100644 --- a/runner/utils/results.py +++ b/runner/utils/results.py @@ -15,6 +15,27 @@ from pathlib import Path from typing import Any +# Single source of truth for the mean/stddev summary CSV's columns, shared by +# `write_csv_headers` and `write_csv_summary_row` so the two can't drift +# apart (mirrors how `csv_record` is the one source of truth for the main +# results CSV's columns). +_MEAN_STDDEV_HEADERS = [ + "Problem", + "Solver", + "Solver Version", + "Solver Release Year", + "Status", + "Termination Condition", + "Runtime Mean (s)", + "Runtime StdDev (s)", + "Memory Mean (MB)", + "Memory StdDev (MB)", + "Objective Value", + "Run ID", + "Timestamp", + "Seed", +] + def csv_record(check: bool = False, **kwargs: Any) -> OrderedDict[str, Any]: """Build one benchmark-results row, mapping kwargs to their CSV column names. @@ -49,6 +70,14 @@ def csv_record(check: bool = False, **kwargs: Any) -> OrderedDict[str, Any]: metadata.yaml is now one row per model+size combination already (see `metadata.load_problems`) -- nothing left to disambiguate. Historical CSVs still have it; `analyze.load_results` knows how to read those. + + "Seed" is appended last (not grouped with the other solver-identifying + columns) so adding it doesn't shift every other column's position -- + see `ensure_csv_schema` for how an existing CSV predating this column + is widened to include it. Empty for a single-iteration run (the + configuration's own fixed seed applies); set to the actual seed used + when `orchestrator.run_benchmark`'s `iterations` > 1 varies it per + iteration. """ record = OrderedDict( [ @@ -71,6 +100,7 @@ def csv_record(check: bool = False, **kwargs: Any) -> OrderedDict[str, Any]: ("VM Instance Type", kwargs.get("vm_instance_type")), ("VM Zone", kwargs.get("vm_zone")), ("Solver benchmark version", kwargs.get("solver_benchmark_version")), + ("Seed", kwargs.get("seed")), ] ) @@ -105,23 +135,92 @@ def write_csv_headers( with open(mean_stddev_csv, mode="w", newline="") as file: writer = csv.writer(file) - writer.writerow( - [ - "Problem", - "Solver", - "Solver Version", - "Solver Release Year", - "Status", - "Termination Condition", - "Runtime Mean (s)", - "Runtime StdDev (s)", - "Memory Mean (MB)", - "Memory StdDev (MB)", - "Objective Value", - "Run ID", - "Timestamp", - ] - ) + writer.writerow(_MEAN_STDDEV_HEADERS) + + +def ensure_csv_schema( + results_csv: Path, + mean_stddev_csv: Path, + append: bool, +) -> None: + """Prepare both result CSVs for a run, without losing `--append` history. + + The single entry point `orchestrator.run_benchmark` should call instead + of `write_csv_headers` directly: it only overwrites when there's nothing + to preserve (`append` is False, or a file doesn't exist yet), and + otherwise widens an existing file to the current schema in place -- see + `_migrate_columns_if_needed` -- so appending to a CSV written by an older + version of this code (missing a column added since, e.g. `Seed`) doesn't + produce a ragged file that `pd.read_csv` can't parse. + + Parameters + ---------- + results_csv : Path + Per-iteration results file. + mean_stddev_csv : Path + Mean/stddev-across-iterations summary file. + append : bool + If False, both files are (re)created with just a header row, + discarding any existing content -- same as `write_csv_headers`. + If True and both files already exist, they're widened in place if + their schema is out of date, and otherwise left untouched. + """ + if not append or not results_csv.exists() or not mean_stddev_csv.exists(): + write_csv_headers(results_csv, mean_stddev_csv) + return + + _migrate_columns_if_needed(results_csv, list(csv_record(check=False).keys())) + _migrate_columns_if_needed(mean_stddev_csv, _MEAN_STDDEV_HEADERS) + + +def _migrate_columns_if_needed(csv_path: Path, expected_headers: list[str]) -> None: + """Widen an existing CSV to `expected_headers`, in place, preserving rows. + + A no-op if `csv_path`'s header already matches `expected_headers` + exactly (the common case, checked cheaply before reading the rest of + the file). Otherwise, only ever *adds* columns: an existing row missing + a column that's new in `expected_headers` gets an empty cell for it, + and every column and value it already had is preserved as-is. Never + reorders or drops a column `csv_path` already has, so no existing data + is silently lost. + + Parameters + ---------- + csv_path : Path + The results or mean/stddev CSV to check, and migrate if needed. + expected_headers : list[str] + The column names this run's code expects, in order (see + `csv_record` and `_MEAN_STDDEV_HEADERS`). + + Raises + ------ + ValueError + If `csv_path` has a column not in `expected_headers` -- silently + dropping it would lose data, so this needs a deliberate decision + (e.g. renaming the column, or updating `expected_headers`) rather + than an automatic one. + """ + with open(csv_path, newline="") as file: + reader = csv.DictReader(file) + current_headers = reader.fieldnames or [] + if list(current_headers) == expected_headers: + return + + unexpected = [h for h in current_headers if h not in expected_headers] + if unexpected: + raise ValueError( + f"{csv_path} has column(s) {unexpected} not in the current " + "schema -- resolve manually rather than risk silently " + "dropping data." + ) + rows = list(reader) + + added = [h for h in expected_headers if h not in current_headers] + print(f"Migrating {csv_path} to the current schema (adding {added})") + with open(csv_path, mode="w", newline="") as file: + writer = csv.DictWriter(file, fieldnames=expected_headers, restval="") + writer.writeheader() + writer.writerows(rows) def write_csv_row( @@ -211,8 +310,10 @@ def write_csv_summary_row( Notes ----- - Column order must match `write_csv_headers`'s hardcoded mean/stddev - header list. + Column order must match `_MEAN_STDDEV_HEADERS` (also used by + `write_csv_headers`). `Seed` (like `status`/`condition`) reflects only + the last iteration's value, not every seed tested across iterations -- + see `orchestrator.run_benchmark`'s own docstring. """ with open(mean_stddev_csv, mode="a", newline="") as file: writer = csv.writer(file) @@ -231,5 +332,6 @@ def write_csv_summary_row( metrics["objective"], run_id, timestamp, + metrics.get("seed"), ] ) From bbdc4d9f001afcbc0dc87e0532093243957b2851 Mon Sep 17 00:00:00 2001 From: Enrico Antonini <50218270+eantonini@users.noreply.github.com> Date: Tue, 1 Sep 2026 13:29:56 +0200 Subject: [PATCH 3/5] Update documentation and fix name --- runner/README.md | 12 ++++++++++- runner/SOLVERS.md | 2 +- runner/benchmark.py | 12 +++++++++++ runner/tests/test_benchmark.py | 34 ++++++++++++++++++++++++++++++ runner/tests/test_orchestrator.py | 18 +++++++--------- runner/utils/orchestrator.py | 35 ++++++++++++++++--------------- runner/utils/results.py | 6 +++--- 7 files changed, 87 insertions(+), 32 deletions(-) diff --git a/runner/README.md b/runner/README.md index 3bc8b81d..961e9cd3 100644 --- a/runner/README.md +++ b/runner/README.md @@ -48,6 +48,7 @@ once per given year. - `-a, --append` - Append to the results CSVs instead of overwriting them for the first year - `-y, --years YEAR` - Solver-version year to run (repeatable), or `tests` for the shared CI smoke-test env. Defaults to every year with a registered solver version - `-s, --solver-configurations CONFIG` - Solver configuration to run (repeatable), e.g. `highs-default` or `highs-hipo`. Defaults to `solver_configurations.yaml`'s `default_configurations` +- `-n, --num-seeds N` - Number of seeds to try per (problem, solver configuration) pair. When greater than 1, each repetition uses a different seed (0, 1, 2, ...) instead of the configuration's own fixed seed, to gauge the solver's sensitivity to it. Default: 1 (no repetition, the configuration's own fixed seed applies) - `-r, --ref-bench-interval SECONDS` - Run a reference benchmark at most once every N seconds. 0 disables it - `-u, --run-id RUN_ID` - Identifier shared by every row from this run. Auto-generated if not given - `--help` - Show this message and exit @@ -70,6 +71,11 @@ pixi run -e runner python -m runner.benchmark --solver-configurations highs-defa pixi run -e runner python -m runner.benchmark --years 2025 results/metadata.yaml ``` +4. Run each problem 3 times per solver configuration, under 3 different seeds, to gauge runtime sensitivity to the seed: +```shell +pixi run -e runner python -m runner.benchmark --num-seeds 3 --years 2025 benchmarks/sample_run/standard-00.yaml +``` + ## Running with Docker Docker is optional. On native Linux with systemd, you can run the scripts directly (see above). Memory limit enforcement via `systemd-run` is skipped automatically when systemd is not available. @@ -124,13 +130,14 @@ docker run --rm \ Use `runner.utils.solver` to test a single solver on a single problem. This is useful for debugging. Since it's a package module (not a standalone script), run it with `-m` **from the repo root**, not from `runner/`: ```bash -python -m runner.utils.solver +python -m runner.utils.solver [--seed N] ``` **Arguments:** - `solver_configuration` - Solver configuration name (e.g., highs-default, highs-hipo, scip-default) - `input_file` - Path to a problem file (.lp or .mps) - `solver_version` - Solver version string (e.g., 1.10.0) +- `--seed N` - Optional. Overrides the configuration's own fixed seed (see `runner/config/solvers.yaml`'s `seed_options`) **Examples:** @@ -140,6 +147,9 @@ pixi run --manifest-path runner/envs/benchmark-highs-2024 python -m runner.utils # Test SCIP pixi run --manifest-path runner/envs/benchmark-scip-2024 python -m runner.utils.solver scip-default runner/benchmarks/pypsa-eur-elec-op-2-1h.lp 9.2.2 + +# Test HiGHS with a specific seed instead of highs-default's own fixed one +pixi run --manifest-path runner/envs/benchmark-highs-2024 python -m runner.utils.solver highs-default runner/benchmarks/pypsa-eur-elec-op-2-1h.lp 1.10.0 --seed 7 ``` **Output:** diff --git a/runner/SOLVERS.md b/runner/SOLVERS.md index bfcacc3b..c4ae562a 100644 --- a/runner/SOLVERS.md +++ b/runner/SOLVERS.md @@ -35,7 +35,7 @@ Onboarding a solver package we don't support at all yet (as opposed to a new ver 1. Confirm [linopy](https://github.com/PyPSA/linopy) already supports it (`linopy.solvers.SolverName` lists every solver it knows how to drive) -- everything below assumes it does. If not, that support has to land in linopy first. 2. Its per-solver-year pixi manifest(s) under `runner/envs/` (step 1 of [Updating Solver Versions](#updating-solver-versions) above). -3. A registry entry in `runner/config/solvers.yaml`'s `solvers` map (version/year/env), plus a `packages` entry if its PyPI package name differs from the solver's own name, plus a `license_env_vars` entry if it needs a license env var forwarded under `systemd-run` (see that file's own header comment for the full schema). +3. A registry entry in `runner/config/solvers.yaml`'s `solvers` map (version/year/env), plus a `packages` entry if its PyPI package name differs from the solver's own name, a `license_env_vars` entry if it needs a license env var forwarded under `systemd-run`, and a `seed_options` entry naming the options key that holds its random seed (used to run a configuration under several different seeds, e.g. via `runner.benchmark`'s `--num-seeds`) -- see that file's own header comment for the full schema. 4. A solver adapter module at `runner/utils/solvers/.py` exporting `is_mip(model)`, `duality_gap(model)`, and `reported_runtime(model)` -- copy any existing module there as a template. It's auto-discovered by filename; nothing else needs to import or register it (see `runner/utils/solvers/__init__.py`'s own docstring). 5. At least a `-default` entry in `runner/config/solver_configurations.yaml` (see [Adding a New Solver Configuration](#adding-a-new-solver-configuration) below), and add it to `default_configurations` there if it should run whenever the CLI is given no explicit `--solver-configurations`. diff --git a/runner/benchmark.py b/runner/benchmark.py index 4d28b5c4..165d44fe 100644 --- a/runner/benchmark.py +++ b/runner/benchmark.py @@ -44,6 +44,17 @@ def run( help="Append to the results CSVs instead of overwriting them for " "the first year.", ), + num_seeds: int = typer.Option( + 1, + "--num-seeds", + "-n", + help="Number of seeds to try per (problem, solver configuration) " + "pair. When greater than 1, each repetition uses a different seed " + "(0, 1, 2, ...) instead of the configuration's own fixed seed, to " + "gauge the solver's sensitivity to it (see " + "`runner/config/solvers.yaml`'s `seed_options`). Default: 1 (the " + "configuration's own fixed seed, no repetition).", + ), ref_bench_interval: int = typer.Option( 0, "--ref-bench-interval", @@ -88,6 +99,7 @@ def run( problems_yaml_path, resolved_solver_configurations, year=year, + num_seeds=num_seeds, reference_interval=ref_bench_interval, append=append or index > 0, run_id=resolved_run_id, diff --git a/runner/tests/test_benchmark.py b/runner/tests/test_benchmark.py index 9f3be05f..952f3899 100644 --- a/runner/tests/test_benchmark.py +++ b/runner/tests/test_benchmark.py @@ -88,6 +88,40 @@ def test_single_year_run_writes_results(self, problems_yaml, tmp_path): assert list(results["Problem"]) == ["tiny-problem"] assert results.iloc[0]["Solver Release Year"] == 2025 + def test_num_seeds_flag_varies_seed_across_repetitions( + self, problems_yaml, tmp_path + ): + result = runner_cli.invoke( + benchmark.app, + [ + str(problems_yaml), + "--years", + "2025", + "--solver-configurations", + "highs-default", + "--num-seeds", + "3", + ], + ) + assert result.exit_code == 0, result.output + results = pd.read_csv(tmp_path / "results" / "benchmark_results.csv") + assert sorted(results["Seed"]) == [0, 1, 2] + + def test_default_num_seeds_leaves_seed_column_empty(self, problems_yaml, tmp_path): + result = runner_cli.invoke( + benchmark.app, + [ + str(problems_yaml), + "--years", + "2025", + "--solver-configurations", + "highs-default", + ], + ) + assert result.exit_code == 0, result.output + results = pd.read_csv(tmp_path / "results" / "benchmark_results.csv") + assert results["Seed"].isna().all() + def test_tests_pseudo_year_runs_against_real_solver_registry( self, problems_yaml, tmp_path ): diff --git a/runner/tests/test_orchestrator.py b/runner/tests/test_orchestrator.py index b4847f9f..d97d89dd 100644 --- a/runner/tests/test_orchestrator.py +++ b/runner/tests/test_orchestrator.py @@ -129,7 +129,7 @@ def test_append_false_overwrites_existing_results( results = pd.read_csv(results_csv) assert "Problem" in results.columns - def test_multiple_iterations_computes_mean_and_stddev( + def test_multiple_seeds_computes_mean_and_stddev( self, problems_yaml, tmp_path, mocker ): mocker.patch.object( @@ -140,14 +140,14 @@ def test_multiple_iterations_computes_mean_and_stddev( ["highs-default"], year="2025", run_id="test-run", - iterations=2, + num_seeds=2, ) summary = pd.read_csv( tmp_path / "results" / "benchmark_results_mean_stddev.csv" ) assert summary.iloc[0]["Runtime StdDev (s)"] == 0.0 - def test_iterations_greater_than_one_varies_seed(self, problems_yaml, mocker): + def test_num_seeds_greater_than_one_varies_seed(self, problems_yaml, mocker): run_solver_mock = mocker.patch.object( orchestrator, "run_solver", return_value=dict(_FAKE_METRICS) ) @@ -156,13 +156,13 @@ def test_iterations_greater_than_one_varies_seed(self, problems_yaml, mocker): ["highs-default"], year="2025", run_id="test-run", - iterations=3, + num_seeds=3, ) seeds = [call.kwargs["seed"] for call in run_solver_mock.call_args_list] assert seeds == [0, 1, 2] - def test_single_iteration_passes_no_seed(self, problems_yaml, mocker): - # Backward compatibility: the default `iterations=1` must not + def test_single_seed_passes_no_seed_override(self, problems_yaml, mocker): + # Backward compatibility: the default `num_seeds=1` must not # override the configuration's own fixed seed. run_solver_mock = mocker.patch.object( orchestrator, "run_solver", return_value=dict(_FAKE_METRICS) @@ -172,9 +172,7 @@ def test_single_iteration_passes_no_seed(self, problems_yaml, mocker): ) assert run_solver_mock.call_args.kwargs["seed"] is None - def test_error_status_stops_further_iterations( - self, problems_yaml, tmp_path, mocker - ): + def test_error_status_stops_further_seeds(self, problems_yaml, tmp_path, mocker): error_metrics = {**_FAKE_METRICS, "status": "ER"} run_solver_mock = mocker.patch.object( orchestrator, "run_solver", return_value=error_metrics @@ -184,7 +182,7 @@ def test_error_status_stops_further_iterations( ["highs-default"], year="2025", run_id="test-run", - iterations=3, + num_seeds=3, ) assert run_solver_mock.call_count == 1 diff --git a/runner/utils/orchestrator.py b/runner/utils/orchestrator.py index 22e9798d..69101f6a 100644 --- a/runner/utils/orchestrator.py +++ b/runner/utils/orchestrator.py @@ -79,7 +79,7 @@ def run_benchmark( problems_yaml_path: str | Path, solver_configurations: list[str], year: str | None = None, - iterations: int = 1, + num_seeds: int = 1, reference_interval: int = 0, # Default: disabled append: bool = False, run_id: str | None = None, @@ -97,15 +97,16 @@ def run_benchmark( registered for `year` at all (see `env.get_registered_solver_versions`). year : str, optional The solver-version year to run, e.g. `"2025"`. - iterations : int, optional - Repetitions per (problem, solver configuration) pair. When greater - than 1, each iteration `i` overrides the configuration's own fixed - seed with `i` (see `execution.run_solver`'s `seed` parameter), so - repeated runs sample the solver's actual sensitivity to its seed - rather than just re-measuring one deterministic solve. A timeout or - error on one iteration skips the rest. Statistics are still recorded - when this is 1 (mean == the single value, stddev == 0), and the seed - is left unset (the configuration's own fixed seed applies). + num_seeds : int, optional + Number of seeds to try per (problem, solver configuration) pair. + When greater than 1, each repetition `i` overrides the + configuration's own fixed seed with `i` (see `execution.run_solver`'s + `seed` parameter), so repeated runs sample the solver's actual + sensitivity to its seed rather than just re-measuring one + deterministic solve. A timeout or error on one repetition skips the + rest. Statistics are still recorded when this is 1 (mean == the + single value, stddev == 0), and the seed is left unset (the + configuration's own fixed seed applies). reference_interval : int, optional Minimum seconds between reference-benchmark runs (see `execution.run_reference_highs_binary`), interleaved between real @@ -197,14 +198,14 @@ def run_benchmark( memory_usages = [] timestamp = "" - for i in range(iterations): - # Vary the seed across iterations so repeated runs sample the + for seed_index in range(num_seeds): + # Vary the seed across repetitions so they sample the # solver's actual sensitivity to it. - seed = i if iterations > 1 else None + seed = seed_index if num_seeds > 1 else None print( f"Running solver {solver_configuration} (version {solver_version}) " - f"on {problem['path']} ({i})" + f"on {problem['path']} ({seed_index})" + (f" with seed {seed}" if seed is not None else "") + "...", flush=True, @@ -243,15 +244,15 @@ def run_benchmark( **environment_metadata, ) - # If solver errors or times out, don't run further iterations + # If solver errors or times out, don't try further seeds if metrics["status"] in {"ER", "TO"}: break # Calculate mean and standard deviation. Guarded by how many # runtimes were actually collected, not the requested - # `iterations`: an error/timeout on the first iteration breaks + # `num_seeds`: an error/timeout on the first repetition breaks # the loop above early, leaving a single-element `runtimes` - # even when `iterations` > 1, and stdev requires 2+ points. + # even when `num_seeds` > 1, and stdev requires 2+ points. if len(runtimes) > 1: metrics["runtime_mean"] = statistics.mean(runtimes) metrics["runtime_stddev"] = statistics.stdev(runtimes) diff --git a/runner/utils/results.py b/runner/utils/results.py index 25d5539e..e390751f 100644 --- a/runner/utils/results.py +++ b/runner/utils/results.py @@ -74,10 +74,10 @@ def csv_record(check: bool = False, **kwargs: Any) -> OrderedDict[str, Any]: "Seed" is appended last (not grouped with the other solver-identifying columns) so adding it doesn't shift every other column's position -- see `ensure_csv_schema` for how an existing CSV predating this column - is widened to include it. Empty for a single-iteration run (the + is widened to include it. Empty for a single-seed run (the configuration's own fixed seed applies); set to the actual seed used - when `orchestrator.run_benchmark`'s `iterations` > 1 varies it per - iteration. + when `orchestrator.run_benchmark`'s `num_seeds` > 1 varies it per + repetition. """ record = OrderedDict( [ From d8bfa04c95d2799b2a9ee922f7e81d4d4fa3941a Mon Sep 17 00:00:00 2001 From: Enrico Antonini <50218270+eantonini@users.noreply.github.com> Date: Tue, 1 Sep 2026 15:04:19 +0200 Subject: [PATCH 4/5] Resolve non-deterministic seed value for CBC --- runner/README.md | 2 +- runner/benchmark.py | 2 +- runner/tests/test_benchmark.py | 2 +- runner/tests/test_orchestrator.py | 2 +- runner/utils/orchestrator.py | 16 ++++++++++++---- 5 files changed, 16 insertions(+), 8 deletions(-) diff --git a/runner/README.md b/runner/README.md index 961e9cd3..fedecf18 100644 --- a/runner/README.md +++ b/runner/README.md @@ -48,7 +48,7 @@ once per given year. - `-a, --append` - Append to the results CSVs instead of overwriting them for the first year - `-y, --years YEAR` - Solver-version year to run (repeatable), or `tests` for the shared CI smoke-test env. Defaults to every year with a registered solver version - `-s, --solver-configurations CONFIG` - Solver configuration to run (repeatable), e.g. `highs-default` or `highs-hipo`. Defaults to `solver_configurations.yaml`'s `default_configurations` -- `-n, --num-seeds N` - Number of seeds to try per (problem, solver configuration) pair. When greater than 1, each repetition uses a different seed (0, 1, 2, ...) instead of the configuration's own fixed seed, to gauge the solver's sensitivity to it. Default: 1 (no repetition, the configuration's own fixed seed applies) +- `-n, --num-seeds N` - Number of seeds to try per (problem, solver configuration) pair. When greater than 1, each repetition uses a different seed (1, 2, 3, ...) instead of the configuration's own fixed seed, to gauge the solver's sensitivity to it. Default: 1 (no repetition, the configuration's own fixed seed applies) - `-r, --ref-bench-interval SECONDS` - Run a reference benchmark at most once every N seconds. 0 disables it - `-u, --run-id RUN_ID` - Identifier shared by every row from this run. Auto-generated if not given - `--help` - Show this message and exit diff --git a/runner/benchmark.py b/runner/benchmark.py index 165d44fe..d9e60b19 100644 --- a/runner/benchmark.py +++ b/runner/benchmark.py @@ -50,7 +50,7 @@ def run( "-n", help="Number of seeds to try per (problem, solver configuration) " "pair. When greater than 1, each repetition uses a different seed " - "(0, 1, 2, ...) instead of the configuration's own fixed seed, to " + "(1, 2, 3, ...) instead of the configuration's own fixed seed, to " "gauge the solver's sensitivity to it (see " "`runner/config/solvers.yaml`'s `seed_options`). Default: 1 (the " "configuration's own fixed seed, no repetition).", diff --git a/runner/tests/test_benchmark.py b/runner/tests/test_benchmark.py index 952f3899..b5f99085 100644 --- a/runner/tests/test_benchmark.py +++ b/runner/tests/test_benchmark.py @@ -105,7 +105,7 @@ def test_num_seeds_flag_varies_seed_across_repetitions( ) assert result.exit_code == 0, result.output results = pd.read_csv(tmp_path / "results" / "benchmark_results.csv") - assert sorted(results["Seed"]) == [0, 1, 2] + assert sorted(results["Seed"]) == [1, 2, 3] def test_default_num_seeds_leaves_seed_column_empty(self, problems_yaml, tmp_path): result = runner_cli.invoke( diff --git a/runner/tests/test_orchestrator.py b/runner/tests/test_orchestrator.py index d97d89dd..ef86f446 100644 --- a/runner/tests/test_orchestrator.py +++ b/runner/tests/test_orchestrator.py @@ -159,7 +159,7 @@ def test_num_seeds_greater_than_one_varies_seed(self, problems_yaml, mocker): num_seeds=3, ) seeds = [call.kwargs["seed"] for call in run_solver_mock.call_args_list] - assert seeds == [0, 1, 2] + assert seeds == [1, 2, 3] def test_single_seed_passes_no_seed_override(self, problems_yaml, mocker): # Backward compatibility: the default `num_seeds=1` must not diff --git a/runner/utils/orchestrator.py b/runner/utils/orchestrator.py index 69101f6a..7d27bdb8 100644 --- a/runner/utils/orchestrator.py +++ b/runner/utils/orchestrator.py @@ -99,9 +99,11 @@ def run_benchmark( The solver-version year to run, e.g. `"2025"`. num_seeds : int, optional Number of seeds to try per (problem, solver configuration) pair. - When greater than 1, each repetition `i` overrides the - configuration's own fixed seed with `i` (see `execution.run_solver`'s - `seed` parameter), so repeated runs sample the solver's actual + When greater than 1, each repetition overrides the configuration's + own fixed seed with 1, 2, 3, ... (see `execution.run_solver`'s + `seed` parameter) -- starting at 1, not 0, since CBC's own seed + option treats 0 as "use the time of day" rather than an actual + fixed seed -- so repeated runs sample the solver's actual sensitivity to its seed rather than just re-measuring one deterministic solve. A timeout or error on one repetition skips the rest. Statistics are still recorded when this is 1 (mean == the @@ -198,7 +200,13 @@ def run_benchmark( memory_usages = [] timestamp = "" - for seed_index in range(num_seeds): + # Seeds start at 1, not 0: CBC's own seed option (randomCbcSeed) + # treats 0 as a sentinel meaning "use the time of day" instead of + # an actual fixed seed (see solver_configurations.yaml's own + # comment on cbc-default), which would make that repetition + # silently non-deterministic. No other solver here gives 0 any + # special meaning, so starting at 1 is safe for all of them. + for seed_index in range(1, num_seeds + 1): # Vary the seed across repetitions so they sample the # solver's actual sensitivity to it. seed = seed_index if num_seeds > 1 else None From 82aeb1e53272d1d8373910e3e16fa6f56f74f519 Mon Sep 17 00:00:00 2001 From: Enrico Antonini <50218270+eantonini@users.noreply.github.com> Date: Tue, 1 Sep 2026 16:04:04 +0200 Subject: [PATCH 5/5] Harmonize random seed values --- runner/config/solver_configurations.yaml | 23 +++++++++++++---------- runner/tests/test_solver.py | 8 ++++---- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/runner/config/solver_configurations.yaml b/runner/config/solver_configurations.yaml index d949bd96..00360a24 100644 --- a/runner/config/solver_configurations.yaml +++ b/runner/config/solver_configurations.yaml @@ -24,10 +24,13 @@ # package's own option-key vocabulary (hence the very different-looking # names for the same underlying setting, e.g. HiGHS's "mip_rel_gap" vs # Gurobi's "MIPGap" vs CPLEX's "mip.tolerances.mipgap" all reference -# `shared.mip_gap`). Seed values are NOT unified across solvers -- -# GLPK/SCIP default to 0, CBC to 1 (0 means "time of day" for CBC -# specifically), the rest to 4 -- preserved as-is rather than -# homogenized here. A non-default configuration's `options` entirely +# `shared.mip_gap`). Seed values default to 0 for every solver except +# CBC, whose own seed option (randomCbcSeed) treats 0 as "use the time +# of day" rather than an actual fixed seed -- confirmed via each +# solver's own option docs that 0 is an ordinary, deterministic value +# everywhere else (some solvers have their own *different* "auto" +# sentinel instead, e.g. Xpress's is -1, GLPK's is the literal string +# "?" -- none of them is 0). A non-default configuration's `options` entirely # replace its solver's default-configuration options (rather than # layering on top), matching how picking a specific algorithm is an # all-or-nothing choice. @@ -68,7 +71,7 @@ configurations: highs-default: solver_package: highs options: - random_seed: 4 + random_seed: 0 mip_rel_gap: { shared: mip_gap } primal_feasibility_tolerance: { shared: lp_tolerance } dual_feasibility_tolerance: { shared: lp_tolerance } @@ -121,7 +124,7 @@ configurations: gurobi-default: solver_package: gurobi options: - seed: 4 + seed: 0 MIPGap: { shared: mip_gap } FeasibilityTol: { shared: lp_tolerance } OptimalityTol: { shared: lp_tolerance } @@ -131,7 +134,7 @@ configurations: cplex-default: solver_package: cplex options: - randomseed: 4 + randomseed: 0 mip.tolerances.mipgap: { shared: mip_gap } simplex.tolerances.feasibility: { shared: lp_tolerance } simplex.tolerances.optimality: { shared: lp_tolerance } @@ -140,7 +143,7 @@ configurations: knitro-default: solver_package: knitro options: - ms_seed: 4 + ms_seed: 0 mip_opt_gap_rel: { shared: mip_gap } feastol: { shared: lp_tolerance } opttol: { shared: lp_tolerance } @@ -149,7 +152,7 @@ configurations: xpress-default: solver_package: xpress options: - randomseed: 4 + randomseed: 0 miprelstop: { shared: mip_gap } FEASTOL: { shared: lp_tolerance } OPTIMALITYTOL: { shared: lp_tolerance } @@ -158,7 +161,7 @@ configurations: mosek-default: solver_package: mosek options: - MSK_IPAR_MIO_SEED: 4 + MSK_IPAR_MIO_SEED: 0 MSK_IPAR_INTPNT_BASIS: MSK_BI_NEVER MSK_DPAR_MIO_TOL_REL_GAP: { shared: mip_gap } MSK_DPAR_INTPNT_TOL_PFEAS: { shared: lp_tolerance } diff --git a/runner/tests/test_solver.py b/runner/tests/test_solver.py index ab57a4c1..d9165d3b 100644 --- a/runner/tests/test_solver.py +++ b/runner/tests/test_solver.py @@ -30,7 +30,7 @@ def test_plain_configuration_uses_its_own_options(self, monkeypatch): captured = self._patch_solver_class(monkeypatch, "Highs") _, solver_package = get_solver("highs-default") assert solver_package == "highs" - assert captured["options"]["random_seed"] == 4 + assert captured["options"]["random_seed"] == 0 assert captured["options"]["mip_rel_gap"] == pytest.approx(1e-4) def test_named_configuration_resolves_to_its_solver(self, monkeypatch): @@ -46,7 +46,7 @@ def test_unregistered_name_falls_back_to_bare_solver_with_no_options( captured = self._patch_solver_class(monkeypatch, "Mosek") _, solver_package = get_solver("mosek-default") assert solver_package == "mosek" - assert captured["options"]["MSK_IPAR_MIO_SEED"] == 4 + assert captured["options"]["MSK_IPAR_MIO_SEED"] == 0 def test_unsupported_solver_name_raises(self): with pytest.raises(ValueError): @@ -62,7 +62,7 @@ def test_seed_overrides_configurations_own_seed(self, monkeypatch): def test_no_seed_keeps_configurations_own_seed(self, monkeypatch): captured = self._patch_solver_class(monkeypatch, "Highs") get_solver("highs-default", seed=None) - assert captured["options"]["random_seed"] == 4 + assert captured["options"]["random_seed"] == 0 def test_seed_ignored_with_warning_when_no_seed_options_entry( self, monkeypatch, capsys @@ -72,7 +72,7 @@ def test_seed_ignored_with_warning_when_no_seed_options_entry( ) captured = self._patch_solver_class(monkeypatch, "Highs") get_solver("highs-default", seed=42) - assert captured["options"]["random_seed"] == 4 + assert captured["options"]["random_seed"] == 0 assert "no seed_options entry" in capsys.readouterr().err