From abe67fcede5964529c4361170762c07728607808 Mon Sep 17 00:00:00 2001 From: Toby Coleman Date: Sun, 29 Jun 2025 14:17:45 +0100 Subject: [PATCH 1/3] Add tolerance to tuner tests --- tests/integration/test_tuner.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/integration/test_tuner.py b/tests/integration/test_tuner.py index a0546a35..61426e46 100644 --- a/tests/integration/test_tuner.py +++ b/tests/integration/test_tuner.py @@ -43,7 +43,7 @@ async def test_tune(config: dict, mode: str, process_type: str, ray_ctx: None) - field_type="arg", field_name="iters", lower=6, - upper=8, + upper=9, ) ], num_samples=5, @@ -57,13 +57,13 @@ async def test_tune(config: dict, mode: str, process_type: str, ray_ctx: None) - result = tuner.result_grid # There must be no failed trials assert not [t for t in result if t.error] - # Correct optimimum must be found + # Correct optimimum must be found (within tolerance) if mode == "min": - assert best_result.config["a.iters"] == 6 - assert best_result.metrics["c.in_1"] == 5 + assert best_result.config["a.iters"] in {6, 7} + assert best_result.metrics["c.in_1"] == best_result.config["a.iters"] - 1 else: - assert best_result.config["a.iters"] == 7 - assert best_result.metrics["c.in_1"] == 6 + assert best_result.config["a.iters"] in {7, 8} + assert best_result.metrics["c.in_1"] == best_result.config["a.iters"] - 1 @pytest.mark.tuner From ab4db664d6e65c9983f38db4b8fb1566ec8e44b9 Mon Sep 17 00:00:00 2001 From: Toby Coleman Date: Sun, 29 Jun 2025 13:54:28 +0100 Subject: [PATCH 2/3] Increase RabbitMQ tolerance --- tests/integration/test_process_stop_event.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_process_stop_event.py b/tests/integration/test_process_stop_event.py index 6261efbb..482686ba 100644 --- a/tests/integration/test_process_stop_event.py +++ b/tests/integration/test_process_stop_event.py @@ -69,7 +69,7 @@ async def test_process_stop_event( max_iters = 25 iters_before_stop = 15 sleep_time = 0.1 - stop_tolerance = 5 if connector_cls in {RabbitMQConnector} else STOP_TOLERANCE + stop_tolerance = 7 if connector_cls in {RabbitMQConnector} else STOP_TOLERANCE comp_a = A(iters=max_iters, sleep_time=sleep_time, name="comp_a") comp_b1, comp_b2, comp_b3, comp_b4, comp_b5 = [B(name=f"comp_b{i}") for i in range(1, 6)] From 0af4509a1b8e774089644031cd00c7d90cae91b7 Mon Sep 17 00:00:00 2001 From: Toby Coleman Date: Sun, 29 Jun 2025 14:37:45 +0100 Subject: [PATCH 3/3] Improvements --- tests/integration/test_tuner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_tuner.py b/tests/integration/test_tuner.py index 61426e46..9dd3189e 100644 --- a/tests/integration/test_tuner.py +++ b/tests/integration/test_tuner.py @@ -59,10 +59,10 @@ async def test_tune(config: dict, mode: str, process_type: str, ray_ctx: None) - assert not [t for t in result if t.error] # Correct optimimum must be found (within tolerance) if mode == "min": - assert best_result.config["a.iters"] in {6, 7} + assert best_result.config["a.iters"] <= tuner._parameters["a.iters"].lower + 1 assert best_result.metrics["c.in_1"] == best_result.config["a.iters"] - 1 else: - assert best_result.config["a.iters"] in {7, 8} + assert best_result.config["a.iters"] >= tuner._parameters["a.iters"].upper - 1 assert best_result.metrics["c.in_1"] == best_result.config["a.iters"] - 1