From 58d8bb70031c3a4557d9f7c6edd975a5259a83bd Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Thu, 25 Jun 2026 22:09:45 +0100 Subject: [PATCH 01/54] feat: relax storage SoC bounds by default Signed-off-by: Mohamed Belhsan Hmida --- .../data/schemas/scheduling/__init__.py | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py index ad9dc79d17..97a2029b2a 100644 --- a/flexmeasures/data/schemas/scheduling/__init__.py +++ b/flexmeasures/data/schemas/scheduling/__init__.py @@ -180,7 +180,7 @@ class FlexContextSchema(Schema): # Dev fields relax_soc_constraints = fields.Bool( data_key="relax-soc-constraints", - load_default=False, + load_default=True, metadata=metadata.RELAX_SOC_CONSTRAINTS.to_dict(), ) relax_capacity_constraints = fields.Bool( @@ -358,7 +358,9 @@ def check_prices(self, data: dict, original_data: dict, **kwargs): for field_var, field in self.declared_fields.items() } if any( - field_map[field] in data and data[field_map[field]] + field in original_data + and field_map[field] in data + and data[field_map[field]] for field in ( "soc-minima-breach-price", "soc-maxima-breach-price", @@ -391,8 +393,7 @@ def check_prices(self, data: dict, original_data: dict, **kwargs): # Fill in default soc breach prices when asked to relax SoC constraints, unless already set explicitly. if ( - data["relax_soc_constraints"] - or data["relax_constraints"] + (data["relax_soc_constraints"] or data["relax_constraints"]) and not data.get("soc_minima_breach_price") and not data.get("soc_maxima_breach_price") ): @@ -404,8 +405,7 @@ def check_prices(self, data: dict, original_data: dict, **kwargs): # Fill in default capacity breach prices when asked to relax capacity constraints, unless already set explicitly. if ( - data["relax_capacity_constraints"] - or data["relax_constraints"] + (data["relax_capacity_constraints"] or data["relax_constraints"]) and not data.get("consumption_breach_price") and not data.get("production_breach_price") ): @@ -417,8 +417,7 @@ def check_prices(self, data: dict, original_data: dict, **kwargs): # Fill in default site capacity breach prices when asked to relax site capacity constraints, unless already set explicitly. if ( - data["relax_site_capacity_constraints"] - or data["relax_constraints"] + (data["relax_site_capacity_constraints"] or data["relax_constraints"]) and not data.get("ems_consumption_breach_price") and not data.get("ems_production_breach_price") ): @@ -764,6 +763,11 @@ def _to_currency_per_mwh(price_unit: str) -> str: class DBFlexContextSchema(FlexContextSchema, NoTimeSeriesSpecs): + relax_soc_constraints = fields.Bool( + data_key="relax-soc-constraints", + load_default=False, + metadata=metadata.RELAX_SOC_CONSTRAINTS.to_dict(), + ) commitments = fields.Nested( DBCommitmentSchema, data_key="commitments", required=False, many=True From 2944dacda1f5f4cc0132cff5cf3a19907f742279 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Thu, 25 Jun 2026 22:09:51 +0100 Subject: [PATCH 02/54] feat: remove storage fallback scheduler Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/data/models/planning/storage.py | 73 -------------------- 1 file changed, 73 deletions(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 4c59a0ffa9..75705f06d6 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -3,7 +3,6 @@ import re import copy from datetime import datetime, timedelta -from typing import Type import pandas as pd import numpy as np @@ -26,7 +25,6 @@ initialize_series, initialize_df, get_power_values, - fallback_charging_policy, get_continuous_series_sensor_or_quantity, ) from flexmeasures.data.models.planning.exceptions import InfeasibleProblemException @@ -1582,81 +1580,10 @@ def _ensure_variable_quantity( return q -class StorageFallbackScheduler(MetaStorageScheduler): - __version__ = "3" - __author__ = "Seita" - - def compute(self, skip_validation: bool = False) -> SchedulerOutputType: - """Schedule a battery or Charge Point by just starting to charge, discharge, or do neither, - depending on the first target state of charge and the capabilities of the Charge Point. - For the resulting consumption schedule, consumption is defined as positive values. - - Note that this ignores any cause of the infeasibility. - - :param skip_validation: If True, skip validation of constraints specified in the data. - :returns: The computed schedule. - """ - - ( - sensors, - start, - end, - resolution, - soc_at_start, - device_constraints, - ems_constraints, - commitments, - ) = self._prepare(skip_validation=skip_validation) - - # Fallback policy if the problem was unsolvable - storage_schedule = { - sensor: fallback_charging_policy( - sensor, device_constraints[d], start, end, resolution - ) - for d, sensor in enumerate(sensors) - if sensor is not None - } - - # Convert each device schedule to the unit of the device's power sensor - storage_schedule = { - sensor: convert_units( - storage_schedule[sensor], - "MW", - sensor.unit, - event_resolution=sensor.event_resolution, - ) - for sensor in sensors - if sensor is not None - } - - # Round schedule - if self.round_to_decimals: - storage_schedule = { - sensor: storage_schedule[sensor].round(self.round_to_decimals) - for sensor in sensors - if sensor is not None - } - - if self.return_multiple: - return [ - { - "name": "storage_schedule", - "sensor": sensor, - "data": storage_schedule[sensor], - } - for sensor in sensors - if sensor is not None - ] - else: - return storage_schedule[sensors[0]] - - class StorageScheduler(MetaStorageScheduler): __version__ = "8" __author__ = "Seita" - fallback_scheduler_class: Type[Scheduler] = StorageFallbackScheduler - @staticmethod def _build_soc_schedule( flex_model: list[dict], From 09e9998b59d9b18bce3a153f82ab7e1438625e13 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Thu, 25 Jun 2026 22:09:56 +0100 Subject: [PATCH 03/54] feat: remove storage fallback policy helper Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/data/models/planning/utils.py | 77 ---------------------- 1 file changed, 77 deletions(-) diff --git a/flexmeasures/data/models/planning/utils.py b/flexmeasures/data/models/planning/utils.py index cfea6eba58..7cd79f7156 100644 --- a/flexmeasures/data/models/planning/utils.py +++ b/flexmeasures/data/models/planning/utils.py @@ -220,83 +220,6 @@ def get_power_values( return -series -def fallback_charging_policy( - sensor: Sensor, - device_constraints: pd.DataFrame, - start: datetime, - end: datetime, - resolution: timedelta, -) -> pd.Series: - """This fallback charging policy is to just start charging or discharging, or do neither, - depending on the first target state of charge and the capabilities of the Charge Point. - Note that this ignores any cause of the infeasibility and, - while probably a decent policy for Charge Points, - should not be considered a robust policy for other asset types. - """ - max_charge_capacity = ( - device_constraints[["derivative max", "derivative equals"]].min().min() - ) - max_discharge_capacity = ( - -device_constraints[["derivative min", "derivative equals"]].max().max() - ) - charge_power = max_charge_capacity if sensor.get_attribute("is_consumer") else 0 - discharge_power = ( - -max_discharge_capacity if sensor.get_attribute("is_producer") else 0 - ) - - charge_schedule = initialize_series(charge_power, start, end, resolution) - discharge_schedule = initialize_series(discharge_power, start, end, resolution) - idle_schedule = initialize_series(0, start, end, resolution) - if ( - device_constraints["equals"].first_valid_index() is not None - and device_constraints["equals"][ - device_constraints["equals"].first_valid_index() - ] - > 0 - ): - # start charging to get as close as possible to the next target - return idle_after_reaching_target(charge_schedule, device_constraints["equals"]) - if ( - device_constraints["equals"].first_valid_index() is not None - and device_constraints["equals"][ - device_constraints["equals"].first_valid_index() - ] - < 0 - ): - # start discharging to get as close as possible to the next target - return idle_after_reaching_target( - discharge_schedule, device_constraints["equals"] - ) - if ( - device_constraints["max"].first_valid_index() is not None - and device_constraints["max"][device_constraints["max"].first_valid_index()] < 0 - ): - # start discharging to try and bring back the soc below the next max constraint - return idle_after_reaching_target(discharge_schedule, device_constraints["max"]) - if ( - device_constraints["min"].first_valid_index() is not None - and device_constraints["min"][device_constraints["min"].first_valid_index()] > 0 - ): - # start charging to try and bring back the soc above the next min constraint - return idle_after_reaching_target(charge_schedule, device_constraints["min"]) - # stand idle - return idle_schedule - - -def idle_after_reaching_target( - schedule: pd.Series, - target: pd.Series, - initial_state: float = 0, -) -> pd.Series: - """Stop planned (dis)charging after target is reached (or constraint is met).""" - first_target = target[target.first_valid_index()] - if first_target > initial_state: - schedule[schedule.cumsum() > first_target] = 0 - else: - schedule[schedule.cumsum() < first_target] = 0 - return schedule - - def get_series_from_quantity_or_sensor( variable_quantity: Sensor | SensorReference | list[dict] | ur.Quantity, unit: ur.Quantity | str, From 11eeb37011b626265325c7a7dfb0cdebe2265650 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Thu, 25 Jun 2026 22:10:01 +0100 Subject: [PATCH 04/54] test: cover SoC relaxation schema defaults Signed-off-by: Mohamed Belhsan Hmida --- .../data/schemas/tests/test_scheduling.py | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/flexmeasures/data/schemas/tests/test_scheduling.py b/flexmeasures/data/schemas/tests/test_scheduling.py index 3d4450580e..4522d97484 100644 --- a/flexmeasures/data/schemas/tests/test_scheduling.py +++ b/flexmeasures/data/schemas/tests/test_scheduling.py @@ -517,6 +517,48 @@ def test_flex_context_schema( check_schema_loads_data(schema=schema, data=flex_context, fails=fails) +def test_flex_context_schema_relaxes_soc_constraints_by_default(): + loaded_flex_context = FlexContextSchema().load({"consumption-price": "1 EUR/MWh"}) + + assert loaded_flex_context["relax_soc_constraints"] is True + assert loaded_flex_context["soc_minima_breach_price"].to( + "EUR/MWh" + ).magnitude == pytest.approx(1_000_000) + assert loaded_flex_context["soc_maxima_breach_price"].to( + "EUR/MWh" + ).magnitude == pytest.approx(1_000_000) + assert "consumption_breach_price" not in loaded_flex_context + assert "production_breach_price" not in loaded_flex_context + assert "ems_consumption_breach_price" not in loaded_flex_context + assert "ems_production_breach_price" not in loaded_flex_context + + +def test_flex_context_schema_preserves_explicit_soc_breach_prices(): + loaded_flex_context = FlexContextSchema().load( + { + "consumption-price": "1 EUR/MWh", + "soc-minima-breach-price": "5 EUR/kWh", + "soc-maxima-breach-price": "7 EUR/kWh", + } + ) + + assert loaded_flex_context["relax_soc_constraints"] is True + assert loaded_flex_context["soc_minima_breach_price"].to( + "EUR/kWh" + ).magnitude == pytest.approx(5) + assert loaded_flex_context["soc_maxima_breach_price"].to( + "EUR/kWh" + ).magnitude == pytest.approx(7) + + +def test_db_flex_context_schema_does_not_relax_soc_constraints_by_default(): + loaded_flex_context = DBFlexContextSchema().load({}) + + assert loaded_flex_context["relax_soc_constraints"] is False + assert "soc_minima_breach_price" not in loaded_flex_context + assert "soc_maxima_breach_price" not in loaded_flex_context + + def check_schema_loads_data(schema, data, fails): if fails: with pytest.raises(ValidationError) as e_info: From 0fd1487b0d8f7fefe5a4f999e986a2449adafbb2 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Thu, 25 Jun 2026 22:10:06 +0100 Subject: [PATCH 05/54] test: use default SoC breach prices Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/data/models/planning/tests/test_storage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flexmeasures/data/models/planning/tests/test_storage.py b/flexmeasures/data/models/planning/tests/test_storage.py index ed8c09be82..a5a47d6bc6 100644 --- a/flexmeasures/data/models/planning/tests/test_storage.py +++ b/flexmeasures/data/models/planning/tests/test_storage.py @@ -174,7 +174,8 @@ def test_battery_relaxation(add_battery_assets, db): """Check that resolving SoC breaches is more important than resolving device power breaches. The battery is still charging with 25 kW between noon and 4 PM, when the consumption capacity is supposed to be 0. - It is still charging because resolving the still unmatched SoC minima takes precedence (via breach prices). + It is still charging because resolving the still unmatched SoC minima takes precedence + through the default SoC breach prices. """ _, battery = get_sensors_from_db( db, add_battery_assets, battery_name="Test battery" @@ -252,7 +253,6 @@ def test_battery_relaxation(add_battery_assets, db): "site-peak-production-price": series_to_ts_specs( pd.Series(260, production_prices.index), unit="EUR/MW" ), - "soc-minima-breach-price": "6000 EUR/kWh", # high breach price (to mimic a hard constraint) "consumption-breach-price": f"{device_power_breach_price} EUR/kW", # lower breach price (thus prioritizing minimizing soc breaches) "production-breach-price": f"{device_power_breach_price} EUR/kW", # lower breach price (thus prioritizing minimizing soc breaches) }, From 18aa13b03b26845adb030dff7e3af4637b96acd3 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Thu, 25 Jun 2026 22:10:12 +0100 Subject: [PATCH 06/54] test: expect storage infeasibility without fallback Signed-off-by: Mohamed Belhsan Hmida --- .../data/models/planning/tests/test_solver.py | 32 +++---------------- 1 file changed, 5 insertions(+), 27 deletions(-) diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py index e9e61da848..8c37ff05c2 100644 --- a/flexmeasures/data/models/planning/tests/test_solver.py +++ b/flexmeasures/data/models/planning/tests/test_solver.py @@ -498,22 +498,18 @@ def test_charging_station_solver_day_2( (15, "Test charging station (bidirectional)"), ], ) -def test_fallback_to_unsolvable_problem( +def test_storage_scheduler_reports_unsolvable_problem_without_fallback( target_soc, charging_station_name, setup_planning_test_data, db ): """Starting with a state of charge 10 kWh, within 2 hours we should be able to reach any state of charge in the range [10, 14] kWh for a unidirectional station, or [6, 14] for a bidirectional station, given a charging capacity of 2 kW. Here we test target states of charge outside that range, ones that we should be able - to get as close to as 1 kWh difference. - We want our scheduler to handle unsolvable problems like these with a sensible fallback policy. - - The StorageScheduler raises an Exception which triggers the creation of a new job to compute a fallback - schedule. + The StorageScheduler should report this infeasible problem without hiding it behind + a fallback schedule. """ soc_at_start = 10 duration_until_target = timedelta(hours=2) - expected_gap = 1 epex_da = get_test_sensor(db) charging_station = setup_planning_test_data[charging_station_name].sensors[0] @@ -578,26 +574,8 @@ def test_fallback_to_unsolvable_problem( # calling the scheduler with an infeasible problem raises an Exception with pytest.raises(InfeasibleProblemException): - consumption_schedule = scheduler.compute(skip_validation=True) - - # check that the fallback scheduler provides a sensible fallback policy - fallback_scheduler = scheduler.fallback_scheduler_class(**kwargs) - fallback_scheduler.config_deserialized = True - consumption_schedule = fallback_scheduler.compute(skip_validation=True) - - soc_schedule = integrate_time_series( - consumption_schedule, soc_at_start, decimal_precision=6 - ) - - # Check if constraints were met - assert min(consumption_schedule.values) >= capacity * -1 - assert max(consumption_schedule.values) <= capacity - print(consumption_schedule.head(12)) - print(soc_schedule.head(12)) - assert ( - abs(abs(soc_schedule.loc[target_soc_datetime] - target_soc) - expected_gap) - < TOLERANCE - ) + scheduler.compute(skip_validation=True) + assert scheduler.fallback_scheduler_class is None @pytest.mark.parametrize( From 783fd8b1e221cb144ba361f5b7be3915ed5514b8 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Thu, 25 Jun 2026 22:10:27 +0100 Subject: [PATCH 07/54] test: assert storage schedules do not fall back Signed-off-by: Mohamed Belhsan Hmida --- .../api/v3_0/tests/test_sensor_schedules.py | 196 ++---------------- 1 file changed, 16 insertions(+), 180 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py index 6928c452ab..37cb47bee7 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py @@ -340,7 +340,8 @@ def test_trigger_and_get_schedule_with_unknown_prices( @pytest.mark.parametrize( "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True ) -def test_get_schedule_fallback( +@pytest.mark.parametrize("fallback_redirect", [True, False]) +def test_get_schedule_infeasible_storage_job_without_fallback( app, add_battery_assets, add_market_prices, @@ -349,13 +350,14 @@ def test_get_schedule_fallback( keep_scheduling_queue_empty, requesting_user, db, + fallback_redirect, ): """ - Test if the fallback job is created after a failing StorageScheduler call. This test - is based on flexmeasures/data/models/planning/tests/test_solver.py + Test that a failing StorageScheduler call reports the failure without creating a fallback job. + + This test is based on flexmeasures/data/models/planning/tests/test_solver.py. """ - assert app.config["FLEXMEASURES_FALLBACK_REDIRECT"] is False - app.config["FLEXMEASURES_FALLBACK_REDIRECT"] = True + app.config["FLEXMEASURES_FALLBACK_REDIRECT"] = fallback_redirect target_soc = 9 charging_station_name = "Test charging station" @@ -375,7 +377,7 @@ def test_get_schedule_fallback( assert capacity == 2 assert charging_station.get_attribute("consumption-price") == {"sensor": epex_da.id} - # check that no Fallback schedule has been saved before + # check that no retired storage fallback schedule has been saved before models = [ source.model for source in charging_station.search_beliefs().sources.unique() ] @@ -386,6 +388,7 @@ def test_get_schedule_fallback( "start": start, "duration": "PT24H", "resolution": "PT15M", # just schedule in the original sensor resolution + "force-new-job-creation": True, "flex-model": { "soc-at-start": 10, "soc-min": charging_station.get_attribute("min_soc_in_mwh", 0), @@ -439,193 +442,26 @@ def test_get_schedule_fallback( # Make sure the resolution shows up in the job kwargs assert job.kwargs.get("resolution") == pd.Timedelta(message["resolution"]) - # the callback creates the fallback job which is still pending - assert len(app.queues["scheduling"]) == 1 - fallback_job_id = Job.fetch( - job_id, connection=app.queues["scheduling"].connection - ).meta.get("fallback_job_id") - - # check that the fallback_job_id is stored on the metadata of the original job - assert app.queues["scheduling"].get_job_ids()[0] == fallback_job_id - assert fallback_job_id != job_id + # no storage fallback job is created + assert len(app.queues["scheduling"]) == 0 + assert job.meta.get("fallback_job_id") is None get_schedule_response = client.get( url_for("SensorAPI:get_schedule", id=charging_station.id, uuid=job_id), ) - # requesting the original job redirects to the fallback job - assert ( - get_schedule_response.status_code == 303 - ) # Status code for redirect ("See other") - assert ( + assert get_schedule_response.status_code == 400 + assert "Scheduling job failed with InfeasibleProblemException: infeasible." in ( get_schedule_response.json["message"] - == "Scheduling job failed with InfeasibleProblemException: infeasible. StorageScheduler was used." ) + assert "StorageScheduler was used." in get_schedule_response.json["message"] assert get_schedule_response.json["status"] == "UNKNOWN_SCHEDULE" assert get_schedule_response.json["result"] == "Rejected" - # check that the redirection location points to the fallback job - assert ( - get_schedule_response.headers["location"] - == f"http://localhost/api/v3_0/sensors/{charging_station.id}/schedules/{fallback_job_id}" - ) - - # run the fallback job - work_on_rq( - app.queues["scheduling"], - exc_handler=handle_scheduling_exception, - max_jobs=1, - ) - - # check that the queue is empty - assert len(app.queues["scheduling"]) == 0 - - # get the fallback schedule - fallback_schedule = client.get( - get_schedule_response.headers["location"], - json={"duration": "PT24H"}, - ).json - - # check that the fallback schedule has the right status and start dates - assert fallback_schedule["status"] == "PROCESSED" - assert parse_datetime(fallback_schedule["start"]) == parse_datetime(start) - models = [ source.model for source in charging_station.search_beliefs().sources.unique() ] - assert "StorageFallbackScheduler" in models - - app.config["FLEXMEASURES_FALLBACK_REDIRECT"] = False - - -@pytest.mark.parametrize( - "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True -) -def test_get_schedule_fallback_not_redirect( - app, - add_battery_assets, - add_market_prices, - battery_soc_sensor, - add_charging_station_assets, - keep_scheduling_queue_empty, - requesting_user, - db, -): - """ - Test if the fallback scheduler is returned directly after a failing StorageScheduler call. This test - is based on flexmeasures/data/models/planning/tests/test_solver.py - """ - app.config["FLEXMEASURES_FALLBACK_REDIRECT"] = False - - target_soc = 9 - charging_station_name = "Test charging station" - - start = "2015-01-02T00:00:00+01:00" - epex_da = get_test_sensor(db) - charging_station = get_sensor_by_name( - add_charging_station_assets[charging_station_name], "power" - ) - - capacity = charging_station.get_attribute( - "capacity_in_mw", - ur.Quantity(charging_station.get_attribute("site-power-capacity")) - .to("MW") - .magnitude, - ) - assert capacity == 2 - assert charging_station.get_attribute("consumption-price") == {"sensor": epex_da.id} - - # create a scenario that yields an infeasible problem (unreachable target SOC at 2am) - message = { - "start": start, - "duration": "PT24H", - "flex-model": { - "soc-at-start": 10, - "soc-min": charging_station.get_attribute("min_soc_in_mwh", 0), - "soc-max": charging_station.get_attribute("max-soc-in-mwh", target_soc), - "roundtrip-efficiency": charging_station.get_attribute( - "roundtrip-efficiency", 1 - ), - "storage-efficiency": charging_station.get_attribute( - "storage-efficiency", 1 - ), - "soc-targets": [ - { - "value": target_soc, - "start": "2015-01-02T02:00:00+01:00", - "duration": "PT0H", - } - ], - }, - } - - with app.test_client() as client: - # trigger storage scheduler - trigger_schedule_response = client.post( - url_for("SensorAPI:trigger_schedule", id=charging_station.id), - json=message, - ) - - # check that the call is successful - assert trigger_schedule_response.status_code == 200 - job_id = trigger_schedule_response.json["schedule"] - - # look for scheduling jobs in queue - assert ( - len(app.queues["scheduling"]) == 1 - ) # only 1 schedule should be made for 1 asset - job = app.queues["scheduling"].jobs[0] - assert job.kwargs["asset_or_sensor"]["id"] == charging_station.id - assert job.kwargs["start"] == parse_datetime(message["start"]) - assert job.id == job_id - - # process only the job that runs the storage scheduler (max_jobs=1) - work_on_rq( - app.queues["scheduling"], - exc_handler=handle_scheduling_exception, - max_jobs=1, - ) - - # check that the job is failing - job = Job.fetch(job_id, connection=app.queues["scheduling"].connection) - assert job.is_failed - - # Make sure that the db flex_context shows up in the job kwargs - assert "flex-context" not in message and job.kwargs.get("flex_context") - - # the callback creates the fallback job which is still pending - assert len(app.queues["scheduling"]) == 1 - - fallback_job_id = Job.fetch( - job_id, connection=app.queues["scheduling"].connection - ).meta.get("fallback_job_id") - - # check that the fallback_job_id is stored on the metadata of the original job - assert app.queues["scheduling"].get_job_ids()[0] == fallback_job_id - assert fallback_job_id != job_id - - get_schedule_response = client.get( - url_for("SensorAPI:get_schedule", id=charging_station.id, uuid=job_id), - ) - - work_on_rq( - app.queues["scheduling"], - exc_handler=handle_scheduling_exception, - max_jobs=1, - ) - - get_schedule_response = client.get( - url_for("SensorAPI:get_schedule", id=charging_station.id, uuid=job_id), - ) - - assert get_schedule_response.status_code == 200 - - schedule = get_schedule_response.json - - # check that the fallback schedule has the right status and start dates - assert schedule["status"] == "PROCESSED" - assert parse_datetime(schedule["start"]) == parse_datetime(start) - assert schedule["scheduler_info"]["scheduler"] == "StorageFallbackScheduler" + assert "StorageFallbackScheduler" not in models app.config["FLEXMEASURES_FALLBACK_REDIRECT"] = False From 48114c12560080e0dd9bae0dbff24251b4df8e0e Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Thu, 25 Jun 2026 22:10:35 +0100 Subject: [PATCH 08/54] test: update sequential scheduling fallback case Signed-off-by: Mohamed Belhsan Hmida --- .../data/tests/test_scheduling_sequential.py | 106 +++++++++--------- 1 file changed, 52 insertions(+), 54 deletions(-) diff --git a/flexmeasures/data/tests/test_scheduling_sequential.py b/flexmeasures/data/tests/test_scheduling_sequential.py index 53f67fa8c9..b36c547aaa 100644 --- a/flexmeasures/data/tests/test_scheduling_sequential.py +++ b/flexmeasures/data/tests/test_scheduling_sequential.py @@ -141,13 +141,13 @@ def test_create_sequential_jobs(db, app, flex_description_sequential, smart_buil assert total_cost == -2.1775, f"Total cost should be -2.1775 €, got {total_cost} €" -def test_create_sequential_jobs_fallback( +def test_create_sequential_jobs_without_storage_fallback( db, app, flex_description_sequential, smart_building ): - """Test fallback scheduler in a chain of sequential scheduling (sub)jobs. + """Test an infeasible first subjob in a chain of sequential scheduling jobs. - Checks execution of a sequential scheduling job, where 1 of the subjobs is set up to fail and trigger its fallback. - The deferred subjobs should still succeed after the fallback succeeds, even though the first subjob fails. + Checks that no storage fallback job is created. The deferred subjobs should remain + deferred because the first subjob failed. """ assets, sensors, _ = smart_building queue = app.queues["scheduling"] @@ -166,53 +166,51 @@ def test_create_sequential_jobs_fallback( storage_module = "flexmeasures.data.models.planning.storage" with patch(f"{storage_module}.StorageScheduler.persist_flex_model"): - with patch(f"{storage_module}.StorageFallbackScheduler.persist_flex_model"): - with patch( - f"{storage_module}.StorageScheduler.compute", - side_effect=iter([InfeasibleProblemException(), [], []]), - ): - create_sequential_scheduling_job( - asset=assets["Test Site"], - scheduler_specs=scheduler_specs, - enqueue=True, - force_new_job_creation=True, # otherwise the cache might kick in due to sub-jobs already created in other tests - **flex_description_sequential, - ) - - # There should be 3 jobs: - # 2 jobs scheduling the 2 flexible devices in the flex-model, plus 1 'done job' to wrap things up - queued_jobs = app.queues["scheduling"].jobs - deferred_jobs = [ - Job.fetch(job_id, connection=queue.connection) - for job_id in app.queues[ - "scheduling" - ].deferred_job_registry.get_job_ids() - ] - # Sort deferred_jobs by their created_at attribute - deferred_jobs = sorted(deferred_jobs, key=lambda job: job.created_at) - assert ( - len(queued_jobs) == 1 - ), "Only the job for scheduling the first device sequentially should be queued." - assert ( - len(deferred_jobs) == 2 - ), "The job for scheduling the second device, and the wrap-up job, should be deferred." - - # Work on jobs - work_on_rq(queue, exc_handler=handle_scheduling_exception) - - # Refresh jobs so that the fallback_job_id (which should be set by now) can be read - for job in queued_jobs: - job.refresh() - - finished_jobs = queue.finished_job_registry.get_job_ids() - failed_jobs = queue.failed_job_registry.get_job_ids() - - # Original job failed - assert queued_jobs[0].id in failed_jobs - - # The fallback job ran successfully - assert queued_jobs[0].meta["fallback_job_id"] in finished_jobs - - # The deferred jobs ran successfully - assert deferred_jobs[0].id in finished_jobs - assert deferred_jobs[1].id in finished_jobs + with patch( + f"{storage_module}.StorageScheduler.compute", + side_effect=InfeasibleProblemException(), + ): + create_sequential_scheduling_job( + asset=assets["Test Site"], + scheduler_specs=scheduler_specs, + enqueue=True, + force_new_job_creation=True, # otherwise the cache might kick in due to sub-jobs already created in other tests + **flex_description_sequential, + ) + + # There should be 3 jobs: + # 2 jobs scheduling the 2 flexible devices in the flex-model, plus 1 'done job' to wrap things up + queued_jobs = app.queues["scheduling"].jobs + deferred_jobs = [ + Job.fetch(job_id, connection=queue.connection) + for job_id in app.queues[ + "scheduling" + ].deferred_job_registry.get_job_ids() + ] + # Sort deferred_jobs by their created_at attribute + deferred_jobs = sorted(deferred_jobs, key=lambda job: job.created_at) + assert ( + len(queued_jobs) == 1 + ), "Only the job for scheduling the first device sequentially should be queued." + assert ( + len(deferred_jobs) == 2 + ), "The job for scheduling the second device, and the wrap-up job, should be deferred." + + # Work on jobs + work_on_rq(queue, exc_handler=handle_scheduling_exception) + + for job in queued_jobs: + job.refresh() + for job in deferred_jobs: + job.refresh() + + finished_jobs = queue.finished_job_registry.get_job_ids() + failed_jobs = queue.failed_job_registry.get_job_ids() + + # Original job failed and no fallback job was created + assert queued_jobs[0].id in failed_jobs + assert queued_jobs[0].meta.get("fallback_job_id") is None + + # The deferred jobs should not run when their dependency fails without fallback + assert deferred_jobs[0].id not in finished_jobs + assert deferred_jobs[1].id not in finished_jobs From 9d90ff917c99d602e33b83d48c3b12bdac88099b Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Thu, 25 Jun 2026 22:10:41 +0100 Subject: [PATCH 09/54] docs: describe default SoC relaxation metadata Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/data/schemas/scheduling/metadata.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/flexmeasures/data/schemas/scheduling/metadata.py b/flexmeasures/data/schemas/scheduling/metadata.py index 21c7765f18..97d2377b13 100644 --- a/flexmeasures/data/schemas/scheduling/metadata.py +++ b/flexmeasures/data/schemas/scheduling/metadata.py @@ -145,13 +145,14 @@ def to_dict(self): 2. Avoid not meeting SoC minima/maxima. 3. Avoid breaching the desired device consumption/production capacity. -We recommend to set this field to ``True`` to enable the default prices and associated priorities as defined by FlexMeasures. +SoC minima/maxima are already relaxed by default through ``relax-soc-constraints``. +Set this field to ``True`` to also enable the default site and device capacity breach prices and associated priorities as defined by FlexMeasures. For tighter control over prices and priorities, the breach prices can also be set explicitly (the relevant fields have ``breach-price`` in their name). """, example=True, ) RELAX_SOC_CONSTRAINTS = MetaData( - description="If True, avoids not meeting SoC minima/maxima as a relaxed constraint.", + description="If True (default), avoids not meeting SoC minima/maxima as relaxed constraints. Set this to False to keep SoC minima/maxima as hard constraints unless breach prices are supplied explicitly.", example=True, ) RELAX_CAPACITY_CONSTRAINTS = MetaData( @@ -246,8 +247,8 @@ def to_dict(self): ) SOC_MINIMA = MetaData( description="""Set points that form lower boundaries, e.g. to target a full car battery in the morning. -If a ``soc-minima-breach-price`` is defined, the ``soc-minima`` become soft constraints in the optimization problem. -Otherwise, they become hard constraints. [#maximum_overlap]_. Both single points in time and ranges are possible, see example.""", +The ``soc-minima`` are soft constraints in the optimization problem by default, because ``relax-soc-constraints`` defaults to ``True`` and supplies a default ``soc-minima-breach-price``. +Set ``relax-soc-constraints`` to ``False`` to keep them as hard constraints unless ``soc-minima-breach-price`` is supplied explicitly. [#maximum_overlap]_. Both single points in time and ranges are possible, see example.""", example=[ {"datetime": "2024-02-05T08:00:00+01:00", "value": "8.2 kWh"}, { @@ -259,8 +260,8 @@ def to_dict(self): ) SOC_MAXIMA = MetaData( description="""Set points that form upper boundaries at certain times, e.g. to target an empty heat buffer before a maintenance window. -If a ``soc-maxima-breach-price`` is defined, the ``soc-maxima`` become soft constraints in the optimization problem. -Otherwise, they become hard constraints. [#minimum_overlap]_""", +The ``soc-maxima`` are soft constraints in the optimization problem by default, because ``relax-soc-constraints`` defaults to ``True`` and supplies a default ``soc-maxima-breach-price``. +Set ``relax-soc-constraints`` to ``False`` to keep them as hard constraints unless ``soc-maxima-breach-price`` is supplied explicitly. [#minimum_overlap]_""", example=[ { "value": "51 kWh", From ed233429d56b8b8e6e2ad8b180bf0a169f5d3af1 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Thu, 25 Jun 2026 22:10:47 +0100 Subject: [PATCH 10/54] docs: update storage scheduling infeasibility guide Signed-off-by: Mohamed Belhsan Hmida --- documentation/features/scheduling.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/documentation/features/scheduling.rst b/documentation/features/scheduling.rst index 1643330a6d..4f53b5f809 100644 --- a/documentation/features/scheduling.rst +++ b/documentation/features/scheduling.rst @@ -269,12 +269,12 @@ However, here are some tips to model a buffer correctly: - Set ``storage-efficiency`` to a value below 100% to model (heat) loss. What happens if the flex model describes an infeasible problem for the storage scheduler? Excellent question! -It is highly important for a robust operation that these situations still lead to a somewhat good outcome. -From our practical experience, we derived a ``StorageFallbackScheduler``. -It simplifies an infeasible situation by just starting to charge, discharge, or do neither, -depending on the first target state of charge and the capabilities of the asset. +It is highly important for robust operation that these situations remain visible. +By default, ``soc-minima`` and ``soc-maxima`` are relaxed into soft constraints, so the scheduler can still return a useful schedule when these boundaries cannot be fully met. +Exact ``soc-targets`` and physical ``soc-min`` / ``soc-max`` bounds remain hard constraints. +If those hard constraints make the problem infeasible, the scheduling job fails instead of producing a fallback schedule. -Of course, we also log a failure in the scheduling job, so it's important to take note of these failures. Often, mis-configured flex models are the reason. +It is important to take note of these failures. Often, mis-configured flex models are the reason. For a hands-on tutorial on using some of the storage flex-model fields, head over to :ref:`tut_v2g` use case and `the API documentation for triggering schedules <../api/v3_0.html#post--api-v3_0-assets-id-schedules-trigger>`_. From 6483a52f72a311956eb699ee8c8e5a855dfae43f Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Thu, 25 Jun 2026 22:10:54 +0100 Subject: [PATCH 11/54] docs: clarify fallback redirects for custom schedulers Signed-off-by: Mohamed Belhsan Hmida --- documentation/api/introduction.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/documentation/api/introduction.rst b/documentation/api/introduction.rst index caa047d716..bac858783c 100644 --- a/documentation/api/introduction.rst +++ b/documentation/api/introduction.rst @@ -116,18 +116,18 @@ See Other (303) --------------- Some API responses return ``HTTP status 303 (See Other)`` to redirect the client to a different resource. -This happens, for example, when a scheduling job fails and a fallback schedule has been computed instead. +This can happen when a custom scheduler defines a fallback scheduler, the original scheduling job fails, and the fallback schedule has been computed instead. In that case, the response includes a ``Location`` header pointing to the fallback schedule endpoint, so clients can automatically retrieve the fallback result. The response body will contain a JSON message with a ``status`` field set to ``"UNKNOWN_SCHEDULE"`` and a ``message`` field explaining the reason for the redirect. .. note:: - The fallback schedule mechanism activates when the main scheduler encounters an infeasible problem (i.e. when constraints cannot be satisfied). - This is less likely to happen when ``"relax-constraints": true`` is set in the ``flex-context``, as constraint relaxation softens most infeasibility-causing constraints. - The hard constraints that remain even after constraint relaxation are ``soc-min``, ``soc-max``, ``soc-targets`` and ``power-capacity`` in the ``flex-model``, and ``site-power-capacity`` in the ``flex-context``. + FlexMeasures' built-in storage scheduler no longer computes a fallback schedule for infeasible problems. + Instead, ``soc-minima`` and ``soc-maxima`` are relaxed by default through ``"relax-soc-constraints": true``, while ``soc-min``, ``soc-max`` and ``soc-targets`` remain hard constraints. + If hard constraints cannot be satisfied, the scheduling job fails and clients receive the failure reason when requesting the schedule. - Server administrators can configure whether clients receive a 303 redirect (``FLEXMEASURES_FALLBACK_REDIRECT = True``) or whether FlexMeasures follows the fallback automatically and returns the fallback schedule directly (``FLEXMEASURES_FALLBACK_REDIRECT = False``, the default). + For custom schedulers that still define a fallback scheduler, server administrators can configure whether clients receive a 303 redirect (``FLEXMEASURES_FALLBACK_REDIRECT = True``) or whether FlexMeasures follows the fallback automatically and returns the fallback schedule directly (``FLEXMEASURES_FALLBACK_REDIRECT = False``, the default). Here is a client-side code example in Python for handling 303 redirects (this merely follows the redirect and should be revised to make use of the client's monitoring tools): From 2191013e9bd43b601f7781e2b6e596e961984f1b Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Thu, 25 Jun 2026 22:10:59 +0100 Subject: [PATCH 12/54] docs: scope fallback redirect configuration Signed-off-by: Mohamed Belhsan Hmida --- documentation/configuration.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/documentation/configuration.rst b/documentation/configuration.rst index 1b823aa585..99f9195ce2 100644 --- a/documentation/configuration.rst +++ b/documentation/configuration.rst @@ -787,7 +787,9 @@ Default: ``None`` (defaults are set internally for each sunset API version, e.g. FLEXMEASURES_FALLBACK_REDIRECT ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Control how the API handles a failed scheduling job when a fallback schedule has been computed. +Control how the API handles a failed scheduling job when a custom scheduler has computed a fallback schedule. + +FlexMeasures' built-in storage scheduler no longer computes fallback schedules, but custom schedulers may still define fallback schedulers. If ``True``, the API returns ``HTTP status 303 (See Other)`` with a ``Location`` header pointing to the fallback schedule endpoint. Clients must follow this redirect themselves to obtain the fallback schedule (see :ref:`api_see_other`). From c08ba18b68ee61beb73a5af6e70f1ed12217ea4f Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Thu, 25 Jun 2026 22:11:04 +0100 Subject: [PATCH 13/54] docs: refresh SoC relaxation OpenAPI text Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/ui/static/openapi-specs.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/flexmeasures/ui/static/openapi-specs.json b/flexmeasures/ui/static/openapi-specs.json index fd62b861ac..89efb79337 100644 --- a/flexmeasures/ui/static/openapi-specs.json +++ b/flexmeasures/ui/static/openapi-specs.json @@ -4583,13 +4583,13 @@ "relax-constraints": { "type": "boolean", "default": false, - "description": "If True (default is False), several constraints are relaxed by setting default breach prices within the optimization problem, leading to the default priority:\n\n1. Avoid breaching the site consumption/production capacity.\n2. Avoid not meeting SoC minima/maxima.\n3. Avoid breaching the desired device consumption/production capacity.\n\nWe recommend to set this field to True to enable the default prices and associated priorities as defined by FlexMeasures.\nFor tighter control over prices and priorities, the breach prices can also be set explicitly (the relevant fields have breach-price in their name).\n", + "description": "If True (default is False), several constraints are relaxed by setting default breach prices within the optimization problem, leading to the default priority:\n\n1. Avoid breaching the site consumption/production capacity.\n2. Avoid not meeting SoC minima/maxima.\n3. Avoid breaching the desired device consumption/production capacity.\n\nSoC minima/maxima are already relaxed by default through relax-soc-constraints.\nSet this field to True to also enable the default site and device capacity breach prices and associated priorities as defined by FlexMeasures.\nFor tighter control over prices and priorities, the breach prices can also be set explicitly (the relevant fields have breach-price in their name).\n", "example": true }, "relax-soc-constraints": { "type": "boolean", - "default": false, - "description": "If True, avoids not meeting SoC minima/maxima as a relaxed constraint.", + "default": true, + "description": "If True (default), avoids not meeting SoC minima/maxima as relaxed constraints. Set this to False to keep SoC minima/maxima as hard constraints unless breach prices are supplied explicitly.", "example": true }, "relax-capacity-constraints": { @@ -6302,4 +6302,4 @@ } } } -} \ No newline at end of file +} From 7135f338da7529e08bf40c886539716e7b305c21 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 26 Jun 2026 01:14:17 +0100 Subject: [PATCH 14/54] docs: add fallback scheduler changelog entry Signed-off-by: Mohamed Belhsan Hmida --- documentation/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 6e8d585933..a0b0e26f84 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -14,6 +14,7 @@ New features * Sensor references in flex-model and flex-context support various ways of filtering by source [see `PR #2209 `_] * Let storage scheduling infer missing ``power-capacity`` from directional device capacities before falling back to site capacity, and default the missing opposite capacity to zero when only a non-zero ``consumption-capacity`` or ``production-capacity`` is configured [see `PR #2222 `_] * CLI support for adding/editing account attributes [see `PR #2242 `_] +* Relax storage SoC constraints by default and report infeasible storage schedules directly instead of saving fallback schedules [see `PR #2252 `_] Infrastructure / Support From 429d63944b8b410858b9d877ad011470e76b490b Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 26 Jun 2026 01:35:41 +0100 Subject: [PATCH 15/54] test: allow small unit conversion drift Signed-off-by: Mohamed Belhsan Hmida --- .../api/v3_0/tests/test_sensors_api_freshdb.py | 4 ++-- flexmeasures/data/schemas/tests/test_sensor.py | 11 +++++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py b/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py index 18596d64aa..2a6a61d34f 100644 --- a/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py +++ b/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py @@ -246,7 +246,7 @@ def test_upload_sensor_data_with_unit_conversion_success( len(beliefs) == expected_num_beliefs ), f"Fetched {len(beliefs)} beliefs from the database, expecting {expected_num_beliefs}." - assert [b.event_value for b in beliefs] == expected_event_values + assert [b.event_value for b in beliefs] == pytest.approx(expected_event_values) @pytest.mark.parametrize( @@ -327,7 +327,7 @@ def test_upload_sensor_data_floors_offclock_datetimes( pd.testing.assert_index_equal( bdf.event_starts, pd.DatetimeIndex(expected_event_starts, name="event_start") ) - assert bdf["event_value"].to_list() == expected_event_values + assert bdf["event_value"].to_list() == pytest.approx(expected_event_values) @pytest.mark.parametrize( diff --git a/flexmeasures/data/schemas/tests/test_sensor.py b/flexmeasures/data/schemas/tests/test_sensor.py index 7d3c06fe4b..a4cd8b1474 100644 --- a/flexmeasures/data/schemas/tests/test_sensor.py +++ b/flexmeasures/data/schemas/tests/test_sensor.py @@ -20,6 +20,11 @@ def serialize_variable_quantity(value): return VariableQuantityDumpSchema().dump({"value": value})["value"] +def assert_quantity_equals(quantity, expected_quantity): + assert str(quantity.units) == str(expected_quantity.units) + assert quantity.magnitude == pytest.approx(expected_quantity.magnitude) + + @pytest.mark.parametrize( "src_quantity, dst_unit, fails, exp_dst_quantity", [ @@ -93,10 +98,12 @@ def test_quantity_or_sensor_deserialize( try: dst_quantity = schema.deserialize(src_quantity) if isinstance(src_quantity, (ur.Quantity, int, float)): - assert dst_quantity == ur.Quantity(exp_dst_quantity) + assert_quantity_equals(dst_quantity, ur.Quantity(exp_dst_quantity)) assert str(dst_quantity) == exp_dst_quantity elif isinstance(src_quantity, list): - assert dst_quantity[0]["value"] == ur.Quantity(exp_dst_quantity) + assert_quantity_equals( + dst_quantity[0]["value"], ur.Quantity(exp_dst_quantity) + ) assert str(dst_quantity[0]["value"]) == exp_dst_quantity assert not fails except ValidationError as e: From aedfa122d40be5b27c1395f7056358de63ba3ccb Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 26 Jun 2026 01:59:52 +0100 Subject: [PATCH 16/54] test: avoid exact quantity string comparisons Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/data/schemas/tests/test_sensor.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/flexmeasures/data/schemas/tests/test_sensor.py b/flexmeasures/data/schemas/tests/test_sensor.py index a4cd8b1474..ec574b0bcf 100644 --- a/flexmeasures/data/schemas/tests/test_sensor.py +++ b/flexmeasures/data/schemas/tests/test_sensor.py @@ -99,12 +99,10 @@ def test_quantity_or_sensor_deserialize( dst_quantity = schema.deserialize(src_quantity) if isinstance(src_quantity, (ur.Quantity, int, float)): assert_quantity_equals(dst_quantity, ur.Quantity(exp_dst_quantity)) - assert str(dst_quantity) == exp_dst_quantity elif isinstance(src_quantity, list): assert_quantity_equals( dst_quantity[0]["value"], ur.Quantity(exp_dst_quantity) ) - assert str(dst_quantity[0]["value"]) == exp_dst_quantity assert not fails except ValidationError as e: assert fails, e From 3a68fa788a572ff2184c437f262276b6a96568bc Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Sun, 28 Jun 2026 21:40:48 +0100 Subject: [PATCH 17/54] docs: clarify storage infeasibility behavior Signed-off-by: Mohamed Belhsan Hmida --- documentation/features/scheduling.rst | 5 ++--- flexmeasures/data/schemas/scheduling/metadata.py | 3 ++- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/documentation/features/scheduling.rst b/documentation/features/scheduling.rst index 4f53b5f809..4fef6110bb 100644 --- a/documentation/features/scheduling.rst +++ b/documentation/features/scheduling.rst @@ -268,13 +268,12 @@ However, here are some tips to model a buffer correctly: - Set ``charging-efficiency`` to the sensor describing the :abbr:`COP (coefficient of performance)` values. - Set ``storage-efficiency`` to a value below 100% to model (heat) loss. -What happens if the flex model describes an infeasible problem for the storage scheduler? Excellent question! -It is highly important for robust operation that these situations remain visible. +If the flex model describes an infeasible problem for the storage scheduler, the failure should remain visible. By default, ``soc-minima`` and ``soc-maxima`` are relaxed into soft constraints, so the scheduler can still return a useful schedule when these boundaries cannot be fully met. Exact ``soc-targets`` and physical ``soc-min`` / ``soc-max`` bounds remain hard constraints. If those hard constraints make the problem infeasible, the scheduling job fails instead of producing a fallback schedule. -It is important to take note of these failures. Often, mis-configured flex models are the reason. +It is important to take note of these failures. Often, misconfigured flex models are the reason. For a hands-on tutorial on using some of the storage flex-model fields, head over to :ref:`tut_v2g` use case and `the API documentation for triggering schedules <../api/v3_0.html#post--api-v3_0-assets-id-schedules-trigger>`_. diff --git a/flexmeasures/data/schemas/scheduling/metadata.py b/flexmeasures/data/schemas/scheduling/metadata.py index 97d2377b13..bfa2db6be7 100644 --- a/flexmeasures/data/schemas/scheduling/metadata.py +++ b/flexmeasures/data/schemas/scheduling/metadata.py @@ -248,7 +248,8 @@ def to_dict(self): SOC_MINIMA = MetaData( description="""Set points that form lower boundaries, e.g. to target a full car battery in the morning. The ``soc-minima`` are soft constraints in the optimization problem by default, because ``relax-soc-constraints`` defaults to ``True`` and supplies a default ``soc-minima-breach-price``. -Set ``relax-soc-constraints`` to ``False`` to keep them as hard constraints unless ``soc-minima-breach-price`` is supplied explicitly. [#maximum_overlap]_. Both single points in time and ranges are possible, see example.""", +Set ``relax-soc-constraints`` to ``False`` to keep them as hard constraints unless ``soc-minima-breach-price`` is supplied explicitly [#maximum_overlap]_. +Both single points in time and ranges are possible, see example.""", example=[ {"datetime": "2024-02-05T08:00:00+01:00", "value": "8.2 kWh"}, { From 79241eed7e24b9588e6c341439b0154249e308dc Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Sun, 28 Jun 2026 21:41:04 +0100 Subject: [PATCH 18/54] test: simplify storage fallback assertion comment Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/api/v3_0/tests/test_sensor_schedules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py index 37cb47bee7..392c2fb12b 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py @@ -377,7 +377,7 @@ def test_get_schedule_infeasible_storage_job_without_fallback( assert capacity == 2 assert charging_station.get_attribute("consumption-price") == {"sensor": epex_da.id} - # check that no retired storage fallback schedule has been saved before + # check that no storage fallback schedule has been saved before models = [ source.model for source in charging_station.search_beliefs().sources.unique() ] From 0f805b83a9168db2a3720150d3f6452e6665743d Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Sun, 28 Jun 2026 21:52:34 +0100 Subject: [PATCH 19/54] docs: regenerate openapi-specs.json Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/ui/static/openapi-specs.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flexmeasures/ui/static/openapi-specs.json b/flexmeasures/ui/static/openapi-specs.json index 89efb79337..9457e828d7 100644 --- a/flexmeasures/ui/static/openapi-specs.json +++ b/flexmeasures/ui/static/openapi-specs.json @@ -6124,7 +6124,7 @@ "example": true }, "soc-maxima": { - "description": "Set points that form upper boundaries at certain times, e.g. to target an empty heat buffer before a maintenance window.\nIf a soc-maxima-breach-price is defined, the soc-maxima become soft constraints in the optimization problem.\nOtherwise, they become hard constraints.", + "description": "Set points that form upper boundaries at certain times, e.g. to target an empty heat buffer before a maintenance window.\nThe soc-maxima are soft constraints in the optimization problem by default, because relax-soc-constraints defaults to True and supplies a default soc-maxima-breach-price.\nSet relax-soc-constraints to False to keep them as hard constraints unless soc-maxima-breach-price is supplied explicitly.", "example": [ { "value": "51 kWh", @@ -6135,7 +6135,7 @@ "$ref": "#/components/schemas/VariableQuantityOpenAPI" }, "soc-minima": { - "description": "Set points that form lower boundaries, e.g. to target a full car battery in the morning.\nIf a soc-minima-breach-price is defined, the soc-minima become soft constraints in the optimization problem.\nOtherwise, they become hard constraints.. Both single points in time and ranges are possible, see example.", + "description": "Set points that form lower boundaries, e.g. to target a full car battery in the morning.\nThe soc-minima are soft constraints in the optimization problem by default, because relax-soc-constraints defaults to True and supplies a default soc-minima-breach-price.\nSet relax-soc-constraints to False to keep them as hard constraints unless soc-minima-breach-price is supplied explicitly.\nBoth single points in time and ranges are possible, see example.", "example": [ { "datetime": "2024-02-05T08:00:00+01:00", @@ -6302,4 +6302,4 @@ } } } -} +} \ No newline at end of file From eaa4a7a8baf12be19c92de62c709fe1dcf14d4fc Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Sun, 5 Jul 2026 14:24:13 +0100 Subject: [PATCH 20/54] fix: preserve explicit zero breach prices Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/data/schemas/scheduling/__init__.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py index 97a2029b2a..ab24b7c621 100644 --- a/flexmeasures/data/schemas/scheduling/__init__.py +++ b/flexmeasures/data/schemas/scheduling/__init__.py @@ -394,8 +394,8 @@ def check_prices(self, data: dict, original_data: dict, **kwargs): # Fill in default soc breach prices when asked to relax SoC constraints, unless already set explicitly. if ( (data["relax_soc_constraints"] or data["relax_constraints"]) - and not data.get("soc_minima_breach_price") - and not data.get("soc_maxima_breach_price") + and data.get("soc_minima_breach_price") is None + and data.get("soc_maxima_breach_price") is None ): self.set_default_breach_prices( data, @@ -406,8 +406,8 @@ def check_prices(self, data: dict, original_data: dict, **kwargs): # Fill in default capacity breach prices when asked to relax capacity constraints, unless already set explicitly. if ( (data["relax_capacity_constraints"] or data["relax_constraints"]) - and not data.get("consumption_breach_price") - and not data.get("production_breach_price") + and data.get("consumption_breach_price") is None + and data.get("production_breach_price") is None ): self.set_default_breach_prices( data, @@ -418,8 +418,8 @@ def check_prices(self, data: dict, original_data: dict, **kwargs): # Fill in default site capacity breach prices when asked to relax site capacity constraints, unless already set explicitly. if ( (data["relax_site_capacity_constraints"] or data["relax_constraints"]) - and not data.get("ems_consumption_breach_price") - and not data.get("ems_production_breach_price") + and data.get("ems_consumption_breach_price") is None + and data.get("ems_production_breach_price") is None ): self.set_default_breach_prices( data, From 492519a7ddf90eb6e80a0edbc417aad05ef94036 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Mon, 6 Jul 2026 01:45:43 +0100 Subject: [PATCH 21/54] feat: support sensor reference defaults Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/data/models/planning/utils.py | 8 ++ flexmeasures/data/schemas/sensors.py | 90 ++++++++++++++++++---- 2 files changed, 83 insertions(+), 15 deletions(-) diff --git a/flexmeasures/data/models/planning/utils.py b/flexmeasures/data/models/planning/utils.py index 7cd79f7156..161d2d3810 100644 --- a/flexmeasures/data/models/planning/utils.py +++ b/flexmeasures/data/models/planning/utils.py @@ -299,6 +299,14 @@ def get_series_from_quantity_or_sensor( time_series = convert_units( time_series, variable_quantity.unit, unit, resolution ) + if variable_quantity.default is not None: + default_value = convert_units( + variable_quantity.default.magnitude, + str(variable_quantity.default.units), + unit, + resolution, + ) + time_series = time_series.fillna(default_value) elif isinstance(variable_quantity, Sensor): bdf: tb.BeliefsDataFrame = TimedBelief.search( variable_quantity, diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py index fc510adf05..ba66e57814 100644 --- a/flexmeasures/data/schemas/sensors.py +++ b/flexmeasures/data/schemas/sensors.py @@ -433,11 +433,28 @@ def _deserialize( ) -> Sensor | list[dict] | ur.Quantity: if isinstance(value, dict): - return self._deserialize_dict(value) + return self._deserialize_dict(value, attr, data, **kwargs) elif isinstance(value, list): return self._deserialize_list(value) elif isinstance(value, str): return self._deserialize_str(value) + elif isinstance(value, ur.Quantity): + return value.to(self.to_unit) + elif isinstance(value, tuple): + try: + return ur.Quantity.from_tuple(value).to(self.to_unit) + except Exception: + if ( + len(value) == 1 + and isinstance(value[0], numbers.Real) + and self.default_src_unit is not None + ): + return self._deserialize_numeric(value[0], attr, data, **kwargs) + if len(value) == 2: + return self._deserialize_str(f"{value[0]} {value[1]}") + raise FMValidationError( + f"Unsupported value type. `{type(value)}` was provided but only dict, list and str are supported." + ) elif isinstance(value, numbers.Real) and self.default_src_unit is not None: return self._deserialize_numeric(value, attr, data, **kwargs) else: @@ -501,13 +518,15 @@ def _deserialize_source_filters(self, value: dict[str, Any]) -> tuple[ return source_types, exclude_source_types, sources, source_account - def _deserialize_dict(self, value: dict[str, Any]) -> Sensor | SensorReference: + def _deserialize_dict( + self, value: dict[str, Any], attr, data, **kwargs + ) -> Sensor | SensorReference: """Deserialize a sensor reference to a Sensor or SensorReference. - Returns a plain :class:`Sensor` when no source filter keys are present - (backward compatible), and a :class:`SensorReference` when any of - ``source-types``, ``exclude-source-types``, ``sources``, or - ``source-account`` are provided. + Returns a plain :class:`Sensor` when no source filter or default keys are + present (backward compatible), and a :class:`SensorReference` when any of + ``source-types``, ``exclude-source-types``, ``sources``, ``source-account`` + or ``default`` are provided. """ if "sensor" not in value: raise FMValidationError("Dictionary provided but `sensor` key not found.") @@ -527,8 +546,12 @@ def _deserialize_dict(self, value: dict[str, Any]) -> Sensor | SensorReference: unit=self.to_unit if not self.to_unit.startswith("/") else None ).deserialize(value["sensor"], None, None) - # If source filter keys are present, return a SensorReference instead of a plain Sensor. - if self._SOURCE_FILTER_KEYS.isdisjoint(value.keys()): + default = None + if "default" in value and value["default"] is not None: + default = self._deserialize_default(value["default"], attr, data, **kwargs) + + # If no source filter or default keys are present, keep returning a plain Sensor. + if self._SOURCE_FILTER_KEYS.isdisjoint(value.keys()) and default is None: return sensor # backward compat: no filters → plain Sensor source_types, exclude_source_types, sources, source_account = ( @@ -540,8 +563,23 @@ def _deserialize_dict(self, value: dict[str, Any]) -> Sensor | SensorReference: exclude_source_types=exclude_source_types, sources=sources, source_account=source_account, + default=default, ) + def _deserialize_default(self, value, attr, data, **kwargs) -> ur.Quantity: + """Deserialize a sensor reference fallback value.""" + if isinstance(value, str): + default = self._deserialize_str(value) + elif isinstance(value, numbers.Real) and self.default_src_unit is not None: + default = self._deserialize_numeric(value, attr, data, **kwargs) + else: + raise FMValidationError( + "Sensor reference `default` must be a quantity string or a numeric value with a known default source unit." + ) + if self.value_validator is not None: + self.value_validator(default) + return default + def _deserialize_list(self, value: list[dict]) -> list[dict]: """Deserialize a time series to a list of timed events.""" if self.return_magnitude is True: @@ -593,6 +631,8 @@ def _serialize( sensor_reference["source-account"] = [ account.id for account in value.source_account ] + if value.default is not None: + sensor_reference["default"] = str(value.default.to(self.to_unit)) return sensor_reference elif isinstance(value, Sensor): return dict(sensor=value.id) @@ -948,13 +988,13 @@ class QuantitySchema(Schema): @dataclass class SensorReference: - """A sensor reference that wraps a Sensor with optional source filters for belief queries. + """A sensor reference that wraps a Sensor with optional query settings. Exposes the same ``unit``, ``id``, and ``event_resolution`` properties as a plain :class:`~flexmeasures.data.models.time_series.Sensor`, so code that reads those - properties works without modification. The source filters are passed through to - :meth:`TimedBelief.search ` - in :func:`~flexmeasures.data.models.planning.utils.get_series_from_quantity_or_sensor`. + properties works without modification. The source filters and optional default + value are passed through to + :func:`~flexmeasures.data.models.planning.utils.get_series_from_quantity_or_sensor`. """ sensor: Sensor @@ -962,6 +1002,7 @@ class SensorReference: exclude_source_types: list[str] | None = field(default=None) sources: list[DataSource] | None = field(default=None) source_account: list[Account] | None = field(default=None) + default: ur.Quantity | None = field(default=None) @property def unit(self) -> str: @@ -980,7 +1021,7 @@ def event_resolution(self) -> timedelta: class SensorReferenceSchema(Schema): - """Sensor reference with optional source filters.""" + """Sensor reference with optional source filters and fallback value.""" class Meta: description = "Sensor reference from which to look up a variable quantity." @@ -1022,6 +1063,13 @@ class Meta: description="Only use beliefs from data sources linked to these account IDs.", ), ) + default = fields.String( + load_default=None, + metadata=dict( + description="Fallback quantity to use when the referenced sensor has missing values.", + example="0 kWh", + ), + ) class TimeSeriesSchema(Schema): @@ -1031,9 +1079,21 @@ class TimeSeriesSchema(Schema): fields.Dict, required=True, metadata=dict( - description="Time series specification containing a list of segments that together describe a variable quantity.", + description=( + "Time series specification containing a list of segments that together " + "describe a variable quantity. Each segment may specify either " + "`datetime`, `start` and `end`, `start` and `duration`, or `end` and " + "`duration`." + ), example=[ - {"value": "23 kW", "start": "2025-11-20T15:15+01", "duration": "PT1H"} + {"value": "23 kW", "datetime": "2025-11-20T15:15+01"}, + { + "value": "24 kW", + "start": "2025-11-20T16:00+01", + "end": "2025-11-20T17:00+01", + }, + {"value": "25 kW", "start": "2025-11-20T17:00+01", "duration": "PT1H"}, + {"value": "26 kW", "end": "2025-11-20T19:00+01", "duration": "PT1H"}, ], ), ) From 07f0963f668b545efa2a437d6e863d9b9cbc2d45 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Mon, 6 Jul 2026 01:45:57 +0100 Subject: [PATCH 22/54] test: cover sensor reference defaults Signed-off-by: Mohamed Belhsan Hmida --- .../planning/tests/test_utils_fresh_db.py | 43 +++++++++++++++++++ .../data/schemas/tests/test_sensor.py | 24 +++++++++++ 2 files changed, 67 insertions(+) diff --git a/flexmeasures/data/models/planning/tests/test_utils_fresh_db.py b/flexmeasures/data/models/planning/tests/test_utils_fresh_db.py index da4a9cbc7b..7068a4763f 100644 --- a/flexmeasures/data/models/planning/tests/test_utils_fresh_db.py +++ b/flexmeasures/data/models/planning/tests/test_utils_fresh_db.py @@ -8,6 +8,7 @@ from flexmeasures.data.models.data_sources import DataSource from flexmeasures.data.schemas.sensors import SensorReference from flexmeasures.data.models.planning.utils import get_series_from_quantity_or_sensor +from flexmeasures.utils.unit_utils import ur def test_get_series_from_sensor_reference_source_filter_integration(fresh_db): @@ -86,6 +87,48 @@ def test_get_series_from_sensor_reference_source_filter_integration(fresh_db): assert result_forecaster.iloc[0] == pytest.approx(200.0) +def test_get_series_from_sensor_reference_default_fills_missing_values(fresh_db): + """A SensorReference default fills query slots with no matching sensor belief.""" + query_window = ( + pd.Timestamp("2025-06-01 08:00:00+02:00"), + pd.Timestamp("2025-06-01 08:30:00+02:00"), + ) + source = DataSource(name="test-default-source", type="scheduler") + fresh_db.session.add(source) + asset_type = GenericAssetType(name="test-asset-type-default") + fresh_db.session.add(asset_type) + asset = GenericAsset(name="test-asset-default", generic_asset_type=asset_type) + fresh_db.session.add(asset) + sensor = Sensor( + name="test-sensor-default", + generic_asset=asset, + event_resolution=timedelta(minutes=15), + unit="MW", + ) + fresh_db.session.add(sensor) + fresh_db.session.flush() + fresh_db.session.add( + TimedBelief( + event_start=query_window[0], + belief_horizon=timedelta(0), + event_value=0.1, + source=source, + sensor=sensor, + ) + ) + fresh_db.session.commit() + + result = get_series_from_quantity_or_sensor( + variable_quantity=SensorReference(sensor=sensor, default=ur.Quantity("1 MW")), + query_window=query_window, + resolution=sensor.event_resolution, + unit="kW", + as_instantaneous_events=False, + ) + + assert list(result) == pytest.approx([100.0, 1000.0]) + + def test_get_series_from_sensor_reference_sources_filter_integration(fresh_db): """A :class:`SensorReference` with ``sources`` returns only beliefs from the specified source. diff --git a/flexmeasures/data/schemas/tests/test_sensor.py b/flexmeasures/data/schemas/tests/test_sensor.py index ec574b0bcf..334e40cb24 100644 --- a/flexmeasures/data/schemas/tests/test_sensor.py +++ b/flexmeasures/data/schemas/tests/test_sensor.py @@ -210,6 +210,18 @@ def test_sensor_reference_backward_compatible(setup_dummy_sensors): assert result.id == sensor1.id +def test_sensor_reference_with_default(setup_dummy_sensors): + """``{"sensor": , "default": ...}`` deserializes to a SensorReference.""" + sensor1, _, _, _ = setup_dummy_sensors + field = VariableQuantityField(to_unit="MWh", return_magnitude=False) + + result = field.deserialize({"sensor": sensor1.id, "default": "500 kWh"}) + + assert isinstance(result, SensorReference) + assert result.sensor == sensor1 + assert result.default == ur.Quantity("0.5 MWh") + + def test_sensor_reference_with_source_types(setup_dummy_sensors): """``{"sensor": , "source-types": [...]}`` deserializes to a :class:`SensorReference`. @@ -332,6 +344,18 @@ def test_sensor_reference_serialization_preserves_source_filters( } +def test_sensor_reference_serialization_preserves_default(setup_dummy_sensors): + sensor1, _, _, _ = setup_dummy_sensors + field = VariableQuantityField(to_unit="MWh", return_magnitude=False) + source_reference = field.deserialize({"sensor": sensor1.id, "default": "500 kWh"}) + + assert isinstance(source_reference, SensorReference) + assert serialize_variable_quantity(source_reference) == { + "sensor": sensor1.id, + "default": "0.5 MWh", + } + + def test_sensor_reference_filters_are_kept_per_reference( setup_dummy_sensors, setup_sources, db ): From 2711f81337213ff26cc39da3468ece0612f7045f Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Mon, 6 Jul 2026 01:46:05 +0100 Subject: [PATCH 23/54] feat: accept dynamic canonical soc bounds Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/data/models/planning/storage.py | 6 +- .../data/schemas/scheduling/storage.py | 61 ++++++++++++++++--- 2 files changed, 55 insertions(+), 12 deletions(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 75705f06d6..57c32dae49 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -1598,9 +1598,9 @@ def _build_soc_schedule( For sensors with a '%' unit, the soc-max flex-model field is used as capacity. If soc-max is missing or zero for a '%' sensor, the schedule is skipped with a warning. - Note: soc-max is a QuantityField (not a VariableQuantityField), so it is always a float - after deserialization and cannot be a sensor reference. The isinstance guard below is - therefore a defensive check for forward-compatibility. + Note: dynamic soc-max values are routed to soc_maxima during deserialization, so + soc_max is still a fixed float after deserialization. The isinstance guard below + is therefore a defensive check for malformed data. """ soc_schedule = {} for d, flex_model_d in enumerate(flex_model): diff --git a/flexmeasures/data/schemas/scheduling/storage.py b/flexmeasures/data/schemas/scheduling/storage.py index 8453c2149d..5f328f6afa 100644 --- a/flexmeasures/data/schemas/scheduling/storage.py +++ b/flexmeasures/data/schemas/scheduling/storage.py @@ -113,19 +113,20 @@ class StorageFlexModelSchema(Schema): metadata=metadata.SOC_AT_START.to_dict(), ) - soc_min = QuantityField( - validate=validate.Range(min=ur.Quantity("0 MWh")), + soc_min = VariableQuantityField( to_unit="MWh", default_src_unit="dimensionless", # placeholder, overridden in __init__ - return_magnitude=False, + timezone="placeholder", data_key="soc-min", + value_validator=validate.Range(min=0), metadata=metadata.SOC_MIN.to_dict(), ) - soc_max = QuantityField( + soc_max = VariableQuantityField( to_unit="MWh", default_src_unit="dimensionless", # placeholder, overridden in __init__ - return_magnitude=False, + timezone="placeholder", data_key="soc-max", + value_validator=validate.Range(min=0), metadata=metadata.SOC_MAX.to_dict(), ) @@ -167,7 +168,8 @@ class StorageFlexModelSchema(Schema): default_src_unit="dimensionless", # placeholder, overridden in __init__ timezone="placeholder", data_key="soc-maxima", - metadata=metadata.SOC_MAXIMA.to_dict(), + value_validator=validate.Range(min=0), + metadata={**metadata.SOC_MAXIMA.to_dict(), "deprecated": True}, ) soc_minima = VariableQuantityField( @@ -176,7 +178,7 @@ class StorageFlexModelSchema(Schema): timezone="placeholder", data_key="soc-minima", value_validator=validate.Range(min=0), - metadata=metadata.SOC_MINIMA.to_dict(), + metadata={**metadata.SOC_MINIMA.to_dict(), "deprecated": True}, ) soc_targets = VariableQuantityField( @@ -276,12 +278,31 @@ def __init__( else: default_soc_unit = "MWh" + self.soc_min = VariableQuantityField( + to_unit="MWh", + default_src_unit=default_soc_unit, + timezone=self.timezone, + event_resolution=self.flooring_resolution, + data_key="soc-min", + value_validator=validate.Range(min=0), + ) + + self.soc_max = VariableQuantityField( + to_unit="MWh", + default_src_unit=default_soc_unit, + timezone=self.timezone, + event_resolution=self.flooring_resolution, + data_key="soc-max", + value_validator=validate.Range(min=0), + ) + self.soc_maxima = VariableQuantityField( to_unit="MWh", default_src_unit=default_soc_unit, timezone=self.timezone, event_resolution=self.flooring_resolution, data_key="soc-maxima", + value_validator=validate.Range(min=0), ) self.soc_minima = VariableQuantityField( @@ -305,6 +326,9 @@ def __init__( for field in self.fields.keys(): if field.startswith("soc_"): setattr(self.fields[field], "default_src_unit", default_soc_unit) + for field in ("soc_min", "soc_max", "soc_minima", "soc_maxima", "soc_targets"): + setattr(self.fields[field], "timezone", self.timezone) + setattr(self.fields[field], "event_resolution", self.flooring_resolution) @validates_schema def check_whether_targets_exceed_max_planning_horizon(self, data: dict, **kwargs): @@ -420,11 +444,28 @@ def post_load_sequence(self, data: dict, **kwargs) -> dict: if data.get("soc_at_start") is not None: data["soc_at_start"] = (data["soc_at_start"] / ur.Quantity("MWh")).magnitude + dynamic_types = (Sensor, SensorReference, list) + if isinstance(data.get("soc_min"), dynamic_types): + if data.get("soc_minima") is not None: + raise ValidationError( + "Fields `soc-min` and `soc-minima` are mutually exclusive.", + field_name="soc-min", + ) + data["soc_minima"] = data.pop("soc_min") + + if isinstance(data.get("soc_max"), dynamic_types): + if data.get("soc_maxima") is not None: + raise ValidationError( + "Fields `soc-max` and `soc-maxima` are mutually exclusive.", + field_name="soc-max", + ) + data["soc_maxima"] = data.pop("soc_max") + # Convert soc_min to dimensionless - if data.get("soc_min") is not None: + if isinstance(data.get("soc_min"), ur.Quantity): data["soc_min"] = (data["soc_min"] / ur.Quantity("MWh")).magnitude # Convert soc_max to dimensionless - if data.get("soc_max") is not None: + if isinstance(data.get("soc_max"), ur.Quantity): data["soc_max"] = (data["soc_max"] / ur.Quantity("MWh")).magnitude return data @@ -459,6 +500,7 @@ class DBStorageFlexModelSchema(Schema): data_key="soc-minima", required=False, value_validator=validate.Range(min=0), + metadata={"deprecated": True}, ) soc_maxima = VariableQuantityField( @@ -466,6 +508,7 @@ class DBStorageFlexModelSchema(Schema): data_key="soc-maxima", required=False, value_validator=validate.Range(min=0), + metadata={"deprecated": True}, ) soc_targets = VariableQuantityField( From bf315d2951715cc2fea7f52c222bfd9725c05744 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Mon, 6 Jul 2026 01:46:10 +0100 Subject: [PATCH 24/54] test: cover dynamic canonical soc bounds Signed-off-by: Mohamed Belhsan Hmida --- .../data/models/planning/tests/test_solver.py | 84 +++++++++-- .../data/schemas/tests/test_scheduling.py | 136 +++++++++++++++++- 2 files changed, 210 insertions(+), 10 deletions(-) diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py index 8c37ff05c2..43f5517494 100644 --- a/flexmeasures/data/models/planning/tests/test_solver.py +++ b/flexmeasures/data/models/planning/tests/test_solver.py @@ -2452,6 +2452,17 @@ def compute_schedule(flex_model): # because soc-maxima = soc-minima = soc-targets assert all(abs(soc[8:].values - expected_soc_schedule) < 1e-5) + # remove legacy soc-minima/soc-maxima and use dynamic canonical soc-min/soc-max + del flex_model["soc-minima"] + del flex_model["soc-maxima"] + flex_model["soc-min"] = {"sensor": soc_minima.id, "default": "0 MWh"} + flex_model["soc-max"] = {"sensor": soc_maxima.id, "default": "10 MWh"} + schedule = compute_schedule(flex_model) + + soc = check_constraints(power, schedule, soc_at_start) + + assert all(abs(soc[8:].values - expected_soc_schedule) < 1e-5) + @pytest.mark.parametrize("unit", [None, "MWh", "kWh"]) @pytest.mark.parametrize("soc_unit", ["kWh", "MWh"]) @@ -2537,7 +2548,7 @@ def test_battery_storage_different_units( @pytest.mark.parametrize( - "ts_field, ts_specs", + "ts_field, ts_specs, expected_charge, expected_discharge", [ # The battery only has time to charge up to 950 kWh halfway ( @@ -2549,6 +2560,8 @@ def test_battery_storage_different_units( "value": "850 kW", } ], + 0.85, + -0.85, ), # Same, but the event time is specified with a duration instead of an end time ( @@ -2560,6 +2573,8 @@ def test_battery_storage_different_units( "value": "850 kW", } ], + 0.85, + -0.85, ), # Can only charge up to 950 kWh halfway ( @@ -2570,6 +2585,20 @@ def test_battery_storage_different_units( "value": "950 kWh", } ], + 0.85, + -0.85, + ), + # Same dynamic maximum through the canonical soc-max field + ( + "soc-max", + [ + { + "datetime": "2015-01-02T16:00+01", + "value": "950 kWh", + } + ], + 0.85, + -0.85, ), # Must end up at a maximum of 200 kWh, for which it is cheapest to charge to 950 and then to discharge to 200 ( @@ -2581,6 +2610,47 @@ def test_battery_storage_different_units( "value": "200 kWh", } ], + 0.85, + -0.85, + ), + # Same dynamic maximum through the canonical soc-max field + ( + "soc-max", + [ + { + "start": "2015-01-02T16:45+01", + "duration": "PT15M", + "value": "200 kWh", + } + ], + 0.85, + -0.85, + ), + # Must end up at a minimum of 200 kWh, so it is cheapest to fill completely and then discharge to 200 + ( + "soc-minima", + [ + { + "start": "2015-01-02T16:45+01", + "duration": "PT15M", + "value": "200 kWh", + } + ], + 0.9, + -0.8, + ), + # Same dynamic minimum through the canonical soc-min field + ( + "soc-min", + [ + { + "start": "2015-01-02T16:45+01", + "duration": "PT15M", + "value": "200 kWh", + } + ], + 0.9, + -0.8, ), ], ) @@ -2589,6 +2659,8 @@ def test_battery_storage_with_time_series_in_flex_model( db, ts_field, ts_specs, + expected_charge, + expected_discharge, ): """ Test scheduling a 1 MWh battery for 2h with a low -> high price transition with @@ -2638,14 +2710,8 @@ def test_battery_storage_with_time_series_in_flex_model( soc_at_start = ur.Quantity(soc_at_start).to("MWh").magnitude check_constraints(battery, schedule, soc_at_start) - # charge 850 kWh in the cheap price period (100 kWh -> 950kWh) - assert schedule[:4].sum() * 0.25 == pytest.approx(0.85) - - # discharge fully or to what's needed in the expensive price period (950 kWh -> 100 or 200 kWh) - if ts_field == "soc-minima": - assert schedule[4:].sum() * 0.25 == pytest.approx(-0.75) - else: - assert schedule[4:].sum() * 0.25 == pytest.approx(-0.85) + assert schedule[:4].sum() * 0.25 == pytest.approx(expected_charge) + assert schedule[4:].sum() * 0.25 == pytest.approx(expected_discharge) def test_unavoidable_capacity_breach(): diff --git a/flexmeasures/data/schemas/tests/test_scheduling.py b/flexmeasures/data/schemas/tests/test_scheduling.py index 4522d97484..88a99419a5 100644 --- a/flexmeasures/data/schemas/tests/test_scheduling.py +++ b/flexmeasures/data/schemas/tests/test_scheduling.py @@ -15,7 +15,12 @@ DBStorageFlexModelSchema, ) from flexmeasures.data.models.time_series import Sensor -from flexmeasures.data.schemas.sensors import TimedEventSchema, VariableQuantityField +from flexmeasures.data.schemas.sensors import ( + SensorReference, + TimedEventSchema, + VariableQuantityField, +) +from flexmeasures.utils.unit_utils import ur @pytest.mark.parametrize( @@ -878,6 +883,38 @@ def test_storage_flex_model_schema_rejects_filtered_production( assert "cannot use source filters" in str(exc_info.value) +def test_soc_min_sensor_reference_with_default_loads_as_dynamic_minimum( + setup_dummy_sensors, +): + energy_sensor, _, _, _ = setup_dummy_sensors + schema = StorageFlexModelSchema(start=datetime(2026, 6, 1), sensor=None) + + loaded_flex_model = schema.load( + {"soc-min": {"sensor": energy_sensor.id, "default": "0 kWh"}} + ) + + assert "soc_min" not in loaded_flex_model + assert isinstance(loaded_flex_model["soc_minima"], SensorReference) + assert loaded_flex_model["soc_minima"].sensor == energy_sensor + assert loaded_flex_model["soc_minima"].default == ur.Quantity("0 MWh") + + +def test_soc_max_sensor_reference_with_default_loads_as_dynamic_maximum( + setup_dummy_sensors, +): + energy_sensor, _, _, _ = setup_dummy_sensors + schema = StorageFlexModelSchema(start=datetime(2026, 6, 1), sensor=None) + + loaded_flex_model = schema.load( + {"soc-max": {"sensor": energy_sensor.id, "default": "1000 kWh"}} + ) + + assert "soc_max" not in loaded_flex_model + assert isinstance(loaded_flex_model["soc_maxima"], SensorReference) + assert loaded_flex_model["soc_maxima"].sensor == energy_sensor + assert loaded_flex_model["soc_maxima"].default == ur.Quantity("1 MWh") + + @pytest.mark.parametrize( ["flex_model", "fails"], [ @@ -889,6 +926,103 @@ def test_storage_flex_model_schema_rejects_filtered_production( {"soc-min": "3500 kWh"}, False, ), + ( + {"soc-max": "3500 kWh"}, + False, + ), + ( + {"soc-max": (1, "MWh")}, + False, + ), + ( + {"soc-max": (1,)}, + [ + False, + { + "soc-max": "Unsupported value type. `` was provided but only dict, list and str are supported." + }, + ], + ), + ( + {"soc-max": ur.Quantity("1 MWh")}, + False, + ), + ( + {"soc-max": ur.Quantity("1 MWh").to_tuple()}, + False, + ), + ( + {"soc-min": {"sensor": "energy-sensor", "default": "0 kWh"}}, + False, + ), + ( + {"soc-max": {"sensor": "energy-sensor", "default": "1 MWh"}}, + False, + ), + ( + {"soc-min": {"sensor": "price-sensor", "default": "0 kWh"}}, + {"soc-min": "Cannot convert EUR/MWh to MWh"}, + ), + ( + {"soc-max": {"sensor": "price-sensor", "default": "1 MWh"}}, + {"soc-max": "Cannot convert EUR/MWh to MWh"}, + ), + ( + { + "soc-min": [ + { + "datetime": "2026-06-01T12:00:00+00:00", + "value": "1 MWh", + } + ] + }, + [ + False, + { + "soc-min": "A time series specification (listing segments) is not supported when storing flex-model fields." + }, + ], + ), + ( + { + "soc-max": [ + { + "datetime": "2026-06-01T12:00:00+00:00", + "value": "2 MWh", + } + ] + }, + [ + False, + { + "soc-max": "A time series specification (listing segments) is not supported when storing flex-model fields." + }, + ], + ), + ( + { + "soc-min": {"sensor": "energy-sensor", "default": "0 kWh"}, + "soc-minima": {"sensor": "energy-sensor"}, + }, + [ + { + "soc-min": "Fields `soc-min` and `soc-minima` are mutually exclusive." + }, + False, + ], + ), + ( + { + "soc-max": {"sensor": "energy-sensor", "default": "1 MWh"}, + "soc-maxima": {"sensor": "energy-sensor"}, + }, + [ + { + "soc-max": "Fields `soc-max` and `soc-maxima` are mutually exclusive." + }, + False, + ], + ), ( {"soc-minima": {"sensor": "energy-sensor"}}, False, From 0291546687fd4ab977b5367c80dd9e13ad9c2d08 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Mon, 6 Jul 2026 01:55:23 +0100 Subject: [PATCH 25/54] docs: update soc bounds schema metadata Signed-off-by: Mohamed Belhsan Hmida --- .../data/schemas/scheduling/__init__.py | 8 ++-- .../data/schemas/scheduling/metadata.py | 38 +++++++++++-------- 2 files changed, 26 insertions(+), 20 deletions(-) diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py index ab24b7c621..e33d9781d5 100644 --- a/flexmeasures/data/schemas/scheduling/__init__.py +++ b/flexmeasures/data/schemas/scheduling/__init__.py @@ -611,7 +611,7 @@ def _to_currency_per_mwh(price_unit: str) -> str: "description": rst_to_openapi(metadata.SOC_MIN.description), "types": { "backend": "typeThree", - "ui": "One fixed value or a dynamic signal (via a sensor).", + "ui": "A fixed lower boundary or a dynamic lower boundary with an optional default fallback.", }, "example-units": EXAMPLE_UNIT_TYPES["energy"], }, @@ -620,7 +620,7 @@ def _to_currency_per_mwh(price_unit: str) -> str: "description": rst_to_openapi(metadata.SOC_MAX.description), "types": { "backend": "typeThree", - "ui": "One fixed value or a dynamic signal (via a sensor).", + "ui": "A fixed upper boundary or a dynamic upper boundary with an optional default fallback.", }, "example-units": EXAMPLE_UNIT_TYPES["energy"], }, @@ -629,7 +629,7 @@ def _to_currency_per_mwh(price_unit: str) -> str: "description": rst_to_openapi(metadata.SOC_MINIMA.description), "types": { "backend": "typeTwo", - "ui": "A sensor which records the state of charge.", + "ui": "Deprecated alias for dynamic soc-min values.", }, "example-units": EXAMPLE_UNIT_TYPES["energy"], }, @@ -638,7 +638,7 @@ def _to_currency_per_mwh(price_unit: str) -> str: "description": rst_to_openapi(metadata.SOC_MAXIMA.description), "types": { "backend": "typeTwo", - "ui": "A sensor which records the state of charge.", + "ui": "Deprecated alias for dynamic soc-max values.", }, "example-units": EXAMPLE_UNIT_TYPES["energy"], }, diff --git a/flexmeasures/data/schemas/scheduling/metadata.py b/flexmeasures/data/schemas/scheduling/metadata.py index bfa2db6be7..f934b015a4 100644 --- a/flexmeasures/data/schemas/scheduling/metadata.py +++ b/flexmeasures/data/schemas/scheduling/metadata.py @@ -105,20 +105,20 @@ def to_dict(self): example="260 EUR/MW", ) SOC_MINIMA_BREACH_PRICE = MetaData( - description="""This **penalty value** is used to discourage the violation of ``soc-minima`` constraints in the flex-model, which the scheduler will attempt to minimize. + description="""This **penalty value** is used to discourage the violation of dynamic lower SoC boundary constraints in the flex-model, which the scheduler will attempt to minimize. It must use the same currency as the other price settings and cannot be negative. While it's an internal nudge to steer the scheduler—and doesn't represent a real-life cost—it should still be chosen in proportion to the actual energy prices at your site. If it's too high, it will overly dominate other constraints; if it's too low, it will have no effect. -Without this value, the soc-minima become hard constraints, which means that any infeasible state-of-charge minima would prevent a complete schedule from being computed. [#penalty_field]_ [#breach_field]_ +Without this value, dynamic ``soc-min`` boundaries and legacy ``soc-minima`` boundaries become hard constraints, which means that any infeasible state-of-charge minima would prevent a complete schedule from being computed. [#penalty_field]_ [#breach_field]_ """, example="120 EUR/kWh", ) SOC_MAXIMA_BREACH_PRICE = MetaData( - description="""This **penalty value** is used to discourage the violation of ``soc-maxima`` constraints in the flex-model, which the scheduler will attempt to minimize. + description="""This **penalty value** is used to discourage the violation of dynamic upper SoC boundary constraints in the flex-model, which the scheduler will attempt to minimize. It must use the same currency as the other price settings and cannot be negative. While it's an **internal nudge** to steer the scheduler—and doesn't represent a real-life cost—it should still be chosen in proportion to the actual energy prices at your site. If it's too high, it will overly dominate other constraints; if it's too low, it will have no effect. -Without this value, the soc-maxima become hard constraints, which means that any infeasible state-of-charge maxima would prevent a complete schedule from being computed. [#penalty_field]_ [#breach_field]_ +Without this value, dynamic ``soc-max`` boundaries and legacy ``soc-maxima`` boundaries become hard constraints, which means that any infeasible state-of-charge maxima would prevent a complete schedule from being computed. [#penalty_field]_ [#breach_field]_ """, example="120 EUR/kWh", ) @@ -230,24 +230,29 @@ def to_dict(self): example="kWh", ) SOC_MIN = MetaData( - description="""A constant and non-negotiable lower boundary for all SoC values in the schedule. + description="""Lower boundary for all SoC values in the schedule. If omitted, no lower boundary is applied. -If used, this is regarded as an unsurpassable physical limitation. -To set softer boundaries, use the ``soc-minima`` flex-model field instead together with the ``soc-minima-breach-price`` field in the flex-context. [#quantity_field]_ +When passed as a fixed quantity, this is regarded as an unsurpassable physical limitation. +When passed as a sensor reference or time series, it defines dynamic lower boundaries. Dynamic boundaries are soft constraints by default, because ``relax-soc-constraints`` defaults to ``True`` and supplies a default ``soc-minima-breach-price``. +Sensor references may include a ``default`` fallback quantity for missing sensor values, for example ``{"sensor": 50, "default": "0 kWh"}``. +Set ``relax-soc-constraints`` to ``False`` to keep dynamic lower boundaries as hard constraints unless ``soc-minima-breach-price`` is supplied explicitly. [#maximum_overlap]_ [#projecting_scheduling_constraints]_ """, - example="2.5 kWh", + example={"sensor": 50, "default": "0 kWh"}, ) SOC_MAX = MetaData( - description="""A constant and non-negotiable upper boundary for all values in the schedule (for storage devices, this defaults to max soc-target, if that is provided). + description="""Upper boundary for all SoC values in the schedule (for storage devices, this defaults to max soc-target, if that is provided). If omitted, no upper boundary is applied. -If used, this is regarded as an unsurpassable physical limitation. -To set softer boundaries, use the ``soc-maxima`` flex-model field instead together with the ``soc-maxima-breach-price`` field in the flex-context. [#quantity_field]_ +When passed as a fixed quantity, this is regarded as an unsurpassable physical limitation. +When passed as a sensor reference or time series, it defines dynamic upper boundaries. Dynamic boundaries are soft constraints by default, because ``relax-soc-constraints`` defaults to ``True`` and supplies a default ``soc-maxima-breach-price``. +Sensor references may include a ``default`` fallback quantity for missing sensor values, for example ``{"sensor": 51, "default": "100 kWh"}``. +Set ``relax-soc-constraints`` to ``False`` to keep dynamic upper boundaries as hard constraints unless ``soc-maxima-breach-price`` is supplied explicitly. [#minimum_overlap]_ [#projecting_scheduling_constraints]_ """, - example="7 kWh", + example={"sensor": 51, "default": "100 kWh"}, ) SOC_MINIMA = MetaData( - description="""Set points that form lower boundaries, e.g. to target a full car battery in the morning. -The ``soc-minima`` are soft constraints in the optimization problem by default, because ``relax-soc-constraints`` defaults to ``True`` and supplies a default ``soc-minima-breach-price``. + description="""[Deprecated field] Use dynamic ``soc-min`` values instead. +Set points that form lower boundaries, e.g. to target a full car battery in the morning. +The ``soc-minima`` legacy alias is soft in the optimization problem by default, because ``relax-soc-constraints`` defaults to ``True`` and supplies a default ``soc-minima-breach-price``. Set ``relax-soc-constraints`` to ``False`` to keep them as hard constraints unless ``soc-minima-breach-price`` is supplied explicitly [#maximum_overlap]_. Both single points in time and ranges are possible, see example.""", example=[ @@ -260,8 +265,9 @@ def to_dict(self): ], ) SOC_MAXIMA = MetaData( - description="""Set points that form upper boundaries at certain times, e.g. to target an empty heat buffer before a maintenance window. -The ``soc-maxima`` are soft constraints in the optimization problem by default, because ``relax-soc-constraints`` defaults to ``True`` and supplies a default ``soc-maxima-breach-price``. + description="""[Deprecated field] Use dynamic ``soc-max`` values instead. +Set points that form upper boundaries at certain times, e.g. to target an empty heat buffer before a maintenance window. +The ``soc-maxima`` legacy alias is soft in the optimization problem by default, because ``relax-soc-constraints`` defaults to ``True`` and supplies a default ``soc-maxima-breach-price``. Set ``relax-soc-constraints`` to ``False`` to keep them as hard constraints unless ``soc-maxima-breach-price`` is supplied explicitly. [#minimum_overlap]_""", example=[ { From 08bf214fcb2f1fa11772b73c8a3aff7e183a4339 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Mon, 6 Jul 2026 01:55:47 +0100 Subject: [PATCH 26/54] docs: update soc bounds api examples Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/api/v3_0/assets.py | 2 +- flexmeasures/api/v3_0/sensors.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/flexmeasures/api/v3_0/assets.py b/flexmeasures/api/v3_0/assets.py index 4c66094d3e..b4e4d774e1 100644 --- a/flexmeasures/api/v3_0/assets.py +++ b/flexmeasures/api/v3_0/assets.py @@ -1433,7 +1433,7 @@ def trigger_schedule( power-capacity: 25 kW consumption-capacity: {sensor: 42} production-capacity: 30 kW - soc-minima: + soc-min: - {start: "2015-06-02T12:00:00+00:00", end: "2015-06-02T13:00:00+00:00", value: 10 kWh} - sensor: 932 consumption-capacity: 0 kW diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index 1c5e53d653..164dfde945 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -940,9 +940,9 @@ def trigger_schedule( soc-targets: - value: "25 kWh" datetime: "2015-06-02T16:00:00+00:00" - soc-minima: + soc-min: sensor: 300 - soc-min: "10 kWh" + default: "10 kWh" soc-max: "25 kWh" charging-efficiency: "120%" discharging-efficiency: From ec2fb40777d922c8be6766f6eb35dfea638c1072 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Mon, 6 Jul 2026 02:01:00 +0100 Subject: [PATCH 27/54] docs: explain canonical dynamic soc bounds Signed-off-by: Mohamed Belhsan Hmida --- documentation/api/introduction.rst | 3 ++- documentation/features/scheduling.rst | 7 ++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/documentation/api/introduction.rst b/documentation/api/introduction.rst index bac858783c..56d9219975 100644 --- a/documentation/api/introduction.rst +++ b/documentation/api/introduction.rst @@ -124,7 +124,8 @@ The response body will contain a JSON message with a ``status`` field set to ``" .. note:: FlexMeasures' built-in storage scheduler no longer computes a fallback schedule for infeasible problems. - Instead, ``soc-minima`` and ``soc-maxima`` are relaxed by default through ``"relax-soc-constraints": true``, while ``soc-min``, ``soc-max`` and ``soc-targets`` remain hard constraints. + Instead, dynamic ``soc-min`` and ``soc-max`` boundaries, including the legacy aliases ``soc-minima`` and ``soc-maxima``, are relaxed by default through ``"relax-soc-constraints": true``. + Fixed ``soc-min`` / ``soc-max`` values and exact ``soc-targets`` remain hard constraints. If hard constraints cannot be satisfied, the scheduling job fails and clients receive the failure reason when requesting the schedule. For custom schedulers that still define a fallback scheduler, server administrators can configure whether clients receive a 303 redirect (``FLEXMEASURES_FALLBACK_REDIRECT = True``) or whether FlexMeasures follows the fallback automatically and returns the fallback schedule directly (``FLEXMEASURES_FALLBACK_REDIRECT = False``, the default). diff --git a/documentation/features/scheduling.rst b/documentation/features/scheduling.rst index 1debe7bb03..706527d937 100644 --- a/documentation/features/scheduling.rst +++ b/documentation/features/scheduling.rst @@ -264,13 +264,14 @@ If you model devices that *buffer* energy (e.g. thermal energy storage systems c However, here are some tips to model a buffer correctly: - Describe the thermal energy content in kWh or MWh. - - Set ``soc-minima`` to the accumulative usage forecast. + - Set dynamic ``soc-min`` values to the accumulative usage forecast. - Set ``charging-efficiency`` to the sensor describing the :abbr:`COP (coefficient of performance)` values. - Set ``storage-efficiency`` to a value below 100% to model (heat) loss. If the flex model describes an infeasible problem for the storage scheduler, the failure should remain visible. -By default, ``soc-minima`` and ``soc-maxima`` are relaxed into soft constraints, so the scheduler can still return a useful schedule when these boundaries cannot be fully met. -Exact ``soc-targets`` and physical ``soc-min`` / ``soc-max`` bounds remain hard constraints. +By default, dynamic ``soc-min`` and ``soc-max`` boundaries are relaxed into soft constraints, so the scheduler can still return a useful schedule when these boundaries cannot be fully met. +The legacy ``soc-minima`` and ``soc-maxima`` aliases follow the same behavior. +Exact ``soc-targets`` and fixed physical ``soc-min`` / ``soc-max`` bounds remain hard constraints. If those hard constraints make the problem infeasible, the scheduling job fails instead of producing a fallback schedule. It is important to take note of these failures. Often, misconfigured flex models are the reason. From 16f8c53c3759c8593c8283f580dbde1bfbba8be9 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Mon, 6 Jul 2026 02:01:22 +0100 Subject: [PATCH 28/54] docs: regenerate openapi specs Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/ui/static/openapi-specs.json | 64 +++++++++++++++++------ 1 file changed, 47 insertions(+), 17 deletions(-) diff --git a/flexmeasures/ui/static/openapi-specs.json b/flexmeasures/ui/static/openapi-specs.json index 6109a45f4d..b36d73c184 100644 --- a/flexmeasures/ui/static/openapi-specs.json +++ b/flexmeasures/ui/static/openapi-specs.json @@ -1456,10 +1456,10 @@ "datetime": "2015-06-02T16:00:00+00:00" } ], - "soc-minima": { - "sensor": 300 + "soc-min": { + "sensor": 300, + "default": "10 kWh" }, - "soc-min": "10 kWh", "soc-max": "25 kWh", "charging-efficiency": "120%", "discharging-efficiency": { @@ -3905,7 +3905,7 @@ "sensor": 42 }, "production-capacity": "30 kW", - "soc-minima": [ + "soc-min": [ { "start": "2015-06-02T12:00:00+00:00", "end": "2015-06-02T13:00:00+00:00", @@ -4509,6 +4509,15 @@ "items": { "type": "integer" } + }, + "default": { + "type": [ + "string", + "null" + ], + "default": null, + "description": "Fallback quantity to use when the referenced sensor has missing values.", + "example": "0 kWh" } }, "required": [ @@ -4518,11 +4527,25 @@ }, "TimeSeries": { "type": "array", - "description": "Time series specification containing a list of segments that together describe a variable quantity.", + "description": "Time series specification containing a list of segments that together describe a variable quantity. Each segment may specify either `datetime`, `start` and `end`, `start` and `duration`, or `end` and `duration`.", "example": [ { "value": "23 kW", - "start": "2025-11-20T15:15+01", + "datetime": "2025-11-20T15:15+01" + }, + { + "value": "24 kW", + "start": "2025-11-20T16:00+01", + "end": "2025-11-20T17:00+01" + }, + { + "value": "25 kW", + "start": "2025-11-20T17:00+01", + "duration": "PT1H" + }, + { + "value": "26 kW", + "end": "2025-11-20T19:00+01", "duration": "PT1H" } ], @@ -4581,12 +4604,12 @@ "$ref": "#/components/schemas/VariableQuantityOpenAPI" }, "soc-minima-breach-price": { - "description": "This penalty value is used to discourage the violation of soc-minima constraints in the flex-model, which the scheduler will attempt to minimize.\nIt must use the same currency as the other price settings and cannot be negative.\nWhile it's an internal nudge to steer the scheduler\u2014and doesn't represent a real-life cost\u2014it should still be chosen in proportion to the actual energy prices at your site.\nIf it's too high, it will overly dominate other constraints; if it's too low, it will have no effect.\nWithout this value, the soc-minima become hard constraints, which means that any infeasible state-of-charge minima would prevent a complete schedule from being computed.\n", + "description": "This penalty value is used to discourage the violation of dynamic lower SoC boundary constraints in the flex-model, which the scheduler will attempt to minimize.\nIt must use the same currency as the other price settings and cannot be negative.\nWhile it's an internal nudge to steer the scheduler\u2014and doesn't represent a real-life cost\u2014it should still be chosen in proportion to the actual energy prices at your site.\nIf it's too high, it will overly dominate other constraints; if it's too low, it will have no effect.\nWithout this value, dynamic soc-min boundaries and legacy soc-minima boundaries become hard constraints, which means that any infeasible state-of-charge minima would prevent a complete schedule from being computed.\n", "example": "120 EUR/kWh", "$ref": "#/components/schemas/VariableQuantityOpenAPI" }, "soc-maxima-breach-price": { - "description": "This penalty value is used to discourage the violation of soc-maxima constraints in the flex-model, which the scheduler will attempt to minimize.\nIt must use the same currency as the other price settings and cannot be negative.\nWhile it's an internal nudge to steer the scheduler\u2014and doesn't represent a real-life cost\u2014it should still be chosen in proportion to the actual energy prices at your site.\nIf it's too high, it will overly dominate other constraints; if it's too low, it will have no effect.\nWithout this value, the soc-maxima become hard constraints, which means that any infeasible state-of-charge maxima would prevent a complete schedule from being computed.\n", + "description": "This penalty value is used to discourage the violation of dynamic upper SoC boundary constraints in the flex-model, which the scheduler will attempt to minimize.\nIt must use the same currency as the other price settings and cannot be negative.\nWhile it's an internal nudge to steer the scheduler\u2014and doesn't represent a real-life cost\u2014it should still be chosen in proportion to the actual energy prices at your site.\nIf it's too high, it will overly dominate other constraints; if it's too low, it will have no effect.\nWithout this value, dynamic soc-max boundaries and legacy soc-maxima boundaries become hard constraints, which means that any infeasible state-of-charge maxima would prevent a complete schedule from being computed.\n", "example": "120 EUR/kWh", "$ref": "#/components/schemas/VariableQuantityOpenAPI" }, @@ -6099,15 +6122,20 @@ "example": "3.1 kWh" }, "soc-min": { - "type": "string", - "x-minimum": "0 MWh", - "description": "A constant and non-negotiable lower boundary for all SoC values in the schedule.\nIf omitted, no lower boundary is applied.\nIf used, this is regarded as an unsurpassable physical limitation.\nTo set softer boundaries, use the soc-minima flex-model field instead together with the soc-minima-breach-price field in the flex-context.\n", - "example": "2.5 kWh" + "description": "Lower boundary for all SoC values in the schedule.\nIf omitted, no lower boundary is applied.\nWhen passed as a fixed quantity, this is regarded as an unsurpassable physical limitation.\nWhen passed as a sensor reference or time series, it defines dynamic lower boundaries. Dynamic boundaries are soft constraints by default, because relax-soc-constraints defaults to True and supplies a default soc-minima-breach-price.\nSensor references may include a default fallback quantity for missing sensor values, for example {\"sensor\": 50, \"default\": \"0 kWh\"}.\nSet relax-soc-constraints to False to keep dynamic lower boundaries as hard constraints unless soc-minima-breach-price is supplied explicitly.\n", + "example": { + "sensor": 50, + "default": "0 kWh" + }, + "$ref": "#/components/schemas/VariableQuantityOpenAPI" }, "soc-max": { - "type": "string", - "description": "A constant and non-negotiable upper boundary for all values in the schedule (for storage devices, this defaults to max soc-target, if that is provided).\nIf omitted, no upper boundary is applied.\nIf used, this is regarded as an unsurpassable physical limitation.\nTo set softer boundaries, use the soc-maxima flex-model field instead together with the soc-maxima-breach-price field in the flex-context.\n", - "example": "7 kWh" + "description": "Upper boundary for all SoC values in the schedule (for storage devices, this defaults to max soc-target, if that is provided).\nIf omitted, no upper boundary is applied.\nWhen passed as a fixed quantity, this is regarded as an unsurpassable physical limitation.\nWhen passed as a sensor reference or time series, it defines dynamic upper boundaries. Dynamic boundaries are soft constraints by default, because relax-soc-constraints defaults to True and supplies a default soc-maxima-breach-price.\nSensor references may include a default fallback quantity for missing sensor values, for example {\"sensor\": 51, \"default\": \"100 kWh\"}.\nSet relax-soc-constraints to False to keep dynamic upper boundaries as hard constraints unless soc-maxima-breach-price is supplied explicitly.\n", + "example": { + "sensor": 51, + "default": "100 kWh" + }, + "$ref": "#/components/schemas/VariableQuantityOpenAPI" }, "power-capacity": { "description": "Symmetric device-level power constraint. How much power can be applied to this asset in either direction.\nIf omitted, the scheduler infers this limit from the greatest of consumption-capacity and production-capacity when either is configured, before falling back to site-power-capacity.\nWhen exactly one of consumption-capacity or production-capacity is configured to non-zero capacity, the missing opposite capacity defaults to zero.", @@ -6139,7 +6167,7 @@ "example": true }, "soc-maxima": { - "description": "Set points that form upper boundaries at certain times, e.g. to target an empty heat buffer before a maintenance window.\nThe soc-maxima are soft constraints in the optimization problem by default, because relax-soc-constraints defaults to True and supplies a default soc-maxima-breach-price.\nSet relax-soc-constraints to False to keep them as hard constraints unless soc-maxima-breach-price is supplied explicitly.", + "description": "[Deprecated field] Use dynamic soc-max values instead.\nSet points that form upper boundaries at certain times, e.g. to target an empty heat buffer before a maintenance window.\nThe soc-maxima legacy alias is soft in the optimization problem by default, because relax-soc-constraints defaults to True and supplies a default soc-maxima-breach-price.\nSet relax-soc-constraints to False to keep them as hard constraints unless soc-maxima-breach-price is supplied explicitly.", "example": [ { "value": "51 kWh", @@ -6147,10 +6175,11 @@ "end": "2024-02-05T13:30:00+01:00" } ], + "deprecated": true, "$ref": "#/components/schemas/VariableQuantityOpenAPI" }, "soc-minima": { - "description": "Set points that form lower boundaries, e.g. to target a full car battery in the morning.\nThe soc-minima are soft constraints in the optimization problem by default, because relax-soc-constraints defaults to True and supplies a default soc-minima-breach-price.\nSet relax-soc-constraints to False to keep them as hard constraints unless soc-minima-breach-price is supplied explicitly.\nBoth single points in time and ranges are possible, see example.", + "description": "[Deprecated field] Use dynamic soc-min values instead.\nSet points that form lower boundaries, e.g. to target a full car battery in the morning.\nThe soc-minima legacy alias is soft in the optimization problem by default, because relax-soc-constraints defaults to True and supplies a default soc-minima-breach-price.\nSet relax-soc-constraints to False to keep them as hard constraints unless soc-minima-breach-price is supplied explicitly.\nBoth single points in time and ranges are possible, see example.", "example": [ { "datetime": "2024-02-05T08:00:00+01:00", @@ -6162,6 +6191,7 @@ "end": "2024-02-05T13:30:00+01:00" } ], + "deprecated": true, "$ref": "#/components/schemas/VariableQuantityOpenAPI" }, "soc-targets": { From c9fe6d60f836213a1335dbcbc9aaf38c02fd9283 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Mon, 6 Jul 2026 09:52:08 +0100 Subject: [PATCH 29/54] fix: narrow tuple quantity parsing errors Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/data/schemas/sensors.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py index ba66e57814..ab64d37ce1 100644 --- a/flexmeasures/data/schemas/sensors.py +++ b/flexmeasures/data/schemas/sensors.py @@ -24,6 +24,7 @@ ) import marshmallow.validate as validate from pandas.api.types import is_numeric_dtype +from pint.errors import PintError import timely_beliefs as tb from werkzeug.datastructures import FileStorage from marshmallow.validate import Validator @@ -443,7 +444,7 @@ def _deserialize( elif isinstance(value, tuple): try: return ur.Quantity.from_tuple(value).to(self.to_unit) - except Exception: + except (PintError, TypeError, ValueError, AttributeError): if ( len(value) == 1 and isinstance(value[0], numbers.Real) From b8d96a0b6bf85faaa33b0fb059633abd8714f2ea Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Mon, 6 Jul 2026 09:52:52 +0100 Subject: [PATCH 30/54] fix: clarify variable quantity type errors Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/data/schemas/sensors.py | 9 +++++++-- flexmeasures/data/schemas/tests/test_scheduling.py | 4 ++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py index ab64d37ce1..d0b60f368e 100644 --- a/flexmeasures/data/schemas/sensors.py +++ b/flexmeasures/data/schemas/sensors.py @@ -361,6 +361,11 @@ def _serialize(self, value: Sensor, attr, obj, **kwargs) -> int: class VariableQuantityField(MarshmallowClickMixin, fields.Field): + _UNSUPPORTED_VALUE_TYPE_MESSAGE = ( + "Unsupported value type. `{value_type}` was provided but only dict, list, " + "str, pint Quantity, tuple, and numeric values with a default source unit are supported." + ) + def __init__( self, to_unit, @@ -454,13 +459,13 @@ def _deserialize( if len(value) == 2: return self._deserialize_str(f"{value[0]} {value[1]}") raise FMValidationError( - f"Unsupported value type. `{type(value)}` was provided but only dict, list and str are supported." + self._UNSUPPORTED_VALUE_TYPE_MESSAGE.format(value_type=type(value)) ) elif isinstance(value, numbers.Real) and self.default_src_unit is not None: return self._deserialize_numeric(value, attr, data, **kwargs) else: raise FMValidationError( - f"Unsupported value type. `{type(value)}` was provided but only dict, list and str are supported." + self._UNSUPPORTED_VALUE_TYPE_MESSAGE.format(value_type=type(value)) ) _SOURCE_FILTER_KEYS = frozenset( diff --git a/flexmeasures/data/schemas/tests/test_scheduling.py b/flexmeasures/data/schemas/tests/test_scheduling.py index 88a99419a5..a182142589 100644 --- a/flexmeasures/data/schemas/tests/test_scheduling.py +++ b/flexmeasures/data/schemas/tests/test_scheduling.py @@ -624,7 +624,7 @@ def check_schema_loads_data(schema, data, fails): ( {"site-power-capacity": 100}, { - "site-power-capacity": f"Unsupported value type. `{type(100)}` was provided but only dict, list and str are supported." + "site-power-capacity": f"Unsupported value type. `{type(100)}` was provided but only dict, list, str, pint Quantity, tuple, and numeric values with a default source unit are supported." }, ), ( @@ -939,7 +939,7 @@ def test_soc_max_sensor_reference_with_default_loads_as_dynamic_maximum( [ False, { - "soc-max": "Unsupported value type. `` was provided but only dict, list and str are supported." + "soc-max": "Unsupported value type. `` was provided but only dict, list, str, pint Quantity, tuple, and numeric values with a default source unit are supported." }, ], ), From e6a24eb0539022952df46eb5fbfebd7b6a71c121 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Mon, 6 Jul 2026 09:53:45 +0100 Subject: [PATCH 31/54] fix: update variable quantity deserialize typing Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/data/schemas/sensors.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py index d0b60f368e..f96f344136 100644 --- a/flexmeasures/data/schemas/sensors.py +++ b/flexmeasures/data/schemas/sensors.py @@ -435,8 +435,19 @@ def __init__( @with_appcontext_if_needed() def _deserialize( - self, value: dict[str, int] | list[dict] | str, attr, data, **kwargs - ) -> Sensor | list[dict] | ur.Quantity: + self, + value: ( + dict[str, Any] + | list[dict] + | str + | ur.Quantity + | tuple[Any, ...] + | numbers.Real + ), + attr, + data, + **kwargs, + ) -> Sensor | SensorReference | list[dict] | ur.Quantity: if isinstance(value, dict): return self._deserialize_dict(value, attr, data, **kwargs) From 7e226b4cc6fda51d90f22d0813a559be52635406 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Mon, 6 Jul 2026 09:54:26 +0100 Subject: [PATCH 32/54] fix: allow null sensor reference defaults Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/data/schemas/sensors.py | 1 + flexmeasures/data/schemas/tests/test_sensor.py | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py index f96f344136..28762907ae 100644 --- a/flexmeasures/data/schemas/sensors.py +++ b/flexmeasures/data/schemas/sensors.py @@ -1082,6 +1082,7 @@ class Meta: ) default = fields.String( load_default=None, + allow_none=True, metadata=dict( description="Fallback quantity to use when the referenced sensor has missing values.", example="0 kWh", diff --git a/flexmeasures/data/schemas/tests/test_sensor.py b/flexmeasures/data/schemas/tests/test_sensor.py index 334e40cb24..7f3ab49d01 100644 --- a/flexmeasures/data/schemas/tests/test_sensor.py +++ b/flexmeasures/data/schemas/tests/test_sensor.py @@ -5,6 +5,7 @@ from flexmeasures.data.schemas.sensors import ( QuantityOrSensor, SensorReference, + SensorReferenceSchema, VariableQuantityField, floor_bdf_event_starts, ) @@ -222,6 +223,15 @@ def test_sensor_reference_with_default(setup_dummy_sensors): assert result.default == ur.Quantity("0.5 MWh") +def test_sensor_reference_schema_accepts_null_default(setup_dummy_sensors): + sensor1, _, _, _ = setup_dummy_sensors + + result = SensorReferenceSchema().load({"sensor": sensor1.id, "default": None}) + + assert result["sensor"] == sensor1 + assert result["default"] is None + + def test_sensor_reference_with_source_types(setup_dummy_sensors): """``{"sensor": , "source-types": [...]}`` deserializes to a :class:`SensorReference`. From 3c82f2d3b386c8984fdb925f0cb6f36ceb86ae37 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Mon, 6 Jul 2026 09:55:02 +0100 Subject: [PATCH 33/54] refactor: remove ineffective storage schema fields Signed-off-by: Mohamed Belhsan Hmida --- .../data/schemas/scheduling/storage.py | 43 ------------------- 1 file changed, 43 deletions(-) diff --git a/flexmeasures/data/schemas/scheduling/storage.py b/flexmeasures/data/schemas/scheduling/storage.py index 5f328f6afa..1f82aae89c 100644 --- a/flexmeasures/data/schemas/scheduling/storage.py +++ b/flexmeasures/data/schemas/scheduling/storage.py @@ -278,49 +278,6 @@ def __init__( else: default_soc_unit = "MWh" - self.soc_min = VariableQuantityField( - to_unit="MWh", - default_src_unit=default_soc_unit, - timezone=self.timezone, - event_resolution=self.flooring_resolution, - data_key="soc-min", - value_validator=validate.Range(min=0), - ) - - self.soc_max = VariableQuantityField( - to_unit="MWh", - default_src_unit=default_soc_unit, - timezone=self.timezone, - event_resolution=self.flooring_resolution, - data_key="soc-max", - value_validator=validate.Range(min=0), - ) - - self.soc_maxima = VariableQuantityField( - to_unit="MWh", - default_src_unit=default_soc_unit, - timezone=self.timezone, - event_resolution=self.flooring_resolution, - data_key="soc-maxima", - value_validator=validate.Range(min=0), - ) - - self.soc_minima = VariableQuantityField( - to_unit="MWh", - default_src_unit=default_soc_unit, - timezone=self.timezone, - event_resolution=self.flooring_resolution, - data_key="soc-minima", - value_validator=validate.Range(min=0), - ) - self.soc_targets = VariableQuantityField( - to_unit="MWh", - default_src_unit=default_soc_unit, - timezone=self.timezone, - event_resolution=self.flooring_resolution, - data_key="soc-targets", - ) - super().__init__(*args, **kwargs) if default_soc_unit is not None: for field in self.fields.keys(): From e20c2cf9d5cd9df977bab9eb0c4e7cbcbd380c41 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Mon, 6 Jul 2026 11:28:11 +0100 Subject: [PATCH 34/54] fix: handle malformed quantity tuples Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/data/schemas/sensors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py index 28762907ae..10d20a6fcb 100644 --- a/flexmeasures/data/schemas/sensors.py +++ b/flexmeasures/data/schemas/sensors.py @@ -460,7 +460,7 @@ def _deserialize( elif isinstance(value, tuple): try: return ur.Quantity.from_tuple(value).to(self.to_unit) - except (PintError, TypeError, ValueError, AttributeError): + except (PintError, TypeError, ValueError, AttributeError, IndexError): if ( len(value) == 1 and isinstance(value[0], numbers.Real) From 73716740718ea086b94ada13b0b2a4c4a3becaac Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Mon, 6 Jul 2026 11:28:39 +0100 Subject: [PATCH 35/54] docs: add changelog entry Signed-off-by: Mohamed Belhsan Hmida --- documentation/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index ea45585897..6d3c88ac89 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -18,6 +18,7 @@ New features * Let storage scheduling infer missing ``power-capacity`` from directional device capacities before falling back to site capacity, and default the missing opposite capacity to zero when only a non-zero ``consumption-capacity`` or ``production-capacity`` is configured [see `PR #2222 `_] * CLI support for adding/editing account attributes [see `PR #2242 `_] * Relax storage SoC constraints by default and report infeasible storage schedules directly instead of saving fallback schedules [see `PR #2252 `_] +* Support defining dynamic storage ``soc-min`` and ``soc-max`` boundaries with sensor references or time series, including sensor ``default`` fallbacks; ``soc-minima`` and ``soc-maxima`` remain supported as legacy aliases [see `PR #2267 `_] Infrastructure / Support ---------------------- From ed9e107c33e0138604cbc40f814ef9e73d95c654 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 10 Jul 2026 12:10:30 +0100 Subject: [PATCH 36/54] fix: reject null sensor reference defaults Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/data/schemas/sensors.py | 6 ++--- .../data/schemas/tests/test_sensor.py | 22 +++++++++++++++---- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py index 10d20a6fcb..9090d9ddf0 100644 --- a/flexmeasures/data/schemas/sensors.py +++ b/flexmeasures/data/schemas/sensors.py @@ -564,7 +564,7 @@ def _deserialize_dict( ).deserialize(value["sensor"], None, None) default = None - if "default" in value and value["default"] is not None: + if "default" in value: default = self._deserialize_default(value["default"], attr, data, **kwargs) # If no source filter or default keys are present, keep returning a plain Sensor. @@ -1081,8 +1081,8 @@ class Meta: ), ) default = fields.String( - load_default=None, - allow_none=True, + required=False, + allow_none=False, metadata=dict( description="Fallback quantity to use when the referenced sensor has missing values.", example="0 kWh", diff --git a/flexmeasures/data/schemas/tests/test_sensor.py b/flexmeasures/data/schemas/tests/test_sensor.py index 7f3ab49d01..6c15014ec2 100644 --- a/flexmeasures/data/schemas/tests/test_sensor.py +++ b/flexmeasures/data/schemas/tests/test_sensor.py @@ -223,13 +223,27 @@ def test_sensor_reference_with_default(setup_dummy_sensors): assert result.default == ur.Quantity("0.5 MWh") -def test_sensor_reference_schema_accepts_null_default(setup_dummy_sensors): +def test_sensor_reference_schema_rejects_null_default(setup_dummy_sensors): sensor1, _, _, _ = setup_dummy_sensors - result = SensorReferenceSchema().load({"sensor": sensor1.id, "default": None}) + with pytest.raises(ValidationError) as exc_info: + SensorReferenceSchema().load({"sensor": sensor1.id, "default": None}) + + assert "default" in exc_info.value.messages + - assert result["sensor"] == sensor1 - assert result["default"] is None +def test_sensor_reference_field_rejects_null_default(setup_dummy_sensors): + """``default`` must be a concrete fallback quantity when provided.""" + sensor1, _, _, _ = setup_dummy_sensors + field = VariableQuantityField(to_unit="MWh", return_magnitude=False) + + with pytest.raises(ValidationError) as exc_info: + field.deserialize({"sensor": sensor1.id, "default": None}) + + assert ( + "Sensor reference `default` must be a quantity string or a numeric value with a known default source unit." + in str(exc_info.value) + ) def test_sensor_reference_with_source_types(setup_dummy_sensors): From a0410a308a4d6d8dc8e18322998fc13fd672d409 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 10 Jul 2026 12:10:58 +0100 Subject: [PATCH 37/54] fix: relax scalar soc bounds by default Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/data/models/planning/storage.py | 72 +++++++++++++++++++ .../models/planning/tests/test_storage.py | 37 ++++++++++ 2 files changed, 109 insertions(+) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 57c32dae49..40ec7789ed 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -521,6 +521,14 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 as_instantaneous_events=True, resolve_overlaps="max", ) + if self.flex_context.get("soc_minima_breach_price") is not None: + soc_min[d], soc_minima[d] = self._relax_scalar_soc_minimum( + soc_min=soc_min[d], + soc_minima=soc_minima[d], + query_window=(start + resolution, end + resolution), + resolution=resolution, + beliefs_before=belief_time, + ) if ( self.flex_context.get("soc_minima_breach_price") is not None and soc_minima[d] is not None @@ -595,6 +603,14 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 as_instantaneous_events=True, resolve_overlaps="min", ) + if self.flex_context.get("soc_maxima_breach_price") is not None: + soc_max[d], soc_maxima[d] = self._relax_scalar_soc_maximum( + soc_max=soc_max[d], + soc_maxima=soc_maxima[d], + query_window=(start + resolution, end + resolution), + resolution=resolution, + beliefs_before=belief_time, + ) if ( self.flex_context.get("soc_maxima_breach_price") is not None and soc_maxima[d] is not None @@ -1026,6 +1042,62 @@ def convert_to_commitments( return commitments + @staticmethod + def _relax_scalar_soc_minimum( + soc_min: float | None, + soc_minima: ( + Sensor | SensorReference | list[dict] | ur.Quantity | pd.Series | None + ), + **timing_kwargs, + ) -> tuple[ + None, Sensor | SensorReference | list[dict] | ur.Quantity | pd.Series | None + ]: + """Move a fixed SoC minimum into the relaxed minima path. + + If legacy dynamic minima are also configured, the fixed minimum still tightens + them, but no longer stays behind as a hard constraint. + """ + if soc_min is None: + return None, soc_minima + if soc_minima is None: + return None, soc_min * ur.Quantity("MWh") + soc_minima = get_continuous_series_sensor_or_quantity( + variable_quantity=soc_minima, + unit="MWh", + as_instantaneous_events=True, + resolve_overlaps="max", + **timing_kwargs, + ) + return None, soc_minima.clip(lower=soc_min) + + @staticmethod + def _relax_scalar_soc_maximum( + soc_max: float | None, + soc_maxima: ( + Sensor | SensorReference | list[dict] | ur.Quantity | pd.Series | None + ), + **timing_kwargs, + ) -> tuple[ + None, Sensor | SensorReference | list[dict] | ur.Quantity | pd.Series | None + ]: + """Move a fixed SoC maximum into the relaxed maxima path. + + If legacy dynamic maxima are also configured, the fixed maximum still tightens + them, but no longer stays behind as a hard constraint. + """ + if soc_max is None: + return None, soc_maxima + if soc_maxima is None: + return None, soc_max * ur.Quantity("MWh") + soc_maxima = get_continuous_series_sensor_or_quantity( + variable_quantity=soc_maxima, + unit="MWh", + as_instantaneous_events=True, + resolve_overlaps="min", + **timing_kwargs, + ) + return None, soc_maxima.clip(upper=soc_max) + def persist_flex_model(self): """Store new soc info as GenericAsset attributes diff --git a/flexmeasures/data/models/planning/tests/test_storage.py b/flexmeasures/data/models/planning/tests/test_storage.py index a5a47d6bc6..4e12c5bf46 100644 --- a/flexmeasures/data/models/planning/tests/test_storage.py +++ b/flexmeasures/data/models/planning/tests/test_storage.py @@ -15,6 +15,7 @@ get_sensors_from_db, series_to_ts_specs, ) +from flexmeasures.utils.unit_utils import ur def test_battery_solver_multi_commitment(add_battery_assets, db): @@ -290,6 +291,42 @@ def test_battery_relaxation(add_battery_assets, db): ) # 100 EUR/(kW*h) * 0.025 MW * 1000 kW/MW * 4 hours +def test_scalar_soc_minimum_moves_to_relaxed_minimum(): + soc_min, soc_minima = StorageScheduler._relax_scalar_soc_minimum( + soc_min=0.2, + soc_minima=None, + ) + + assert soc_min is None + assert soc_minima == 0.2 * ur.Quantity("MWh") + + +def test_scalar_soc_bounds_tighten_legacy_dynamic_bounds(): + index = pd.date_range( + "2015-01-01T00:00:00+01:00", + periods=2, + freq="15min", + ) + timing_kwargs = { + "query_window": (index[0], index[-1] + timedelta(minutes=15)), + "resolution": timedelta(minutes=15), + } + + _, soc_minima = StorageScheduler._relax_scalar_soc_minimum( + soc_min=0.4, + soc_minima=pd.Series([0.1, 0.5], index=index), + **timing_kwargs, + ) + _, soc_maxima = StorageScheduler._relax_scalar_soc_maximum( + soc_max=0.8, + soc_maxima=pd.Series([0.7, 0.9], index=index), + **timing_kwargs, + ) + + assert list(soc_minima) == pytest.approx([0.4, 0.5]) + assert list(soc_maxima) == pytest.approx([0.7, 0.8]) + + def test_deserialize_storage_soc_at_start_from_state_of_charge_sensor( add_charging_station_assets, setup_markets, setup_sources, db ): From fb06d92f6789c34ba8205af02ed6ef48bb69273a Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 10 Jul 2026 12:11:33 +0100 Subject: [PATCH 38/54] docs: clarify sensor defaults and soc relaxation Signed-off-by: Mohamed Belhsan Hmida --- documentation/api/introduction.rst | 5 +++-- documentation/changelog.rst | 2 +- documentation/concepts/commitments.rst | 4 ++-- documentation/features/scheduling.rst | 4 ++-- flexmeasures/data/schemas/scheduling/metadata.py | 12 ++++++------ flexmeasures/ui/static/openapi-specs.json | 10 +++------- 6 files changed, 17 insertions(+), 20 deletions(-) diff --git a/documentation/api/introduction.rst b/documentation/api/introduction.rst index 56d9219975..53754812f6 100644 --- a/documentation/api/introduction.rst +++ b/documentation/api/introduction.rst @@ -124,8 +124,9 @@ The response body will contain a JSON message with a ``status`` field set to ``" .. note:: FlexMeasures' built-in storage scheduler no longer computes a fallback schedule for infeasible problems. - Instead, dynamic ``soc-min`` and ``soc-max`` boundaries, including the legacy aliases ``soc-minima`` and ``soc-maxima``, are relaxed by default through ``"relax-soc-constraints": true``. - Fixed ``soc-min`` / ``soc-max`` values and exact ``soc-targets`` remain hard constraints. + Instead, ``soc-min`` and ``soc-max`` boundaries, including the legacy aliases ``soc-minima`` and ``soc-maxima``, are relaxed by default through ``"relax-soc-constraints": true``. + To keep ``soc-min`` / ``soc-max`` hard, explicitly set ``"relax-soc-constraints": false``. + Exact ``soc-targets`` remain hard constraints. If hard constraints cannot be satisfied, the scheduling job fails and clients receive the failure reason when requesting the schedule. For custom schedulers that still define a fallback scheduler, server administrators can configure whether clients receive a 303 redirect (``FLEXMEASURES_FALLBACK_REDIRECT = True``) or whether FlexMeasures follows the fallback automatically and returns the fallback schedule directly (``FLEXMEASURES_FALLBACK_REDIRECT = False``, the default). diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 6d3c88ac89..687819dc21 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -18,7 +18,7 @@ New features * Let storage scheduling infer missing ``power-capacity`` from directional device capacities before falling back to site capacity, and default the missing opposite capacity to zero when only a non-zero ``consumption-capacity`` or ``production-capacity`` is configured [see `PR #2222 `_] * CLI support for adding/editing account attributes [see `PR #2242 `_] * Relax storage SoC constraints by default and report infeasible storage schedules directly instead of saving fallback schedules [see `PR #2252 `_] -* Support defining dynamic storage ``soc-min`` and ``soc-max`` boundaries with sensor references or time series, including sensor ``default`` fallbacks; ``soc-minima`` and ``soc-maxima`` remain supported as legacy aliases [see `PR #2267 `_] +* Support defining ``default`` fallbacks on sensor references, and support dynamic storage ``soc-min`` and ``soc-max`` boundaries with sensor references or time series; ``soc-minima`` and ``soc-maxima`` remain supported as legacy aliases, and scalar ``soc-min`` / ``soc-max`` follow the default SoC relaxation behavior [see `PR #2267 `_] Infrastructure / Support ---------------------- diff --git a/documentation/concepts/commitments.rst b/documentation/concepts/commitments.rst index 82cb1d4f9c..db89fcf607 100644 --- a/documentation/concepts/commitments.rst +++ b/documentation/concepts/commitments.rst @@ -182,9 +182,9 @@ commitments the scheduler constructs. 6. **SOC minima / maxima (storage preferences)** - - *Fields used*: ``soc-minima``, ``soc-minima-breach-price``, ``soc-maxima`` and ``soc-maxima-breach-price``. + - *Fields used*: ``soc-min``, ``soc-minima``, ``soc-minima-breach-price``, ``soc-max``, ``soc-maxima`` and ``soc-maxima-breach-price``. - *Commitment*: StockCommitment(s) that price deviations below minima or - above maxima. Hard storage capacities are set through ``soc-min`` and ``soc-max`` instead and are modelled as Pyomo constraints. + above maxima. Set ``relax-soc-constraints`` to ``False`` to keep these SoC bounds as hard Pyomo constraints instead. 7. **Power bands per device** diff --git a/documentation/features/scheduling.rst b/documentation/features/scheduling.rst index 706527d937..dd661b76bb 100644 --- a/documentation/features/scheduling.rst +++ b/documentation/features/scheduling.rst @@ -269,9 +269,9 @@ However, here are some tips to model a buffer correctly: - Set ``storage-efficiency`` to a value below 100% to model (heat) loss. If the flex model describes an infeasible problem for the storage scheduler, the failure should remain visible. -By default, dynamic ``soc-min`` and ``soc-max`` boundaries are relaxed into soft constraints, so the scheduler can still return a useful schedule when these boundaries cannot be fully met. +By default, ``soc-min`` and ``soc-max`` boundaries are relaxed into soft constraints, so the scheduler can still return a useful schedule when these boundaries cannot be fully met. The legacy ``soc-minima`` and ``soc-maxima`` aliases follow the same behavior. -Exact ``soc-targets`` and fixed physical ``soc-min`` / ``soc-max`` bounds remain hard constraints. +Exact ``soc-targets`` remain hard constraints, and ``soc-min`` / ``soc-max`` can be kept hard by setting ``relax-soc-constraints`` to ``False``. If those hard constraints make the problem infeasible, the scheduling job fails instead of producing a fallback schedule. It is important to take note of these failures. Often, misconfigured flex models are the reason. diff --git a/flexmeasures/data/schemas/scheduling/metadata.py b/flexmeasures/data/schemas/scheduling/metadata.py index f934b015a4..006c227d1a 100644 --- a/flexmeasures/data/schemas/scheduling/metadata.py +++ b/flexmeasures/data/schemas/scheduling/metadata.py @@ -232,20 +232,20 @@ def to_dict(self): SOC_MIN = MetaData( description="""Lower boundary for all SoC values in the schedule. If omitted, no lower boundary is applied. -When passed as a fixed quantity, this is regarded as an unsurpassable physical limitation. -When passed as a sensor reference or time series, it defines dynamic lower boundaries. Dynamic boundaries are soft constraints by default, because ``relax-soc-constraints`` defaults to ``True`` and supplies a default ``soc-minima-breach-price``. +This boundary is soft in the optimization problem by default, because ``relax-soc-constraints`` defaults to ``True`` and supplies a default ``soc-minima-breach-price``. +When passed as a sensor reference or time series, it defines dynamic lower boundaries. Sensor references may include a ``default`` fallback quantity for missing sensor values, for example ``{"sensor": 50, "default": "0 kWh"}``. -Set ``relax-soc-constraints`` to ``False`` to keep dynamic lower boundaries as hard constraints unless ``soc-minima-breach-price`` is supplied explicitly. [#maximum_overlap]_ [#projecting_scheduling_constraints]_ +Set ``relax-soc-constraints`` to ``False`` to keep lower boundaries as hard constraints unless ``soc-minima-breach-price`` is supplied explicitly. [#maximum_overlap]_ [#projecting_scheduling_constraints]_ """, example={"sensor": 50, "default": "0 kWh"}, ) SOC_MAX = MetaData( description="""Upper boundary for all SoC values in the schedule (for storage devices, this defaults to max soc-target, if that is provided). If omitted, no upper boundary is applied. -When passed as a fixed quantity, this is regarded as an unsurpassable physical limitation. -When passed as a sensor reference or time series, it defines dynamic upper boundaries. Dynamic boundaries are soft constraints by default, because ``relax-soc-constraints`` defaults to ``True`` and supplies a default ``soc-maxima-breach-price``. +This boundary is soft in the optimization problem by default, because ``relax-soc-constraints`` defaults to ``True`` and supplies a default ``soc-maxima-breach-price``. +When passed as a sensor reference or time series, it defines dynamic upper boundaries. Sensor references may include a ``default`` fallback quantity for missing sensor values, for example ``{"sensor": 51, "default": "100 kWh"}``. -Set ``relax-soc-constraints`` to ``False`` to keep dynamic upper boundaries as hard constraints unless ``soc-maxima-breach-price`` is supplied explicitly. [#minimum_overlap]_ [#projecting_scheduling_constraints]_ +Set ``relax-soc-constraints`` to ``False`` to keep upper boundaries as hard constraints unless ``soc-maxima-breach-price`` is supplied explicitly. [#minimum_overlap]_ [#projecting_scheduling_constraints]_ """, example={"sensor": 51, "default": "100 kWh"}, ) diff --git a/flexmeasures/ui/static/openapi-specs.json b/flexmeasures/ui/static/openapi-specs.json index b36d73c184..6d752c5bd6 100644 --- a/flexmeasures/ui/static/openapi-specs.json +++ b/flexmeasures/ui/static/openapi-specs.json @@ -4511,11 +4511,7 @@ } }, "default": { - "type": [ - "string", - "null" - ], - "default": null, + "type": "string", "description": "Fallback quantity to use when the referenced sensor has missing values.", "example": "0 kWh" } @@ -6122,7 +6118,7 @@ "example": "3.1 kWh" }, "soc-min": { - "description": "Lower boundary for all SoC values in the schedule.\nIf omitted, no lower boundary is applied.\nWhen passed as a fixed quantity, this is regarded as an unsurpassable physical limitation.\nWhen passed as a sensor reference or time series, it defines dynamic lower boundaries. Dynamic boundaries are soft constraints by default, because relax-soc-constraints defaults to True and supplies a default soc-minima-breach-price.\nSensor references may include a default fallback quantity for missing sensor values, for example {\"sensor\": 50, \"default\": \"0 kWh\"}.\nSet relax-soc-constraints to False to keep dynamic lower boundaries as hard constraints unless soc-minima-breach-price is supplied explicitly.\n", + "description": "Lower boundary for all SoC values in the schedule.\nIf omitted, no lower boundary is applied.\nThis boundary is soft in the optimization problem by default, because relax-soc-constraints defaults to True and supplies a default soc-minima-breach-price.\nWhen passed as a sensor reference or time series, it defines dynamic lower boundaries.\nSensor references may include a default fallback quantity for missing sensor values, for example {\"sensor\": 50, \"default\": \"0 kWh\"}.\nSet relax-soc-constraints to False to keep lower boundaries as hard constraints unless soc-minima-breach-price is supplied explicitly.\n", "example": { "sensor": 50, "default": "0 kWh" @@ -6130,7 +6126,7 @@ "$ref": "#/components/schemas/VariableQuantityOpenAPI" }, "soc-max": { - "description": "Upper boundary for all SoC values in the schedule (for storage devices, this defaults to max soc-target, if that is provided).\nIf omitted, no upper boundary is applied.\nWhen passed as a fixed quantity, this is regarded as an unsurpassable physical limitation.\nWhen passed as a sensor reference or time series, it defines dynamic upper boundaries. Dynamic boundaries are soft constraints by default, because relax-soc-constraints defaults to True and supplies a default soc-maxima-breach-price.\nSensor references may include a default fallback quantity for missing sensor values, for example {\"sensor\": 51, \"default\": \"100 kWh\"}.\nSet relax-soc-constraints to False to keep dynamic upper boundaries as hard constraints unless soc-maxima-breach-price is supplied explicitly.\n", + "description": "Upper boundary for all SoC values in the schedule (for storage devices, this defaults to max soc-target, if that is provided).\nIf omitted, no upper boundary is applied.\nThis boundary is soft in the optimization problem by default, because relax-soc-constraints defaults to True and supplies a default soc-maxima-breach-price.\nWhen passed as a sensor reference or time series, it defines dynamic upper boundaries.\nSensor references may include a default fallback quantity for missing sensor values, for example {\"sensor\": 51, \"default\": \"100 kWh\"}.\nSet relax-soc-constraints to False to keep upper boundaries as hard constraints unless soc-maxima-breach-price is supplied explicitly.\n", "example": { "sensor": 51, "default": "100 kWh" From 6874c64d1a6e28134ca9e5fb96b7e824834e9ffb Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 10 Jul 2026 23:20:54 +0100 Subject: [PATCH 39/54] test: preserve hard soc bound expectations Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/data/models/planning/tests/test_solver.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py index 43f5517494..1cb90dd633 100644 --- a/flexmeasures/data/models/planning/tests/test_solver.py +++ b/flexmeasures/data/models/planning/tests/test_solver.py @@ -783,6 +783,7 @@ def compute_schedule(flex_model): end, resolution, flex_model=flex_model, + flex_context={"relax-soc-constraints": False}, ) schedule = scheduler.compute() @@ -1268,6 +1269,7 @@ def compute_schedule(flex_model): end, resolution, flex_model=flex_model, + flex_context={"relax-soc-constraints": False}, ) schedule = scheduler.compute() @@ -1349,6 +1351,7 @@ def test_numerical_errors(app_with_each_solver, setup_planning_test_data, db): ], "soc-unit": "MWh", }, + flex_context={"relax-soc-constraints": False}, ) ( @@ -2070,6 +2073,7 @@ def test_battery_stock_delta_sensor( end, resolution, flex_model=flex_model, + flex_context={"relax-soc-constraints": False}, ) if stock_delta_sensor == "delta fails": @@ -2427,6 +2431,7 @@ def compute_schedule(flex_model): "site-power-capacity": "100 MW", "production-price": {"sensor": epex_da.id}, "consumption-price": {"sensor": epex_da.id}, + "relax-soc-constraints": False, }, ) return scheduler.compute() From 779d61e08512e972bb9761cc71d44ba49e95eb00 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 10 Jul 2026 23:22:30 +0100 Subject: [PATCH 40/54] feat: support sensor reference fallbacks Signed-off-by: Mohamed Belhsan Hmida --- .../ui/templates/assets/asset_properties.html | 57 ++++++++++++++++++- 1 file changed, 54 insertions(+), 3 deletions(-) diff --git a/flexmeasures/ui/templates/assets/asset_properties.html b/flexmeasures/ui/templates/assets/asset_properties.html index 4564e7ef45..b0fb113684 100644 --- a/flexmeasures/ui/templates/assets/asset_properties.html +++ b/flexmeasures/ui/templates/assets/asset_properties.html @@ -572,14 +572,18 @@ const card = activeCard(); const name = card.id.replace('-control', ''); const flexModel = getFlexModel() + const sensorReference = { "sensor": sensorId }; + const defaultInput = document.getElementById('flexSensorDefaultInput'); + if (defaultInput && defaultInput.value.trim()) { + sensorReference.default = defaultInput.value.trim(); + } if (FlexModelFieldValidTypes[name].includes(Array)) { const valueIndex = selectedIndex() const fieldValues = flexModel[name]; - fieldValues[valueIndex] = { "sensor": sensorId }; + fieldValues[valueIndex] = sensorReference; flexModel[name] = fieldValues; } else { - const value = flexModel[name]; - flexModel[name] = { "sensor": sensorId }; + flexModel[name] = sensorReference; } setFlexModel(flexModel); @@ -1017,6 +1021,7 @@ tabContent.querySelector('.flex-input-group').appendChild(boolTabContentSaveBtn); } else if (dataType == Object) { // Object/Senosr Tab ================= btn.textContent = 'A sensor'; + const supportsSensorDefault = FlexModelFieldValidTypes[name].includes(Object); tabContent.innerHTML = `
@@ -1051,8 +1056,54 @@
+ ${supportsSensorDefault ? ` + + + ` : ''} `; + if (supportsSensorDefault) { + const defaultInput = tabContent.querySelector('#flexSensorDefaultInput'); + defaultInput.value = value && typeof value === 'object' && value.default + ? value.default + : ''; + + const saveDefaultButton = document.createElement('button'); + saveDefaultButton.className = 'btn btn-secondary btn-sm me-2 mt-2'; + saveDefaultButton.textContent = 'Use fallback'; + saveDefaultButton.onclick = function () { + const currentFlexModel = getFlexModel(); + let sensorReference = currentFlexModel[name]; + if (Array.isArray(sensorReference)) { + sensorReference = sensorReference[selectedIndex()]; + } + if (!sensorReference || typeof sensorReference !== 'object' || !sensorReference.sensor) { + showToast("Select a sensor before setting a fallback", "info"); + return; + } + + const fallback = defaultInput.value.trim(); + if (fallback) { + sensorReference.default = fallback; + } else { + delete sensorReference.default; + } + setFlexModel(currentFlexModel); + }; + tabContent.querySelector('.flex-input-group').appendChild(saveDefaultButton); + } } else if (dataType == String) { // String Tab ================= btn.textContent = 'Fixed value'; From 3db0f879fd885c29bef3833dc03e47d65aa275ae Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 10 Jul 2026 23:27:06 +0100 Subject: [PATCH 41/54] docs: clarify canonical soc bounds Signed-off-by: Mohamed Belhsan Hmida --- documentation/concepts/commitments.rst | 2 +- documentation/tut/flex-model-v2g.rst | 25 +++++++++++-------- .../tut/toy-example-from-scratch.rst | 4 +++ 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/documentation/concepts/commitments.rst b/documentation/concepts/commitments.rst index db89fcf607..22b38df912 100644 --- a/documentation/concepts/commitments.rst +++ b/documentation/concepts/commitments.rst @@ -141,7 +141,7 @@ Typical translations include: - tariffs (``consumption-price``, ``production-price``) → an ``"energy"`` FlowCommitment with zero baseline so net consumption/production is priced; - peak/excess limits (``site-peak-production``, ``site-peak-production-price``, etc.) → dedicated peak FlowCommitment(s); -- storage-related fields (``soc-minima``, ``soc-minima-breach-price``, etc.) → StockCommitment(s). +- storage-related fields (``soc-min``, ``soc-max``, legacy ``soc-minima`` and ``soc-maxima``, and the SoC breach prices) → StockCommitment(s). Let us look at some concrete examples. diff --git a/documentation/tut/flex-model-v2g.rst b/documentation/tut/flex-model-v2g.rst index 25b0ebd040..23a8260d5b 100644 --- a/documentation/tut/flex-model-v2g.rst +++ b/documentation/tut/flex-model-v2g.rst @@ -40,10 +40,11 @@ Constraining the cycling to occur within a static 25-85% SoC range can be modell A starting SoC below 15 kWh (25%) will lead to immediate charging to get within limits (as shown above). Likewise, a starting SoC above 51 kWh (85%) would lead to immediate discharging. -Setting a SoC target outside of the static range leads to an infeasible problem and will be rejected by the FlexMeasures API. +By default, these boundaries are soft constraints, so the scheduler may breach them when necessary and will penalize the breach. +To enforce them as hard limits, set ``"relax-soc-constraints": false`` in the flex-context. +An exact ``soc-targets`` value remains hard and can still make a schedule infeasible when it conflicts with hard limits. -The soc-min and soc-max settings are constant constraints. -To enable a temporary target SoC of more than 85% (for car reservations, see the next section), it is necessary to relax the ``soc-max`` field to 60 kWh (100%), and to instead use the ``soc-maxima`` field to convey the desired upper limit for regular cycling: +To enable a temporary target SoC of more than 85% (for car reservations, see the next section), this example keeps a global ``soc-max`` of 60 kWh and uses the legacy ``soc-maxima`` alias for the desired upper limit during regular cycling: .. code-block:: json @@ -61,7 +62,9 @@ To enable a temporary target SoC of more than 85% (for car reservations, see the } } -The maxima constraints should be relaxed—or withheld entirely—within some time window before any SoC target (as shown above). +The ``soc-maxima`` field is retained for backwards compatibility and is deprecated for new configurations. +When no separate global ``soc-max`` is needed, use a dynamic ``soc-max`` time series instead. +The dynamic maximum should be relaxed—or withheld entirely—within some time window before any SoC target (as shown above). This time window should be at least wide enough to allow the target to be reached in time, and can be made wider to allow the scheduler to take advantage of favourable market prices along the way. @@ -70,13 +73,13 @@ This time window should be at least wide enough to allow the target to be reache Car reservations ================ -Given a reservation for 8 AM on February 5th, constraint 2 can be modelled through the following (additional) ``soc-minima`` constraint: +Given a reservation for 8 AM on February 5th, constraint 2 can be modelled through the following dynamic ``soc-min`` constraint: .. code-block:: json { "flex-model": { - "soc-minima": [ + "soc-min": [ { "value": "57 kWh", "datetime": "2024-02-05T08:00:00+01:00" @@ -86,13 +89,13 @@ Given a reservation for 8 AM on February 5th, constraint 2 can be modelled throu } This constraint also signals that if the car is not plugged out of the Charge Point at 8 AM, the scheduler is in principle allowed to start discharging immediately afterwards. -To make sure the car remains at or above 95% SoC for some time, additional soc-minima constraints should be set accordingly, taking into account the scheduling resolution (here, 5 minutes). For example, to keep it charged (nearly) fully until 8.15 AM: +To make sure the car remains at or above 95% SoC for some time, additional dynamic ``soc-min`` constraints should be set accordingly, taking into account the scheduling resolution (here, 5 minutes). For example, to keep it charged (nearly) fully until 8.15 AM: .. code-block:: json { "flex-model": { - "soc-minima": [ + "soc-min": [ { "value": "57 kWh", "start": "2024-02-05T08:00:00+01:00", @@ -109,7 +112,7 @@ Alternatively, to keep the car from discharging altogether during that time, lim { "flex-model": { - "soc-minima": [ + "soc-min": [ { "value": "57 kWh", "datetime": "2024-02-05T08:00:00+01:00" @@ -125,7 +128,7 @@ Alternatively, to keep the car from discharging altogether during that time, lim } } -.. note:: In case the ``soc-minima`` field defines partially overlapping time periods, FlexMeasures automatically resolves this by selecting the maximum. Likewise, the minimum is selected for partially overlapping time periods in the ``soc-maxima``, ``power-capacity``, ``production-capacity`` and ``consumption-capacity`` flex-model fields, and also in the ``site-power-capacity``, ``site-production-capacity`` and ``site-consumption-capacity`` flex-context fields. +.. note:: In case the dynamic ``soc-min`` field defines partially overlapping time periods, FlexMeasures automatically resolves this by selecting the maximum. The legacy ``soc-minima`` and ``soc-maxima`` aliases remain supported. Likewise, the minimum is selected for partially overlapping time periods in the dynamic ``soc-max`` field, ``power-capacity``, ``production-capacity`` and ``consumption-capacity`` flex-model fields, and also in the ``site-power-capacity``, ``site-production-capacity`` and ``site-consumption-capacity`` flex-context fields. .. _earning_by_cycling: @@ -145,4 +148,4 @@ To provide an incentive for cycling the battery in response to market prices, th We hope this demonstration helped to illustrate the flex-model of the storage scheduler. Until now, optimizing storage (like batteries) has been the sole focus of these tutorial series. -In :ref:`tut_toy_schedule_process`, we'll turn to something different: the optimal timing of processes with fixed energy work and duration. \ No newline at end of file +In :ref:`tut_toy_schedule_process`, we'll turn to something different: the optimal timing of processes with fixed energy work and duration. diff --git a/documentation/tut/toy-example-from-scratch.rst b/documentation/tut/toy-example-from-scratch.rst index e4a9d40d04..078f8ff0ee 100644 --- a/documentation/tut/toy-example-from-scratch.rst +++ b/documentation/tut/toy-example-from-scratch.rst @@ -64,6 +64,10 @@ There is more information being used by the scheduler, such as the battery's cap ], "soc-usage": [{"sensor": 73}] } + + The ``soc-maxima`` field in this combined fixed-plus-dynamic example is a + supported legacy alias. Prefer dynamic ``soc-max`` when no separate global + ``soc-max`` value is needed. $ flexmeasures add schedule \ --sensor 2 \ From 49ec4a0a9ab71ef853aa009868b8d8abfc2fe353 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 10 Jul 2026 23:36:36 +0100 Subject: [PATCH 42/54] docs: explain canonical soc routing Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/data/schemas/scheduling/storage.py | 1 + 1 file changed, 1 insertion(+) diff --git a/flexmeasures/data/schemas/scheduling/storage.py b/flexmeasures/data/schemas/scheduling/storage.py index 1f82aae89c..90c8cc4330 100644 --- a/flexmeasures/data/schemas/scheduling/storage.py +++ b/flexmeasures/data/schemas/scheduling/storage.py @@ -401,6 +401,7 @@ def post_load_sequence(self, data: dict, **kwargs) -> dict: if data.get("soc_at_start") is not None: data["soc_at_start"] = (data["soc_at_start"] / ur.Quantity("MWh")).magnitude + # Canonical dynamic bounds reuse the scheduler's existing minima/maxima path internally. dynamic_types = (Sensor, SensorReference, list) if isinstance(data.get("soc_min"), dynamic_types): if data.get("soc_minima") is not None: From b7955bbab365a0c63636e45efdecebe9bb1411fb Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Sat, 11 Jul 2026 00:24:50 +0100 Subject: [PATCH 43/54] test: preserve hard soc scheduling scenarios Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/api/v3_0/tests/test_jobs_api.py | 1 + flexmeasures/data/tests/conftest.py | 2 ++ flexmeasures/data/tests/test_scheduling_jobs.py | 1 + 3 files changed, 4 insertions(+) diff --git a/flexmeasures/api/v3_0/tests/test_jobs_api.py b/flexmeasures/api/v3_0/tests/test_jobs_api.py index 28454db63f..f02fea0faa 100644 --- a/flexmeasures/api/v3_0/tests/test_jobs_api.py +++ b/flexmeasures/api/v3_0/tests/test_jobs_api.py @@ -296,6 +296,7 @@ def test_get_job_status_failed_infeasible_schedule_includes_exc_info( ): charging_station = add_charging_station_assets["Test charging station"].sensors[0] message = message_for_trigger_schedule(with_targets=True, realistic_targets=False) + message["flex-context"] = {"relax-soc-constraints": False} with app.test_client() as client: trigger_response = client.post( diff --git a/flexmeasures/data/tests/conftest.py b/flexmeasures/data/tests/conftest.py index c901aa9683..fab4c1571f 100644 --- a/flexmeasures/data/tests/conftest.py +++ b/flexmeasures/data/tests/conftest.py @@ -413,6 +413,8 @@ def flex_description_sequential( ], "site-production-capacity": "2kW", "site-consumption-capacity": "5kW", + # These tests exercise the scheduling pipeline with hard physical SoC bounds. + "relax-soc-constraints": False, # Cheap commitments that are not expected to affect the resulting schedule "commitments": [ { diff --git a/flexmeasures/data/tests/test_scheduling_jobs.py b/flexmeasures/data/tests/test_scheduling_jobs.py index e9bc6ded95..e20328ace2 100644 --- a/flexmeasures/data/tests/test_scheduling_jobs.py +++ b/flexmeasures/data/tests/test_scheduling_jobs.py @@ -457,6 +457,7 @@ def test_save_state_of_charge_percent_sensor( "production-price": "0 EUR/MWh", "site-production-capacity": "1MW", "site-consumption-capacity": "1MW", + "relax-soc-constraints": False, } create_scheduling_job( From 468bdd9c4d19a959e60f48871e4e04b82406b227 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Mon, 13 Jul 2026 09:28:19 +0100 Subject: [PATCH 44/54] chore: align agent instructions with main Signed-off-by: Mohamed Belhsan Hmida --- .github/agents/architecture-domain-specialist.md | 6 +++--- .github/instructions/feature-branch-sync.instructions.md | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/agents/architecture-domain-specialist.md b/.github/agents/architecture-domain-specialist.md index 9616b83bec..465265cd5b 100644 --- a/.github/agents/architecture-domain-specialist.md +++ b/.github/agents/architecture-domain-specialist.md @@ -129,13 +129,13 @@ differently-cleaned parameters instead of raising an error. - **Location**: `flexmeasures/data/models/generic_assets.py` - **Purpose**: Represents economic value (tangible/intangible) - **Key fields**: `id`, `name`, `account_id`, `parent_asset_id`, `attributes`, `flex_context`, `flex_model`, `sensors_to_show` -- **Relationships**: +- **Relationships**: - `owner` → Account (via account_id) - `parent_asset` → GenericAsset (via parent_asset_id) - `child_assets` ← GenericAsset (reverse of parent) - `sensors` ← Sensor (one-to-many) - **Invariant**: `db.CheckConstraint("parent_asset_id != id", name="generic_asset_self_reference_ck")` -- **Methods**: +- **Methods**: - `get_flex_context()` - Walks parent tree to reconstitute full context - `great_circle_distance()` - Geographic distance calculations - **Path representation**: Account > Asset > ... > Asset @@ -159,7 +159,7 @@ differently-cleaned parameters instead of raising an error. #### Scheduler - **Location**: `flexmeasures/data/models/planning/__init__.py` - **Purpose**: Base class for other schedulers (incl. from plugins) -- **Inputs**: +- **Inputs**: - Asset (more modern way) or Sensor (older approach) - Time window: start, end, resolution, belief_time - flex_model + flex_context diff --git a/.github/instructions/feature-branch-sync.instructions.md b/.github/instructions/feature-branch-sync.instructions.md index 97abfb9c56..926d917fac 100644 --- a/.github/instructions/feature-branch-sync.instructions.md +++ b/.github/instructions/feature-branch-sync.instructions.md @@ -34,7 +34,7 @@ git commit -m "Merge origin/main into feature branch" ``` This ensures your implementation starts from the latest state of the repository. - + ## Why this matters - Merging later causes merge conflicts to compound From 919e28d3e2eea8af97f87e375ad578f883f0c776 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Mon, 13 Jul 2026 09:28:37 +0100 Subject: [PATCH 45/54] docs: fix jobs OpenAPI description indentation Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/api/v3_0/jobs.py | 6 +++--- flexmeasures/ui/static/openapi-specs.json | 7 ++++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/flexmeasures/api/v3_0/jobs.py b/flexmeasures/api/v3_0/jobs.py index 384732d710..74dccc884f 100644 --- a/flexmeasures/api/v3_0/jobs.py +++ b/flexmeasures/api/v3_0/jobs.py @@ -141,9 +141,9 @@ def get_job_status(self, job_id: str, **kwargs): constraint analysis (empty arrays when the flex model defines no ``soc-minima``/``soc-maxima``, or when a scheduler other than ``StorageScheduler`` was used). - The ``num-beliefs`` field holds the total number of - beliefs (scheduled values) saved to the database. - nullable: true + The ``num-beliefs`` field holds the total number of + beliefs (scheduled values) saved to the database. + nullable: true func_name: type: string description: Fully-qualified name of the function executed by this job. diff --git a/flexmeasures/ui/static/openapi-specs.json b/flexmeasures/ui/static/openapi-specs.json index ed8b1d16b9..cfe65f0bb4 100644 --- a/flexmeasures/ui/static/openapi-specs.json +++ b/flexmeasures/ui/static/openapi-specs.json @@ -4270,7 +4270,7 @@ "/api/v3_0/jobs/{uuid}": { "get": { "summary": "Get the status of a background job", - "description": "Look up a background job by its UUID and see whether it is\nqueued, running, finished, or failed.\n\nThe response includes a status message plus job metadata such\nas the queue name, function name, timestamps, and the job\nresult when available.\n\nFailed jobs also include traceback information when the worker\nstored it with the job result.\n\nFor a finished scheduling job, result is an object. For a\nStorageScheduler job it holds soft state-of-charge constraint\nanalysis: unresolved lists constraints the scheduler could not\nsatisfy, and resolved lists constraints that were satisfied\nwith some margin. Each device entry's soc-minima/soc-maxima\nvalue is a list, holding one entry per violated slot (for\nunresolved) or per met slot with its margin (for resolved),\nordered chronologically. Both arrays are empty when the flex model\ndefines no soc-minima/soc-maxima, or when a scheduler other\nthan StorageScheduler was used. This is the only place\nconstraint analysis is available \u2014 the sensor schedule endpoint\n(GET /api/v3_0/sensors//schedules/) returns power\nvalues only.\n", + "description": "Look up a background job by its UUID and see whether it is\nqueued, running, finished, or failed.\n\nThe response includes a status message plus job metadata such\nas the queue name, function name, timestamps, and the job\nresult when available.\n\nFailed jobs also include traceback information when the worker\nstored it with the job result.\n\nFor a finished scheduling job, result is an object. For a\nStorageScheduler job it holds soft state-of-charge constraint\nanalysis: unresolved lists constraints the scheduler could not\nsatisfy, and resolved lists constraints that were satisfied\nwith some margin. Each device entry's soc-minima/soc-maxima\nvalue is a list, holding one entry per violated slot (for\nunresolved) or per met slot with its margin (for resolved),\nordered chronologically. Both arrays are empty when the flex model\ndefines no soc-minima/soc-maxima, or when a scheduler other\nthan StorageScheduler was used. The num-beliefs field holds\nthe total number of beliefs (scheduled values) saved to the database.\nThis is the only place constraint analysis is available \u2014 the sensor\nschedule endpoint (GET /api/v3_0/sensors//schedules/)\nreturns power values only.\n", "security": [ { "ApiKeyAuth": [] @@ -4315,7 +4315,7 @@ "description": "Human-readable description of the job status." }, "result": { - "description": "Return value of the job function, or null when not yet available. For a finished scheduling job, this is an object; a StorageScheduler job populates it with unresolved/resolved soft state-of-charge constraint analysis (empty arrays when the flex model defines no soc-minima/soc-maxima, or when a scheduler other than StorageScheduler was used).\n", + "description": "Return value of the job function, or null when not yet available. For a finished scheduling job, this is an object; a StorageScheduler job populates it with unresolved/resolved soft state-of-charge constraint analysis (empty arrays when the flex model defines no soc-minima/soc-maxima, or when a scheduler other than StorageScheduler was used). The num-beliefs field holds the total number of beliefs (scheduled values) saved to the database.\n", "nullable": true }, "func_name": { @@ -4387,7 +4387,8 @@ ] } ], - "resolved": [] + "resolved": [], + "num-beliefs": 96 }, "func_name": "flexmeasures.data.services.scheduling.create_schedule", "origin": "scheduling", From f17a7c65fbc2ffafc2d1c6bdac66243072d99a44 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Mon, 13 Jul 2026 11:02:19 +0100 Subject: [PATCH 46/54] fix: honor explicit soc relaxation disablement Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/data/schemas/scheduling/__init__.py | 10 +++++++++- .../data/schemas/tests/test_scheduling.py | 16 ++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py index 9e86404ffc..9133bdcff3 100644 --- a/flexmeasures/data/schemas/scheduling/__init__.py +++ b/flexmeasures/data/schemas/scheduling/__init__.py @@ -602,9 +602,17 @@ def check_prices(self, data: dict, original_data: dict, **kwargs): ): return data + # An explicit false value for either relaxation switch disables the related + # default SoC breach prices, even when the other switch remains enabled. + soc_relaxation_disabled = ( + original_data.get("relax-soc-constraints") is False + or original_data.get("relax-constraints") is False + ) + # Fill in default soc breach prices when asked to relax SoC constraints, unless already set explicitly. if ( - (data["relax_soc_constraints"] or data["relax_constraints"]) + not soc_relaxation_disabled + and (data["relax_soc_constraints"] or data["relax_constraints"]) and data.get("soc_minima_breach_price") is None and data.get("soc_maxima_breach_price") is None ): diff --git a/flexmeasures/data/schemas/tests/test_scheduling.py b/flexmeasures/data/schemas/tests/test_scheduling.py index 9fb171f968..bbfa8be01c 100644 --- a/flexmeasures/data/schemas/tests/test_scheduling.py +++ b/flexmeasures/data/schemas/tests/test_scheduling.py @@ -553,6 +553,22 @@ def test_flex_context_schema_preserves_explicit_soc_breach_prices(): ).magnitude == pytest.approx(7) +@pytest.mark.parametrize( + "disabled_field", + ["relax-soc-constraints", "relax-constraints"], +) +def test_flex_context_schema_disables_default_soc_breach_prices(disabled_field): + loaded_flex_context = FlexContextSchema().load( + { + "consumption-price": "1 EUR/MWh", + disabled_field: False, + } + ) + + assert "soc_minima_breach_price" not in loaded_flex_context + assert "soc_maxima_breach_price" not in loaded_flex_context + + def test_db_flex_context_schema_does_not_relax_soc_constraints_by_default(): loaded_flex_context = DBFlexContextSchema().load({}) From 4f8ce42df954c9be30b71c780e4b45f01fcf3172 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Mon, 13 Jul 2026 11:02:39 +0100 Subject: [PATCH 47/54] fix: skip soc relaxation without initial state Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/data/models/planning/storage.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 60e049afab..c083a5293e 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -802,7 +802,10 @@ def device_list_series( as_instantaneous_events=True, resolve_overlaps="max", ) - if self.flex_context.get("soc_minima_breach_price") is not None: + if ( + self.flex_context.get("soc_minima_breach_price") is not None + and soc_at_start[d] is not None + ): soc_min[d], soc_minima[d] = self._relax_scalar_soc_minimum( soc_min=soc_min[d], soc_minima=soc_minima[d], @@ -813,6 +816,7 @@ def device_list_series( if ( self.flex_context.get("soc_minima_breach_price") is not None and soc_minima[d] is not None + and soc_at_start[d] is not None ): soc_minima_breach_price = self.flex_context["soc_minima_breach_price"] any_soc_minima_breach_price = get_continuous_series_sensor_or_quantity( @@ -884,7 +888,10 @@ def device_list_series( as_instantaneous_events=True, resolve_overlaps="min", ) - if self.flex_context.get("soc_maxima_breach_price") is not None: + if ( + self.flex_context.get("soc_maxima_breach_price") is not None + and soc_at_start[d] is not None + ): soc_max[d], soc_maxima[d] = self._relax_scalar_soc_maximum( soc_max=soc_max[d], soc_maxima=soc_maxima[d], @@ -895,6 +902,7 @@ def device_list_series( if ( self.flex_context.get("soc_maxima_breach_price") is not None and soc_maxima[d] is not None + and soc_at_start[d] is not None ): soc_maxima_breach_price = self.flex_context["soc_maxima_breach_price"] any_soc_maxima_breach_price = get_continuous_series_sensor_or_quantity( From 2defb6fbcc17afca39fed95589686e9e087dc4fe Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Mon, 13 Jul 2026 12:08:08 +0100 Subject: [PATCH 48/54] test: match infeasible validation error Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/api/v3_0/tests/test_jobs_api.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/flexmeasures/api/v3_0/tests/test_jobs_api.py b/flexmeasures/api/v3_0/tests/test_jobs_api.py index 03a0843daf..9bd65f0890 100644 --- a/flexmeasures/api/v3_0/tests/test_jobs_api.py +++ b/flexmeasures/api/v3_0/tests/test_jobs_api.py @@ -403,7 +403,9 @@ def test_get_job_status_failed_infeasible_schedule_includes_exc_info( data = response.json assert data["status"] == "FAILED" assert "infeasible" in data["message"].lower() - assert "InfeasibleProblemException" in data["exc_info"] + assert ( + "ValueError: The input data yields an infeasible problem." in data["exc_info"] + ) def test_get_job_status_unauthenticated( From f354561972dcc8dbb694dd7c39024d37e3ed072e Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Sat, 18 Jul 2026 00:12:38 +0100 Subject: [PATCH 49/54] fix(schema): let relax-constraints=false keep SoC constraints hard Both relax-soc-constraints and relax-constraints default to True, so the former's default swallowed an explicit opt-out through the umbrella flag. Now an explicit relax-soc-constraints wins and otherwise the umbrella relax-constraints decides, per the semantics discussed in PR #2267: setting either flag to False keeps SoC minima/maxima hard. Also document why DBFlexContextSchema turns the relaxation defaults off. Co-Authored-By: Claude Fable 5 --- .../data/schemas/scheduling/__init__.py | 14 +++++++- .../data/schemas/scheduling/metadata.py | 2 +- .../data/schemas/tests/test_scheduling.py | 34 +++++++++++++++++++ flexmeasures/ui/static/openapi-specs.json | 6 ++-- 4 files changed, 51 insertions(+), 5 deletions(-) diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py index a59efb31e9..d02bafcdb2 100644 --- a/flexmeasures/data/schemas/scheduling/__init__.py +++ b/flexmeasures/data/schemas/scheduling/__init__.py @@ -875,8 +875,16 @@ def check_prices(self, data: dict, original_data: dict, **kwargs): return data # Fill in default soc breach prices when asked to relax SoC constraints, unless already set explicitly. + # Both relax-soc-constraints and relax-constraints default to True, so an + # explicit relax-soc-constraints wins and otherwise the umbrella + # relax-constraints decides: setting either flag to False keeps + # SoC minima/maxima as hard constraints. + if "relax-soc-constraints" in original_data: + relax_soc_constraints = data["relax_soc_constraints"] + else: + relax_soc_constraints = data["relax_constraints"] if ( - (data["relax_soc_constraints"] or data["relax_constraints"]) + relax_soc_constraints and data.get("soc_minima_breach_price") is None and data.get("soc_maxima_breach_price") is None ): @@ -1215,6 +1223,10 @@ def check_prices(self, data: dict, original_data: dict, **kwargs): class DBFlexContextSchema(FlexContextSchema, NoTimeSeriesSpecs): + # The relaxation defaults are turned off here, so validating a stored asset + # flex-context does not fill in default breach prices. The API-side defaults + # (which are True) are applied at scheduling time instead, after the stored + # flex-context is merged with the one passed in the scheduling request. relax_constraints = fields.Bool( data_key="relax-constraints", load_default=False, diff --git a/flexmeasures/data/schemas/scheduling/metadata.py b/flexmeasures/data/schemas/scheduling/metadata.py index 45d15591e7..895c22fe51 100644 --- a/flexmeasures/data/schemas/scheduling/metadata.py +++ b/flexmeasures/data/schemas/scheduling/metadata.py @@ -185,7 +185,7 @@ def to_dict(self): example=True, ) RELAX_SOC_CONSTRAINTS = MetaData( - description="If True (default), avoids not meeting SoC minima/maxima as relaxed constraints. Set this to False to keep SoC minima/maxima as hard constraints unless breach prices are supplied explicitly.", + description="If True (default), avoids not meeting SoC minima/maxima as relaxed constraints. Setting this field (or ``relax-constraints``) to False keeps SoC minima/maxima as hard constraints unless breach prices are supplied explicitly; an explicit ``relax-soc-constraints`` takes precedence over ``relax-constraints``.", example=True, ) RELAX_CAPACITY_CONSTRAINTS = MetaData( diff --git a/flexmeasures/data/schemas/tests/test_scheduling.py b/flexmeasures/data/schemas/tests/test_scheduling.py index 855e188edd..04c2893db0 100644 --- a/flexmeasures/data/schemas/tests/test_scheduling.py +++ b/flexmeasures/data/schemas/tests/test_scheduling.py @@ -552,6 +552,40 @@ def test_flex_context_schema_preserves_explicit_soc_breach_prices(): ).magnitude == pytest.approx(7) +def test_flex_context_schema_umbrella_opt_out_disables_soc_relaxation(): + """Setting relax-constraints to False alone keeps SoC minima/maxima hard.""" + loaded_flex_context = FlexContextSchema().load( + {"consumption-price": "1 EUR/MWh", "relax-constraints": False} + ) + + assert "soc_minima_breach_price" not in loaded_flex_context + assert "soc_maxima_breach_price" not in loaded_flex_context + assert "consumption_breach_price" not in loaded_flex_context + assert "ems_consumption_breach_price" not in loaded_flex_context + + +def test_flex_context_schema_explicit_soc_relaxation_overrides_umbrella_opt_out(): + """An explicit relax-soc-constraints wins over an explicit relax-constraints.""" + loaded_flex_context = FlexContextSchema().load( + { + "consumption-price": "1 EUR/MWh", + "relax-constraints": False, + "relax-soc-constraints": True, + } + ) + + assert loaded_flex_context["soc_minima_breach_price"].to( + "EUR/MWh" + ).magnitude == pytest.approx(1_000_000) + + loaded_flex_context = FlexContextSchema().load( + {"consumption-price": "1 EUR/MWh", "relax-soc-constraints": False} + ) + + assert "soc_minima_breach_price" not in loaded_flex_context + assert "soc_maxima_breach_price" not in loaded_flex_context + + def test_db_flex_context_schema_does_not_relax_soc_constraints_by_default(): loaded_flex_context = DBFlexContextSchema().load({}) diff --git a/flexmeasures/ui/static/openapi-specs.json b/flexmeasures/ui/static/openapi-specs.json index b4b13bd9e5..59830b94e6 100644 --- a/flexmeasures/ui/static/openapi-specs.json +++ b/flexmeasures/ui/static/openapi-specs.json @@ -4752,7 +4752,7 @@ "relax-soc-constraints": { "type": "boolean", "default": true, - "description": "If True (default), avoids not meeting SoC minima/maxima as relaxed constraints. Set this to False to keep SoC minima/maxima as hard constraints unless breach prices are supplied explicitly.", + "description": "If True (default), avoids not meeting SoC minima/maxima as relaxed constraints. Setting this field (or relax-constraints) to False keeps SoC minima/maxima as hard constraints unless breach prices are supplied explicitly; an explicit relax-soc-constraints takes precedence over relax-constraints.", "example": true }, "relax-capacity-constraints": { @@ -6276,7 +6276,7 @@ "example": true }, "soc-maxima": { - "description": "Set points that form upper boundaries at certain times, e.g. to target an empty heat buffer before a maintenance window.\nThe soc-maxima are soft constraints in the optimization problem by default, because relax-soc-constraints defaults to True and supplies a default soc-maxima-breach-price.\nSet relax-soc-constraints to False to keep them as hard constraints unless soc-maxima-breach-price is supplied explicitly.", + "description": "Set points that form upper boundaries at certain times, e.g. to target an empty heat buffer before a maintenance window.\nThe soc-maxima are soft constraints in the optimization problem by default, because relax-soc-constraints defaults to True and supplies a default soc-maxima-breach-price.\nSet relax-soc-constraints (or relax-constraints) to False to keep them as hard constraints unless soc-maxima-breach-price is supplied explicitly.", "example": [ { "value": "51 kWh", @@ -6287,7 +6287,7 @@ "$ref": "#/components/schemas/VariableQuantityOpenAPI" }, "soc-minima": { - "description": "Set points that form lower boundaries, e.g. to target a full car battery in the morning.\nThe soc-minima are soft constraints in the optimization problem by default, because relax-soc-constraints defaults to True and supplies a default soc-minima-breach-price.\nSet relax-soc-constraints to False to keep them as hard constraints unless soc-minima-breach-price is supplied explicitly.\nBoth single points in time and ranges are possible, see example.", + "description": "Set points that form lower boundaries, e.g. to target a full car battery in the morning.\nThe soc-minima are soft constraints in the optimization problem by default, because relax-soc-constraints defaults to True and supplies a default soc-minima-breach-price.\nSet relax-soc-constraints (or relax-constraints) to False to keep them as hard constraints unless soc-minima-breach-price is supplied explicitly.\nBoth single points in time and ranges are possible, see example.", "example": [ { "datetime": "2024-02-05T08:00:00+01:00", From 1bafc44f4ce23d0834d2728e0a22d7e885ca46f7 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Sat, 18 Jul 2026 00:13:08 +0100 Subject: [PATCH 50/54] docs: restore hard-constraints list and describe the SoC relaxation opt-out The capacity bounds (power-capacity and site-power-capacity) still remain hard constraints after relaxation; mention them again alongside soc-min, soc-max and soc-targets, and explain that setting either relax-soc-constraints or relax-constraints to false keeps SoC minima/maxima hard. Co-Authored-By: Claude Fable 5 --- documentation/api/introduction.rst | 3 ++- documentation/features/scheduling.rst | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/documentation/api/introduction.rst b/documentation/api/introduction.rst index bac858783c..0e4517c935 100644 --- a/documentation/api/introduction.rst +++ b/documentation/api/introduction.rst @@ -124,7 +124,8 @@ The response body will contain a JSON message with a ``status`` field set to ``" .. note:: FlexMeasures' built-in storage scheduler no longer computes a fallback schedule for infeasible problems. - Instead, ``soc-minima`` and ``soc-maxima`` are relaxed by default through ``"relax-soc-constraints": true``, while ``soc-min``, ``soc-max`` and ``soc-targets`` remain hard constraints. + Instead, ``soc-minima`` and ``soc-maxima`` are relaxed by default through ``"relax-soc-constraints": true`` (setting either it or ``"relax-constraints"`` to ``false`` keeps them hard). + The hard constraints that remain even after constraint relaxation are ``soc-min``, ``soc-max``, ``soc-targets`` and ``power-capacity`` in the ``flex-model``, and ``site-power-capacity`` in the ``flex-context``. If hard constraints cannot be satisfied, the scheduling job fails and clients receive the failure reason when requesting the schedule. For custom schedulers that still define a fallback scheduler, server administrators can configure whether clients receive a 303 redirect (``FLEXMEASURES_FALLBACK_REDIRECT = True``) or whether FlexMeasures follows the fallback automatically and returns the fallback schedule directly (``FLEXMEASURES_FALLBACK_REDIRECT = False``, the default). diff --git a/documentation/features/scheduling.rst b/documentation/features/scheduling.rst index 264793a055..8d438dacd7 100644 --- a/documentation/features/scheduling.rst +++ b/documentation/features/scheduling.rst @@ -314,6 +314,7 @@ However, here are some tips to model a buffer correctly: If the flex model describes an infeasible problem for the storage scheduler, the failure should remain visible. By default, ``soc-minima`` and ``soc-maxima`` are relaxed into soft constraints, so the scheduler can still return a useful schedule when these boundaries cannot be fully met. +Setting either ``relax-soc-constraints`` or ``relax-constraints`` to ``false`` in the flex-context keeps them as hard constraints. Exact ``soc-targets`` and physical ``soc-min`` / ``soc-max`` bounds remain hard constraints. If those hard constraints make the problem infeasible, the scheduling job fails instead of producing a fallback schedule. From bf2faf37bc3e757597cd8a7a12971d0b2bdeeabd Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Sat, 18 Jul 2026 00:13:08 +0100 Subject: [PATCH 51/54] test: restore config via monkeypatch and fix truncated docstring Co-Authored-By: Claude Fable 5 --- flexmeasures/api/v3_0/tests/test_sensor_schedules.py | 5 ++--- flexmeasures/data/models/planning/tests/test_solver.py | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py index 392c2fb12b..3d464121e6 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py @@ -351,13 +351,14 @@ def test_get_schedule_infeasible_storage_job_without_fallback( requesting_user, db, fallback_redirect, + monkeypatch, ): """ Test that a failing StorageScheduler call reports the failure without creating a fallback job. This test is based on flexmeasures/data/models/planning/tests/test_solver.py. """ - app.config["FLEXMEASURES_FALLBACK_REDIRECT"] = fallback_redirect + monkeypatch.setitem(app.config, "FLEXMEASURES_FALLBACK_REDIRECT", fallback_redirect) target_soc = 9 charging_station_name = "Test charging station" @@ -463,8 +464,6 @@ def test_get_schedule_infeasible_storage_job_without_fallback( ] assert "StorageFallbackScheduler" not in models - app.config["FLEXMEASURES_FALLBACK_REDIRECT"] = False - @pytest.mark.parametrize( "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py index a238d00b3c..b7adfcb1d1 100644 --- a/flexmeasures/data/models/planning/tests/test_solver.py +++ b/flexmeasures/data/models/planning/tests/test_solver.py @@ -507,7 +507,7 @@ def test_storage_scheduler_reports_unsolvable_problem_without_fallback( """Starting with a state of charge 10 kWh, within 2 hours we should be able to reach any state of charge in the range [10, 14] kWh for a unidirectional station, or [6, 14] for a bidirectional station, given a charging capacity of 2 kW. - Here we test target states of charge outside that range, ones that we should be able + Here we test target states of charge outside that range. The StorageScheduler should report this infeasible problem without hiding it behind a fallback schedule. """ From 3e0c6ae57caf3d96d1ea4544600547a581bd7d15 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Sat, 18 Jul 2026 18:13:02 +0100 Subject: [PATCH 52/54] fix: complete the merge reconciliation left out of the previous commit The previous merge commit accidentally captured a stale index, missing the actual reconciliation work: - Remove the broken flooring_resolution loop from StorageFlexModelSchema (main's off-tick design projects SoC datetimes instead of flooring them). - Excise the pre-projection soc-minima/maxima commitments blocks that the merge resurrected; fold scalar soc-min/soc-max into the projection-scoped relaxation path instead (hard bounds are kept when relaxation was auto-enabled purely for off-tick projection). - Restore the explicit-specific-wins opt-out precedence in check_prices, which the off-tick projection machinery relies on. - Guard the %-SoC-sensor capacity lookup against raw dicts/lists. - Update tests and the OpenAPI specs accordingly. Co-Authored-By: Claude Fable 5 --- flexmeasures/data/models/planning/storage.py | 211 +++--------------- .../models/planning/tests/test_storage.py | 6 +- .../data/schemas/scheduling/__init__.py | 23 +- .../data/schemas/scheduling/metadata.py | 2 +- .../data/schemas/scheduling/storage.py | 3 - .../data/schemas/tests/test_scheduling.py | 13 +- flexmeasures/ui/static/openapi-specs.json | 2 +- 7 files changed, 59 insertions(+), 201 deletions(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 9e2cf3be43..957f45b432 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -745,82 +745,6 @@ def device_list_series( as_instantaneous_events=True, resolve_overlaps="max", ) - if ( - self.flex_context.get("soc_minima_breach_price") is not None - and soc_at_start[d] is not None - ): - soc_min[d], soc_minima[d] = self._relax_scalar_soc_minimum( - soc_min=soc_min[d], - soc_minima=soc_minima[d], - query_window=(start + resolution, end + resolution), - resolution=resolution, - beliefs_before=belief_time, - ) - if ( - self.flex_context.get("soc_minima_breach_price") is not None - and soc_minima[d] is not None - and soc_at_start[d] is not None - ): - soc_minima_breach_price = self.flex_context["soc_minima_breach_price"] - any_soc_minima_breach_price = get_continuous_series_sensor_or_quantity( - variable_quantity=soc_minima_breach_price, - unit=self.flex_context["shared_currency_unit"] + "/MWh", - query_window=(start + resolution, end + resolution), - resolution=resolution, - beliefs_before=belief_time, - fill_sides=True, - ).shift(-1, freq=resolution) - all_soc_minima_breach_price = get_continuous_series_sensor_or_quantity( - variable_quantity=soc_minima_breach_price, - unit=self.flex_context["shared_currency_unit"] - + "/MWh*h", # from EUR/MWh² to EUR/MWh/resolution - query_window=(start + resolution, end + resolution), - resolution=resolution, - beliefs_before=belief_time, - fill_sides=True, - ).shift(-1, freq=resolution) - # Set up commitments DataFrame - # soc_minima_d is a temp variable because add_storage_constraints can't deal with Series yet - soc_minima_d = get_continuous_series_sensor_or_quantity( - variable_quantity=soc_minima[d], - unit="MWh", - query_window=(start + resolution, end + resolution), - resolution=resolution, - beliefs_before=belief_time, - as_instantaneous_events=True, - resolve_overlaps="max", - ) - # shift soc minima by one resolution (they define a state at a certain time, - # while the commitment defines what the total stock should be at the end of a time slot, - # where the time slot is indexed by its starting time) - soc_minima_d = soc_minima_d.shift(-1, freq=resolution) * ( - timedelta(hours=1) / resolution - ) - soc_at_start[d] * (timedelta(hours=1) / resolution) - - commitment = StockCommitment( - name="any soc minima", - quantity=soc_minima_d, - # negative price because breaching in the downwards (shortage) direction is penalized - downwards_deviation_price=-any_soc_minima_breach_price, - index=index, - _type="any", - device=d, - ) - commitments.append(commitment) - - commitment = StockCommitment( - name="all soc minima", - quantity=soc_minima_d, - # negative price because breaching in the downwards (shortage) direction is penalized - downwards_deviation_price=-all_soc_minima_breach_price, - index=index, - device=d, - ) - commitments.append(commitment) - - # soc-minima will become a soft constraint (modelled as stock commitments), so remove hard constraint - soc_minima[d] = None - if isinstance(soc_maxima[d], (Sensor, SensorReference)): soc_maxima[d] = get_continuous_series_sensor_or_quantity( variable_quantity=soc_maxima[d], @@ -831,105 +755,6 @@ def device_list_series( as_instantaneous_events=True, resolve_overlaps="min", ) - if ( - self.flex_context.get("soc_maxima_breach_price") is not None - and soc_at_start[d] is not None - ): - soc_max[d], soc_maxima[d] = self._relax_scalar_soc_maximum( - soc_max=soc_max[d], - soc_maxima=soc_maxima[d], - query_window=(start + resolution, end + resolution), - resolution=resolution, - beliefs_before=belief_time, - ) - if ( - self.flex_context.get("soc_maxima_breach_price") is not None - and soc_maxima[d] is not None - and soc_at_start[d] is not None - ): - soc_maxima_breach_price = self.flex_context["soc_maxima_breach_price"] - any_soc_maxima_breach_price = get_continuous_series_sensor_or_quantity( - variable_quantity=soc_maxima_breach_price, - unit=self.flex_context["shared_currency_unit"] + "/MWh", - query_window=(start + resolution, end + resolution), - resolution=resolution, - beliefs_before=belief_time, - fill_sides=True, - ).shift(-1, freq=resolution) - all_soc_maxima_breach_price = get_continuous_series_sensor_or_quantity( - variable_quantity=soc_maxima_breach_price, - unit=self.flex_context["shared_currency_unit"] - + "/MWh*h", # from EUR/MWh² to EUR/MWh/resolution - query_window=(start + resolution, end + resolution), - resolution=resolution, - beliefs_before=belief_time, - fill_sides=True, - ).shift(-1, freq=resolution) - # Set up commitments DataFrame - # soc_maxima_d is a temp variable because add_storage_constraints can't deal with Series yet - soc_maxima_d = get_continuous_series_sensor_or_quantity( - variable_quantity=soc_maxima[d], - unit="MWh", - query_window=(start + resolution, end + resolution), - resolution=resolution, - beliefs_before=belief_time, - as_instantaneous_events=True, - resolve_overlaps="min", - ) - # shift soc maxima by one resolution (they define a state at a certain time, - # while the commitment defines what the total stock should be at the end of a time slot, - # where the time slot is indexed by its starting time) - soc_maxima_d = soc_maxima_d.shift(-1, freq=resolution) * ( - timedelta(hours=1) / resolution - ) - soc_at_start[d] * (timedelta(hours=1) / resolution) - - commitment = StockCommitment( - name="any soc maxima", - quantity=soc_maxima_d, - # positive price because breaching in the upwards (surplus) direction is penalized - upwards_deviation_price=any_soc_maxima_breach_price, - index=index, - _type="any", - device=d, - ) - commitments.append(commitment) - - commitment = StockCommitment( - name="all soc maxima", - quantity=soc_maxima_d, - # positive price because breaching in the upwards (surplus) direction is penalized - upwards_deviation_price=all_soc_maxima_breach_price, - index=index, - device=d, - ) - commitments.append(commitment) - - # soc-maxima will become a soft constraint (modelled as stock commitments), so remove hard constraint - soc_maxima[d] = None - - # only apply SOC constraints to the first device of a shared stock - apply_soc_constraints = True - - for stock_id, devices in self.stock_groups.items(): - if d in devices and d != devices[0]: - apply_soc_constraints = False - break - - if soc_at_start[d] is not None and apply_soc_constraints: - device_constraints[d] = add_storage_constraints( - start, - end, - resolution, - soc_at_start[d], - soc_targets[d], - soc_maxima[d], - soc_minima[d], - soc_max[d], - soc_min[d], - ) - else: - # No need to validate non-existing storage constraints - skip_validation = True power_capacity_in_mw[d] = get_continuous_series_sensor_or_quantity( variable_quantity=power_capacity_in_mw[d], @@ -1165,6 +990,24 @@ def device_list_series( discharging_efficiency=discharging_efficiency[d], ) + # Fold a fixed soc-min into the relaxed minima path (after off-tick + # projection, which needs the scalar bound), so it is softened along + # with any dynamic minima instead of staying behind as a hard bound. + # When relaxation was auto-enabled purely for off-tick projection + # (the user explicitly opted out), the scalar bound stays hard. + if ( + self.flex_context.get("soc_minima_breach_price") is not None + and soc_at_start[d] is not None + and not getattr(self, "scope_soc_relaxation_to_off_tick_devices", False) + and self._soc_relaxation_applies_to(device_stock_key.get(d), sensor_d) + ): + soc_min[d], soc_minima[d] = self._relax_scalar_soc_minimum( + soc_min=soc_min[d], + soc_minima=soc_minima[d], + query_window=(start + resolution, end + resolution), + resolution=resolution, + beliefs_before=belief_time, + ) if ( self.flex_context.get("soc_minima_breach_price") is not None and soc_minima[d] is not None @@ -1232,6 +1075,24 @@ def device_list_series( # soc-minima will become a soft constraint (modelled as stock commitments), so remove hard constraint soc_minima[d] = None + # Fold a fixed soc-max into the relaxed maxima path (after off-tick + # projection, which needs the scalar bound), so it is softened along + # with any dynamic maxima instead of staying behind as a hard bound. + # When relaxation was auto-enabled purely for off-tick projection + # (the user explicitly opted out), the scalar bound stays hard. + if ( + self.flex_context.get("soc_maxima_breach_price") is not None + and soc_at_start[d] is not None + and not getattr(self, "scope_soc_relaxation_to_off_tick_devices", False) + and self._soc_relaxation_applies_to(device_stock_key.get(d), sensor_d) + ): + soc_max[d], soc_maxima[d] = self._relax_scalar_soc_maximum( + soc_max=soc_max[d], + soc_maxima=soc_maxima[d], + query_window=(start + resolution, end + resolution), + resolution=resolution, + beliefs_before=belief_time, + ) if ( self.flex_context.get("soc_maxima_breach_price") is not None and soc_maxima[d] is not None diff --git a/flexmeasures/data/models/planning/tests/test_storage.py b/flexmeasures/data/models/planning/tests/test_storage.py index 83bcf7e39d..7ee0409d97 100644 --- a/flexmeasures/data/models/planning/tests/test_storage.py +++ b/flexmeasures/data/models/planning/tests/test_storage.py @@ -1483,10 +1483,8 @@ def test_off_tick_soc_minima_are_projected_into_soft_commitments( storage_constraints = device_constraints[0].tz_convert(tz) assert ( - storage_constraints["min"] == 0 - ).all(), ( - "with a breach price, only the global soc-min should remain a hard constraint" - ) + storage_constraints["min"].isna().all() + ), "with a breach price, the global soc-min is folded into the soft commitments instead of staying a hard constraint" soc_minima_commitments = [ c for c in commitments if getattr(c, "name", "") == "any soc minima" diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py index cdb6fdc3a3..f51b9084c2 100644 --- a/flexmeasures/data/schemas/scheduling/__init__.py +++ b/flexmeasures/data/schemas/scheduling/__init__.py @@ -874,19 +874,20 @@ def check_prices(self, data: dict, original_data: dict, **kwargs): ): return data - # An explicit False for either relaxation switch keeps SoC minima/maxima - # hard, even when the other switch is explicitly enabled. - # todo: confirm precedence for the relax-constraints=false + - # relax-soc-constraints=true combo with the maintainers. - soc_relaxation_disabled = ( - original_data.get("relax-soc-constraints") is False - or original_data.get("relax-constraints") is False - ) - # Fill in default soc breach prices when asked to relax SoC constraints, unless already set explicitly. + # Both relax-soc-constraints and relax-constraints default to True, so an + # explicit relax-soc-constraints wins and otherwise the umbrella + # relax-constraints decides: setting either flag to False keeps + # SoC minima/maxima as hard constraints. Note that off-tick SoC constraint + # projection relies on this precedence: it injects an explicit + # relax-soc-constraints=true, which must win over an explicit + # relax-constraints=false. + if "relax-soc-constraints" in original_data: + relax_soc_constraints = data["relax_soc_constraints"] + else: + relax_soc_constraints = data["relax_constraints"] if ( - not soc_relaxation_disabled - and (data["relax_soc_constraints"] or data["relax_constraints"]) + relax_soc_constraints and data.get("soc_minima_breach_price") is None and data.get("soc_maxima_breach_price") is None ): diff --git a/flexmeasures/data/schemas/scheduling/metadata.py b/flexmeasures/data/schemas/scheduling/metadata.py index d76fcddd7f..542d3942e1 100644 --- a/flexmeasures/data/schemas/scheduling/metadata.py +++ b/flexmeasures/data/schemas/scheduling/metadata.py @@ -185,7 +185,7 @@ def to_dict(self): example=True, ) RELAX_SOC_CONSTRAINTS = MetaData( - description="If True (default), avoids not meeting SoC boundaries as relaxed constraints. Explicitly setting either this field or ``relax-constraints`` to False keeps SoC boundaries as hard constraints unless breach prices are supplied explicitly.", + description="If True (default), avoids not meeting SoC boundaries as relaxed constraints. Setting this field (or ``relax-constraints``) to False keeps SoC boundaries as hard constraints unless breach prices are supplied explicitly; an explicit ``relax-soc-constraints`` takes precedence over ``relax-constraints``.", example=True, ) RELAX_CAPACITY_CONSTRAINTS = MetaData( diff --git a/flexmeasures/data/schemas/scheduling/storage.py b/flexmeasures/data/schemas/scheduling/storage.py index b875c36943..17dd949c55 100644 --- a/flexmeasures/data/schemas/scheduling/storage.py +++ b/flexmeasures/data/schemas/scheduling/storage.py @@ -281,9 +281,6 @@ def __init__( setattr(self.fields[field], "timezone", self.timezone) if default_soc_unit is not None: setattr(self.fields[field], "default_src_unit", default_soc_unit) - for field in ("soc_min", "soc_max", "soc_minima", "soc_maxima", "soc_targets"): - setattr(self.fields[field], "timezone", self.timezone) - setattr(self.fields[field], "event_resolution", self.flooring_resolution) @validates_schema def check_whether_targets_exceed_max_planning_horizon(self, data: dict, **kwargs): diff --git a/flexmeasures/data/schemas/tests/test_scheduling.py b/flexmeasures/data/schemas/tests/test_scheduling.py index 975e689359..8782d277d3 100644 --- a/flexmeasures/data/schemas/tests/test_scheduling.py +++ b/flexmeasures/data/schemas/tests/test_scheduling.py @@ -572,11 +572,11 @@ def test_flex_context_schema_disables_default_soc_breach_prices(disabled_field): assert "soc_maxima_breach_price" not in loaded_flex_context -def test_flex_context_schema_umbrella_opt_out_wins_over_explicit_soc_opt_in(): - """Any explicit False keeps SoC constraints hard, even next to an explicit opt-in. +def test_flex_context_schema_explicit_soc_relaxation_overrides_umbrella_opt_out(): + """An explicit relax-soc-constraints wins over an explicit relax-constraints. - Note: the precedence for this particular combination still awaits - maintainer confirmation (see the discussion in PR #2267). + Off-tick SoC constraint projection relies on this precedence: it injects an + explicit relax-soc-constraints=true next to whatever the user configured. """ loaded_flex_context = FlexContextSchema().load( { @@ -586,8 +586,9 @@ def test_flex_context_schema_umbrella_opt_out_wins_over_explicit_soc_opt_in(): } ) - assert "soc_minima_breach_price" not in loaded_flex_context - assert "soc_maxima_breach_price" not in loaded_flex_context + assert loaded_flex_context["soc_minima_breach_price"].to( + "EUR/MWh" + ).magnitude == pytest.approx(1_000_000) def test_flex_context_schema_umbrella_opt_out_disables_capacity_relaxation(): diff --git a/flexmeasures/ui/static/openapi-specs.json b/flexmeasures/ui/static/openapi-specs.json index 1b29451702..323daab490 100644 --- a/flexmeasures/ui/static/openapi-specs.json +++ b/flexmeasures/ui/static/openapi-specs.json @@ -4771,7 +4771,7 @@ "relax-soc-constraints": { "type": "boolean", "default": true, - "description": "If True (default), avoids not meeting SoC boundaries as relaxed constraints. Explicitly setting either this field or relax-constraints to False keeps SoC boundaries as hard constraints unless breach prices are supplied explicitly.", + "description": "If True (default), avoids not meeting SoC boundaries as relaxed constraints. Setting this field (or relax-constraints) to False keeps SoC boundaries as hard constraints unless breach prices are supplied explicitly; an explicit relax-soc-constraints takes precedence over relax-constraints.", "example": true }, "relax-capacity-constraints": { From b932527a588bab58da998f0eebb98bf73662fbe2 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Wed, 22 Jul 2026 13:07:06 +0100 Subject: [PATCH 53/54] docs+test: fix review follow-ups for relax-soc-constraints default - Correct RELAX_CONSTRAINTS metadata text: it now defaults to True, not False. - Give DBFlexContextSchema's relax flags their own description, since a stored flex-context defaults to False, unlike the True scheduling-time default applied after merging with the request. - Align the hard-constraints list in scheduling.rst with introduction.rst (both now mention power-capacity / site-power-capacity). - Add the missing v3.0-32 API changelog entry for the relax-soc-constraints default flip and the retired fallback scheduler's effect on GET schedule. - Drop now-tautological assertions checking for "StorageFallbackScheduler" in belief sources, since that class no longer exists in the codebase. Co-Authored-By: Claude Sonnet 5 --- documentation/api/change_log.rst | 1 + documentation/features/scheduling.rst | 2 +- .../api/v3_0/tests/test_sensor_schedules.py | 12 ---------- .../data/schemas/scheduling/__init__.py | 22 +++++++++++++++++-- .../data/schemas/scheduling/metadata.py | 2 +- flexmeasures/ui/static/openapi-specs.json | 4 ++-- 6 files changed, 25 insertions(+), 18 deletions(-) diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst index c75a260f70..80efeefc5b 100644 --- a/documentation/api/change_log.rst +++ b/documentation/api/change_log.rst @@ -9,6 +9,7 @@ v3.0-32 | July XX, 2026 """""""""""""""""""""""" - Extended ``GET /api/v3_0/jobs/`` with a ``result`` field containing ``unresolved`` and ``resolved`` arrays, each keyed by asset ID. For scheduling jobs, this surfaces soft state-of-charge constraint analysis: ``soc-minima`` and ``soc-maxima`` violations (with a ``violation`` magnitude) or satisfied constraints (with a ``margin`` headroom). Both arrays are empty when no SoC constraints were defined. +- Breaking: ``relax-soc-constraints`` now defaults to ``True`` (set it or ``relax-constraints`` to ``False`` to keep ``soc-minima``/``soc-maxima`` hard), and the built-in storage fallback scheduler has been retired. ``GET /api/v3_0/sensors//schedules/`` now returns ``400`` with the failure reason for an infeasible storage schedule instead of a ``303`` redirect to (or an automatic follow of) a fallback schedule. ``FLEXMEASURES_FALLBACK_REDIRECT`` remains relevant only for custom schedulers that still define a fallback scheduler. v3.0-31 | 2026-06-01 """""""""""""""""""" diff --git a/documentation/features/scheduling.rst b/documentation/features/scheduling.rst index 8d438dacd7..ec698e1ea6 100644 --- a/documentation/features/scheduling.rst +++ b/documentation/features/scheduling.rst @@ -315,7 +315,7 @@ However, here are some tips to model a buffer correctly: If the flex model describes an infeasible problem for the storage scheduler, the failure should remain visible. By default, ``soc-minima`` and ``soc-maxima`` are relaxed into soft constraints, so the scheduler can still return a useful schedule when these boundaries cannot be fully met. Setting either ``relax-soc-constraints`` or ``relax-constraints`` to ``false`` in the flex-context keeps them as hard constraints. -Exact ``soc-targets`` and physical ``soc-min`` / ``soc-max`` bounds remain hard constraints. +Exact ``soc-targets``, physical ``soc-min`` / ``soc-max`` bounds, and ``power-capacity`` (in the flex-model) and ``site-power-capacity`` (in the flex-context) remain hard constraints. If those hard constraints make the problem infeasible, the scheduling job fails instead of producing a fallback schedule. It is important to take note of these failures. Often, misconfigured flex models are the reason. diff --git a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py index 3d464121e6..4fbfdd7464 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py @@ -378,12 +378,6 @@ def test_get_schedule_infeasible_storage_job_without_fallback( assert capacity == 2 assert charging_station.get_attribute("consumption-price") == {"sensor": epex_da.id} - # check that no storage fallback schedule has been saved before - models = [ - source.model for source in charging_station.search_beliefs().sources.unique() - ] - assert "StorageFallbackScheduler" not in models - # create a scenario that yields an infeasible problem (unreachable target SOC at 2am) message = { "start": start, @@ -458,12 +452,6 @@ def test_get_schedule_infeasible_storage_job_without_fallback( assert get_schedule_response.json["status"] == "UNKNOWN_SCHEDULE" assert get_schedule_response.json["result"] == "Rejected" - models = [ - source.model - for source in charging_station.search_beliefs().sources.unique() - ] - assert "StorageFallbackScheduler" not in models - @pytest.mark.parametrize( "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py index d02bafcdb2..d02c10ba95 100644 --- a/flexmeasures/data/schemas/scheduling/__init__.py +++ b/flexmeasures/data/schemas/scheduling/__init__.py @@ -1230,12 +1230,30 @@ class DBFlexContextSchema(FlexContextSchema, NoTimeSeriesSpecs): relax_constraints = fields.Bool( data_key="relax-constraints", load_default=False, - metadata=metadata.RELAX_CONSTRAINTS.to_dict(), + metadata={ + **metadata.RELAX_CONSTRAINTS.to_dict(), + "description": ( + "Defaults to False when stored on an asset (unlike the True default " + "used when scheduling): a stored flex-context should not silently bake " + "in default breach prices. The scheduling-time default of True is " + "applied after this stored flex-context is merged with the one passed " + "in the scheduling request." + ), + }, ) relax_soc_constraints = fields.Bool( data_key="relax-soc-constraints", load_default=False, - metadata=metadata.RELAX_SOC_CONSTRAINTS.to_dict(), + metadata={ + **metadata.RELAX_SOC_CONSTRAINTS.to_dict(), + "description": ( + "Defaults to False when stored on an asset (unlike the True default " + "used when scheduling): a stored flex-context should not silently bake " + "in default breach prices. The scheduling-time default of True is " + "applied after this stored flex-context is merged with the one passed " + "in the scheduling request." + ), + }, ) commitments = fields.Nested( diff --git a/flexmeasures/data/schemas/scheduling/metadata.py b/flexmeasures/data/schemas/scheduling/metadata.py index 895c22fe51..d8d2b46ca8 100644 --- a/flexmeasures/data/schemas/scheduling/metadata.py +++ b/flexmeasures/data/schemas/scheduling/metadata.py @@ -172,7 +172,7 @@ def to_dict(self): example="10 EUR/kW", ) RELAX_CONSTRAINTS = MetaData( - description="""If True (default is ``False``), several constraints are relaxed by setting default breach prices within the optimization problem, leading to the default priority: + description="""If True (default), several constraints are relaxed by setting default breach prices within the optimization problem, leading to the default priority: 1. Avoid breaching the site consumption/production capacity. 2. Avoid not meeting SoC minima/maxima. diff --git a/flexmeasures/ui/static/openapi-specs.json b/flexmeasures/ui/static/openapi-specs.json index 59830b94e6..695eca9eda 100644 --- a/flexmeasures/ui/static/openapi-specs.json +++ b/flexmeasures/ui/static/openapi-specs.json @@ -7,7 +7,7 @@ }, "termsOfService": null, "title": "FlexMeasures", - "version": "1.0.0" + "version": "0.33.2" }, "externalDocs": { "description": "FlexMeasures runs on the open source FlexMeasures technology. Read the docs here.", @@ -4746,7 +4746,7 @@ "relax-constraints": { "type": "boolean", "default": true, - "description": "If True (default is False), several constraints are relaxed by setting default breach prices within the optimization problem, leading to the default priority:\n\n1. Avoid breaching the site consumption/production capacity.\n2. Avoid not meeting SoC minima/maxima.\n3. Avoid breaching the desired device consumption/production capacity.\n\nSoC minima/maxima are already relaxed by default through relax-soc-constraints.\nSet this field to True to also enable the default site and device capacity breach prices and associated priorities as defined by FlexMeasures.\nFor tighter control over prices and priorities, the breach prices can also be set explicitly (the relevant fields have breach-price in their name).\n", + "description": "If True (default), several constraints are relaxed by setting default breach prices within the optimization problem, leading to the default priority:\n\n1. Avoid breaching the site consumption/production capacity.\n2. Avoid not meeting SoC minima/maxima.\n3. Avoid breaching the desired device consumption/production capacity.\n\nSoC minima/maxima are already relaxed by default through relax-soc-constraints.\nSet this field to True to also enable the default site and device capacity breach prices and associated priorities as defined by FlexMeasures.\nFor tighter control over prices and priorities, the breach prices can also be set explicitly (the relevant fields have breach-price in their name).\n", "example": true }, "relax-soc-constraints": { From 8dd90b6466c79040158bfc3019e43b7fffdf0cb5 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Wed, 22 Jul 2026 13:12:02 +0100 Subject: [PATCH 54/54] fix+docs: review follow-ups for canonical soc-min/soc-max - Harden the '%'-unit soc-max guard in _build_soc_schedule to also reject dict/list values, matching the sibling guard in _get_soc_capacity_for_percent_conversion that was already hardened for raw DB flex-model shapes. - Fix VariableQuantityField._serialize crashing when dumping a SensorReference's `default` on a denominator-only to_unit (e.g. "/MWh"): the default was already resolved to a concrete unit at deserialization time, so there's nothing left to convert to in that case. - Add the missing v3.0-32 API changelog entry for canonical dynamic soc-min/soc-max, the generic sensor-reference `default` fallback, and the soc-minima/soc-maxima deprecation. Co-Authored-By: Claude Sonnet 5 --- documentation/api/change_log.rst | 1 + flexmeasures/data/models/planning/storage.py | 9 ++++++--- flexmeasures/data/schemas/sensors.py | 9 ++++++++- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst index c75a260f70..978130c535 100644 --- a/documentation/api/change_log.rst +++ b/documentation/api/change_log.rst @@ -9,6 +9,7 @@ v3.0-32 | July XX, 2026 """""""""""""""""""""""" - Extended ``GET /api/v3_0/jobs/`` with a ``result`` field containing ``unresolved`` and ``resolved`` arrays, each keyed by asset ID. For scheduling jobs, this surfaces soft state-of-charge constraint analysis: ``soc-minima`` and ``soc-maxima`` violations (with a ``violation`` magnitude) or satisfied constraints (with a ``margin`` headroom). Both arrays are empty when no SoC constraints were defined. +- ``soc-min`` and ``soc-max`` in the storage flex-model are now canonical: besides a fixed quantity, they also accept a sensor reference or time series, making them dynamic storage SoC boundaries. Sensor references (on any field that accepts them) may now include a ``default`` fallback quantity to fill time slots where the referenced sensor has no value. ``soc-minima`` and ``soc-maxima`` remain supported as deprecated legacy aliases, and a fixed ``soc-min``/``soc-max`` now follows the default SoC relaxation behavior instead of always being a hard constraint. v3.0-31 | 2026-06-01 """""""""""""""""""" diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 957f45b432..d263bf4118 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -2426,11 +2426,14 @@ def _build_soc_schedule( # noqa: C901 capacity = None if soc_unit == "%": soc_max = flex_model_d0.get("soc_max") - if isinstance(soc_max, (Sensor, SensorReference)): + # A dict or list can arrive from the raw DB flex-model (e.g. a + # stored sensor reference or time series spec), so guard against + # those as well. + if isinstance(soc_max, (Sensor, SensorReference, dict, list)): raise ValueError( f"Cannot convert state-of-charge schedule to '%' unit for sensor " - f"{state_of_charge_sensor.id}: soc-max as a sensor reference is " - "not supported for '%' unit conversion." + f"{state_of_charge_sensor.id}: soc-max as a sensor reference or " + "time series is not supported for '%' unit conversion." ) if not soc_max: raise ValueError( diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py index 5ff262b34c..45da8b97ed 100644 --- a/flexmeasures/data/schemas/sensors.py +++ b/flexmeasures/data/schemas/sensors.py @@ -649,7 +649,14 @@ def _serialize( account.id for account in value.source_account ] if value.default is not None: - sensor_reference["default"] = str(value.default.to(self.to_unit)) + # `default` was already resolved to a concrete, compatible unit at + # deserialization time (see _deserialize_default), so for a + # denominator-only to_unit (e.g. "/MWh", not a valid pint unit on + # its own) there is nothing left to convert to. + if self.to_unit.startswith("/"): + sensor_reference["default"] = str(value.default) + else: + sensor_reference["default"] = str(value.default.to(self.to_unit)) return sensor_reference elif isinstance(value, Sensor): return dict(sensor=value.id)