From 58d8bb70031c3a4557d9f7c6edd975a5259a83bd Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Thu, 25 Jun 2026 22:09:45 +0100 Subject: [PATCH 01/30] feat: relax storage SoC bounds by default Signed-off-by: Mohamed Belhsan Hmida --- .../data/schemas/scheduling/__init__.py | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py index ad9dc79d17..97a2029b2a 100644 --- a/flexmeasures/data/schemas/scheduling/__init__.py +++ b/flexmeasures/data/schemas/scheduling/__init__.py @@ -180,7 +180,7 @@ class FlexContextSchema(Schema): # Dev fields relax_soc_constraints = fields.Bool( data_key="relax-soc-constraints", - load_default=False, + load_default=True, metadata=metadata.RELAX_SOC_CONSTRAINTS.to_dict(), ) relax_capacity_constraints = fields.Bool( @@ -358,7 +358,9 @@ def check_prices(self, data: dict, original_data: dict, **kwargs): for field_var, field in self.declared_fields.items() } if any( - field_map[field] in data and data[field_map[field]] + field in original_data + and field_map[field] in data + and data[field_map[field]] for field in ( "soc-minima-breach-price", "soc-maxima-breach-price", @@ -391,8 +393,7 @@ def check_prices(self, data: dict, original_data: dict, **kwargs): # Fill in default soc breach prices when asked to relax SoC constraints, unless already set explicitly. if ( - data["relax_soc_constraints"] - or data["relax_constraints"] + (data["relax_soc_constraints"] or data["relax_constraints"]) and not data.get("soc_minima_breach_price") and not data.get("soc_maxima_breach_price") ): @@ -404,8 +405,7 @@ def check_prices(self, data: dict, original_data: dict, **kwargs): # Fill in default capacity breach prices when asked to relax capacity constraints, unless already set explicitly. if ( - data["relax_capacity_constraints"] - or data["relax_constraints"] + (data["relax_capacity_constraints"] or data["relax_constraints"]) and not data.get("consumption_breach_price") and not data.get("production_breach_price") ): @@ -417,8 +417,7 @@ def check_prices(self, data: dict, original_data: dict, **kwargs): # Fill in default site capacity breach prices when asked to relax site capacity constraints, unless already set explicitly. if ( - data["relax_site_capacity_constraints"] - or data["relax_constraints"] + (data["relax_site_capacity_constraints"] or data["relax_constraints"]) and not data.get("ems_consumption_breach_price") and not data.get("ems_production_breach_price") ): @@ -764,6 +763,11 @@ def _to_currency_per_mwh(price_unit: str) -> str: class DBFlexContextSchema(FlexContextSchema, NoTimeSeriesSpecs): + relax_soc_constraints = fields.Bool( + data_key="relax-soc-constraints", + load_default=False, + metadata=metadata.RELAX_SOC_CONSTRAINTS.to_dict(), + ) commitments = fields.Nested( DBCommitmentSchema, data_key="commitments", required=False, many=True From 2944dacda1f5f4cc0132cff5cf3a19907f742279 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Thu, 25 Jun 2026 22:09:51 +0100 Subject: [PATCH 02/30] feat: remove storage fallback scheduler Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/data/models/planning/storage.py | 73 -------------------- 1 file changed, 73 deletions(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 4c59a0ffa9..75705f06d6 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -3,7 +3,6 @@ import re import copy from datetime import datetime, timedelta -from typing import Type import pandas as pd import numpy as np @@ -26,7 +25,6 @@ initialize_series, initialize_df, get_power_values, - fallback_charging_policy, get_continuous_series_sensor_or_quantity, ) from flexmeasures.data.models.planning.exceptions import InfeasibleProblemException @@ -1582,81 +1580,10 @@ def _ensure_variable_quantity( return q -class StorageFallbackScheduler(MetaStorageScheduler): - __version__ = "3" - __author__ = "Seita" - - def compute(self, skip_validation: bool = False) -> SchedulerOutputType: - """Schedule a battery or Charge Point by just starting to charge, discharge, or do neither, - depending on the first target state of charge and the capabilities of the Charge Point. - For the resulting consumption schedule, consumption is defined as positive values. - - Note that this ignores any cause of the infeasibility. - - :param skip_validation: If True, skip validation of constraints specified in the data. - :returns: The computed schedule. - """ - - ( - sensors, - start, - end, - resolution, - soc_at_start, - device_constraints, - ems_constraints, - commitments, - ) = self._prepare(skip_validation=skip_validation) - - # Fallback policy if the problem was unsolvable - storage_schedule = { - sensor: fallback_charging_policy( - sensor, device_constraints[d], start, end, resolution - ) - for d, sensor in enumerate(sensors) - if sensor is not None - } - - # Convert each device schedule to the unit of the device's power sensor - storage_schedule = { - sensor: convert_units( - storage_schedule[sensor], - "MW", - sensor.unit, - event_resolution=sensor.event_resolution, - ) - for sensor in sensors - if sensor is not None - } - - # Round schedule - if self.round_to_decimals: - storage_schedule = { - sensor: storage_schedule[sensor].round(self.round_to_decimals) - for sensor in sensors - if sensor is not None - } - - if self.return_multiple: - return [ - { - "name": "storage_schedule", - "sensor": sensor, - "data": storage_schedule[sensor], - } - for sensor in sensors - if sensor is not None - ] - else: - return storage_schedule[sensors[0]] - - class StorageScheduler(MetaStorageScheduler): __version__ = "8" __author__ = "Seita" - fallback_scheduler_class: Type[Scheduler] = StorageFallbackScheduler - @staticmethod def _build_soc_schedule( flex_model: list[dict], From 09e9998b59d9b18bce3a153f82ab7e1438625e13 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Thu, 25 Jun 2026 22:09:56 +0100 Subject: [PATCH 03/30] feat: remove storage fallback policy helper Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/data/models/planning/utils.py | 77 ---------------------- 1 file changed, 77 deletions(-) diff --git a/flexmeasures/data/models/planning/utils.py b/flexmeasures/data/models/planning/utils.py index cfea6eba58..7cd79f7156 100644 --- a/flexmeasures/data/models/planning/utils.py +++ b/flexmeasures/data/models/planning/utils.py @@ -220,83 +220,6 @@ def get_power_values( return -series -def fallback_charging_policy( - sensor: Sensor, - device_constraints: pd.DataFrame, - start: datetime, - end: datetime, - resolution: timedelta, -) -> pd.Series: - """This fallback charging policy is to just start charging or discharging, or do neither, - depending on the first target state of charge and the capabilities of the Charge Point. - Note that this ignores any cause of the infeasibility and, - while probably a decent policy for Charge Points, - should not be considered a robust policy for other asset types. - """ - max_charge_capacity = ( - device_constraints[["derivative max", "derivative equals"]].min().min() - ) - max_discharge_capacity = ( - -device_constraints[["derivative min", "derivative equals"]].max().max() - ) - charge_power = max_charge_capacity if sensor.get_attribute("is_consumer") else 0 - discharge_power = ( - -max_discharge_capacity if sensor.get_attribute("is_producer") else 0 - ) - - charge_schedule = initialize_series(charge_power, start, end, resolution) - discharge_schedule = initialize_series(discharge_power, start, end, resolution) - idle_schedule = initialize_series(0, start, end, resolution) - if ( - device_constraints["equals"].first_valid_index() is not None - and device_constraints["equals"][ - device_constraints["equals"].first_valid_index() - ] - > 0 - ): - # start charging to get as close as possible to the next target - return idle_after_reaching_target(charge_schedule, device_constraints["equals"]) - if ( - device_constraints["equals"].first_valid_index() is not None - and device_constraints["equals"][ - device_constraints["equals"].first_valid_index() - ] - < 0 - ): - # start discharging to get as close as possible to the next target - return idle_after_reaching_target( - discharge_schedule, device_constraints["equals"] - ) - if ( - device_constraints["max"].first_valid_index() is not None - and device_constraints["max"][device_constraints["max"].first_valid_index()] < 0 - ): - # start discharging to try and bring back the soc below the next max constraint - return idle_after_reaching_target(discharge_schedule, device_constraints["max"]) - if ( - device_constraints["min"].first_valid_index() is not None - and device_constraints["min"][device_constraints["min"].first_valid_index()] > 0 - ): - # start charging to try and bring back the soc above the next min constraint - return idle_after_reaching_target(charge_schedule, device_constraints["min"]) - # stand idle - return idle_schedule - - -def idle_after_reaching_target( - schedule: pd.Series, - target: pd.Series, - initial_state: float = 0, -) -> pd.Series: - """Stop planned (dis)charging after target is reached (or constraint is met).""" - first_target = target[target.first_valid_index()] - if first_target > initial_state: - schedule[schedule.cumsum() > first_target] = 0 - else: - schedule[schedule.cumsum() < first_target] = 0 - return schedule - - def get_series_from_quantity_or_sensor( variable_quantity: Sensor | SensorReference | list[dict] | ur.Quantity, unit: ur.Quantity | str, From 11eeb37011b626265325c7a7dfb0cdebe2265650 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Thu, 25 Jun 2026 22:10:01 +0100 Subject: [PATCH 04/30] test: cover SoC relaxation schema defaults Signed-off-by: Mohamed Belhsan Hmida --- .../data/schemas/tests/test_scheduling.py | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/flexmeasures/data/schemas/tests/test_scheduling.py b/flexmeasures/data/schemas/tests/test_scheduling.py index 3d4450580e..4522d97484 100644 --- a/flexmeasures/data/schemas/tests/test_scheduling.py +++ b/flexmeasures/data/schemas/tests/test_scheduling.py @@ -517,6 +517,48 @@ def test_flex_context_schema( check_schema_loads_data(schema=schema, data=flex_context, fails=fails) +def test_flex_context_schema_relaxes_soc_constraints_by_default(): + loaded_flex_context = FlexContextSchema().load({"consumption-price": "1 EUR/MWh"}) + + assert loaded_flex_context["relax_soc_constraints"] is True + assert loaded_flex_context["soc_minima_breach_price"].to( + "EUR/MWh" + ).magnitude == pytest.approx(1_000_000) + assert loaded_flex_context["soc_maxima_breach_price"].to( + "EUR/MWh" + ).magnitude == pytest.approx(1_000_000) + assert "consumption_breach_price" not in loaded_flex_context + assert "production_breach_price" not in loaded_flex_context + assert "ems_consumption_breach_price" not in loaded_flex_context + assert "ems_production_breach_price" not in loaded_flex_context + + +def test_flex_context_schema_preserves_explicit_soc_breach_prices(): + loaded_flex_context = FlexContextSchema().load( + { + "consumption-price": "1 EUR/MWh", + "soc-minima-breach-price": "5 EUR/kWh", + "soc-maxima-breach-price": "7 EUR/kWh", + } + ) + + assert loaded_flex_context["relax_soc_constraints"] is True + assert loaded_flex_context["soc_minima_breach_price"].to( + "EUR/kWh" + ).magnitude == pytest.approx(5) + assert loaded_flex_context["soc_maxima_breach_price"].to( + "EUR/kWh" + ).magnitude == pytest.approx(7) + + +def test_db_flex_context_schema_does_not_relax_soc_constraints_by_default(): + loaded_flex_context = DBFlexContextSchema().load({}) + + assert loaded_flex_context["relax_soc_constraints"] is False + assert "soc_minima_breach_price" not in loaded_flex_context + assert "soc_maxima_breach_price" not in loaded_flex_context + + def check_schema_loads_data(schema, data, fails): if fails: with pytest.raises(ValidationError) as e_info: From 0fd1487b0d8f7fefe5a4f999e986a2449adafbb2 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Thu, 25 Jun 2026 22:10:06 +0100 Subject: [PATCH 05/30] test: use default SoC breach prices Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/data/models/planning/tests/test_storage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flexmeasures/data/models/planning/tests/test_storage.py b/flexmeasures/data/models/planning/tests/test_storage.py index ed8c09be82..a5a47d6bc6 100644 --- a/flexmeasures/data/models/planning/tests/test_storage.py +++ b/flexmeasures/data/models/planning/tests/test_storage.py @@ -174,7 +174,8 @@ def test_battery_relaxation(add_battery_assets, db): """Check that resolving SoC breaches is more important than resolving device power breaches. The battery is still charging with 25 kW between noon and 4 PM, when the consumption capacity is supposed to be 0. - It is still charging because resolving the still unmatched SoC minima takes precedence (via breach prices). + It is still charging because resolving the still unmatched SoC minima takes precedence + through the default SoC breach prices. """ _, battery = get_sensors_from_db( db, add_battery_assets, battery_name="Test battery" @@ -252,7 +253,6 @@ def test_battery_relaxation(add_battery_assets, db): "site-peak-production-price": series_to_ts_specs( pd.Series(260, production_prices.index), unit="EUR/MW" ), - "soc-minima-breach-price": "6000 EUR/kWh", # high breach price (to mimic a hard constraint) "consumption-breach-price": f"{device_power_breach_price} EUR/kW", # lower breach price (thus prioritizing minimizing soc breaches) "production-breach-price": f"{device_power_breach_price} EUR/kW", # lower breach price (thus prioritizing minimizing soc breaches) }, From 18aa13b03b26845adb030dff7e3af4637b96acd3 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Thu, 25 Jun 2026 22:10:12 +0100 Subject: [PATCH 06/30] test: expect storage infeasibility without fallback Signed-off-by: Mohamed Belhsan Hmida --- .../data/models/planning/tests/test_solver.py | 32 +++---------------- 1 file changed, 5 insertions(+), 27 deletions(-) diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py index e9e61da848..8c37ff05c2 100644 --- a/flexmeasures/data/models/planning/tests/test_solver.py +++ b/flexmeasures/data/models/planning/tests/test_solver.py @@ -498,22 +498,18 @@ def test_charging_station_solver_day_2( (15, "Test charging station (bidirectional)"), ], ) -def test_fallback_to_unsolvable_problem( +def test_storage_scheduler_reports_unsolvable_problem_without_fallback( target_soc, charging_station_name, setup_planning_test_data, db ): """Starting with a state of charge 10 kWh, within 2 hours we should be able to reach any state of charge in the range [10, 14] kWh for a unidirectional station, or [6, 14] for a bidirectional station, given a charging capacity of 2 kW. Here we test target states of charge outside that range, ones that we should be able - to get as close to as 1 kWh difference. - We want our scheduler to handle unsolvable problems like these with a sensible fallback policy. - - The StorageScheduler raises an Exception which triggers the creation of a new job to compute a fallback - schedule. + The StorageScheduler should report this infeasible problem without hiding it behind + a fallback schedule. """ soc_at_start = 10 duration_until_target = timedelta(hours=2) - expected_gap = 1 epex_da = get_test_sensor(db) charging_station = setup_planning_test_data[charging_station_name].sensors[0] @@ -578,26 +574,8 @@ def test_fallback_to_unsolvable_problem( # calling the scheduler with an infeasible problem raises an Exception with pytest.raises(InfeasibleProblemException): - consumption_schedule = scheduler.compute(skip_validation=True) - - # check that the fallback scheduler provides a sensible fallback policy - fallback_scheduler = scheduler.fallback_scheduler_class(**kwargs) - fallback_scheduler.config_deserialized = True - consumption_schedule = fallback_scheduler.compute(skip_validation=True) - - soc_schedule = integrate_time_series( - consumption_schedule, soc_at_start, decimal_precision=6 - ) - - # Check if constraints were met - assert min(consumption_schedule.values) >= capacity * -1 - assert max(consumption_schedule.values) <= capacity - print(consumption_schedule.head(12)) - print(soc_schedule.head(12)) - assert ( - abs(abs(soc_schedule.loc[target_soc_datetime] - target_soc) - expected_gap) - < TOLERANCE - ) + scheduler.compute(skip_validation=True) + assert scheduler.fallback_scheduler_class is None @pytest.mark.parametrize( From 783fd8b1e221cb144ba361f5b7be3915ed5514b8 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Thu, 25 Jun 2026 22:10:27 +0100 Subject: [PATCH 07/30] test: assert storage schedules do not fall back Signed-off-by: Mohamed Belhsan Hmida --- .../api/v3_0/tests/test_sensor_schedules.py | 196 ++---------------- 1 file changed, 16 insertions(+), 180 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py index 6928c452ab..37cb47bee7 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py @@ -340,7 +340,8 @@ def test_trigger_and_get_schedule_with_unknown_prices( @pytest.mark.parametrize( "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True ) -def test_get_schedule_fallback( +@pytest.mark.parametrize("fallback_redirect", [True, False]) +def test_get_schedule_infeasible_storage_job_without_fallback( app, add_battery_assets, add_market_prices, @@ -349,13 +350,14 @@ def test_get_schedule_fallback( keep_scheduling_queue_empty, requesting_user, db, + fallback_redirect, ): """ - Test if the fallback job is created after a failing StorageScheduler call. This test - is based on flexmeasures/data/models/planning/tests/test_solver.py + Test that a failing StorageScheduler call reports the failure without creating a fallback job. + + This test is based on flexmeasures/data/models/planning/tests/test_solver.py. """ - assert app.config["FLEXMEASURES_FALLBACK_REDIRECT"] is False - app.config["FLEXMEASURES_FALLBACK_REDIRECT"] = True + app.config["FLEXMEASURES_FALLBACK_REDIRECT"] = fallback_redirect target_soc = 9 charging_station_name = "Test charging station" @@ -375,7 +377,7 @@ def test_get_schedule_fallback( assert capacity == 2 assert charging_station.get_attribute("consumption-price") == {"sensor": epex_da.id} - # check that no Fallback schedule has been saved before + # check that no retired storage fallback schedule has been saved before models = [ source.model for source in charging_station.search_beliefs().sources.unique() ] @@ -386,6 +388,7 @@ def test_get_schedule_fallback( "start": start, "duration": "PT24H", "resolution": "PT15M", # just schedule in the original sensor resolution + "force-new-job-creation": True, "flex-model": { "soc-at-start": 10, "soc-min": charging_station.get_attribute("min_soc_in_mwh", 0), @@ -439,193 +442,26 @@ def test_get_schedule_fallback( # Make sure the resolution shows up in the job kwargs assert job.kwargs.get("resolution") == pd.Timedelta(message["resolution"]) - # the callback creates the fallback job which is still pending - assert len(app.queues["scheduling"]) == 1 - fallback_job_id = Job.fetch( - job_id, connection=app.queues["scheduling"].connection - ).meta.get("fallback_job_id") - - # check that the fallback_job_id is stored on the metadata of the original job - assert app.queues["scheduling"].get_job_ids()[0] == fallback_job_id - assert fallback_job_id != job_id + # no storage fallback job is created + assert len(app.queues["scheduling"]) == 0 + assert job.meta.get("fallback_job_id") is None get_schedule_response = client.get( url_for("SensorAPI:get_schedule", id=charging_station.id, uuid=job_id), ) - # requesting the original job redirects to the fallback job - assert ( - get_schedule_response.status_code == 303 - ) # Status code for redirect ("See other") - assert ( + assert get_schedule_response.status_code == 400 + assert "Scheduling job failed with InfeasibleProblemException: infeasible." in ( get_schedule_response.json["message"] - == "Scheduling job failed with InfeasibleProblemException: infeasible. StorageScheduler was used." ) + assert "StorageScheduler was used." in get_schedule_response.json["message"] assert get_schedule_response.json["status"] == "UNKNOWN_SCHEDULE" assert get_schedule_response.json["result"] == "Rejected" - # check that the redirection location points to the fallback job - assert ( - get_schedule_response.headers["location"] - == f"http://localhost/api/v3_0/sensors/{charging_station.id}/schedules/{fallback_job_id}" - ) - - # run the fallback job - work_on_rq( - app.queues["scheduling"], - exc_handler=handle_scheduling_exception, - max_jobs=1, - ) - - # check that the queue is empty - assert len(app.queues["scheduling"]) == 0 - - # get the fallback schedule - fallback_schedule = client.get( - get_schedule_response.headers["location"], - json={"duration": "PT24H"}, - ).json - - # check that the fallback schedule has the right status and start dates - assert fallback_schedule["status"] == "PROCESSED" - assert parse_datetime(fallback_schedule["start"]) == parse_datetime(start) - models = [ source.model for source in charging_station.search_beliefs().sources.unique() ] - assert "StorageFallbackScheduler" in models - - app.config["FLEXMEASURES_FALLBACK_REDIRECT"] = False - - -@pytest.mark.parametrize( - "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True -) -def test_get_schedule_fallback_not_redirect( - app, - add_battery_assets, - add_market_prices, - battery_soc_sensor, - add_charging_station_assets, - keep_scheduling_queue_empty, - requesting_user, - db, -): - """ - Test if the fallback scheduler is returned directly after a failing StorageScheduler call. This test - is based on flexmeasures/data/models/planning/tests/test_solver.py - """ - app.config["FLEXMEASURES_FALLBACK_REDIRECT"] = False - - target_soc = 9 - charging_station_name = "Test charging station" - - start = "2015-01-02T00:00:00+01:00" - epex_da = get_test_sensor(db) - charging_station = get_sensor_by_name( - add_charging_station_assets[charging_station_name], "power" - ) - - capacity = charging_station.get_attribute( - "capacity_in_mw", - ur.Quantity(charging_station.get_attribute("site-power-capacity")) - .to("MW") - .magnitude, - ) - assert capacity == 2 - assert charging_station.get_attribute("consumption-price") == {"sensor": epex_da.id} - - # create a scenario that yields an infeasible problem (unreachable target SOC at 2am) - message = { - "start": start, - "duration": "PT24H", - "flex-model": { - "soc-at-start": 10, - "soc-min": charging_station.get_attribute("min_soc_in_mwh", 0), - "soc-max": charging_station.get_attribute("max-soc-in-mwh", target_soc), - "roundtrip-efficiency": charging_station.get_attribute( - "roundtrip-efficiency", 1 - ), - "storage-efficiency": charging_station.get_attribute( - "storage-efficiency", 1 - ), - "soc-targets": [ - { - "value": target_soc, - "start": "2015-01-02T02:00:00+01:00", - "duration": "PT0H", - } - ], - }, - } - - with app.test_client() as client: - # trigger storage scheduler - trigger_schedule_response = client.post( - url_for("SensorAPI:trigger_schedule", id=charging_station.id), - json=message, - ) - - # check that the call is successful - assert trigger_schedule_response.status_code == 200 - job_id = trigger_schedule_response.json["schedule"] - - # look for scheduling jobs in queue - assert ( - len(app.queues["scheduling"]) == 1 - ) # only 1 schedule should be made for 1 asset - job = app.queues["scheduling"].jobs[0] - assert job.kwargs["asset_or_sensor"]["id"] == charging_station.id - assert job.kwargs["start"] == parse_datetime(message["start"]) - assert job.id == job_id - - # process only the job that runs the storage scheduler (max_jobs=1) - work_on_rq( - app.queues["scheduling"], - exc_handler=handle_scheduling_exception, - max_jobs=1, - ) - - # check that the job is failing - job = Job.fetch(job_id, connection=app.queues["scheduling"].connection) - assert job.is_failed - - # Make sure that the db flex_context shows up in the job kwargs - assert "flex-context" not in message and job.kwargs.get("flex_context") - - # the callback creates the fallback job which is still pending - assert len(app.queues["scheduling"]) == 1 - - fallback_job_id = Job.fetch( - job_id, connection=app.queues["scheduling"].connection - ).meta.get("fallback_job_id") - - # check that the fallback_job_id is stored on the metadata of the original job - assert app.queues["scheduling"].get_job_ids()[0] == fallback_job_id - assert fallback_job_id != job_id - - get_schedule_response = client.get( - url_for("SensorAPI:get_schedule", id=charging_station.id, uuid=job_id), - ) - - work_on_rq( - app.queues["scheduling"], - exc_handler=handle_scheduling_exception, - max_jobs=1, - ) - - get_schedule_response = client.get( - url_for("SensorAPI:get_schedule", id=charging_station.id, uuid=job_id), - ) - - assert get_schedule_response.status_code == 200 - - schedule = get_schedule_response.json - - # check that the fallback schedule has the right status and start dates - assert schedule["status"] == "PROCESSED" - assert parse_datetime(schedule["start"]) == parse_datetime(start) - assert schedule["scheduler_info"]["scheduler"] == "StorageFallbackScheduler" + assert "StorageFallbackScheduler" not in models app.config["FLEXMEASURES_FALLBACK_REDIRECT"] = False From 48114c12560080e0dd9bae0dbff24251b4df8e0e Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Thu, 25 Jun 2026 22:10:35 +0100 Subject: [PATCH 08/30] test: update sequential scheduling fallback case Signed-off-by: Mohamed Belhsan Hmida --- .../data/tests/test_scheduling_sequential.py | 106 +++++++++--------- 1 file changed, 52 insertions(+), 54 deletions(-) diff --git a/flexmeasures/data/tests/test_scheduling_sequential.py b/flexmeasures/data/tests/test_scheduling_sequential.py index 53f67fa8c9..b36c547aaa 100644 --- a/flexmeasures/data/tests/test_scheduling_sequential.py +++ b/flexmeasures/data/tests/test_scheduling_sequential.py @@ -141,13 +141,13 @@ def test_create_sequential_jobs(db, app, flex_description_sequential, smart_buil assert total_cost == -2.1775, f"Total cost should be -2.1775 €, got {total_cost} €" -def test_create_sequential_jobs_fallback( +def test_create_sequential_jobs_without_storage_fallback( db, app, flex_description_sequential, smart_building ): - """Test fallback scheduler in a chain of sequential scheduling (sub)jobs. + """Test an infeasible first subjob in a chain of sequential scheduling jobs. - Checks execution of a sequential scheduling job, where 1 of the subjobs is set up to fail and trigger its fallback. - The deferred subjobs should still succeed after the fallback succeeds, even though the first subjob fails. + Checks that no storage fallback job is created. The deferred subjobs should remain + deferred because the first subjob failed. """ assets, sensors, _ = smart_building queue = app.queues["scheduling"] @@ -166,53 +166,51 @@ def test_create_sequential_jobs_fallback( storage_module = "flexmeasures.data.models.planning.storage" with patch(f"{storage_module}.StorageScheduler.persist_flex_model"): - with patch(f"{storage_module}.StorageFallbackScheduler.persist_flex_model"): - with patch( - f"{storage_module}.StorageScheduler.compute", - side_effect=iter([InfeasibleProblemException(), [], []]), - ): - create_sequential_scheduling_job( - asset=assets["Test Site"], - scheduler_specs=scheduler_specs, - enqueue=True, - force_new_job_creation=True, # otherwise the cache might kick in due to sub-jobs already created in other tests - **flex_description_sequential, - ) - - # There should be 3 jobs: - # 2 jobs scheduling the 2 flexible devices in the flex-model, plus 1 'done job' to wrap things up - queued_jobs = app.queues["scheduling"].jobs - deferred_jobs = [ - Job.fetch(job_id, connection=queue.connection) - for job_id in app.queues[ - "scheduling" - ].deferred_job_registry.get_job_ids() - ] - # Sort deferred_jobs by their created_at attribute - deferred_jobs = sorted(deferred_jobs, key=lambda job: job.created_at) - assert ( - len(queued_jobs) == 1 - ), "Only the job for scheduling the first device sequentially should be queued." - assert ( - len(deferred_jobs) == 2 - ), "The job for scheduling the second device, and the wrap-up job, should be deferred." - - # Work on jobs - work_on_rq(queue, exc_handler=handle_scheduling_exception) - - # Refresh jobs so that the fallback_job_id (which should be set by now) can be read - for job in queued_jobs: - job.refresh() - - finished_jobs = queue.finished_job_registry.get_job_ids() - failed_jobs = queue.failed_job_registry.get_job_ids() - - # Original job failed - assert queued_jobs[0].id in failed_jobs - - # The fallback job ran successfully - assert queued_jobs[0].meta["fallback_job_id"] in finished_jobs - - # The deferred jobs ran successfully - assert deferred_jobs[0].id in finished_jobs - assert deferred_jobs[1].id in finished_jobs + with patch( + f"{storage_module}.StorageScheduler.compute", + side_effect=InfeasibleProblemException(), + ): + create_sequential_scheduling_job( + asset=assets["Test Site"], + scheduler_specs=scheduler_specs, + enqueue=True, + force_new_job_creation=True, # otherwise the cache might kick in due to sub-jobs already created in other tests + **flex_description_sequential, + ) + + # There should be 3 jobs: + # 2 jobs scheduling the 2 flexible devices in the flex-model, plus 1 'done job' to wrap things up + queued_jobs = app.queues["scheduling"].jobs + deferred_jobs = [ + Job.fetch(job_id, connection=queue.connection) + for job_id in app.queues[ + "scheduling" + ].deferred_job_registry.get_job_ids() + ] + # Sort deferred_jobs by their created_at attribute + deferred_jobs = sorted(deferred_jobs, key=lambda job: job.created_at) + assert ( + len(queued_jobs) == 1 + ), "Only the job for scheduling the first device sequentially should be queued." + assert ( + len(deferred_jobs) == 2 + ), "The job for scheduling the second device, and the wrap-up job, should be deferred." + + # Work on jobs + work_on_rq(queue, exc_handler=handle_scheduling_exception) + + for job in queued_jobs: + job.refresh() + for job in deferred_jobs: + job.refresh() + + finished_jobs = queue.finished_job_registry.get_job_ids() + failed_jobs = queue.failed_job_registry.get_job_ids() + + # Original job failed and no fallback job was created + assert queued_jobs[0].id in failed_jobs + assert queued_jobs[0].meta.get("fallback_job_id") is None + + # The deferred jobs should not run when their dependency fails without fallback + assert deferred_jobs[0].id not in finished_jobs + assert deferred_jobs[1].id not in finished_jobs From 9d90ff917c99d602e33b83d48c3b12bdac88099b Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Thu, 25 Jun 2026 22:10:41 +0100 Subject: [PATCH 09/30] docs: describe default SoC relaxation metadata Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/data/schemas/scheduling/metadata.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/flexmeasures/data/schemas/scheduling/metadata.py b/flexmeasures/data/schemas/scheduling/metadata.py index 21c7765f18..97d2377b13 100644 --- a/flexmeasures/data/schemas/scheduling/metadata.py +++ b/flexmeasures/data/schemas/scheduling/metadata.py @@ -145,13 +145,14 @@ def to_dict(self): 2. Avoid not meeting SoC minima/maxima. 3. Avoid breaching the desired device consumption/production capacity. -We recommend to set this field to ``True`` to enable the default prices and associated priorities as defined by FlexMeasures. +SoC minima/maxima are already relaxed by default through ``relax-soc-constraints``. +Set this field to ``True`` to also enable the default site and device capacity breach prices and associated priorities as defined by FlexMeasures. For tighter control over prices and priorities, the breach prices can also be set explicitly (the relevant fields have ``breach-price`` in their name). """, example=True, ) RELAX_SOC_CONSTRAINTS = MetaData( - description="If True, avoids not meeting SoC minima/maxima as a relaxed constraint.", + description="If True (default), avoids not meeting SoC minima/maxima as relaxed constraints. Set this to False to keep SoC minima/maxima as hard constraints unless breach prices are supplied explicitly.", example=True, ) RELAX_CAPACITY_CONSTRAINTS = MetaData( @@ -246,8 +247,8 @@ def to_dict(self): ) SOC_MINIMA = MetaData( description="""Set points that form lower boundaries, e.g. to target a full car battery in the morning. -If a ``soc-minima-breach-price`` is defined, the ``soc-minima`` become soft constraints in the optimization problem. -Otherwise, they become hard constraints. [#maximum_overlap]_. Both single points in time and ranges are possible, see example.""", +The ``soc-minima`` are soft constraints in the optimization problem by default, because ``relax-soc-constraints`` defaults to ``True`` and supplies a default ``soc-minima-breach-price``. +Set ``relax-soc-constraints`` to ``False`` to keep them as hard constraints unless ``soc-minima-breach-price`` is supplied explicitly. [#maximum_overlap]_. Both single points in time and ranges are possible, see example.""", example=[ {"datetime": "2024-02-05T08:00:00+01:00", "value": "8.2 kWh"}, { @@ -259,8 +260,8 @@ def to_dict(self): ) SOC_MAXIMA = MetaData( description="""Set points that form upper boundaries at certain times, e.g. to target an empty heat buffer before a maintenance window. -If a ``soc-maxima-breach-price`` is defined, the ``soc-maxima`` become soft constraints in the optimization problem. -Otherwise, they become hard constraints. [#minimum_overlap]_""", +The ``soc-maxima`` are soft constraints in the optimization problem by default, because ``relax-soc-constraints`` defaults to ``True`` and supplies a default ``soc-maxima-breach-price``. +Set ``relax-soc-constraints`` to ``False`` to keep them as hard constraints unless ``soc-maxima-breach-price`` is supplied explicitly. [#minimum_overlap]_""", example=[ { "value": "51 kWh", From ed233429d56b8b8e6e2ad8b180bf0a169f5d3af1 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Thu, 25 Jun 2026 22:10:47 +0100 Subject: [PATCH 10/30] docs: update storage scheduling infeasibility guide Signed-off-by: Mohamed Belhsan Hmida --- documentation/features/scheduling.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/documentation/features/scheduling.rst b/documentation/features/scheduling.rst index 1643330a6d..4f53b5f809 100644 --- a/documentation/features/scheduling.rst +++ b/documentation/features/scheduling.rst @@ -269,12 +269,12 @@ However, here are some tips to model a buffer correctly: - Set ``storage-efficiency`` to a value below 100% to model (heat) loss. What happens if the flex model describes an infeasible problem for the storage scheduler? Excellent question! -It is highly important for a robust operation that these situations still lead to a somewhat good outcome. -From our practical experience, we derived a ``StorageFallbackScheduler``. -It simplifies an infeasible situation by just starting to charge, discharge, or do neither, -depending on the first target state of charge and the capabilities of the asset. +It is highly important for robust operation that these situations remain visible. +By default, ``soc-minima`` and ``soc-maxima`` are relaxed into soft constraints, so the scheduler can still return a useful schedule when these boundaries cannot be fully met. +Exact ``soc-targets`` and physical ``soc-min`` / ``soc-max`` bounds remain hard constraints. +If those hard constraints make the problem infeasible, the scheduling job fails instead of producing a fallback schedule. -Of course, we also log a failure in the scheduling job, so it's important to take note of these failures. Often, mis-configured flex models are the reason. +It is important to take note of these failures. Often, mis-configured flex models are the reason. For a hands-on tutorial on using some of the storage flex-model fields, head over to :ref:`tut_v2g` use case and `the API documentation for triggering schedules <../api/v3_0.html#post--api-v3_0-assets-id-schedules-trigger>`_. From 6483a52f72a311956eb699ee8c8e5a855dfae43f Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Thu, 25 Jun 2026 22:10:54 +0100 Subject: [PATCH 11/30] docs: clarify fallback redirects for custom schedulers Signed-off-by: Mohamed Belhsan Hmida --- documentation/api/introduction.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/documentation/api/introduction.rst b/documentation/api/introduction.rst index caa047d716..bac858783c 100644 --- a/documentation/api/introduction.rst +++ b/documentation/api/introduction.rst @@ -116,18 +116,18 @@ See Other (303) --------------- Some API responses return ``HTTP status 303 (See Other)`` to redirect the client to a different resource. -This happens, for example, when a scheduling job fails and a fallback schedule has been computed instead. +This can happen when a custom scheduler defines a fallback scheduler, the original scheduling job fails, and the fallback schedule has been computed instead. In that case, the response includes a ``Location`` header pointing to the fallback schedule endpoint, so clients can automatically retrieve the fallback result. The response body will contain a JSON message with a ``status`` field set to ``"UNKNOWN_SCHEDULE"`` and a ``message`` field explaining the reason for the redirect. .. note:: - The fallback schedule mechanism activates when the main scheduler encounters an infeasible problem (i.e. when constraints cannot be satisfied). - This is less likely to happen when ``"relax-constraints": true`` is set in the ``flex-context``, as constraint relaxation softens most infeasibility-causing constraints. - The hard constraints that remain even after constraint relaxation are ``soc-min``, ``soc-max``, ``soc-targets`` and ``power-capacity`` in the ``flex-model``, and ``site-power-capacity`` in the ``flex-context``. + FlexMeasures' built-in storage scheduler no longer computes a fallback schedule for infeasible problems. + Instead, ``soc-minima`` and ``soc-maxima`` are relaxed by default through ``"relax-soc-constraints": true``, while ``soc-min``, ``soc-max`` and ``soc-targets`` remain hard constraints. + If hard constraints cannot be satisfied, the scheduling job fails and clients receive the failure reason when requesting the schedule. - Server administrators can configure whether clients receive a 303 redirect (``FLEXMEASURES_FALLBACK_REDIRECT = True``) or whether FlexMeasures follows the fallback automatically and returns the fallback schedule directly (``FLEXMEASURES_FALLBACK_REDIRECT = False``, the default). + For custom schedulers that still define a fallback scheduler, server administrators can configure whether clients receive a 303 redirect (``FLEXMEASURES_FALLBACK_REDIRECT = True``) or whether FlexMeasures follows the fallback automatically and returns the fallback schedule directly (``FLEXMEASURES_FALLBACK_REDIRECT = False``, the default). Here is a client-side code example in Python for handling 303 redirects (this merely follows the redirect and should be revised to make use of the client's monitoring tools): From 2191013e9bd43b601f7781e2b6e596e961984f1b Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Thu, 25 Jun 2026 22:10:59 +0100 Subject: [PATCH 12/30] docs: scope fallback redirect configuration Signed-off-by: Mohamed Belhsan Hmida --- documentation/configuration.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/documentation/configuration.rst b/documentation/configuration.rst index 1b823aa585..99f9195ce2 100644 --- a/documentation/configuration.rst +++ b/documentation/configuration.rst @@ -787,7 +787,9 @@ Default: ``None`` (defaults are set internally for each sunset API version, e.g. FLEXMEASURES_FALLBACK_REDIRECT ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Control how the API handles a failed scheduling job when a fallback schedule has been computed. +Control how the API handles a failed scheduling job when a custom scheduler has computed a fallback schedule. + +FlexMeasures' built-in storage scheduler no longer computes fallback schedules, but custom schedulers may still define fallback schedulers. If ``True``, the API returns ``HTTP status 303 (See Other)`` with a ``Location`` header pointing to the fallback schedule endpoint. Clients must follow this redirect themselves to obtain the fallback schedule (see :ref:`api_see_other`). From c08ba18b68ee61beb73a5af6e70f1ed12217ea4f Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Thu, 25 Jun 2026 22:11:04 +0100 Subject: [PATCH 13/30] docs: refresh SoC relaxation OpenAPI text Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/ui/static/openapi-specs.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/flexmeasures/ui/static/openapi-specs.json b/flexmeasures/ui/static/openapi-specs.json index fd62b861ac..89efb79337 100644 --- a/flexmeasures/ui/static/openapi-specs.json +++ b/flexmeasures/ui/static/openapi-specs.json @@ -4583,13 +4583,13 @@ "relax-constraints": { "type": "boolean", "default": false, - "description": "If True (default is False), several constraints are relaxed by setting default breach prices within the optimization problem, leading to the default priority:\n\n1. Avoid breaching the site consumption/production capacity.\n2. Avoid not meeting SoC minima/maxima.\n3. Avoid breaching the desired device consumption/production capacity.\n\nWe recommend to set this field to True to enable the default prices and associated priorities as defined by FlexMeasures.\nFor tighter control over prices and priorities, the breach prices can also be set explicitly (the relevant fields have breach-price in their name).\n", + "description": "If True (default is False), several constraints are relaxed by setting default breach prices within the optimization problem, leading to the default priority:\n\n1. Avoid breaching the site consumption/production capacity.\n2. Avoid not meeting SoC minima/maxima.\n3. Avoid breaching the desired device consumption/production capacity.\n\nSoC minima/maxima are already relaxed by default through relax-soc-constraints.\nSet this field to True to also enable the default site and device capacity breach prices and associated priorities as defined by FlexMeasures.\nFor tighter control over prices and priorities, the breach prices can also be set explicitly (the relevant fields have breach-price in their name).\n", "example": true }, "relax-soc-constraints": { "type": "boolean", - "default": false, - "description": "If True, avoids not meeting SoC minima/maxima as a relaxed constraint.", + "default": true, + "description": "If True (default), avoids not meeting SoC minima/maxima as relaxed constraints. Set this to False to keep SoC minima/maxima as hard constraints unless breach prices are supplied explicitly.", "example": true }, "relax-capacity-constraints": { @@ -6302,4 +6302,4 @@ } } } -} \ No newline at end of file +} From 7135f338da7529e08bf40c886539716e7b305c21 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 26 Jun 2026 01:14:17 +0100 Subject: [PATCH 14/30] docs: add fallback scheduler changelog entry Signed-off-by: Mohamed Belhsan Hmida --- documentation/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 6e8d585933..a0b0e26f84 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -14,6 +14,7 @@ New features * Sensor references in flex-model and flex-context support various ways of filtering by source [see `PR #2209 `_] * Let storage scheduling infer missing ``power-capacity`` from directional device capacities before falling back to site capacity, and default the missing opposite capacity to zero when only a non-zero ``consumption-capacity`` or ``production-capacity`` is configured [see `PR #2222 `_] * CLI support for adding/editing account attributes [see `PR #2242 `_] +* Relax storage SoC constraints by default and report infeasible storage schedules directly instead of saving fallback schedules [see `PR #2252 `_] Infrastructure / Support From 429d63944b8b410858b9d877ad011470e76b490b Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 26 Jun 2026 01:35:41 +0100 Subject: [PATCH 15/30] test: allow small unit conversion drift Signed-off-by: Mohamed Belhsan Hmida --- .../api/v3_0/tests/test_sensors_api_freshdb.py | 4 ++-- flexmeasures/data/schemas/tests/test_sensor.py | 11 +++++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py b/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py index 18596d64aa..2a6a61d34f 100644 --- a/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py +++ b/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py @@ -246,7 +246,7 @@ def test_upload_sensor_data_with_unit_conversion_success( len(beliefs) == expected_num_beliefs ), f"Fetched {len(beliefs)} beliefs from the database, expecting {expected_num_beliefs}." - assert [b.event_value for b in beliefs] == expected_event_values + assert [b.event_value for b in beliefs] == pytest.approx(expected_event_values) @pytest.mark.parametrize( @@ -327,7 +327,7 @@ def test_upload_sensor_data_floors_offclock_datetimes( pd.testing.assert_index_equal( bdf.event_starts, pd.DatetimeIndex(expected_event_starts, name="event_start") ) - assert bdf["event_value"].to_list() == expected_event_values + assert bdf["event_value"].to_list() == pytest.approx(expected_event_values) @pytest.mark.parametrize( diff --git a/flexmeasures/data/schemas/tests/test_sensor.py b/flexmeasures/data/schemas/tests/test_sensor.py index 7d3c06fe4b..a4cd8b1474 100644 --- a/flexmeasures/data/schemas/tests/test_sensor.py +++ b/flexmeasures/data/schemas/tests/test_sensor.py @@ -20,6 +20,11 @@ def serialize_variable_quantity(value): return VariableQuantityDumpSchema().dump({"value": value})["value"] +def assert_quantity_equals(quantity, expected_quantity): + assert str(quantity.units) == str(expected_quantity.units) + assert quantity.magnitude == pytest.approx(expected_quantity.magnitude) + + @pytest.mark.parametrize( "src_quantity, dst_unit, fails, exp_dst_quantity", [ @@ -93,10 +98,12 @@ def test_quantity_or_sensor_deserialize( try: dst_quantity = schema.deserialize(src_quantity) if isinstance(src_quantity, (ur.Quantity, int, float)): - assert dst_quantity == ur.Quantity(exp_dst_quantity) + assert_quantity_equals(dst_quantity, ur.Quantity(exp_dst_quantity)) assert str(dst_quantity) == exp_dst_quantity elif isinstance(src_quantity, list): - assert dst_quantity[0]["value"] == ur.Quantity(exp_dst_quantity) + assert_quantity_equals( + dst_quantity[0]["value"], ur.Quantity(exp_dst_quantity) + ) assert str(dst_quantity[0]["value"]) == exp_dst_quantity assert not fails except ValidationError as e: From aedfa122d40be5b27c1395f7056358de63ba3ccb Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 26 Jun 2026 01:59:52 +0100 Subject: [PATCH 16/30] test: avoid exact quantity string comparisons Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/data/schemas/tests/test_sensor.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/flexmeasures/data/schemas/tests/test_sensor.py b/flexmeasures/data/schemas/tests/test_sensor.py index a4cd8b1474..ec574b0bcf 100644 --- a/flexmeasures/data/schemas/tests/test_sensor.py +++ b/flexmeasures/data/schemas/tests/test_sensor.py @@ -99,12 +99,10 @@ def test_quantity_or_sensor_deserialize( dst_quantity = schema.deserialize(src_quantity) if isinstance(src_quantity, (ur.Quantity, int, float)): assert_quantity_equals(dst_quantity, ur.Quantity(exp_dst_quantity)) - assert str(dst_quantity) == exp_dst_quantity elif isinstance(src_quantity, list): assert_quantity_equals( dst_quantity[0]["value"], ur.Quantity(exp_dst_quantity) ) - assert str(dst_quantity[0]["value"]) == exp_dst_quantity assert not fails except ValidationError as e: assert fails, e From 3a68fa788a572ff2184c437f262276b6a96568bc Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Sun, 28 Jun 2026 21:40:48 +0100 Subject: [PATCH 17/30] docs: clarify storage infeasibility behavior Signed-off-by: Mohamed Belhsan Hmida --- documentation/features/scheduling.rst | 5 ++--- flexmeasures/data/schemas/scheduling/metadata.py | 3 ++- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/documentation/features/scheduling.rst b/documentation/features/scheduling.rst index 4f53b5f809..4fef6110bb 100644 --- a/documentation/features/scheduling.rst +++ b/documentation/features/scheduling.rst @@ -268,13 +268,12 @@ However, here are some tips to model a buffer correctly: - Set ``charging-efficiency`` to the sensor describing the :abbr:`COP (coefficient of performance)` values. - Set ``storage-efficiency`` to a value below 100% to model (heat) loss. -What happens if the flex model describes an infeasible problem for the storage scheduler? Excellent question! -It is highly important for robust operation that these situations remain visible. +If the flex model describes an infeasible problem for the storage scheduler, the failure should remain visible. By default, ``soc-minima`` and ``soc-maxima`` are relaxed into soft constraints, so the scheduler can still return a useful schedule when these boundaries cannot be fully met. Exact ``soc-targets`` and physical ``soc-min`` / ``soc-max`` bounds remain hard constraints. If those hard constraints make the problem infeasible, the scheduling job fails instead of producing a fallback schedule. -It is important to take note of these failures. Often, mis-configured flex models are the reason. +It is important to take note of these failures. Often, misconfigured flex models are the reason. For a hands-on tutorial on using some of the storage flex-model fields, head over to :ref:`tut_v2g` use case and `the API documentation for triggering schedules <../api/v3_0.html#post--api-v3_0-assets-id-schedules-trigger>`_. diff --git a/flexmeasures/data/schemas/scheduling/metadata.py b/flexmeasures/data/schemas/scheduling/metadata.py index 97d2377b13..bfa2db6be7 100644 --- a/flexmeasures/data/schemas/scheduling/metadata.py +++ b/flexmeasures/data/schemas/scheduling/metadata.py @@ -248,7 +248,8 @@ def to_dict(self): SOC_MINIMA = MetaData( description="""Set points that form lower boundaries, e.g. to target a full car battery in the morning. The ``soc-minima`` are soft constraints in the optimization problem by default, because ``relax-soc-constraints`` defaults to ``True`` and supplies a default ``soc-minima-breach-price``. -Set ``relax-soc-constraints`` to ``False`` to keep them as hard constraints unless ``soc-minima-breach-price`` is supplied explicitly. [#maximum_overlap]_. Both single points in time and ranges are possible, see example.""", +Set ``relax-soc-constraints`` to ``False`` to keep them as hard constraints unless ``soc-minima-breach-price`` is supplied explicitly [#maximum_overlap]_. +Both single points in time and ranges are possible, see example.""", example=[ {"datetime": "2024-02-05T08:00:00+01:00", "value": "8.2 kWh"}, { From 79241eed7e24b9588e6c341439b0154249e308dc Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Sun, 28 Jun 2026 21:41:04 +0100 Subject: [PATCH 18/30] test: simplify storage fallback assertion comment Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/api/v3_0/tests/test_sensor_schedules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py index 37cb47bee7..392c2fb12b 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py @@ -377,7 +377,7 @@ def test_get_schedule_infeasible_storage_job_without_fallback( assert capacity == 2 assert charging_station.get_attribute("consumption-price") == {"sensor": epex_da.id} - # check that no retired storage fallback schedule has been saved before + # check that no storage fallback schedule has been saved before models = [ source.model for source in charging_station.search_beliefs().sources.unique() ] From 0f805b83a9168db2a3720150d3f6452e6665743d Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Sun, 28 Jun 2026 21:52:34 +0100 Subject: [PATCH 19/30] docs: regenerate openapi-specs.json Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/ui/static/openapi-specs.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flexmeasures/ui/static/openapi-specs.json b/flexmeasures/ui/static/openapi-specs.json index 89efb79337..9457e828d7 100644 --- a/flexmeasures/ui/static/openapi-specs.json +++ b/flexmeasures/ui/static/openapi-specs.json @@ -6124,7 +6124,7 @@ "example": true }, "soc-maxima": { - "description": "Set points that form upper boundaries at certain times, e.g. to target an empty heat buffer before a maintenance window.\nIf a soc-maxima-breach-price is defined, the soc-maxima become soft constraints in the optimization problem.\nOtherwise, they become hard constraints.", + "description": "Set points that form upper boundaries at certain times, e.g. to target an empty heat buffer before a maintenance window.\nThe soc-maxima are soft constraints in the optimization problem by default, because relax-soc-constraints defaults to True and supplies a default soc-maxima-breach-price.\nSet relax-soc-constraints to False to keep them as hard constraints unless soc-maxima-breach-price is supplied explicitly.", "example": [ { "value": "51 kWh", @@ -6135,7 +6135,7 @@ "$ref": "#/components/schemas/VariableQuantityOpenAPI" }, "soc-minima": { - "description": "Set points that form lower boundaries, e.g. to target a full car battery in the morning.\nIf a soc-minima-breach-price is defined, the soc-minima become soft constraints in the optimization problem.\nOtherwise, they become hard constraints.. Both single points in time and ranges are possible, see example.", + "description": "Set points that form lower boundaries, e.g. to target a full car battery in the morning.\nThe soc-minima are soft constraints in the optimization problem by default, because relax-soc-constraints defaults to True and supplies a default soc-minima-breach-price.\nSet relax-soc-constraints to False to keep them as hard constraints unless soc-minima-breach-price is supplied explicitly.\nBoth single points in time and ranges are possible, see example.", "example": [ { "datetime": "2024-02-05T08:00:00+01:00", @@ -6302,4 +6302,4 @@ } } } -} +} \ No newline at end of file From eaa4a7a8baf12be19c92de62c709fe1dcf14d4fc Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Sun, 5 Jul 2026 14:24:13 +0100 Subject: [PATCH 20/30] fix: preserve explicit zero breach prices Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/data/schemas/scheduling/__init__.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py index 97a2029b2a..ab24b7c621 100644 --- a/flexmeasures/data/schemas/scheduling/__init__.py +++ b/flexmeasures/data/schemas/scheduling/__init__.py @@ -394,8 +394,8 @@ def check_prices(self, data: dict, original_data: dict, **kwargs): # Fill in default soc breach prices when asked to relax SoC constraints, unless already set explicitly. if ( (data["relax_soc_constraints"] or data["relax_constraints"]) - and not data.get("soc_minima_breach_price") - and not data.get("soc_maxima_breach_price") + and data.get("soc_minima_breach_price") is None + and data.get("soc_maxima_breach_price") is None ): self.set_default_breach_prices( data, @@ -406,8 +406,8 @@ def check_prices(self, data: dict, original_data: dict, **kwargs): # Fill in default capacity breach prices when asked to relax capacity constraints, unless already set explicitly. if ( (data["relax_capacity_constraints"] or data["relax_constraints"]) - and not data.get("consumption_breach_price") - and not data.get("production_breach_price") + and data.get("consumption_breach_price") is None + and data.get("production_breach_price") is None ): self.set_default_breach_prices( data, @@ -418,8 +418,8 @@ def check_prices(self, data: dict, original_data: dict, **kwargs): # Fill in default site capacity breach prices when asked to relax site capacity constraints, unless already set explicitly. if ( (data["relax_site_capacity_constraints"] or data["relax_constraints"]) - and not data.get("ems_consumption_breach_price") - and not data.get("ems_production_breach_price") + and data.get("ems_consumption_breach_price") is None + and data.get("ems_production_breach_price") is None ): self.set_default_breach_prices( data, From 468bdd9c4d19a959e60f48871e4e04b82406b227 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Mon, 13 Jul 2026 09:28:19 +0100 Subject: [PATCH 21/30] chore: align agent instructions with main Signed-off-by: Mohamed Belhsan Hmida --- .github/agents/architecture-domain-specialist.md | 6 +++--- .github/instructions/feature-branch-sync.instructions.md | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/agents/architecture-domain-specialist.md b/.github/agents/architecture-domain-specialist.md index 9616b83bec..465265cd5b 100644 --- a/.github/agents/architecture-domain-specialist.md +++ b/.github/agents/architecture-domain-specialist.md @@ -129,13 +129,13 @@ differently-cleaned parameters instead of raising an error. - **Location**: `flexmeasures/data/models/generic_assets.py` - **Purpose**: Represents economic value (tangible/intangible) - **Key fields**: `id`, `name`, `account_id`, `parent_asset_id`, `attributes`, `flex_context`, `flex_model`, `sensors_to_show` -- **Relationships**: +- **Relationships**: - `owner` → Account (via account_id) - `parent_asset` → GenericAsset (via parent_asset_id) - `child_assets` ← GenericAsset (reverse of parent) - `sensors` ← Sensor (one-to-many) - **Invariant**: `db.CheckConstraint("parent_asset_id != id", name="generic_asset_self_reference_ck")` -- **Methods**: +- **Methods**: - `get_flex_context()` - Walks parent tree to reconstitute full context - `great_circle_distance()` - Geographic distance calculations - **Path representation**: Account > Asset > ... > Asset @@ -159,7 +159,7 @@ differently-cleaned parameters instead of raising an error. #### Scheduler - **Location**: `flexmeasures/data/models/planning/__init__.py` - **Purpose**: Base class for other schedulers (incl. from plugins) -- **Inputs**: +- **Inputs**: - Asset (more modern way) or Sensor (older approach) - Time window: start, end, resolution, belief_time - flex_model + flex_context diff --git a/.github/instructions/feature-branch-sync.instructions.md b/.github/instructions/feature-branch-sync.instructions.md index 97abfb9c56..926d917fac 100644 --- a/.github/instructions/feature-branch-sync.instructions.md +++ b/.github/instructions/feature-branch-sync.instructions.md @@ -34,7 +34,7 @@ git commit -m "Merge origin/main into feature branch" ``` This ensures your implementation starts from the latest state of the repository. - + ## Why this matters - Merging later causes merge conflicts to compound From 919e28d3e2eea8af97f87e375ad578f883f0c776 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Mon, 13 Jul 2026 09:28:37 +0100 Subject: [PATCH 22/30] docs: fix jobs OpenAPI description indentation Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/api/v3_0/jobs.py | 6 +++--- flexmeasures/ui/static/openapi-specs.json | 7 ++++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/flexmeasures/api/v3_0/jobs.py b/flexmeasures/api/v3_0/jobs.py index 384732d710..74dccc884f 100644 --- a/flexmeasures/api/v3_0/jobs.py +++ b/flexmeasures/api/v3_0/jobs.py @@ -141,9 +141,9 @@ def get_job_status(self, job_id: str, **kwargs): constraint analysis (empty arrays when the flex model defines no ``soc-minima``/``soc-maxima``, or when a scheduler other than ``StorageScheduler`` was used). - The ``num-beliefs`` field holds the total number of - beliefs (scheduled values) saved to the database. - nullable: true + The ``num-beliefs`` field holds the total number of + beliefs (scheduled values) saved to the database. + nullable: true func_name: type: string description: Fully-qualified name of the function executed by this job. diff --git a/flexmeasures/ui/static/openapi-specs.json b/flexmeasures/ui/static/openapi-specs.json index ed8b1d16b9..cfe65f0bb4 100644 --- a/flexmeasures/ui/static/openapi-specs.json +++ b/flexmeasures/ui/static/openapi-specs.json @@ -4270,7 +4270,7 @@ "/api/v3_0/jobs/{uuid}": { "get": { "summary": "Get the status of a background job", - "description": "Look up a background job by its UUID and see whether it is\nqueued, running, finished, or failed.\n\nThe response includes a status message plus job metadata such\nas the queue name, function name, timestamps, and the job\nresult when available.\n\nFailed jobs also include traceback information when the worker\nstored it with the job result.\n\nFor a finished scheduling job, result is an object. For a\nStorageScheduler job it holds soft state-of-charge constraint\nanalysis: unresolved lists constraints the scheduler could not\nsatisfy, and resolved lists constraints that were satisfied\nwith some margin. Each device entry's soc-minima/soc-maxima\nvalue is a list, holding one entry per violated slot (for\nunresolved) or per met slot with its margin (for resolved),\nordered chronologically. Both arrays are empty when the flex model\ndefines no soc-minima/soc-maxima, or when a scheduler other\nthan StorageScheduler was used. This is the only place\nconstraint analysis is available \u2014 the sensor schedule endpoint\n(GET /api/v3_0/sensors//schedules/) returns power\nvalues only.\n", + "description": "Look up a background job by its UUID and see whether it is\nqueued, running, finished, or failed.\n\nThe response includes a status message plus job metadata such\nas the queue name, function name, timestamps, and the job\nresult when available.\n\nFailed jobs also include traceback information when the worker\nstored it with the job result.\n\nFor a finished scheduling job, result is an object. For a\nStorageScheduler job it holds soft state-of-charge constraint\nanalysis: unresolved lists constraints the scheduler could not\nsatisfy, and resolved lists constraints that were satisfied\nwith some margin. Each device entry's soc-minima/soc-maxima\nvalue is a list, holding one entry per violated slot (for\nunresolved) or per met slot with its margin (for resolved),\nordered chronologically. Both arrays are empty when the flex model\ndefines no soc-minima/soc-maxima, or when a scheduler other\nthan StorageScheduler was used. The num-beliefs field holds\nthe total number of beliefs (scheduled values) saved to the database.\nThis is the only place constraint analysis is available \u2014 the sensor\nschedule endpoint (GET /api/v3_0/sensors//schedules/)\nreturns power values only.\n", "security": [ { "ApiKeyAuth": [] @@ -4315,7 +4315,7 @@ "description": "Human-readable description of the job status." }, "result": { - "description": "Return value of the job function, or null when not yet available. For a finished scheduling job, this is an object; a StorageScheduler job populates it with unresolved/resolved soft state-of-charge constraint analysis (empty arrays when the flex model defines no soc-minima/soc-maxima, or when a scheduler other than StorageScheduler was used).\n", + "description": "Return value of the job function, or null when not yet available. For a finished scheduling job, this is an object; a StorageScheduler job populates it with unresolved/resolved soft state-of-charge constraint analysis (empty arrays when the flex model defines no soc-minima/soc-maxima, or when a scheduler other than StorageScheduler was used). The num-beliefs field holds the total number of beliefs (scheduled values) saved to the database.\n", "nullable": true }, "func_name": { @@ -4387,7 +4387,8 @@ ] } ], - "resolved": [] + "resolved": [], + "num-beliefs": 96 }, "func_name": "flexmeasures.data.services.scheduling.create_schedule", "origin": "scheduling", From f354561972dcc8dbb694dd7c39024d37e3ed072e Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Sat, 18 Jul 2026 00:12:38 +0100 Subject: [PATCH 23/30] fix(schema): let relax-constraints=false keep SoC constraints hard Both relax-soc-constraints and relax-constraints default to True, so the former's default swallowed an explicit opt-out through the umbrella flag. Now an explicit relax-soc-constraints wins and otherwise the umbrella relax-constraints decides, per the semantics discussed in PR #2267: setting either flag to False keeps SoC minima/maxima hard. Also document why DBFlexContextSchema turns the relaxation defaults off. Co-Authored-By: Claude Fable 5 --- .../data/schemas/scheduling/__init__.py | 14 +++++++- .../data/schemas/scheduling/metadata.py | 2 +- .../data/schemas/tests/test_scheduling.py | 34 +++++++++++++++++++ flexmeasures/ui/static/openapi-specs.json | 6 ++-- 4 files changed, 51 insertions(+), 5 deletions(-) diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py index a59efb31e9..d02bafcdb2 100644 --- a/flexmeasures/data/schemas/scheduling/__init__.py +++ b/flexmeasures/data/schemas/scheduling/__init__.py @@ -875,8 +875,16 @@ def check_prices(self, data: dict, original_data: dict, **kwargs): return data # Fill in default soc breach prices when asked to relax SoC constraints, unless already set explicitly. + # Both relax-soc-constraints and relax-constraints default to True, so an + # explicit relax-soc-constraints wins and otherwise the umbrella + # relax-constraints decides: setting either flag to False keeps + # SoC minima/maxima as hard constraints. + if "relax-soc-constraints" in original_data: + relax_soc_constraints = data["relax_soc_constraints"] + else: + relax_soc_constraints = data["relax_constraints"] if ( - (data["relax_soc_constraints"] or data["relax_constraints"]) + relax_soc_constraints and data.get("soc_minima_breach_price") is None and data.get("soc_maxima_breach_price") is None ): @@ -1215,6 +1223,10 @@ def check_prices(self, data: dict, original_data: dict, **kwargs): class DBFlexContextSchema(FlexContextSchema, NoTimeSeriesSpecs): + # The relaxation defaults are turned off here, so validating a stored asset + # flex-context does not fill in default breach prices. The API-side defaults + # (which are True) are applied at scheduling time instead, after the stored + # flex-context is merged with the one passed in the scheduling request. relax_constraints = fields.Bool( data_key="relax-constraints", load_default=False, diff --git a/flexmeasures/data/schemas/scheduling/metadata.py b/flexmeasures/data/schemas/scheduling/metadata.py index 45d15591e7..895c22fe51 100644 --- a/flexmeasures/data/schemas/scheduling/metadata.py +++ b/flexmeasures/data/schemas/scheduling/metadata.py @@ -185,7 +185,7 @@ def to_dict(self): example=True, ) RELAX_SOC_CONSTRAINTS = MetaData( - description="If True (default), avoids not meeting SoC minima/maxima as relaxed constraints. Set this to False to keep SoC minima/maxima as hard constraints unless breach prices are supplied explicitly.", + description="If True (default), avoids not meeting SoC minima/maxima as relaxed constraints. Setting this field (or ``relax-constraints``) to False keeps SoC minima/maxima as hard constraints unless breach prices are supplied explicitly; an explicit ``relax-soc-constraints`` takes precedence over ``relax-constraints``.", example=True, ) RELAX_CAPACITY_CONSTRAINTS = MetaData( diff --git a/flexmeasures/data/schemas/tests/test_scheduling.py b/flexmeasures/data/schemas/tests/test_scheduling.py index 855e188edd..04c2893db0 100644 --- a/flexmeasures/data/schemas/tests/test_scheduling.py +++ b/flexmeasures/data/schemas/tests/test_scheduling.py @@ -552,6 +552,40 @@ def test_flex_context_schema_preserves_explicit_soc_breach_prices(): ).magnitude == pytest.approx(7) +def test_flex_context_schema_umbrella_opt_out_disables_soc_relaxation(): + """Setting relax-constraints to False alone keeps SoC minima/maxima hard.""" + loaded_flex_context = FlexContextSchema().load( + {"consumption-price": "1 EUR/MWh", "relax-constraints": False} + ) + + assert "soc_minima_breach_price" not in loaded_flex_context + assert "soc_maxima_breach_price" not in loaded_flex_context + assert "consumption_breach_price" not in loaded_flex_context + assert "ems_consumption_breach_price" not in loaded_flex_context + + +def test_flex_context_schema_explicit_soc_relaxation_overrides_umbrella_opt_out(): + """An explicit relax-soc-constraints wins over an explicit relax-constraints.""" + loaded_flex_context = FlexContextSchema().load( + { + "consumption-price": "1 EUR/MWh", + "relax-constraints": False, + "relax-soc-constraints": True, + } + ) + + assert loaded_flex_context["soc_minima_breach_price"].to( + "EUR/MWh" + ).magnitude == pytest.approx(1_000_000) + + loaded_flex_context = FlexContextSchema().load( + {"consumption-price": "1 EUR/MWh", "relax-soc-constraints": False} + ) + + assert "soc_minima_breach_price" not in loaded_flex_context + assert "soc_maxima_breach_price" not in loaded_flex_context + + def test_db_flex_context_schema_does_not_relax_soc_constraints_by_default(): loaded_flex_context = DBFlexContextSchema().load({}) diff --git a/flexmeasures/ui/static/openapi-specs.json b/flexmeasures/ui/static/openapi-specs.json index b4b13bd9e5..59830b94e6 100644 --- a/flexmeasures/ui/static/openapi-specs.json +++ b/flexmeasures/ui/static/openapi-specs.json @@ -4752,7 +4752,7 @@ "relax-soc-constraints": { "type": "boolean", "default": true, - "description": "If True (default), avoids not meeting SoC minima/maxima as relaxed constraints. Set this to False to keep SoC minima/maxima as hard constraints unless breach prices are supplied explicitly.", + "description": "If True (default), avoids not meeting SoC minima/maxima as relaxed constraints. Setting this field (or relax-constraints) to False keeps SoC minima/maxima as hard constraints unless breach prices are supplied explicitly; an explicit relax-soc-constraints takes precedence over relax-constraints.", "example": true }, "relax-capacity-constraints": { @@ -6276,7 +6276,7 @@ "example": true }, "soc-maxima": { - "description": "Set points that form upper boundaries at certain times, e.g. to target an empty heat buffer before a maintenance window.\nThe soc-maxima are soft constraints in the optimization problem by default, because relax-soc-constraints defaults to True and supplies a default soc-maxima-breach-price.\nSet relax-soc-constraints to False to keep them as hard constraints unless soc-maxima-breach-price is supplied explicitly.", + "description": "Set points that form upper boundaries at certain times, e.g. to target an empty heat buffer before a maintenance window.\nThe soc-maxima are soft constraints in the optimization problem by default, because relax-soc-constraints defaults to True and supplies a default soc-maxima-breach-price.\nSet relax-soc-constraints (or relax-constraints) to False to keep them as hard constraints unless soc-maxima-breach-price is supplied explicitly.", "example": [ { "value": "51 kWh", @@ -6287,7 +6287,7 @@ "$ref": "#/components/schemas/VariableQuantityOpenAPI" }, "soc-minima": { - "description": "Set points that form lower boundaries, e.g. to target a full car battery in the morning.\nThe soc-minima are soft constraints in the optimization problem by default, because relax-soc-constraints defaults to True and supplies a default soc-minima-breach-price.\nSet relax-soc-constraints to False to keep them as hard constraints unless soc-minima-breach-price is supplied explicitly.\nBoth single points in time and ranges are possible, see example.", + "description": "Set points that form lower boundaries, e.g. to target a full car battery in the morning.\nThe soc-minima are soft constraints in the optimization problem by default, because relax-soc-constraints defaults to True and supplies a default soc-minima-breach-price.\nSet relax-soc-constraints (or relax-constraints) to False to keep them as hard constraints unless soc-minima-breach-price is supplied explicitly.\nBoth single points in time and ranges are possible, see example.", "example": [ { "datetime": "2024-02-05T08:00:00+01:00", From 1bafc44f4ce23d0834d2728e0a22d7e885ca46f7 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Sat, 18 Jul 2026 00:13:08 +0100 Subject: [PATCH 24/30] docs: restore hard-constraints list and describe the SoC relaxation opt-out The capacity bounds (power-capacity and site-power-capacity) still remain hard constraints after relaxation; mention them again alongside soc-min, soc-max and soc-targets, and explain that setting either relax-soc-constraints or relax-constraints to false keeps SoC minima/maxima hard. Co-Authored-By: Claude Fable 5 --- documentation/api/introduction.rst | 3 ++- documentation/features/scheduling.rst | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/documentation/api/introduction.rst b/documentation/api/introduction.rst index bac858783c..0e4517c935 100644 --- a/documentation/api/introduction.rst +++ b/documentation/api/introduction.rst @@ -124,7 +124,8 @@ The response body will contain a JSON message with a ``status`` field set to ``" .. note:: FlexMeasures' built-in storage scheduler no longer computes a fallback schedule for infeasible problems. - Instead, ``soc-minima`` and ``soc-maxima`` are relaxed by default through ``"relax-soc-constraints": true``, while ``soc-min``, ``soc-max`` and ``soc-targets`` remain hard constraints. + Instead, ``soc-minima`` and ``soc-maxima`` are relaxed by default through ``"relax-soc-constraints": true`` (setting either it or ``"relax-constraints"`` to ``false`` keeps them hard). + The hard constraints that remain even after constraint relaxation are ``soc-min``, ``soc-max``, ``soc-targets`` and ``power-capacity`` in the ``flex-model``, and ``site-power-capacity`` in the ``flex-context``. If hard constraints cannot be satisfied, the scheduling job fails and clients receive the failure reason when requesting the schedule. For custom schedulers that still define a fallback scheduler, server administrators can configure whether clients receive a 303 redirect (``FLEXMEASURES_FALLBACK_REDIRECT = True``) or whether FlexMeasures follows the fallback automatically and returns the fallback schedule directly (``FLEXMEASURES_FALLBACK_REDIRECT = False``, the default). diff --git a/documentation/features/scheduling.rst b/documentation/features/scheduling.rst index 264793a055..8d438dacd7 100644 --- a/documentation/features/scheduling.rst +++ b/documentation/features/scheduling.rst @@ -314,6 +314,7 @@ However, here are some tips to model a buffer correctly: If the flex model describes an infeasible problem for the storage scheduler, the failure should remain visible. By default, ``soc-minima`` and ``soc-maxima`` are relaxed into soft constraints, so the scheduler can still return a useful schedule when these boundaries cannot be fully met. +Setting either ``relax-soc-constraints`` or ``relax-constraints`` to ``false`` in the flex-context keeps them as hard constraints. Exact ``soc-targets`` and physical ``soc-min`` / ``soc-max`` bounds remain hard constraints. If those hard constraints make the problem infeasible, the scheduling job fails instead of producing a fallback schedule. From bf2faf37bc3e757597cd8a7a12971d0b2bdeeabd Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Sat, 18 Jul 2026 00:13:08 +0100 Subject: [PATCH 25/30] test: restore config via monkeypatch and fix truncated docstring Co-Authored-By: Claude Fable 5 --- flexmeasures/api/v3_0/tests/test_sensor_schedules.py | 5 ++--- flexmeasures/data/models/planning/tests/test_solver.py | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py index 392c2fb12b..3d464121e6 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py @@ -351,13 +351,14 @@ def test_get_schedule_infeasible_storage_job_without_fallback( requesting_user, db, fallback_redirect, + monkeypatch, ): """ Test that a failing StorageScheduler call reports the failure without creating a fallback job. This test is based on flexmeasures/data/models/planning/tests/test_solver.py. """ - app.config["FLEXMEASURES_FALLBACK_REDIRECT"] = fallback_redirect + monkeypatch.setitem(app.config, "FLEXMEASURES_FALLBACK_REDIRECT", fallback_redirect) target_soc = 9 charging_station_name = "Test charging station" @@ -463,8 +464,6 @@ def test_get_schedule_infeasible_storage_job_without_fallback( ] assert "StorageFallbackScheduler" not in models - app.config["FLEXMEASURES_FALLBACK_REDIRECT"] = False - @pytest.mark.parametrize( "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py index a238d00b3c..b7adfcb1d1 100644 --- a/flexmeasures/data/models/planning/tests/test_solver.py +++ b/flexmeasures/data/models/planning/tests/test_solver.py @@ -507,7 +507,7 @@ def test_storage_scheduler_reports_unsolvable_problem_without_fallback( """Starting with a state of charge 10 kWh, within 2 hours we should be able to reach any state of charge in the range [10, 14] kWh for a unidirectional station, or [6, 14] for a bidirectional station, given a charging capacity of 2 kW. - Here we test target states of charge outside that range, ones that we should be able + Here we test target states of charge outside that range. The StorageScheduler should report this infeasible problem without hiding it behind a fallback schedule. """ From b932527a588bab58da998f0eebb98bf73662fbe2 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Wed, 22 Jul 2026 13:07:06 +0100 Subject: [PATCH 26/30] docs+test: fix review follow-ups for relax-soc-constraints default - Correct RELAX_CONSTRAINTS metadata text: it now defaults to True, not False. - Give DBFlexContextSchema's relax flags their own description, since a stored flex-context defaults to False, unlike the True scheduling-time default applied after merging with the request. - Align the hard-constraints list in scheduling.rst with introduction.rst (both now mention power-capacity / site-power-capacity). - Add the missing v3.0-32 API changelog entry for the relax-soc-constraints default flip and the retired fallback scheduler's effect on GET schedule. - Drop now-tautological assertions checking for "StorageFallbackScheduler" in belief sources, since that class no longer exists in the codebase. Co-Authored-By: Claude Sonnet 5 --- documentation/api/change_log.rst | 1 + documentation/features/scheduling.rst | 2 +- .../api/v3_0/tests/test_sensor_schedules.py | 12 ---------- .../data/schemas/scheduling/__init__.py | 22 +++++++++++++++++-- .../data/schemas/scheduling/metadata.py | 2 +- flexmeasures/ui/static/openapi-specs.json | 4 ++-- 6 files changed, 25 insertions(+), 18 deletions(-) diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst index c75a260f70..80efeefc5b 100644 --- a/documentation/api/change_log.rst +++ b/documentation/api/change_log.rst @@ -9,6 +9,7 @@ v3.0-32 | July XX, 2026 """""""""""""""""""""""" - Extended ``GET /api/v3_0/jobs/`` with a ``result`` field containing ``unresolved`` and ``resolved`` arrays, each keyed by asset ID. For scheduling jobs, this surfaces soft state-of-charge constraint analysis: ``soc-minima`` and ``soc-maxima`` violations (with a ``violation`` magnitude) or satisfied constraints (with a ``margin`` headroom). Both arrays are empty when no SoC constraints were defined. +- Breaking: ``relax-soc-constraints`` now defaults to ``True`` (set it or ``relax-constraints`` to ``False`` to keep ``soc-minima``/``soc-maxima`` hard), and the built-in storage fallback scheduler has been retired. ``GET /api/v3_0/sensors//schedules/`` now returns ``400`` with the failure reason for an infeasible storage schedule instead of a ``303`` redirect to (or an automatic follow of) a fallback schedule. ``FLEXMEASURES_FALLBACK_REDIRECT`` remains relevant only for custom schedulers that still define a fallback scheduler. v3.0-31 | 2026-06-01 """""""""""""""""""" diff --git a/documentation/features/scheduling.rst b/documentation/features/scheduling.rst index 8d438dacd7..ec698e1ea6 100644 --- a/documentation/features/scheduling.rst +++ b/documentation/features/scheduling.rst @@ -315,7 +315,7 @@ However, here are some tips to model a buffer correctly: If the flex model describes an infeasible problem for the storage scheduler, the failure should remain visible. By default, ``soc-minima`` and ``soc-maxima`` are relaxed into soft constraints, so the scheduler can still return a useful schedule when these boundaries cannot be fully met. Setting either ``relax-soc-constraints`` or ``relax-constraints`` to ``false`` in the flex-context keeps them as hard constraints. -Exact ``soc-targets`` and physical ``soc-min`` / ``soc-max`` bounds remain hard constraints. +Exact ``soc-targets``, physical ``soc-min`` / ``soc-max`` bounds, and ``power-capacity`` (in the flex-model) and ``site-power-capacity`` (in the flex-context) remain hard constraints. If those hard constraints make the problem infeasible, the scheduling job fails instead of producing a fallback schedule. It is important to take note of these failures. Often, misconfigured flex models are the reason. diff --git a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py index 3d464121e6..4fbfdd7464 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py @@ -378,12 +378,6 @@ def test_get_schedule_infeasible_storage_job_without_fallback( assert capacity == 2 assert charging_station.get_attribute("consumption-price") == {"sensor": epex_da.id} - # check that no storage fallback schedule has been saved before - models = [ - source.model for source in charging_station.search_beliefs().sources.unique() - ] - assert "StorageFallbackScheduler" not in models - # create a scenario that yields an infeasible problem (unreachable target SOC at 2am) message = { "start": start, @@ -458,12 +452,6 @@ def test_get_schedule_infeasible_storage_job_without_fallback( assert get_schedule_response.json["status"] == "UNKNOWN_SCHEDULE" assert get_schedule_response.json["result"] == "Rejected" - models = [ - source.model - for source in charging_station.search_beliefs().sources.unique() - ] - assert "StorageFallbackScheduler" not in models - @pytest.mark.parametrize( "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py index d02bafcdb2..d02c10ba95 100644 --- a/flexmeasures/data/schemas/scheduling/__init__.py +++ b/flexmeasures/data/schemas/scheduling/__init__.py @@ -1230,12 +1230,30 @@ class DBFlexContextSchema(FlexContextSchema, NoTimeSeriesSpecs): relax_constraints = fields.Bool( data_key="relax-constraints", load_default=False, - metadata=metadata.RELAX_CONSTRAINTS.to_dict(), + metadata={ + **metadata.RELAX_CONSTRAINTS.to_dict(), + "description": ( + "Defaults to False when stored on an asset (unlike the True default " + "used when scheduling): a stored flex-context should not silently bake " + "in default breach prices. The scheduling-time default of True is " + "applied after this stored flex-context is merged with the one passed " + "in the scheduling request." + ), + }, ) relax_soc_constraints = fields.Bool( data_key="relax-soc-constraints", load_default=False, - metadata=metadata.RELAX_SOC_CONSTRAINTS.to_dict(), + metadata={ + **metadata.RELAX_SOC_CONSTRAINTS.to_dict(), + "description": ( + "Defaults to False when stored on an asset (unlike the True default " + "used when scheduling): a stored flex-context should not silently bake " + "in default breach prices. The scheduling-time default of True is " + "applied after this stored flex-context is merged with the one passed " + "in the scheduling request." + ), + }, ) commitments = fields.Nested( diff --git a/flexmeasures/data/schemas/scheduling/metadata.py b/flexmeasures/data/schemas/scheduling/metadata.py index 895c22fe51..d8d2b46ca8 100644 --- a/flexmeasures/data/schemas/scheduling/metadata.py +++ b/flexmeasures/data/schemas/scheduling/metadata.py @@ -172,7 +172,7 @@ def to_dict(self): example="10 EUR/kW", ) RELAX_CONSTRAINTS = MetaData( - description="""If True (default is ``False``), several constraints are relaxed by setting default breach prices within the optimization problem, leading to the default priority: + description="""If True (default), several constraints are relaxed by setting default breach prices within the optimization problem, leading to the default priority: 1. Avoid breaching the site consumption/production capacity. 2. Avoid not meeting SoC minima/maxima. diff --git a/flexmeasures/ui/static/openapi-specs.json b/flexmeasures/ui/static/openapi-specs.json index 59830b94e6..695eca9eda 100644 --- a/flexmeasures/ui/static/openapi-specs.json +++ b/flexmeasures/ui/static/openapi-specs.json @@ -7,7 +7,7 @@ }, "termsOfService": null, "title": "FlexMeasures", - "version": "1.0.0" + "version": "0.33.2" }, "externalDocs": { "description": "FlexMeasures runs on the open source FlexMeasures technology. Read the docs here.", @@ -4746,7 +4746,7 @@ "relax-constraints": { "type": "boolean", "default": true, - "description": "If True (default is False), several constraints are relaxed by setting default breach prices within the optimization problem, leading to the default priority:\n\n1. Avoid breaching the site consumption/production capacity.\n2. Avoid not meeting SoC minima/maxima.\n3. Avoid breaching the desired device consumption/production capacity.\n\nSoC minima/maxima are already relaxed by default through relax-soc-constraints.\nSet this field to True to also enable the default site and device capacity breach prices and associated priorities as defined by FlexMeasures.\nFor tighter control over prices and priorities, the breach prices can also be set explicitly (the relevant fields have breach-price in their name).\n", + "description": "If True (default), several constraints are relaxed by setting default breach prices within the optimization problem, leading to the default priority:\n\n1. Avoid breaching the site consumption/production capacity.\n2. Avoid not meeting SoC minima/maxima.\n3. Avoid breaching the desired device consumption/production capacity.\n\nSoC minima/maxima are already relaxed by default through relax-soc-constraints.\nSet this field to True to also enable the default site and device capacity breach prices and associated priorities as defined by FlexMeasures.\nFor tighter control over prices and priorities, the breach prices can also be set explicitly (the relevant fields have breach-price in their name).\n", "example": true }, "relax-soc-constraints": { From 42406c924bc3b279e244a57540f8b5c2f2b8f923 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 7 Aug 2026 00:07:43 +0100 Subject: [PATCH 27/30] feat: support a `default` fallback on sensor references A sensor reference on any flex-model or flex-context field may now carry a `default` quantity, e.g. {"sensor": 50, "default": "0 kWh"}. It fills the time slots for which the referenced sensor holds no value, and is settable from the flex-model UI. This is the uncontroversial half of #2267, split out on Felix's suggestion so it can land while the SoC constraint hardness question is settled across the whole flex-model (see #2395). The canonical soc-min/soc-max work stays on feat/dynamic-soc-bounds-defaults. Note that a default fills *every* slot the sensor leaves empty, so a sensor recording only occasional setpoints becomes densely constrained; the field documentation says so. Also omit `default` from serialized sensor references when it is unset, rather than emitting `default: None`, which is not valid input on the way back in and broke the forecaster config round-trips. Signed-off-by: Mohamed Belhsan Hmida Co-Authored-By: Claude Fable 5 Signed-off-by: Mohamed Belhsan Hmida --- documentation/api/change_log.rst | 1 + documentation/changelog.rst | 1 + .../planning/tests/test_utils_fresh_db.py | 43 ++++++ flexmeasures/data/models/planning/utils.py | 8 ++ flexmeasures/data/schemas/sensors.py | 133 +++++++++++++++--- .../data/schemas/tests/test_scheduling.py | 2 +- .../data/schemas/tests/test_sensor.py | 48 +++++++ flexmeasures/ui/static/openapi-specs.json | 28 +++- .../ui/templates/assets/asset_properties.html | 57 +++++++- 9 files changed, 297 insertions(+), 24 deletions(-) diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst index 01a5438417..cac446d71f 100644 --- a/documentation/api/change_log.rst +++ b/documentation/api/change_log.rst @@ -10,6 +10,7 @@ v3.0-32 | July XX, 2026 - API endpoints are now rate-limited. A request which exceeds a limit is answered with a ``429 (Too Many Requests)`` status code and a ``Retry-After`` header stating how many seconds to wait. Responses also carry ``X-RateLimit-*`` headers, describing the limit that applied, how much of it is left, and when it resets. A stricter limit applies to ``POST /assets//schedules/trigger``, ``POST /sensors//schedules/trigger`` and ``POST /sensors//forecasts/trigger`` than to other endpoints; the health endpoints are exempt. Per-account overrides are set by assigning the account a plan (a ``Plan`` database row), rather than through an account attribute. - Introduced the ``inflexible-consumption`` and ``inflexible-production`` flex-context fields, which make explicit how the sign of each inflexible device's power data should be read: positive values denote consumption resp. production. Each entry is a sensor reference (``{"sensor": }``), optionally with source filters (``source-types``, ``exclude-source-types``, ``sources``, ``source-account``). Deprecated the ``inflexible-device-sensors`` field (a list of bare sensor IDs, whose sign convention is read from each sensor's ``consumption_is_positive`` attribute); it remains supported, but cannot be combined with the new fields in one flex-context. - Added a ``role`` query parameter to ``GET /api/v3_0/accounts`` for filtering accessible organisations by account role. +- Sensor references (on any flex-model or flex-context field that accepts them) may now include a ``default`` fallback quantity, e.g. ``{"sensor": 50, "default": "0 kWh"}``. It fills the time slots for which the referenced sensor holds no value. Note that this fills *every* such slot, so a sensor recording only occasional setpoints becomes densely constrained. - Extended ``GET /api/v3_0/jobs/`` with a ``result`` field containing ``unresolved`` and ``resolved`` arrays, each keyed by asset ID. For scheduling jobs, this surfaces soft state-of-charge constraint analysis: ``soc-minima`` and ``soc-maxima`` violations (with a ``violation`` magnitude) or satisfied constraints (with a ``margin`` headroom). Both arrays are empty when no SoC constraints were defined. - **Field canonicalization** for background job tracking: * The ``job`` field is now the canonical way to identify background jobs returned by `/sensors//schedules/trigger`, `/assets//schedules/trigger`, and `/sensors//forecasts/trigger` endpoints. If applicable, the triggered response now also returns a ``results-url`` pointing to the sensor-specific results endpoint, alongside the generic ``job-url``. diff --git a/documentation/changelog.rst b/documentation/changelog.rst index bc26b9dc45..3a174a2434 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -41,6 +41,7 @@ New features * CLI support for adding/editing account attributes [see `PR #2242 `_] * Improve chart axis domain for event values not around zero, with a per-sub-chart ``y-axis`` option in ``sensors_to_show`` (default ``zero``, which pads the axis out to include zero) that can be set to ``data`` to fit a sub-chart's y-axis to the values shown, to an explicit ``[min, max]`` domain that the axis will cover at least (expanding to fit data beyond it), or to a strict ``{"min": min, "max": max}`` domain that the axis will never exceed (clamping data beyond it, with a warning when that happens), editable from the graph editor [see `PR #2244 `_] * Breaking behaviour change: storage SoC constraints (``soc-minima``/``soc-maxima``) are now relaxed by default (``relax-soc-constraints`` defaults to ``True``; set it or ``relax-constraints`` to ``False`` to keep them hard), and the built-in storage fallback scheduler has been retired, so infeasible storage problems now fail with their failure reason instead of silently saving a fallback schedule; ``FLEXMEASURES_FALLBACK_REDIRECT`` is now only relevant for custom schedulers that define a fallback scheduler [see `PR #2252 `_] +* Sensor references now accept a ``default`` fallback quantity, e.g. ``{"sensor": 50, "default": "0 kWh"}``, filling the time slots for which the referenced sensor holds no value; also settable from the flex-model UI [see `PR #2267 `_] * Extended ``GET /api/v3_0/jobs/`` with a ``result`` field containing ``unresolved`` and ``resolved`` soft state-of-charge constraint analysis (``soc-minima``/``soc-maxima`` violations or satisfied constraints, keyed by asset ID) for scheduling jobs; both arrays are empty when no SoC constraints were defined [see `PR #2072 `_] * New storage flex-model field ``operation-modes`` confines a device's power to one of several power bands, following the S2 standard's operation modes — for example, a device that is either off or running at one fixed power [see `PR #2278 `_] * New ``FLEXMEASURES_LP_SOLVER_OPTIONS`` config setting to pass solver options to the scheduling solver, validated against the installed HiGHS build so that unknown or unsupported options raise instead of being silently ignored [see `PR #2283 `_] diff --git a/flexmeasures/data/models/planning/tests/test_utils_fresh_db.py b/flexmeasures/data/models/planning/tests/test_utils_fresh_db.py index d6d7fa3000..a4473409cb 100644 --- a/flexmeasures/data/models/planning/tests/test_utils_fresh_db.py +++ b/flexmeasures/data/models/planning/tests/test_utils_fresh_db.py @@ -10,6 +10,7 @@ from flexmeasures.data.schemas.sensors import SensorReference from flexmeasures.data.models.planning.storage import StorageScheduler from flexmeasures.data.models.planning.utils import get_series_from_quantity_or_sensor +from flexmeasures.utils.unit_utils import ur def test_get_series_from_sensor_reference_source_filter_integration(fresh_db): @@ -88,6 +89,48 @@ def test_get_series_from_sensor_reference_source_filter_integration(fresh_db): assert result_forecaster.iloc[0] == pytest.approx(200.0) +def test_get_series_from_sensor_reference_default_fills_missing_values(fresh_db): + """A SensorReference default fills query slots with no matching sensor belief.""" + query_window = ( + pd.Timestamp("2025-06-01 08:00:00+02:00"), + pd.Timestamp("2025-06-01 08:30:00+02:00"), + ) + source = DataSource(name="test-default-source", type="scheduler") + fresh_db.session.add(source) + asset_type = GenericAssetType(name="test-asset-type-default") + fresh_db.session.add(asset_type) + asset = GenericAsset(name="test-asset-default", generic_asset_type=asset_type) + fresh_db.session.add(asset) + sensor = Sensor( + name="test-sensor-default", + generic_asset=asset, + event_resolution=timedelta(minutes=15), + unit="MW", + ) + fresh_db.session.add(sensor) + fresh_db.session.flush() + fresh_db.session.add( + TimedBelief( + event_start=query_window[0], + belief_horizon=timedelta(0), + event_value=0.1, + source=source, + sensor=sensor, + ) + ) + fresh_db.session.commit() + + result = get_series_from_quantity_or_sensor( + variable_quantity=SensorReference(sensor=sensor, default=ur.Quantity("1 MW")), + query_window=query_window, + resolution=sensor.event_resolution, + unit="kW", + as_instantaneous_events=False, + ) + + assert list(result) == pytest.approx([100.0, 1000.0]) + + def test_get_series_from_sensor_reference_sources_filter_integration(fresh_db): """A :class:`SensorReference` with ``sources`` returns only beliefs from the specified source. diff --git a/flexmeasures/data/models/planning/utils.py b/flexmeasures/data/models/planning/utils.py index ce54cd3df8..87fdfe4944 100644 --- a/flexmeasures/data/models/planning/utils.py +++ b/flexmeasures/data/models/planning/utils.py @@ -326,6 +326,14 @@ def get_series_from_quantity_or_sensor( time_series = convert_units( time_series, variable_quantity.unit, unit, resolution ) + if variable_quantity.default is not None: + default_value = convert_units( + variable_quantity.default.magnitude, + str(variable_quantity.default.units), + unit, + resolution, + ) + time_series = time_series.fillna(default_value) elif isinstance(variable_quantity, Sensor): bdf: tb.BeliefsDataFrame = TimedBelief.search( variable_quantity, diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py index ecfde3e581..e9d54a8e0d 100644 --- a/flexmeasures/data/schemas/sensors.py +++ b/flexmeasures/data/schemas/sensors.py @@ -17,6 +17,7 @@ Schema, ValidationError, fields, + post_dump, post_load, pre_load, validates, @@ -24,6 +25,7 @@ ) import marshmallow.validate as validate from pandas.api.types import is_numeric_dtype +from pint.errors import PintError import timely_beliefs as tb from werkzeug.datastructures import FileStorage from marshmallow.validate import Validator @@ -365,6 +367,11 @@ def _serialize(self, value: Sensor, attr, obj, **kwargs) -> int: class VariableQuantityField(MarshmallowClickMixin, fields.Field): + _UNSUPPORTED_VALUE_TYPE_MESSAGE = ( + "Unsupported value type. `{value_type}` was provided but only dict, list, " + "str, pint Quantity, tuple, and numeric values with a default source unit are supported." + ) + def __init__( self, to_unit, @@ -434,20 +441,48 @@ def __init__( @with_appcontext_if_needed() def _deserialize( - self, value: dict[str, int] | list[dict] | str, attr, data, **kwargs - ) -> Sensor | list[dict] | ur.Quantity: + self, + value: ( + dict[str, Any] + | list[dict] + | str + | ur.Quantity + | tuple[Any, ...] + | numbers.Real + ), + attr, + data, + **kwargs, + ) -> Sensor | SensorReference | list[dict] | ur.Quantity: if isinstance(value, dict): - return self._deserialize_dict(value) + return self._deserialize_dict(value, attr, data, **kwargs) elif isinstance(value, list): return self._deserialize_list(value) elif isinstance(value, str): return self._deserialize_str(value) + elif isinstance(value, ur.Quantity): + return value.to(self.to_unit) + elif isinstance(value, tuple): + try: + return ur.Quantity.from_tuple(value).to(self.to_unit) + except (PintError, TypeError, ValueError, AttributeError, IndexError): + if ( + len(value) == 1 + and isinstance(value[0], numbers.Real) + and self.default_src_unit is not None + ): + return self._deserialize_numeric(value[0], attr, data, **kwargs) + if len(value) == 2: + return self._deserialize_str(f"{value[0]} {value[1]}") + raise FMValidationError( + self._UNSUPPORTED_VALUE_TYPE_MESSAGE.format(value_type=type(value)) + ) elif isinstance(value, numbers.Real) and self.default_src_unit is not None: return self._deserialize_numeric(value, attr, data, **kwargs) else: raise FMValidationError( - f"Unsupported value type. `{type(value)}` was provided but only dict, list and str are supported." + self._UNSUPPORTED_VALUE_TYPE_MESSAGE.format(value_type=type(value)) ) _SOURCE_FILTER_KEYS = SENSOR_REFERENCE_SOURCE_FILTER_KEYS @@ -504,13 +539,15 @@ def _deserialize_source_filters(self, value: dict[str, Any]) -> tuple[ return source_types, exclude_source_types, sources, source_account - def _deserialize_dict(self, value: dict[str, Any]) -> Sensor | SensorReference: + def _deserialize_dict( + self, value: dict[str, Any], attr, data, **kwargs + ) -> Sensor | SensorReference: """Deserialize a sensor reference to a Sensor or SensorReference. - Returns a plain :class:`Sensor` when no source filter keys are present - (backward compatible), and a :class:`SensorReference` when any of - ``source-types``, ``exclude-source-types``, ``sources``, or - ``source-account`` are provided. + Returns a plain :class:`Sensor` when no source filter or default keys are + present (backward compatible), and a :class:`SensorReference` when any of + ``source-types``, ``exclude-source-types``, ``sources``, ``source-account`` + or ``default`` are provided. """ if "sensor" not in value: raise FMValidationError("Dictionary provided but `sensor` key not found.") @@ -530,8 +567,12 @@ def _deserialize_dict(self, value: dict[str, Any]) -> Sensor | SensorReference: unit=self.to_unit if not self.to_unit.startswith("/") else None ).deserialize(value["sensor"], None, None) - # If source filter keys are present, return a SensorReference instead of a plain Sensor. - if self._SOURCE_FILTER_KEYS.isdisjoint(value.keys()): + default = None + if "default" in value: + default = self._deserialize_default(value["default"], attr, data, **kwargs) + + # If no source filter or default keys are present, keep returning a plain Sensor. + if self._SOURCE_FILTER_KEYS.isdisjoint(value.keys()) and default is None: return sensor # backward compat: no filters → plain Sensor source_types, exclude_source_types, sources, source_account = ( @@ -543,8 +584,23 @@ def _deserialize_dict(self, value: dict[str, Any]) -> Sensor | SensorReference: exclude_source_types=exclude_source_types, sources=sources, source_account=source_account, + default=default, ) + def _deserialize_default(self, value, attr, data, **kwargs) -> ur.Quantity: + """Deserialize a sensor reference fallback value.""" + if isinstance(value, str): + default = self._deserialize_str(value) + elif isinstance(value, numbers.Real) and self.default_src_unit is not None: + default = self._deserialize_numeric(value, attr, data, **kwargs) + else: + raise FMValidationError( + "Sensor reference `default` must be a quantity string or a numeric value with a known default source unit." + ) + if self.value_validator is not None: + self.value_validator(default) + return default + def _deserialize_list(self, value: list[dict]) -> list[dict]: """Deserialize a time series to a list of timed events.""" if self.return_magnitude is True: @@ -596,6 +652,15 @@ def _serialize( sensor_reference["source-account"] = [ account.id for account in value.source_account ] + if value.default is not None: + # `default` was already resolved to a concrete, compatible unit at + # deserialization time (see _deserialize_default), so for a + # denominator-only to_unit (e.g. "/MWh", not a valid pint unit on + # its own) there is nothing left to convert to. + if self.to_unit.startswith("/"): + sensor_reference["default"] = str(value.default) + else: + sensor_reference["default"] = str(value.default.to(self.to_unit)) return sensor_reference elif isinstance(value, Sensor): return dict(sensor=value.id) @@ -963,13 +1028,13 @@ class QuantitySchema(Schema): @dataclass class SensorReference: - """A sensor reference that wraps a Sensor with optional source filters for belief queries. + """A sensor reference that wraps a Sensor with optional query settings. Exposes the same ``unit``, ``id``, and ``event_resolution`` properties as a plain :class:`~flexmeasures.data.models.time_series.Sensor`, so code that reads those - properties works without modification. The source filters are passed through to - :meth:`TimedBelief.search ` - in :func:`~flexmeasures.data.models.planning.utils.get_series_from_quantity_or_sensor`. + properties works without modification. The source filters and optional default + value are passed through to + :func:`~flexmeasures.data.models.planning.utils.get_series_from_quantity_or_sensor`. """ sensor: Sensor @@ -977,6 +1042,7 @@ class SensorReference: exclude_source_types: list[str] | None = field(default=None) sources: list[DataSource] | None = field(default=None) source_account: list[Account] | None = field(default=None) + default: ur.Quantity | None = field(default=None) @property def unit(self) -> str: @@ -1011,7 +1077,7 @@ class OutputSensorReferenceSchema(SharedSensorReferenceSchema): class SensorReferenceSchema(SharedSensorReferenceSchema): - """Sensor reference with optional source filters.""" + """Sensor reference with optional source filters and fallback value.""" class Meta: description = "Sensor reference from which to look up a variable quantity." @@ -1047,6 +1113,25 @@ class Meta: description="Only use beliefs from data sources linked to these account IDs.", ), ) + default = fields.String( + required=False, + allow_none=False, + metadata=dict( + description="Fallback quantity to use when the referenced sensor has missing values. Note that every time slot the sensor leaves empty is filled with this value, so a sparse setpoint sensor becomes densely constrained.", + example="0 kWh", + ), + ) + + @post_dump + def remove_unset_default(self, data: dict, **kwargs) -> dict: + """Leave out `default` entirely when the reference does not define one. + + Without this, references that set no fallback would serialize a + `default: None` key, which is not valid input on the way back in. + """ + if data.get("default") is None: + data.pop("default", None) + return data class InflexibleDeviceSchema(SensorReferenceSchema): @@ -1127,9 +1212,21 @@ class TimeSeriesSchema(Schema): fields.Dict, required=True, metadata=dict( - description="Time series specification containing a list of segments that together describe a variable quantity.", + description=( + "Time series specification containing a list of segments that together " + "describe a variable quantity. Each segment may specify either " + "`datetime`, `start` and `end`, `start` and `duration`, or `end` and " + "`duration`." + ), example=[ - {"value": "23 kW", "start": "2025-11-20T15:15+01", "duration": "PT1H"} + {"value": "23 kW", "datetime": "2025-11-20T15:15+01"}, + { + "value": "24 kW", + "start": "2025-11-20T16:00+01", + "end": "2025-11-20T17:00+01", + }, + {"value": "25 kW", "start": "2025-11-20T17:00+01", "duration": "PT1H"}, + {"value": "26 kW", "end": "2025-11-20T19:00+01", "duration": "PT1H"}, ], ), ) diff --git a/flexmeasures/data/schemas/tests/test_scheduling.py b/flexmeasures/data/schemas/tests/test_scheduling.py index c9142b07bf..57232e4d59 100644 --- a/flexmeasures/data/schemas/tests/test_scheduling.py +++ b/flexmeasures/data/schemas/tests/test_scheduling.py @@ -745,7 +745,7 @@ def check_schema_loads_data(schema, data, fails): ( {"site-power-capacity": 100}, { - "site-power-capacity": f"Unsupported value type. `{type(100)}` was provided but only dict, list and str are supported." + "site-power-capacity": f"Unsupported value type. `{type(100)}` was provided but only dict, list, str, pint Quantity, tuple, and numeric values with a default source unit are supported." }, ), ( diff --git a/flexmeasures/data/schemas/tests/test_sensor.py b/flexmeasures/data/schemas/tests/test_sensor.py index f41805cb9a..93df41616c 100644 --- a/flexmeasures/data/schemas/tests/test_sensor.py +++ b/flexmeasures/data/schemas/tests/test_sensor.py @@ -5,6 +5,7 @@ from flexmeasures.data.schemas.sensors import ( QuantityOrSensor, SensorReference, + SensorReferenceSchema, VariableQuantityField, floor_bdf_event_starts, ) @@ -220,6 +221,41 @@ def test_sensor_reference_backward_compatible(setup_dummy_sensors): assert result.id == sensor1.id +def test_sensor_reference_with_default(setup_dummy_sensors): + """``{"sensor": , "default": ...}`` deserializes to a SensorReference.""" + sensor1, _, _, _ = setup_dummy_sensors + field = VariableQuantityField(to_unit="MWh", return_magnitude=False) + + result = field.deserialize({"sensor": sensor1.id, "default": "500 kWh"}) + + assert isinstance(result, SensorReference) + assert result.sensor == sensor1 + assert result.default == ur.Quantity("0.5 MWh") + + +def test_sensor_reference_schema_rejects_null_default(setup_dummy_sensors): + sensor1, _, _, _ = setup_dummy_sensors + + with pytest.raises(ValidationError) as exc_info: + SensorReferenceSchema().load({"sensor": sensor1.id, "default": None}) + + assert "default" in exc_info.value.messages + + +def test_sensor_reference_field_rejects_null_default(setup_dummy_sensors): + """``default`` must be a concrete fallback quantity when provided.""" + sensor1, _, _, _ = setup_dummy_sensors + field = VariableQuantityField(to_unit="MWh", return_magnitude=False) + + with pytest.raises(ValidationError) as exc_info: + field.deserialize({"sensor": sensor1.id, "default": None}) + + assert ( + "Sensor reference `default` must be a quantity string or a numeric value with a known default source unit." + in str(exc_info.value) + ) + + def test_sensor_reference_with_source_types(setup_dummy_sensors): """``{"sensor": , "source-types": [...]}`` deserializes to a :class:`SensorReference`. @@ -342,6 +378,18 @@ def test_sensor_reference_serialization_preserves_source_filters( } +def test_sensor_reference_serialization_preserves_default(setup_dummy_sensors): + sensor1, _, _, _ = setup_dummy_sensors + field = VariableQuantityField(to_unit="MWh", return_magnitude=False) + source_reference = field.deserialize({"sensor": sensor1.id, "default": "500 kWh"}) + + assert isinstance(source_reference, SensorReference) + assert serialize_variable_quantity(source_reference) == { + "sensor": sensor1.id, + "default": "0.5 MWh", + } + + def test_sensor_reference_filters_are_kept_per_reference( setup_dummy_sensors, setup_sources, db ): diff --git a/flexmeasures/ui/static/openapi-specs.json b/flexmeasures/ui/static/openapi-specs.json index 7f793d5cee..e051ecc022 100644 --- a/flexmeasures/ui/static/openapi-specs.json +++ b/flexmeasures/ui/static/openapi-specs.json @@ -5030,6 +5030,11 @@ "items": { "type": "integer" } + }, + "default": { + "type": "string", + "description": "Fallback quantity to use when the referenced sensor has missing values. Note that every time slot the sensor leaves empty is filled with this value, so a sparse setpoint sensor becomes densely constrained.", + "example": "0 kWh" } }, "required": [ @@ -5039,11 +5044,25 @@ }, "TimeSeries": { "type": "array", - "description": "Time series specification containing a list of segments that together describe a variable quantity.", + "description": "Time series specification containing a list of segments that together describe a variable quantity. Each segment may specify either `datetime`, `start` and `end`, `start` and `duration`, or `end` and `duration`.", "example": [ { "value": "23 kW", - "start": "2025-11-20T15:15+01", + "datetime": "2025-11-20T15:15+01" + }, + { + "value": "24 kW", + "start": "2025-11-20T16:00+01", + "end": "2025-11-20T17:00+01" + }, + { + "value": "25 kW", + "start": "2025-11-20T17:00+01", + "duration": "PT1H" + }, + { + "value": "26 kW", + "end": "2025-11-20T19:00+01", "duration": "PT1H" } ], @@ -5165,6 +5184,11 @@ "items": { "type": "integer" } + }, + "default": { + "type": "string", + "description": "Fallback quantity to use when the referenced sensor has missing values. Note that every time slot the sensor leaves empty is filled with this value, so a sparse setpoint sensor becomes densely constrained.", + "example": "0 kWh" } }, "required": [ diff --git a/flexmeasures/ui/templates/assets/asset_properties.html b/flexmeasures/ui/templates/assets/asset_properties.html index 61a141eb82..7fd32d79cd 100644 --- a/flexmeasures/ui/templates/assets/asset_properties.html +++ b/flexmeasures/ui/templates/assets/asset_properties.html @@ -607,14 +607,18 @@
Edit {{ asset.name }}'s flex-model
const card = activeCard(); const name = card.id.replace('-control', ''); const flexModel = getFlexModel() + const sensorReference = { "sensor": sensorId }; + const defaultInput = document.getElementById('flexSensorDefaultInput'); + if (defaultInput && defaultInput.value.trim()) { + sensorReference.default = defaultInput.value.trim(); + } if (FlexModelFieldValidTypes[name].includes(Array)) { const valueIndex = selectedIndex() const fieldValues = flexModel[name]; - fieldValues[valueIndex] = { "sensor": sensorId }; + fieldValues[valueIndex] = sensorReference; flexModel[name] = fieldValues; } else { - const value = flexModel[name]; - flexModel[name] = { "sensor": sensorId }; + flexModel[name] = sensorReference; } setFlexModel(flexModel); @@ -1120,6 +1124,7 @@
Edit {{ asset.name }}'s flex-model
tabContent.querySelector('.flex-input-group').appendChild(boolTabContentSaveBtn); } else if (dataType == Object) { // Object/Senosr Tab ================= btn.textContent = 'A sensor'; + const supportsSensorDefault = FlexModelFieldValidTypes[name].includes(Object); tabContent.innerHTML = `
@@ -1154,8 +1159,54 @@
Edit {{ asset.name }}'s flex-model
+ ${supportsSensorDefault ? ` + + + ` : ''} `; + if (supportsSensorDefault) { + const defaultInput = tabContent.querySelector('#flexSensorDefaultInput'); + defaultInput.value = value && typeof value === 'object' && value.default + ? value.default + : ''; + + const saveDefaultButton = document.createElement('button'); + saveDefaultButton.className = 'btn btn-secondary btn-sm me-2 mt-2'; + saveDefaultButton.textContent = 'Use fallback'; + saveDefaultButton.onclick = function () { + const currentFlexModel = getFlexModel(); + let sensorReference = currentFlexModel[name]; + if (Array.isArray(sensorReference)) { + sensorReference = sensorReference[selectedIndex()]; + } + if (!sensorReference || typeof sensorReference !== 'object' || !sensorReference.sensor) { + showToast("Select a sensor before setting a fallback", "info"); + return; + } + + const fallback = defaultInput.value.trim(); + if (fallback) { + sensorReference.default = fallback; + } else { + delete sensorReference.default; + } + setFlexModel(currentFlexModel); + }; + tabContent.querySelector('.flex-input-group').appendChild(saveDefaultButton); + } } else if (dataType == String) { // String Tab ================= btn.textContent = 'Fixed value'; From bbf299bf2289f3f66d97a38c87f8c9ad407c23c4 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 7 Aug 2026 00:47:03 +0100 Subject: [PATCH 28/30] docs: scope the sensor-reference `default` to variable-quantity fields MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `default` is declared on the shared SensorReferenceSchema, so it is accepted on every sensor reference in the API — but it is only applied in get_series_from_quantity_or_sensor. Sensor references resolved by get_power_values (inflexible devices) and by the forecasting pipelines (regressors) ignore it, silently. Rather than claim more than the code does, say where the field takes effect. Applying it in those two paths is worth a follow-up. Signed-off-by: Mohamed Belhsan Hmida Co-Authored-By: Claude Fable 5 Signed-off-by: Mohamed Belhsan Hmida --- documentation/api/change_log.rst | 2 +- documentation/changelog.rst | 2 +- flexmeasures/data/schemas/sensors.py | 2 +- flexmeasures/ui/static/openapi-specs.json | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst index cac446d71f..3e7c2a0b69 100644 --- a/documentation/api/change_log.rst +++ b/documentation/api/change_log.rst @@ -10,7 +10,7 @@ v3.0-32 | July XX, 2026 - API endpoints are now rate-limited. A request which exceeds a limit is answered with a ``429 (Too Many Requests)`` status code and a ``Retry-After`` header stating how many seconds to wait. Responses also carry ``X-RateLimit-*`` headers, describing the limit that applied, how much of it is left, and when it resets. A stricter limit applies to ``POST /assets//schedules/trigger``, ``POST /sensors//schedules/trigger`` and ``POST /sensors//forecasts/trigger`` than to other endpoints; the health endpoints are exempt. Per-account overrides are set by assigning the account a plan (a ``Plan`` database row), rather than through an account attribute. - Introduced the ``inflexible-consumption`` and ``inflexible-production`` flex-context fields, which make explicit how the sign of each inflexible device's power data should be read: positive values denote consumption resp. production. Each entry is a sensor reference (``{"sensor": }``), optionally with source filters (``source-types``, ``exclude-source-types``, ``sources``, ``source-account``). Deprecated the ``inflexible-device-sensors`` field (a list of bare sensor IDs, whose sign convention is read from each sensor's ``consumption_is_positive`` attribute); it remains supported, but cannot be combined with the new fields in one flex-context. - Added a ``role`` query parameter to ``GET /api/v3_0/accounts`` for filtering accessible organisations by account role. -- Sensor references (on any flex-model or flex-context field that accepts them) may now include a ``default`` fallback quantity, e.g. ``{"sensor": 50, "default": "0 kWh"}``. It fills the time slots for which the referenced sensor holds no value. Note that this fills *every* such slot, so a sensor recording only occasional setpoints becomes densely constrained. +- Sensor references on variable-quantity flex-model and flex-context fields (such as ``soc-minima``, ``soc-maxima``, the capacity fields and the price fields) may now include a ``default`` fallback quantity, e.g. ``{"sensor": 50, "default": "0 kWh"}``. It fills the time slots for which the referenced sensor holds no value. Note that this fills *every* such slot, so a sensor recording only occasional setpoints becomes densely constrained. The field is not (yet) applied to sensor references on ``inflexible-consumption``/``inflexible-production`` or to forecaster regressors. - Extended ``GET /api/v3_0/jobs/`` with a ``result`` field containing ``unresolved`` and ``resolved`` arrays, each keyed by asset ID. For scheduling jobs, this surfaces soft state-of-charge constraint analysis: ``soc-minima`` and ``soc-maxima`` violations (with a ``violation`` magnitude) or satisfied constraints (with a ``margin`` headroom). Both arrays are empty when no SoC constraints were defined. - **Field canonicalization** for background job tracking: * The ``job`` field is now the canonical way to identify background jobs returned by `/sensors//schedules/trigger`, `/assets//schedules/trigger`, and `/sensors//forecasts/trigger` endpoints. If applicable, the triggered response now also returns a ``results-url`` pointing to the sensor-specific results endpoint, alongside the generic ``job-url``. diff --git a/documentation/changelog.rst b/documentation/changelog.rst index c33195dfcb..61cc158820 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -43,7 +43,7 @@ New features * CLI support for adding/editing account attributes [see `PR #2242 `_] * Improve chart axis domain for event values not around zero, with a per-sub-chart ``y-axis`` option in ``sensors_to_show`` (default ``zero``, which pads the axis out to include zero) that can be set to ``data`` to fit a sub-chart's y-axis to the values shown, to an explicit ``[min, max]`` domain that the axis will cover at least (expanding to fit data beyond it), or to a strict ``{"min": min, "max": max}`` domain that the axis will never exceed (clamping data beyond it, with a warning when that happens), editable from the graph editor [see `PR #2244 `_] * Breaking behaviour change: storage SoC constraints (``soc-minima``/``soc-maxima``) are now relaxed by default (``relax-soc-constraints`` defaults to ``True``; set it or ``relax-constraints`` to ``False`` to keep them hard), and the built-in storage fallback scheduler has been retired, so infeasible storage problems now fail with their failure reason instead of silently saving a fallback schedule; ``FLEXMEASURES_FALLBACK_REDIRECT`` is now only relevant for custom schedulers that define a fallback scheduler [see `PR #2252 `_] -* Sensor references now accept a ``default`` fallback quantity, e.g. ``{"sensor": 50, "default": "0 kWh"}``, filling the time slots for which the referenced sensor holds no value; also settable from the flex-model UI [see `PR #2267 `_] +* Sensor references on variable-quantity flex-model and flex-context fields now accept a ``default`` fallback quantity, e.g. ``{"sensor": 50, "default": "0 kWh"}``, filling the time slots for which the referenced sensor holds no value; also settable from the flex-model UI [see `PR #2405 `_] * Extended ``GET /api/v3_0/jobs/`` with a ``result`` field containing ``unresolved`` and ``resolved`` soft state-of-charge constraint analysis (``soc-minima``/``soc-maxima`` violations or satisfied constraints, keyed by asset ID) for scheduling jobs; both arrays are empty when no SoC constraints were defined [see `PR #2072 `_] * New storage flex-model field ``operation-modes`` confines a device's power to one of several power bands, following the S2 standard's operation modes — for example, a device that is either off or running at one fixed power [see `PR #2278 `_] * New ``FLEXMEASURES_LP_SOLVER_OPTIONS`` config setting to pass solver options to the scheduling solver, validated against the installed HiGHS build so that unknown or unsupported options raise instead of being silently ignored [see `PR #2283 `_] diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py index e9d54a8e0d..e2804134fc 100644 --- a/flexmeasures/data/schemas/sensors.py +++ b/flexmeasures/data/schemas/sensors.py @@ -1117,7 +1117,7 @@ class Meta: required=False, allow_none=False, metadata=dict( - description="Fallback quantity to use when the referenced sensor has missing values. Note that every time slot the sensor leaves empty is filled with this value, so a sparse setpoint sensor becomes densely constrained.", + description="Fallback quantity to use when the referenced sensor has missing values, on variable-quantity flex-model and flex-context fields (such as soc-minima, soc-maxima, the capacity fields and the price fields). Note that every time slot the sensor leaves empty is filled with this value, so a sparse setpoint sensor becomes densely constrained. This field is not (yet) applied to inflexible-device references or to forecaster regressors.", example="0 kWh", ), ) diff --git a/flexmeasures/ui/static/openapi-specs.json b/flexmeasures/ui/static/openapi-specs.json index e051ecc022..eb2e0e23c8 100644 --- a/flexmeasures/ui/static/openapi-specs.json +++ b/flexmeasures/ui/static/openapi-specs.json @@ -5033,7 +5033,7 @@ }, "default": { "type": "string", - "description": "Fallback quantity to use when the referenced sensor has missing values. Note that every time slot the sensor leaves empty is filled with this value, so a sparse setpoint sensor becomes densely constrained.", + "description": "Fallback quantity to use when the referenced sensor has missing values, on variable-quantity flex-model and flex-context fields (such as soc-minima, soc-maxima, the capacity fields and the price fields). Note that every time slot the sensor leaves empty is filled with this value, so a sparse setpoint sensor becomes densely constrained. This field is not (yet) applied to inflexible-device references or to forecaster regressors.", "example": "0 kWh" } }, @@ -5187,7 +5187,7 @@ }, "default": { "type": "string", - "description": "Fallback quantity to use when the referenced sensor has missing values. Note that every time slot the sensor leaves empty is filled with this value, so a sparse setpoint sensor becomes densely constrained.", + "description": "Fallback quantity to use when the referenced sensor has missing values, on variable-quantity flex-model and flex-context fields (such as soc-minima, soc-maxima, the capacity fields and the price fields). Note that every time slot the sensor leaves empty is filled with this value, so a sparse setpoint sensor becomes densely constrained. This field is not (yet) applied to inflexible-device references or to forecaster regressors.", "example": "0 kWh" } }, From 95d2fac1dbf4648f79aa3ca3c93485074d100040 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 7 Aug 2026 09:27:11 +0100 Subject: [PATCH 29/30] revert the fallback-scheduler content from this branch This branch was carved out of #2267, which was stacked on feat/retire-fallback-scheduler, so #2252's commits travelled along with it. Now that this PR targets main, drop that content here: the tree of this commit equals main plus the sensor-reference `default` work alone. No behaviour of this PR changes; #2252 lands on its own. Signed-off-by: Mohamed Belhsan Hmida --- documentation/api/change_log.rst | 1 - documentation/api/introduction.rst | 9 +- documentation/changelog.rst | 1 - documentation/configuration.rst | 4 +- documentation/features/scheduling.rst | 22 +- .../api/v3_0/tests/test_sensor_schedules.py | 207 ++++++++++++++++-- flexmeasures/data/models/planning/storage.py | 75 +++++++ .../data/models/planning/tests/test_solver.py | 34 ++- .../models/planning/tests/test_storage.py | 4 +- flexmeasures/data/models/planning/utils.py | 77 +++++++ .../data/schemas/scheduling/__init__.py | 63 ++---- .../data/schemas/scheduling/metadata.py | 16 +- .../data/schemas/tests/test_scheduling.py | 74 ------- .../data/tests/test_scheduling_sequential.py | 112 +++++----- flexmeasures/ui/static/openapi-specs.json | 10 +- 15 files changed, 469 insertions(+), 240 deletions(-) diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst index 3e7c2a0b69..6c5ce319ab 100644 --- a/documentation/api/change_log.rst +++ b/documentation/api/change_log.rst @@ -17,7 +17,6 @@ v3.0-32 | July XX, 2026 * Legacy ``schedule`` field (in scheduling endpoints) and ``forecast`` field (in forecasting endpoints) remain in responses, unchanged, for backward compatibility. New clients should prefer ``job``; see :ref:`api_background_jobs` for the full response format. These fields are not (yet) formally deprecated — see the "Planned API v4" discussion linked from :ref:`api_deprecation` for where and when their removal is being tracked. - ``GET /api/v3_0/jobs/`` now returns ``202 Accepted`` while a job is queued or running, ``422 Unprocessable Entity`` for failed jobs, and ``200 OK`` for finished jobs. See :ref:`api_background_jobs` for the response format and polling flow. - ``GET /api/v3_0/jobs/`` now also returns kebab-case metadata fields such as ``func-name`` and ``enqueued-at``, alongside the existing snake_case fields (``func_name``, ``enqueued_at``, etc.), which remain unchanged for backward compatibility. New clients should prefer the kebab-case fields. -- Breaking: ``relax-soc-constraints`` now defaults to ``True`` (set it or ``relax-constraints`` to ``False`` to keep ``soc-minima``/``soc-maxima`` hard), and the built-in storage fallback scheduler has been retired. ``GET /api/v3_0/sensors//schedules/`` now returns ``400`` with the failure reason for an infeasible storage schedule instead of a ``303`` redirect to (or an automatic follow of) a fallback schedule. ``FLEXMEASURES_FALLBACK_REDIRECT`` remains relevant only for custom schedulers that still define a fallback scheduler. - Added a ``group`` field to the storage flex-model, accepted by the `/assets/(id)/schedules/trigger <../api/v3_0.html#post--api-v3_0-assets-id-schedules-trigger>`_ (POST) endpoint, referencing a power sensor representing a group of devices (e.g. a shared inverter or feeder). The group's ``power-capacity`` is enforced as a hard constraint on the group's aggregate power, while its ``consumption-capacity``/``production-capacity`` are enforced as soft constraints with default breach prices; the group's scheduled aggregate power is saved to the group sensor. - The ``group`` field also accepts a ``{"asset": }`` reference (besides ``{"sensor": }``), pointing at an asset whose own (DB-stored) flex-model defines the group's constraints. Such a group defines no power sensor of its own; its aggregate schedule is instead saved via its ``consumption``/``production`` output sensor references, following the same conventions as any other asset-only flex-model entry. This lets the entire flex-model for a device tree (including groups) live in the DB, with ``flex-model`` omitted or empty on the trigger request. - Fixed: `/assets/(id)/schedules/trigger <../api/v3_0.html#post--api-v3_0-assets-id-schedules-trigger>`_ (POST) now also accepts the legacy ``force_new_job_creation`` field name (in addition to ``force-new-job-creation``), matching the sensor-level trigger endpoint. Previously, only the sensor-level endpoint accepted both spellings. diff --git a/documentation/api/introduction.rst b/documentation/api/introduction.rst index 7d289cdfd2..760d902ee8 100644 --- a/documentation/api/introduction.rst +++ b/documentation/api/introduction.rst @@ -116,19 +116,18 @@ See Other (303) --------------- Some API responses return ``HTTP status 303 (See Other)`` to redirect the client to a different resource. -This can happen when a custom scheduler defines a fallback scheduler, the original scheduling job fails, and the fallback schedule has been computed instead. +This happens, for example, when a scheduling job fails and a fallback schedule has been computed instead. In that case, the response includes a ``Location`` header pointing to the fallback schedule endpoint, so clients can automatically retrieve the fallback result. The response body will contain a JSON message with a ``status`` field set to ``"UNKNOWN_SCHEDULE"`` and a ``message`` field explaining the reason for the redirect. .. note:: - FlexMeasures' built-in storage scheduler no longer computes a fallback schedule for infeasible problems. - Instead, ``soc-minima`` and ``soc-maxima`` are relaxed by default through ``"relax-soc-constraints": true`` (setting either it or ``"relax-constraints"`` to ``false`` keeps them hard). + The fallback schedule mechanism activates when the main scheduler encounters an infeasible problem (i.e. when constraints cannot be satisfied). + This is less likely to happen when ``"relax-constraints": true`` is set in the ``flex-context``, as constraint relaxation softens most infeasibility-causing constraints. The hard constraints that remain even after constraint relaxation are ``soc-min``, ``soc-max``, ``soc-targets`` and ``power-capacity`` in the ``flex-model``, and ``site-power-capacity`` in the ``flex-context``. - If hard constraints cannot be satisfied, the scheduling job fails and clients receive the failure reason when requesting the schedule. - For custom schedulers that still define a fallback scheduler, server administrators can configure whether clients receive a 303 redirect (``FLEXMEASURES_FALLBACK_REDIRECT = True``) or whether FlexMeasures follows the fallback automatically and returns the fallback schedule directly (``FLEXMEASURES_FALLBACK_REDIRECT = False``, the default). + Server administrators can configure whether clients receive a 303 redirect (``FLEXMEASURES_FALLBACK_REDIRECT = True``) or whether FlexMeasures follows the fallback automatically and returns the fallback schedule directly (``FLEXMEASURES_FALLBACK_REDIRECT = False``, the default). Here is a client-side code example in Python for handling 303 redirects (this merely follows the redirect and should be revised to make use of the client's monitoring tools): diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 61cc158820..e4a4b9137c 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -42,7 +42,6 @@ New features * Commodity contexts that omit grid-connection fields (prices and site capacities) now get smart defaults instead of failing or silently leaving the grid unconstrained — for instance, a bare ``{"commodity": "gas"}`` is treated as having no grid connection; see :ref:`commodity_context_defaults` for the full rules [see `PR #2272 `_] * CLI support for adding/editing account attributes [see `PR #2242 `_] * Improve chart axis domain for event values not around zero, with a per-sub-chart ``y-axis`` option in ``sensors_to_show`` (default ``zero``, which pads the axis out to include zero) that can be set to ``data`` to fit a sub-chart's y-axis to the values shown, to an explicit ``[min, max]`` domain that the axis will cover at least (expanding to fit data beyond it), or to a strict ``{"min": min, "max": max}`` domain that the axis will never exceed (clamping data beyond it, with a warning when that happens), editable from the graph editor [see `PR #2244 `_] -* Breaking behaviour change: storage SoC constraints (``soc-minima``/``soc-maxima``) are now relaxed by default (``relax-soc-constraints`` defaults to ``True``; set it or ``relax-constraints`` to ``False`` to keep them hard), and the built-in storage fallback scheduler has been retired, so infeasible storage problems now fail with their failure reason instead of silently saving a fallback schedule; ``FLEXMEASURES_FALLBACK_REDIRECT`` is now only relevant for custom schedulers that define a fallback scheduler [see `PR #2252 `_] * Sensor references on variable-quantity flex-model and flex-context fields now accept a ``default`` fallback quantity, e.g. ``{"sensor": 50, "default": "0 kWh"}``, filling the time slots for which the referenced sensor holds no value; also settable from the flex-model UI [see `PR #2405 `_] * Extended ``GET /api/v3_0/jobs/`` with a ``result`` field containing ``unresolved`` and ``resolved`` soft state-of-charge constraint analysis (``soc-minima``/``soc-maxima`` violations or satisfied constraints, keyed by asset ID) for scheduling jobs; both arrays are empty when no SoC constraints were defined [see `PR #2072 `_] * New storage flex-model field ``operation-modes`` confines a device's power to one of several power bands, following the S2 standard's operation modes — for example, a device that is either off or running at one fixed power [see `PR #2278 `_] diff --git a/documentation/configuration.rst b/documentation/configuration.rst index de1b6156b1..79afcdce86 100644 --- a/documentation/configuration.rst +++ b/documentation/configuration.rst @@ -994,9 +994,7 @@ Default: ``None`` (defaults are set internally for each sunset API version, e.g. FLEXMEASURES_FALLBACK_REDIRECT ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Control how the API handles a failed scheduling job when a custom scheduler has computed a fallback schedule. - -FlexMeasures' built-in storage scheduler no longer computes fallback schedules, but custom schedulers may still define fallback schedulers. +Control how the API handles a failed scheduling job when a fallback schedule has been computed. If ``True``, the API returns ``HTTP status 303 (See Other)`` with a ``Location`` header pointing to the fallback schedule endpoint. Clients must follow this redirect themselves to obtain the fallback schedule (see :ref:`api_see_other`). diff --git a/documentation/features/scheduling.rst b/documentation/features/scheduling.rst index c9561a3f0a..6284a26537 100644 --- a/documentation/features/scheduling.rst +++ b/documentation/features/scheduling.rst @@ -1,6 +1,6 @@ .. _scheduling: -Scheduling +Scheduling =========== Scheduling is the main value-drive of FlexMeasures. We have two major types of schedulers built-in, for storage devices (usually batteries or hot water storage) and processes (usually in industry). @@ -393,13 +393,13 @@ However, here are some tips to model a buffer correctly: For a hands-on example of a heat buffer fed by multiple devices, see :ref:`tut_multi_feed_storage`. -If the flex model describes an infeasible problem for the storage scheduler, the failure should remain visible. -By default, ``soc-minima`` and ``soc-maxima`` are relaxed into soft constraints, so the scheduler can still return a useful schedule when these boundaries cannot be fully met. -Setting either ``relax-soc-constraints`` or ``relax-constraints`` to ``false`` in the flex-context keeps them as hard constraints. -Exact ``soc-targets``, physical ``soc-min`` / ``soc-max`` bounds, and ``power-capacity`` (in the flex-model) and ``site-power-capacity`` (in the flex-context) remain hard constraints. -If those hard constraints make the problem infeasible, the scheduling job fails instead of producing a fallback schedule. +What happens if the flex model describes an infeasible problem for the storage scheduler? Excellent question! +It is highly important for a robust operation that these situations still lead to a somewhat good outcome. +From our practical experience, we derived a ``StorageFallbackScheduler``. +It simplifies an infeasible situation by just starting to charge, discharge, or do neither, +depending on the first target state of charge and the capabilities of the asset. -It is important to take note of these failures. Often, misconfigured flex models are the reason. +Of course, we also log a failure in the scheduling job, so it's important to take note of these failures. Often, mis-configured flex models are the reason. For a hands-on tutorial on using some of the storage flex-model fields, head over to :ref:`tut_v2g` use case and `the API documentation for triggering schedules <../api/v3_0.html#post--api-v3_0-assets-id-schedules-trigger>`_. For further hands-on examples, see :ref:`tut_multi_feed_storage` (multiple devices feeding one shared storage) and :ref:`tut_multi_commodity` (devices on different commodities scheduled together). @@ -417,15 +417,15 @@ Some examples from practice (usually industry) could be: - A centrifuge's daily work of combing through sludge water. Depends on amount of sludge present. - Production processes with a target amount of output until the end of the current shift. The target usually comes out of production planning. -- Application of coating under hot temperature, with fixed number of times it needs to happen before some deadline. - +- Application of coating under hot temperature, with fixed number of times it needs to happen before some deadline. + .. list-table:: :header-rows: 1 :widths: 20 25 90 * - Field - Example value - - Description + - Description * - ``power`` - ``"15kW"`` - Nominal power of the load. @@ -436,7 +436,7 @@ Some examples from practice (usually industry) could be: - ``"MAX"`` - Objective of the scheduler, to maximize (``"MAX"``) or minimize (``"MIN"``). * - ``time_restrictions`` - - ``[{"start": "2015-01-02T08:00:00+01:00", "duration": "PT2H"}]`` + - ``[{"start": "2015-01-02T08:00:00+01:00", "duration": "PT2H"}]`` - Time periods in which the load cannot be scheduled to run. * - ``process_type`` - ``"INFLEXIBLE"``, ``"SHIFTABLE"`` or ``"BREAKABLE"`` diff --git a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py index a2b37566d4..0eb7d95b6f 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py @@ -387,8 +387,7 @@ def test_get_schedule_unfinished_job_returns_202_when_sunset_active( @pytest.mark.parametrize( "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True ) -@pytest.mark.parametrize("fallback_redirect", [True, False]) -def test_get_schedule_infeasible_storage_job_without_fallback( +def test_get_schedule_fallback( app, add_battery_assets, add_market_prices, @@ -397,15 +396,13 @@ def test_get_schedule_infeasible_storage_job_without_fallback( keep_scheduling_queue_empty, requesting_user, db, - fallback_redirect, - monkeypatch, ): """ - Test that a failing StorageScheduler call reports the failure without creating a fallback job. - - This test is based on flexmeasures/data/models/planning/tests/test_solver.py. + Test if the fallback job is created after a failing StorageScheduler call. This test + is based on flexmeasures/data/models/planning/tests/test_solver.py """ - monkeypatch.setitem(app.config, "FLEXMEASURES_FALLBACK_REDIRECT", fallback_redirect) + assert app.config["FLEXMEASURES_FALLBACK_REDIRECT"] is False + app.config["FLEXMEASURES_FALLBACK_REDIRECT"] = True target_soc = 9 charging_station_name = "Test charging station" @@ -425,12 +422,17 @@ def test_get_schedule_infeasible_storage_job_without_fallback( assert capacity == 2 assert charging_station.get_attribute("consumption-price") == {"sensor": epex_da.id} + # check that no Fallback schedule has been saved before + models = [ + source.model for source in charging_station.search_beliefs().sources.unique() + ] + assert "StorageFallbackScheduler" not in models + # create a scenario that yields an infeasible problem (unreachable target SOC at 2am) message = { "start": start, "duration": "PT24H", "resolution": "PT15M", # just schedule in the original sensor resolution - "force-new-job-creation": True, "flex-model": { "soc-at-start": 10, "soc-min": charging_station.get_attribute("min_soc_in_mwh", 0), @@ -484,21 +486,196 @@ def test_get_schedule_infeasible_storage_job_without_fallback( # Make sure the resolution shows up in the job kwargs assert job.kwargs.get("resolution") == pd.Timedelta(message["resolution"]) - # no storage fallback job is created - assert len(app.queues["scheduling"]) == 0 - assert job.meta.get("fallback_job_id") is None + # the callback creates the fallback job which is still pending + assert len(app.queues["scheduling"]) == 1 + fallback_job_id = Job.fetch( + job_id, connection=app.queues["scheduling"].connection + ).meta.get("fallback_job_id") + + # check that the fallback_job_id is stored on the metadata of the original job + assert app.queues["scheduling"].get_job_ids()[0] == fallback_job_id + assert fallback_job_id != job_id get_schedule_response = client.get( url_for("SensorAPI:get_schedule", id=charging_station.id, uuid=job_id), ) - assert get_schedule_response.status_code == 400 - assert "Scheduling job failed with InfeasibleProblemException: infeasible." in ( + # requesting the original job redirects to the fallback job + assert ( + get_schedule_response.status_code == 303 + ) # Status code for redirect ("See other") + assert ( get_schedule_response.json["message"] + == "Scheduling job failed with InfeasibleProblemException: infeasible. StorageScheduler was used." ) - assert "StorageScheduler was used." in get_schedule_response.json["message"] assert get_schedule_response.json["status"] == "UNKNOWN_SCHEDULE" assert get_schedule_response.json["result"] == "Rejected" + # check that the redirection location points to the fallback job + assert ( + get_schedule_response.headers["location"] + == f"http://localhost/api/v3_0/sensors/{charging_station.id}/schedules/{fallback_job_id}" + ) + + # run the fallback job + work_on_rq( + app.queues["scheduling"], + exc_handler=handle_scheduling_exception, + max_jobs=1, + ) + + # check that the queue is empty + assert len(app.queues["scheduling"]) == 0 + + # get the fallback schedule + fallback_schedule = client.get( + get_schedule_response.headers["location"], + json={"duration": "PT24H"}, + ).json + + # check that the fallback schedule has the right status and start dates + assert fallback_schedule["status"] == "PROCESSED" + assert parse_datetime(fallback_schedule["start"]) == parse_datetime(start) + + models = [ + source.model + for source in charging_station.search_beliefs().sources.unique() + ] + assert "StorageFallbackScheduler" in models + + app.config["FLEXMEASURES_FALLBACK_REDIRECT"] = False + + +@pytest.mark.parametrize( + "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True +) +def test_get_schedule_fallback_not_redirect( + app, + add_battery_assets, + add_market_prices, + battery_soc_sensor, + add_charging_station_assets, + keep_scheduling_queue_empty, + requesting_user, + db, +): + """ + Test if the fallback scheduler is returned directly after a failing StorageScheduler call. This test + is based on flexmeasures/data/models/planning/tests/test_solver.py + """ + app.config["FLEXMEASURES_FALLBACK_REDIRECT"] = False + + target_soc = 9 + charging_station_name = "Test charging station" + + start = "2015-01-02T00:00:00+01:00" + epex_da = get_test_sensor(db) + charging_station = get_sensor_by_name( + add_charging_station_assets[charging_station_name], "power" + ) + + capacity = charging_station.get_attribute( + "capacity_in_mw", + ur.Quantity(charging_station.get_attribute("site-power-capacity")) + .to("MW") + .magnitude, + ) + assert capacity == 2 + assert charging_station.get_attribute("consumption-price") == {"sensor": epex_da.id} + + # create a scenario that yields an infeasible problem (unreachable target SOC at 2am) + message = { + "start": start, + "duration": "PT24H", + "flex-model": { + "soc-at-start": 10, + "soc-min": charging_station.get_attribute("min_soc_in_mwh", 0), + "soc-max": charging_station.get_attribute("max-soc-in-mwh", target_soc), + "roundtrip-efficiency": charging_station.get_attribute( + "roundtrip-efficiency", 1 + ), + "storage-efficiency": charging_station.get_attribute( + "storage-efficiency", 1 + ), + "soc-targets": [ + { + "value": target_soc, + "start": "2015-01-02T02:00:00+01:00", + "duration": "PT0H", + } + ], + }, + } + + with app.test_client() as client: + # trigger storage scheduler + trigger_schedule_response = client.post( + url_for("SensorAPI:trigger_schedule", id=charging_station.id), + json=message, + ) + + # check that the call is successful + assert trigger_schedule_response.status_code == 202 + job_id = trigger_schedule_response.json["schedule"] + + # look for scheduling jobs in queue + assert ( + len(app.queues["scheduling"]) == 1 + ) # only 1 schedule should be made for 1 asset + job = app.queues["scheduling"].jobs[0] + assert job.kwargs["asset_or_sensor"]["id"] == charging_station.id + assert job.kwargs["start"] == parse_datetime(message["start"]) + assert job.id == job_id + + # process only the job that runs the storage scheduler (max_jobs=1) + work_on_rq( + app.queues["scheduling"], + exc_handler=handle_scheduling_exception, + max_jobs=1, + ) + + # check that the job is failing + job = Job.fetch(job_id, connection=app.queues["scheduling"].connection) + assert job.is_failed + + # Make sure that the db flex_context shows up in the job kwargs + assert "flex-context" not in message and job.kwargs.get("flex_context") + + # the callback creates the fallback job which is still pending + assert len(app.queues["scheduling"]) == 1 + + fallback_job_id = Job.fetch( + job_id, connection=app.queues["scheduling"].connection + ).meta.get("fallback_job_id") + + # check that the fallback_job_id is stored on the metadata of the original job + assert app.queues["scheduling"].get_job_ids()[0] == fallback_job_id + assert fallback_job_id != job_id + + get_schedule_response = client.get( + url_for("SensorAPI:get_schedule", id=charging_station.id, uuid=job_id), + ) + + work_on_rq( + app.queues["scheduling"], + exc_handler=handle_scheduling_exception, + max_jobs=1, + ) + + get_schedule_response = client.get( + url_for("SensorAPI:get_schedule", id=charging_station.id, uuid=job_id), + ) + + assert get_schedule_response.status_code == 200 + + schedule = get_schedule_response.json + + # check that the fallback schedule has the right status and start dates + assert schedule["status"] == "PROCESSED" + assert parse_datetime(schedule["start"]) == parse_datetime(start) + assert schedule["scheduler_info"]["scheduler"] == "StorageFallbackScheduler" + + app.config["FLEXMEASURES_FALLBACK_REDIRECT"] = False + @pytest.mark.parametrize( "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index d412665ab5..c3ecc45bd2 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -3,6 +3,7 @@ import re import copy from datetime import datetime, timedelta +from typing import Type import pandas as pd import numpy as np @@ -31,6 +32,7 @@ initialize_series, initialize_df, get_power_values, + fallback_charging_policy, get_continuous_series_sensor_or_quantity, ) from flexmeasures.data.models.planning.exceptions import InfeasibleProblemException @@ -2607,10 +2609,83 @@ def _ensure_variable_quantity( return q +class StorageFallbackScheduler(MetaStorageScheduler): + __version__ = "3" + __author__ = "Seita" + + def compute(self, skip_validation: bool = False) -> SchedulerOutputType: + """Schedule a battery or Charge Point by just starting to charge, discharge, or do neither, + depending on the first target state of charge and the capabilities of the Charge Point. + For the resulting consumption schedule, consumption is defined as positive values. + + Note that this ignores any cause of the infeasibility. + + :param skip_validation: If True, skip validation of constraints specified in the data. + :returns: The computed schedule. + """ + + ( + sensors, + start, + end, + resolution, + soc_at_start, + device_constraints, + ems_constraints, + commitments, + ) = self._prepare(skip_validation=skip_validation) + + # Fallback policy if the problem was unsolvable + storage_schedule = { + sensor: fallback_charging_policy( + sensor, device_constraints[d], start, end, resolution + ) + for d, sensor in enumerate(sensors) + if sensor is not None + } + + # Convert each device schedule to the unit of the device's power sensor + storage_schedule = { + sensor: convert_units( + storage_schedule[sensor], + "MW", + sensor.unit, + event_resolution=sensor.event_resolution, + ) + for sensor in sensors + if sensor is not None + } + + # Round schedule + if self.round_to_decimals: + storage_schedule = { + sensor: storage_schedule[sensor].round(self.round_to_decimals) + for sensor in sensors + if sensor is not None + } + + if self.return_multiple: + # Iterate over the dict keys (not the sensors list, which may hold the + # same sensor for multiple devices), so no sensor is emitted twice. + return [ + { + "name": "storage_schedule", + "sensor": sensor, + "data": storage_schedule[sensor], + } + for sensor in storage_schedule.keys() + if sensor is not None + ] + else: + return storage_schedule[sensors[0]] + + class StorageScheduler(MetaStorageScheduler): __version__ = "8" __author__ = "Seita" + fallback_scheduler_class: Type[Scheduler] = StorageFallbackScheduler + @staticmethod def _build_soc_schedule( # noqa: C901 flex_model: list[dict], diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py index 62c2c8e791..57cf8840b3 100644 --- a/flexmeasures/data/models/planning/tests/test_solver.py +++ b/flexmeasures/data/models/planning/tests/test_solver.py @@ -502,18 +502,22 @@ def test_charging_station_solver_day_2( (15, "Test charging station (bidirectional)"), ], ) -def test_storage_scheduler_reports_unsolvable_problem_without_fallback( +def test_fallback_to_unsolvable_problem( target_soc, charging_station_name, setup_planning_test_data, db ): """Starting with a state of charge 10 kWh, within 2 hours we should be able to reach any state of charge in the range [10, 14] kWh for a unidirectional station, or [6, 14] for a bidirectional station, given a charging capacity of 2 kW. - Here we test target states of charge outside that range. - The StorageScheduler should report this infeasible problem without hiding it behind - a fallback schedule. + Here we test target states of charge outside that range, ones that we should be able + to get as close to as 1 kWh difference. + We want our scheduler to handle unsolvable problems like these with a sensible fallback policy. + + The StorageScheduler raises an Exception which triggers the creation of a new job to compute a fallback + schedule. """ soc_at_start = 10 duration_until_target = timedelta(hours=2) + expected_gap = 1 epex_da = get_test_sensor(db) charging_station = setup_planning_test_data[charging_station_name].sensors[0] @@ -578,8 +582,26 @@ def test_storage_scheduler_reports_unsolvable_problem_without_fallback( # calling the scheduler with an infeasible problem raises an Exception with pytest.raises(InfeasibleProblemException): - scheduler.compute(skip_validation=True) - assert scheduler.fallback_scheduler_class is None + consumption_schedule = scheduler.compute(skip_validation=True) + + # check that the fallback scheduler provides a sensible fallback policy + fallback_scheduler = scheduler.fallback_scheduler_class(**kwargs) + fallback_scheduler.config_deserialized = True + consumption_schedule = fallback_scheduler.compute(skip_validation=True) + + soc_schedule = integrate_time_series( + consumption_schedule, soc_at_start, decimal_precision=6 + ) + + # Check if constraints were met + assert min(consumption_schedule.values) >= capacity * -1 + assert max(consumption_schedule.values) <= capacity + print(consumption_schedule.head(12)) + print(soc_schedule.head(12)) + assert ( + abs(abs(soc_schedule.loc[target_soc_datetime] - target_soc) - expected_gap) + < TOLERANCE + ) @pytest.mark.parametrize( diff --git a/flexmeasures/data/models/planning/tests/test_storage.py b/flexmeasures/data/models/planning/tests/test_storage.py index 645944aa1f..4bf74a73a1 100644 --- a/flexmeasures/data/models/planning/tests/test_storage.py +++ b/flexmeasures/data/models/planning/tests/test_storage.py @@ -198,8 +198,7 @@ def test_battery_relaxation(add_battery_assets, db): """Check that resolving SoC breaches is more important than resolving device power breaches. The battery is still charging with 25 kW between noon and 4 PM, when the consumption capacity is supposed to be 0. - It is still charging because resolving the still unmatched SoC minima takes precedence - through the default SoC breach prices. + It is still charging because resolving the still unmatched SoC minima takes precedence (via breach prices). """ _, battery = get_sensors_from_db( db, add_battery_assets, battery_name="Test battery" @@ -277,6 +276,7 @@ def test_battery_relaxation(add_battery_assets, db): "site-peak-production-price": series_to_ts_specs( pd.Series(260, production_prices.index), unit="EUR/MW" ), + "soc-minima-breach-price": "6000 EUR/kWh", # high breach price (to mimic a hard constraint) "consumption-breach-price": f"{device_power_breach_price} EUR/kW", # lower breach price (thus prioritizing minimizing soc breaches) "production-breach-price": f"{device_power_breach_price} EUR/kW", # lower breach price (thus prioritizing minimizing soc breaches) }, diff --git a/flexmeasures/data/models/planning/utils.py b/flexmeasures/data/models/planning/utils.py index 87fdfe4944..9d9d146437 100644 --- a/flexmeasures/data/models/planning/utils.py +++ b/flexmeasures/data/models/planning/utils.py @@ -247,6 +247,83 @@ def get_power_values( return -series +def fallback_charging_policy( + sensor: Sensor, + device_constraints: pd.DataFrame, + start: datetime, + end: datetime, + resolution: timedelta, +) -> pd.Series: + """This fallback charging policy is to just start charging or discharging, or do neither, + depending on the first target state of charge and the capabilities of the Charge Point. + Note that this ignores any cause of the infeasibility and, + while probably a decent policy for Charge Points, + should not be considered a robust policy for other asset types. + """ + max_charge_capacity = ( + device_constraints[["derivative max", "derivative equals"]].min().min() + ) + max_discharge_capacity = ( + -device_constraints[["derivative min", "derivative equals"]].max().max() + ) + charge_power = max_charge_capacity if sensor.get_attribute("is_consumer") else 0 + discharge_power = ( + -max_discharge_capacity if sensor.get_attribute("is_producer") else 0 + ) + + charge_schedule = initialize_series(charge_power, start, end, resolution) + discharge_schedule = initialize_series(discharge_power, start, end, resolution) + idle_schedule = initialize_series(0, start, end, resolution) + if ( + device_constraints["equals"].first_valid_index() is not None + and device_constraints["equals"][ + device_constraints["equals"].first_valid_index() + ] + > 0 + ): + # start charging to get as close as possible to the next target + return idle_after_reaching_target(charge_schedule, device_constraints["equals"]) + if ( + device_constraints["equals"].first_valid_index() is not None + and device_constraints["equals"][ + device_constraints["equals"].first_valid_index() + ] + < 0 + ): + # start discharging to get as close as possible to the next target + return idle_after_reaching_target( + discharge_schedule, device_constraints["equals"] + ) + if ( + device_constraints["max"].first_valid_index() is not None + and device_constraints["max"][device_constraints["max"].first_valid_index()] < 0 + ): + # start discharging to try and bring back the soc below the next max constraint + return idle_after_reaching_target(discharge_schedule, device_constraints["max"]) + if ( + device_constraints["min"].first_valid_index() is not None + and device_constraints["min"][device_constraints["min"].first_valid_index()] > 0 + ): + # start charging to try and bring back the soc above the next min constraint + return idle_after_reaching_target(charge_schedule, device_constraints["min"]) + # stand idle + return idle_schedule + + +def idle_after_reaching_target( + schedule: pd.Series, + target: pd.Series, + initial_state: float = 0, +) -> pd.Series: + """Stop planned (dis)charging after target is reached (or constraint is met).""" + first_target = target[target.first_valid_index()] + if first_target > initial_state: + schedule[schedule.cumsum() > first_target] = 0 + else: + schedule[schedule.cumsum() < first_target] = 0 + return schedule + + def get_series_from_quantity_or_sensor( variable_quantity: Sensor | SensorReference | list[dict] | ur.Quantity, unit: ur.Quantity | str, diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py index 26b7bd113b..7ed14ae0dc 100644 --- a/flexmeasures/data/schemas/scheduling/__init__.py +++ b/flexmeasures/data/schemas/scheduling/__init__.py @@ -341,7 +341,7 @@ class SharedSchema(Schema): ) relax_soc_constraints = fields.Bool( data_key="relax-soc-constraints", - load_default=True, + load_default=False, metadata=metadata.RELAX_SOC_CONSTRAINTS.to_dict(), ) relax_capacity_constraints = fields.Bool( @@ -1042,18 +1042,11 @@ def check_prices(self, data: dict, original_data: dict, **kwargs): return data # Fill in default soc breach prices when asked to relax SoC constraints, unless already set explicitly. - # Both relax-soc-constraints and relax-constraints default to True, so an - # explicit relax-soc-constraints wins and otherwise the umbrella - # relax-constraints decides: setting either flag to False keeps - # SoC minima/maxima as hard constraints. - if "relax-soc-constraints" in original_data: - relax_soc_constraints = data["relax_soc_constraints"] - else: - relax_soc_constraints = data["relax_constraints"] if ( - relax_soc_constraints - and data.get("soc_minima_breach_price") is None - and data.get("soc_maxima_breach_price") is None + data["relax_soc_constraints"] + or data["relax_constraints"] + and not data.get("soc_minima_breach_price") + and not data.get("soc_maxima_breach_price") ): self.set_default_breach_prices( data, @@ -1063,9 +1056,10 @@ def check_prices(self, data: dict, original_data: dict, **kwargs): # Fill in default capacity breach prices when asked to relax capacity constraints, unless already set explicitly. if ( - (data["relax_capacity_constraints"] or data["relax_constraints"]) - and data.get("consumption_breach_price") is None - and data.get("production_breach_price") is None + data["relax_capacity_constraints"] + or data["relax_constraints"] + and not data.get("consumption_breach_price") + and not data.get("production_breach_price") ): self.set_default_breach_prices( data, @@ -1075,9 +1069,10 @@ def check_prices(self, data: dict, original_data: dict, **kwargs): # Fill in default site capacity breach prices when asked to relax site capacity constraints, unless already set explicitly. if ( - (data["relax_site_capacity_constraints"] or data["relax_constraints"]) - and data.get("ems_consumption_breach_price") is None - and data.get("ems_production_breach_price") is None + data["relax_site_capacity_constraints"] + or data["relax_constraints"] + and not data.get("ems_consumption_breach_price") + and not data.get("ems_production_breach_price") ): self.set_default_breach_prices( data, @@ -1496,38 +1491,6 @@ def _build_ui_flex_model_schema() -> Dict[str, Dict[str, Any]]: class DBFlexContextSchema(FlexContextSchema, NoTimeSeriesSpecs): - # The relaxation defaults are turned off here, so validating a stored asset - # flex-context does not fill in default breach prices. The API-side defaults - # (which are True) are applied at scheduling time instead, after the stored - # flex-context is merged with the one passed in the scheduling request. - relax_constraints = fields.Bool( - data_key="relax-constraints", - load_default=False, - metadata={ - **metadata.RELAX_CONSTRAINTS.to_dict(), - "description": ( - "Defaults to False when stored on an asset (unlike the True default " - "used when scheduling): a stored flex-context should not silently bake " - "in default breach prices. The scheduling-time default of True is " - "applied after this stored flex-context is merged with the one passed " - "in the scheduling request." - ), - }, - ) - relax_soc_constraints = fields.Bool( - data_key="relax-soc-constraints", - load_default=False, - metadata={ - **metadata.RELAX_SOC_CONSTRAINTS.to_dict(), - "description": ( - "Defaults to False when stored on an asset (unlike the True default " - "used when scheduling): a stored flex-context should not silently bake " - "in default breach prices. The scheduling-time default of True is " - "applied after this stored flex-context is merged with the one passed " - "in the scheduling request." - ), - }, - ) commitments = fields.Nested( DBCommitmentSchema, data_key="commitments", required=False, many=True diff --git a/flexmeasures/data/schemas/scheduling/metadata.py b/flexmeasures/data/schemas/scheduling/metadata.py index 74f4351b7e..c4db606f3e 100644 --- a/flexmeasures/data/schemas/scheduling/metadata.py +++ b/flexmeasures/data/schemas/scheduling/metadata.py @@ -204,20 +204,19 @@ def to_dict(self): example="10 EUR/kW", ) RELAX_CONSTRAINTS = MetaData( - description="""If True (default), several constraints are relaxed by setting default breach prices within the optimization problem, leading to the default priority: + description="""If True (default is ``False``), several constraints are relaxed by setting default breach prices within the optimization problem, leading to the default priority: 1. Avoid breaching the site consumption/production capacity. 2. Avoid not meeting SoC minima/maxima. 3. Avoid breaching the desired device consumption/production capacity. -SoC minima/maxima are already relaxed by default through ``relax-soc-constraints``. -Set this field to ``True`` to also enable the default site and device capacity breach prices and associated priorities as defined by FlexMeasures. +We recommend to set this field to ``True`` to enable the default prices and associated priorities as defined by FlexMeasures. For tighter control over prices and priorities, the breach prices can also be set explicitly (the relevant fields have ``breach-price`` in their name). """, example=True, ) RELAX_SOC_CONSTRAINTS = MetaData( - description="If True (default), avoids not meeting SoC minima/maxima as relaxed constraints. Setting this field (or ``relax-constraints``) to False keeps SoC minima/maxima as hard constraints unless breach prices are supplied explicitly; an explicit ``relax-soc-constraints`` takes precedence over ``relax-constraints``.", + description="If True, avoids not meeting SoC minima/maxima as a relaxed constraint.", example=True, ) RELAX_CAPACITY_CONSTRAINTS = MetaData( @@ -336,9 +335,8 @@ def to_dict(self): ) SOC_MINIMA = MetaData( description="""Set points that form lower boundaries, e.g. to target a full car battery in the morning. -The ``soc-minima`` are soft constraints in the optimization problem by default, because ``relax-soc-constraints`` defaults to ``True`` and supplies a default ``soc-minima-breach-price``. -Set ``relax-soc-constraints`` (or ``relax-constraints``) to ``False`` to keep them as hard constraints unless ``soc-minima-breach-price`` is supplied explicitly [#maximum_overlap]_. -Both single points in time and ranges are possible, see example. [#projecting_scheduling_constraints]_""", +If a ``soc-minima-breach-price`` is defined, the ``soc-minima`` become soft constraints in the optimization problem. +Otherwise, they become hard constraints. [#maximum_overlap]_. Both single points in time and ranges are possible, see example. [#projecting_scheduling_constraints]_""", example=[ {"datetime": "2024-02-05T08:00:00+01:00", "value": "8.2 kWh"}, { @@ -350,8 +348,8 @@ def to_dict(self): ) SOC_MAXIMA = MetaData( description="""Set points that form upper boundaries at certain times, e.g. to target an empty heat buffer before a maintenance window. -The ``soc-maxima`` are soft constraints in the optimization problem by default, because ``relax-soc-constraints`` defaults to ``True`` and supplies a default ``soc-maxima-breach-price``. -Set ``relax-soc-constraints`` (or ``relax-constraints``) to ``False`` to keep them as hard constraints unless ``soc-maxima-breach-price`` is supplied explicitly. [#minimum_overlap]_ [#projecting_scheduling_constraints]_""", +If a ``soc-maxima-breach-price`` is defined, the ``soc-maxima`` become soft constraints in the optimization problem. +Otherwise, they become hard constraints. [#minimum_overlap]_ [#projecting_scheduling_constraints]_""", example=[ { "value": "51 kWh", diff --git a/flexmeasures/data/schemas/tests/test_scheduling.py b/flexmeasures/data/schemas/tests/test_scheduling.py index 57232e4d59..7f2ed33897 100644 --- a/flexmeasures/data/schemas/tests/test_scheduling.py +++ b/flexmeasures/data/schemas/tests/test_scheduling.py @@ -611,80 +611,6 @@ def test_flex_context_schema( check_schema_loads_data(schema=schema, data=flex_context, fails=fails) -def test_flex_context_schema_relaxes_soc_constraints_by_default(): - loaded_flex_context = FlexContextSchema().load({"consumption-price": "1 EUR/MWh"}) - - assert loaded_flex_context["relax_constraints"] is True - assert loaded_flex_context["relax_soc_constraints"] is True - assert loaded_flex_context["soc_minima_breach_price"].to( - "EUR/MWh" - ).magnitude == pytest.approx(1_000_000) - assert loaded_flex_context["soc_maxima_breach_price"].to( - "EUR/MWh" - ).magnitude == pytest.approx(1_000_000) - - -def test_flex_context_schema_preserves_explicit_soc_breach_prices(): - loaded_flex_context = FlexContextSchema().load( - { - "consumption-price": "1 EUR/MWh", - "soc-minima-breach-price": "5 EUR/kWh", - "soc-maxima-breach-price": "7 EUR/kWh", - } - ) - - assert loaded_flex_context["relax_soc_constraints"] is True - assert loaded_flex_context["soc_minima_breach_price"].to( - "EUR/kWh" - ).magnitude == pytest.approx(5) - assert loaded_flex_context["soc_maxima_breach_price"].to( - "EUR/kWh" - ).magnitude == pytest.approx(7) - - -def test_flex_context_schema_umbrella_opt_out_disables_soc_relaxation(): - """Setting relax-constraints to False alone keeps SoC minima/maxima hard.""" - loaded_flex_context = FlexContextSchema().load( - {"consumption-price": "1 EUR/MWh", "relax-constraints": False} - ) - - assert "soc_minima_breach_price" not in loaded_flex_context - assert "soc_maxima_breach_price" not in loaded_flex_context - assert "consumption_breach_price" not in loaded_flex_context - assert "ems_consumption_breach_price" not in loaded_flex_context - - -def test_flex_context_schema_explicit_soc_relaxation_overrides_umbrella_opt_out(): - """An explicit relax-soc-constraints wins over an explicit relax-constraints.""" - loaded_flex_context = FlexContextSchema().load( - { - "consumption-price": "1 EUR/MWh", - "relax-constraints": False, - "relax-soc-constraints": True, - } - ) - - assert loaded_flex_context["soc_minima_breach_price"].to( - "EUR/MWh" - ).magnitude == pytest.approx(1_000_000) - - loaded_flex_context = FlexContextSchema().load( - {"consumption-price": "1 EUR/MWh", "relax-soc-constraints": False} - ) - - assert "soc_minima_breach_price" not in loaded_flex_context - assert "soc_maxima_breach_price" not in loaded_flex_context - - -def test_db_flex_context_schema_does_not_relax_soc_constraints_by_default(): - loaded_flex_context = DBFlexContextSchema().load({}) - - assert loaded_flex_context["relax_constraints"] is False - assert loaded_flex_context["relax_soc_constraints"] is False - assert "soc_minima_breach_price" not in loaded_flex_context - assert "soc_maxima_breach_price" not in loaded_flex_context - - def check_schema_loads_data(schema, data, fails): if fails: with pytest.raises(ValidationError) as e_info: diff --git a/flexmeasures/data/tests/test_scheduling_sequential.py b/flexmeasures/data/tests/test_scheduling_sequential.py index f219c62b60..aea1d5317f 100644 --- a/flexmeasures/data/tests/test_scheduling_sequential.py +++ b/flexmeasures/data/tests/test_scheduling_sequential.py @@ -156,13 +156,13 @@ def test_create_sequential_jobs(db, app, flex_description_sequential, smart_buil # ) -def test_create_sequential_jobs_without_storage_fallback( +def test_create_sequential_jobs_fallback( db, app, flex_description_sequential, smart_building ): - """Test an infeasible first subjob in a chain of sequential scheduling jobs. + """Test fallback scheduler in a chain of sequential scheduling (sub)jobs. - Checks that no storage fallback job is created. The deferred subjobs should remain - deferred because the first subjob failed. + Checks execution of a sequential scheduling job, where 1 of the subjobs is set up to fail and trigger its fallback. + The deferred subjobs should still succeed after the fallback succeeds, even though the first subjob fails. """ assets, sensors, _ = smart_building queue = app.queues["scheduling"] @@ -181,60 +181,56 @@ def test_create_sequential_jobs_without_storage_fallback( storage_module = "flexmeasures.data.models.planning.storage" with patch(f"{storage_module}.StorageScheduler.persist_flex_model"): - with patch( - f"{storage_module}.StorageScheduler.compute", - side_effect=InfeasibleProblemException(), - ): - create_sequential_scheduling_job( - asset=assets["Test Site"], - scheduler_specs=scheduler_specs, - enqueue=True, - force_new_job_creation=True, # otherwise the cache might kick in due to sub-jobs already created in other tests - **flex_description_sequential, - ) - - # There should be 3 jobs: - # 2 jobs scheduling the 2 flexible devices in the flex-model, plus 1 'done job' to wrap things up - queued_jobs = app.queues["scheduling"].jobs - deferred_jobs = [ - Job.fetch(job_id, connection=queue.connection) - for job_id in app.queues[ - "scheduling" - ].deferred_job_registry.get_job_ids() - ] - # Sort deferred_jobs by their created_at attribute - deferred_jobs = sorted(deferred_jobs, key=lambda job: job.created_at) - assert ( - len(queued_jobs) == 1 - ), "Only the job for scheduling the first device sequentially should be queued." - assert ( - len(deferred_jobs) == 2 - ), "The job for scheduling the second device, and the wrap-up job, should be deferred." - - # Work on jobs - work_on_rq(queue, exc_handler=handle_scheduling_exception) - - for job in queued_jobs: - job.refresh() - for job in deferred_jobs: - job.refresh() - - finished_jobs = queue.finished_job_registry.get_job_ids() - failed_jobs = queue.failed_job_registry.get_job_ids() - - # Original job failed and no fallback job was created - assert queued_jobs[0].id in failed_jobs - assert queued_jobs[0].meta.get("fallback_job_id") is None - - # The deferred jobs should not run when their dependency fails without fallback - assert deferred_jobs[0].id not in finished_jobs - assert deferred_jobs[1].id not in finished_jobs - - # Without a fallback to unblock the chain, the deferred subjobs stay deferred - # for good, so clear them here rather than leaking them into the next test. - for deferred_job_id in queue.deferred_job_registry.get_job_ids(): - queue.deferred_job_registry.remove(deferred_job_id) - queue.empty() + with patch(f"{storage_module}.StorageFallbackScheduler.persist_flex_model"): + with patch( + f"{storage_module}.StorageScheduler.compute", + side_effect=iter([InfeasibleProblemException(), [], []]), + ): + create_sequential_scheduling_job( + asset=assets["Test Site"], + scheduler_specs=scheduler_specs, + enqueue=True, + force_new_job_creation=True, # otherwise the cache might kick in due to sub-jobs already created in other tests + **flex_description_sequential, + ) + + # There should be 3 jobs: + # 2 jobs scheduling the 2 flexible devices in the flex-model, plus 1 'done job' to wrap things up + queued_jobs = app.queues["scheduling"].jobs + deferred_jobs = [ + Job.fetch(job_id, connection=queue.connection) + for job_id in app.queues[ + "scheduling" + ].deferred_job_registry.get_job_ids() + ] + # Sort deferred_jobs by their created_at attribute + deferred_jobs = sorted(deferred_jobs, key=lambda job: job.created_at) + assert ( + len(queued_jobs) == 1 + ), "Only the job for scheduling the first device sequentially should be queued." + assert ( + len(deferred_jobs) == 2 + ), "The job for scheduling the second device, and the wrap-up job, should be deferred." + + # Work on jobs + work_on_rq(queue, exc_handler=handle_scheduling_exception) + + # Refresh jobs so that the fallback_job_id (which should be set by now) can be read + for job in queued_jobs: + job.refresh() + + finished_jobs = queue.finished_job_registry.get_job_ids() + failed_jobs = queue.failed_job_registry.get_job_ids() + + # Original job failed + assert queued_jobs[0].id in failed_jobs + + # The fallback job ran successfully + assert queued_jobs[0].meta["fallback_job_id"] in finished_jobs + + # The deferred jobs ran successfully + assert deferred_jobs[0].id in finished_jobs + assert deferred_jobs[1].id in finished_jobs def test_create_sequential_jobs_with_sign_explicit_context( diff --git a/flexmeasures/ui/static/openapi-specs.json b/flexmeasures/ui/static/openapi-specs.json index eb2e0e23c8..7363b3edae 100644 --- a/flexmeasures/ui/static/openapi-specs.json +++ b/flexmeasures/ui/static/openapi-specs.json @@ -5308,13 +5308,13 @@ "relax-constraints": { "type": "boolean", "default": true, - "description": "If True (default), several constraints are relaxed by setting default breach prices within the optimization problem, leading to the default priority:\n\n1. Avoid breaching the site consumption/production capacity.\n2. Avoid not meeting SoC minima/maxima.\n3. Avoid breaching the desired device consumption/production capacity.\n\nSoC minima/maxima are already relaxed by default through relax-soc-constraints.\nSet this field to True to also enable the default site and device capacity breach prices and associated priorities as defined by FlexMeasures.\nFor tighter control over prices and priorities, the breach prices can also be set explicitly (the relevant fields have breach-price in their name).\n", + "description": "If True (default is False), several constraints are relaxed by setting default breach prices within the optimization problem, leading to the default priority:\n\n1. Avoid breaching the site consumption/production capacity.\n2. Avoid not meeting SoC minima/maxima.\n3. Avoid breaching the desired device consumption/production capacity.\n\nWe recommend to set this field to True to enable the default prices and associated priorities as defined by FlexMeasures.\nFor tighter control over prices and priorities, the breach prices can also be set explicitly (the relevant fields have breach-price in their name).\n", "example": true }, "relax-soc-constraints": { "type": "boolean", - "default": true, - "description": "If True (default), avoids not meeting SoC minima/maxima as relaxed constraints. Setting this field (or relax-constraints) to False keeps SoC minima/maxima as hard constraints unless breach prices are supplied explicitly; an explicit relax-soc-constraints takes precedence over relax-constraints.", + "default": false, + "description": "If True, avoids not meeting SoC minima/maxima as a relaxed constraint.", "example": true }, "relax-capacity-constraints": { @@ -7074,7 +7074,7 @@ "example": true }, "soc-maxima": { - "description": "Set points that form upper boundaries at certain times, e.g. to target an empty heat buffer before a maintenance window.\nThe soc-maxima are soft constraints in the optimization problem by default, because relax-soc-constraints defaults to True and supplies a default soc-maxima-breach-price.\nSet relax-soc-constraints (or relax-constraints) to False to keep them as hard constraints unless soc-maxima-breach-price is supplied explicitly.", + "description": "Set points that form upper boundaries at certain times, e.g. to target an empty heat buffer before a maintenance window.\nIf a soc-maxima-breach-price is defined, the soc-maxima become soft constraints in the optimization problem.\nOtherwise, they become hard constraints.", "example": [ { "value": "51 kWh", @@ -7085,7 +7085,7 @@ "$ref": "#/components/schemas/VariableQuantityOpenAPI" }, "soc-minima": { - "description": "Set points that form lower boundaries, e.g. to target a full car battery in the morning.\nThe soc-minima are soft constraints in the optimization problem by default, because relax-soc-constraints defaults to True and supplies a default soc-minima-breach-price.\nSet relax-soc-constraints (or relax-constraints) to False to keep them as hard constraints unless soc-minima-breach-price is supplied explicitly.\nBoth single points in time and ranges are possible, see example.", + "description": "Set points that form lower boundaries, e.g. to target a full car battery in the morning.\nIf a soc-minima-breach-price is defined, the soc-minima become soft constraints in the optimization problem.\nOtherwise, they become hard constraints.. Both single points in time and ranges are possible, see example.", "example": [ { "datetime": "2024-02-05T08:00:00+01:00", From ec5fe72b4cfc891f4ff9bbf7513cc35c226029ef Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 7 Aug 2026 21:32:19 +0200 Subject: [PATCH 30/30] Warn about a zero fallback on a directional capacity Since #2345, a consumption-capacity or production-capacity that is zero throughout the scheduling window is read as a physical statement about the device and enforced strictly. A default fills every slot the sensor leaves empty, so a fallback of 0 on one of those fields turns a silent sensor into a hard bound, which is not what "use this value when the sensor has nothing to say" sounds like it does. Regenerating the specs here also restores their version to 1.0.0. The generator takes it from the installed FlexMeasures, so the 0.33.2 in the previous revision records a stale environment rather than an intended change. Co-Authored-By: Claude Opus 5 Signed-off-by: F.N. Claessen --- documentation/api/change_log.rst | 2 +- documentation/changelog.rst | 2 +- flexmeasures/data/schemas/sensors.py | 2 +- flexmeasures/ui/static/openapi-specs.json | 6 +++--- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst index 71504341cd..ab2a779cf7 100644 --- a/documentation/api/change_log.rst +++ b/documentation/api/change_log.rst @@ -10,7 +10,7 @@ v3.0-32 | July XX, 2026 - API endpoints are now rate-limited. A request which exceeds a limit is answered with a ``429 (Too Many Requests)`` status code and a ``Retry-After`` header stating how many seconds to wait. Responses also carry ``X-RateLimit-*`` headers, describing the limit that applied, how much of it is left, and when it resets. A stricter limit applies to ``POST /assets//schedules/trigger``, ``POST /sensors//schedules/trigger`` and ``POST /sensors//forecasts/trigger`` than to other endpoints; the health endpoints are exempt. Per-account overrides are set by assigning the account a plan (a ``Plan`` database row), rather than through an account attribute. - Introduced the ``inflexible-consumption`` and ``inflexible-production`` flex-context fields, which make explicit how the sign of each inflexible device's power data should be read: positive values denote consumption resp. production. Each entry is a sensor reference (``{"sensor": }``), optionally with source filters (``source-types``, ``exclude-source-types``, ``sources``, ``source-account``). Deprecated the ``inflexible-device-sensors`` field (a list of bare sensor IDs, whose sign convention is read from each sensor's ``consumption_is_positive`` attribute); it remains supported, but cannot be combined with the new fields in one flex-context. - Added a ``role`` query parameter to ``GET /api/v3_0/accounts`` for filtering accessible organisations by account role. -- Sensor references on variable-quantity flex-model and flex-context fields (such as ``soc-minima``, ``soc-maxima``, the capacity fields and the price fields) may now include a ``default`` fallback quantity, e.g. ``{"sensor": 50, "default": "0 kWh"}``. It fills the time slots for which the referenced sensor holds no value. Note that this fills *every* such slot, so a sensor recording only occasional setpoints becomes densely constrained. The field is not (yet) applied to sensor references on ``inflexible-consumption``/``inflexible-production`` or to forecaster regressors. +- Sensor references on variable-quantity flex-model and flex-context fields (such as ``soc-minima``, ``soc-maxima``, the capacity fields and the price fields) may now include a ``default`` fallback quantity, e.g. ``{"sensor": 50, "default": "0 kWh"}``. It fills the time slots for which the referenced sensor holds no value. Note that this fills *every* such slot, so a sensor recording only occasional setpoints becomes densely constrained. The field is not (yet) applied to sensor references on ``inflexible-consumption``/``inflexible-production`` or to forecaster regressors. Take particular care with a fallback of ``0`` on a ``consumption-capacity`` or ``production-capacity``: if the sensor holds no value for the whole scheduling window, the resulting all-zero capacity is read as a physical statement about the device and enforced strictly (see :ref:`the flex-model capacity fields `), rather than as a limit that may be breached at a price. - Extended ``GET /api/v3_0/jobs/`` with a ``result`` field containing ``unresolved`` and ``resolved`` arrays, each keyed by asset ID. For scheduling jobs, this surfaces soft state-of-charge constraint analysis: ``soc-minima`` and ``soc-maxima`` violations (with a ``violation`` magnitude) or satisfied constraints (with a ``margin`` headroom). Both arrays are empty when no SoC constraints were defined. - **Field canonicalization** for background job tracking: * The ``job`` field is now the canonical way to identify background jobs returned by `/sensors//schedules/trigger`, `/assets//schedules/trigger`, and `/sensors//forecasts/trigger` endpoints. If applicable, the triggered response now also returns a ``results-url`` pointing to the sensor-specific results endpoint, alongside the generic ``job-url``. diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 83e4f60ac6..fc4cc18d7e 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -58,7 +58,7 @@ New features * Commodity contexts that omit grid-connection fields (prices and site capacities) now get smart defaults instead of failing or silently leaving the grid unconstrained — for instance, a bare ``{"commodity": "gas"}`` is treated as having no grid connection; see :ref:`commodity_context_defaults` for the full rules [see `PR #2272 `_] * CLI support for adding/editing account attributes [see `PR #2242 `_] * Improve chart axis domain for event values not around zero, with a per-sub-chart ``y-axis`` option in ``sensors_to_show`` (default ``zero``, which pads the axis out to include zero) that can be set to ``data`` to fit a sub-chart's y-axis to the values shown, to an explicit ``[min, max]`` domain that the axis will cover at least (expanding to fit data beyond it), or to a strict ``{"min": min, "max": max}`` domain that the axis will never exceed (clamping data beyond it, with a warning when that happens), editable from the graph editor [see `PR #2244 `_] -* Sensor references on variable-quantity flex-model and flex-context fields now accept a ``default`` fallback quantity, e.g. ``{"sensor": 50, "default": "0 kWh"}``, filling the time slots for which the referenced sensor holds no value; also settable from the flex-model UI [see `PR #2405 `_] +* Sensor references on variable-quantity flex-model and flex-context fields now accept a ``default`` fallback quantity, e.g. ``{"sensor": 50, "default": "0 kWh"}``, filling the time slots for which the referenced sensor holds no value; also settable from the flex-model UI. Note that a fallback of ``0`` on a directional capacity makes that capacity a hard bound wherever the sensor is silent for the whole scheduling window [see `PR #2405 `_] * Breaking behaviour change: the built-in storage fallback scheduler has been retired, so an infeasible storage problem now fails with its failure reason instead of silently saving a fallback schedule; ``FLEXMEASURES_FALLBACK_REDIRECT`` is now only relevant for custom schedulers that define a fallback scheduler [see `PR #2252 `_] * Breaking behaviour change: the specific ``relax-soc-constraints`` and ``relax-site-capacity-constraints`` flags now follow the umbrella ``relax-constraints`` flag (which now defaults to true, see above) unless they are set explicitly, in which case they take precedence for their respective constraints, in either direction (previously, these flags were independent opt-ins defaulting to false); also fixed: explicitly set breach prices (including zero prices) are no longer overwritten with default breach prices when relaxation is enabled, and when only one breach price of the ``soc-minima``/``soc-maxima`` (or site capacity) pair is set explicitly, the other one still gets its default [see `PR #2252 `_] * Extended ``GET /api/v3_0/jobs/`` with a ``result`` field containing ``unresolved`` and ``resolved`` soft state-of-charge constraint analysis (``soc-minima``/``soc-maxima`` violations or satisfied constraints, keyed by asset ID) for scheduling jobs; both arrays are empty when no SoC constraints were defined [see `PR #2072 `_] diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py index e2804134fc..db4926e93b 100644 --- a/flexmeasures/data/schemas/sensors.py +++ b/flexmeasures/data/schemas/sensors.py @@ -1117,7 +1117,7 @@ class Meta: required=False, allow_none=False, metadata=dict( - description="Fallback quantity to use when the referenced sensor has missing values, on variable-quantity flex-model and flex-context fields (such as soc-minima, soc-maxima, the capacity fields and the price fields). Note that every time slot the sensor leaves empty is filled with this value, so a sparse setpoint sensor becomes densely constrained. This field is not (yet) applied to inflexible-device references or to forecaster regressors.", + description="Fallback quantity to use when the referenced sensor has missing values, on variable-quantity flex-model and flex-context fields (such as soc-minima, soc-maxima, the capacity fields and the price fields). Note that every time slot the sensor leaves empty is filled with this value, so a sparse setpoint sensor becomes densely constrained. Take particular care with a fallback of 0 on a consumption-capacity or production-capacity: if the sensor holds no value for the whole scheduling window, the resulting all-zero capacity is read as a physical statement about the device and enforced strictly, rather than as a limit that may be breached at a price. This field is not (yet) applied to inflexible-device references or to forecaster regressors.", example="0 kWh", ), ) diff --git a/flexmeasures/ui/static/openapi-specs.json b/flexmeasures/ui/static/openapi-specs.json index 096c6be47e..006541b145 100644 --- a/flexmeasures/ui/static/openapi-specs.json +++ b/flexmeasures/ui/static/openapi-specs.json @@ -7,7 +7,7 @@ }, "termsOfService": null, "title": "FlexMeasures", - "version": "0.33.2" + "version": "1.0.0" }, "externalDocs": { "description": "FlexMeasures runs on the open source FlexMeasures technology. Read the docs here.", @@ -5033,7 +5033,7 @@ }, "default": { "type": "string", - "description": "Fallback quantity to use when the referenced sensor has missing values, on variable-quantity flex-model and flex-context fields (such as soc-minima, soc-maxima, the capacity fields and the price fields). Note that every time slot the sensor leaves empty is filled with this value, so a sparse setpoint sensor becomes densely constrained. This field is not (yet) applied to inflexible-device references or to forecaster regressors.", + "description": "Fallback quantity to use when the referenced sensor has missing values, on variable-quantity flex-model and flex-context fields (such as soc-minima, soc-maxima, the capacity fields and the price fields). Note that every time slot the sensor leaves empty is filled with this value, so a sparse setpoint sensor becomes densely constrained. Take particular care with a fallback of 0 on a consumption-capacity or production-capacity: if the sensor holds no value for the whole scheduling window, the resulting all-zero capacity is read as a physical statement about the device and enforced strictly, rather than as a limit that may be breached at a price. This field is not (yet) applied to inflexible-device references or to forecaster regressors.", "example": "0 kWh" } }, @@ -5187,7 +5187,7 @@ }, "default": { "type": "string", - "description": "Fallback quantity to use when the referenced sensor has missing values, on variable-quantity flex-model and flex-context fields (such as soc-minima, soc-maxima, the capacity fields and the price fields). Note that every time slot the sensor leaves empty is filled with this value, so a sparse setpoint sensor becomes densely constrained. This field is not (yet) applied to inflexible-device references or to forecaster regressors.", + "description": "Fallback quantity to use when the referenced sensor has missing values, on variable-quantity flex-model and flex-context fields (such as soc-minima, soc-maxima, the capacity fields and the price fields). Note that every time slot the sensor leaves empty is filled with this value, so a sparse setpoint sensor becomes densely constrained. Take particular care with a fallback of 0 on a consumption-capacity or production-capacity: if the sensor holds no value for the whole scheduling window, the resulting all-zero capacity is read as a physical statement about the device and enforced strictly, rather than as a limit that may be breached at a price. This field is not (yet) applied to inflexible-device references or to forecaster regressors.", "example": "0 kWh" } },