From 9e4cb72aaeeaf89c9d959c3f6e266f6122318cf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Thu, 13 Aug 2026 12:42:30 +0200 Subject: [PATCH 01/21] add explicit deletion/cleanup script and docs; user does not have to be admin: account-admin suffices - improve docs and function names as all assets are in account MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- docs/HEMS.rst | 31 ++++++++- examples/HEMS/HEMS_cleanup.py | 28 ++++++++ examples/HEMS/HEMS_setup.py | 14 ++-- examples/HEMS/assets_setup.py | 30 ++++---- examples/HEMS/const.py | 4 +- examples/HEMS/utils/asset_utils.py | 108 +++++++++++++---------------- 6 files changed, 128 insertions(+), 87 deletions(-) create mode 100644 examples/HEMS/HEMS_cleanup.py diff --git a/docs/HEMS.rst b/docs/HEMS.rst index ed24a9af..3cad002a 100644 --- a/docs/HEMS.rst +++ b/docs/HEMS.rst @@ -43,12 +43,17 @@ Or, alternatively, to install released versions into a fresh project: Next steps: - Follow instructions to set up flexmeasures (fresh database, etc). -- Create an organisation account and an admin with: +- Create an organisation account and a user with the ``account-admin`` role: .. code-block:: bash - flexmeasures add account - flexmeasures add user --roles admin + flexmeasures add account --name "HEMS tutorial" + flexmeasures add user --username hems-admin --email hems-admin@example.com \ + --account 2 --roles account-admin + +Replace ``2`` with the account ID printed by the first command. The tutorial +creates all assets and sensors in this organisation account. It does not create +public assets, so a site-wide ``admin`` role is not required. - Update the credentials in the ``examples/HEMS/const.py`` script accordingly. @@ -83,3 +88,23 @@ In the third terminal, run the client script using the `/examples/HEMS` folder a - ``FLEXMEASURES_CLI_CMD``: the command used to invoke the CLI, e.g. ``"docker compose exec -T server flexmeasures"``. - ``FLEXMEASURES_CLI_CONFIG_DIR``: the directory the CLI process sees the ``examples/HEMS/configs/`` files at, if different from their local path (e.g. because that directory is bind-mounted into a container at a different path). + + +Delete the tutorial assets and data +=================================== + +To remove the HEMS setup from the configured account, run the cleanup script +from the same directory: + +.. code-block:: bash + + python3 HEMS_cleanup.py + +The script shows the matching top-level assets and asks for confirmation. It +deletes the community asset, the energy market, and the weather station. Asset +deletion also removes their child assets, sensors, and time-series data. + +.. warning:: + Deletion is permanent. The energy market and weather station are separate + top-level assets; do not continue if other systems in the account share them. + The configured user needs the ``account-admin`` role to delete assets. diff --git a/examples/HEMS/HEMS_cleanup.py b/examples/HEMS/HEMS_cleanup.py new file mode 100644 index 00000000..bf831810 --- /dev/null +++ b/examples/HEMS/HEMS_cleanup.py @@ -0,0 +1,28 @@ +"""Delete the assets and data created by the HEMS tutorial.""" + +import asyncio + +from const import COMMUNITY_NAME, host, pwd, usr +from utils.asset_utils import delete_hems_assets + +from flexmeasures_client import FlexMeasuresClient + + +async def main() -> None: + client = FlexMeasuresClient(email=usr, password=pwd, host=host) + try: + account = await client.get_account() + if not account: + raise RuntimeError("No account found for the configured user.") + print(f"Connected to account: {account['name']} (ID: {account['id']})") + await delete_hems_assets( + client=client, + account_id=account["id"], + community_name=COMMUNITY_NAME, + ) + finally: + await client.close() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/HEMS/HEMS_setup.py b/examples/HEMS/HEMS_setup.py index 7a1bcf07..13cd5489 100644 --- a/examples/HEMS/HEMS_setup.py +++ b/examples/HEMS/HEMS_setup.py @@ -12,7 +12,7 @@ from forecasting import generate_forecasts from reporters import create_reports from scheduling import just_continue, run_scheduling_simulation -from utils.asset_utils import cleanup_existing_assets, upload_data_for_first_two_weeks +from utils.asset_utils import delete_hems_assets, upload_data_for_first_two_weeks from flexmeasures_client import FlexMeasuresClient @@ -24,7 +24,7 @@ async def main( Complete HEMS setup using FlexMeasures client. Creates a comprehensive home energy management structure including: - - Public price sensor for electricity costs + - Price sensor for electricity costs - Building asset with consumption and energy cost KPI sensors - PV asset (child of building) with production sensor - Battery asset (child of building) with power and SoC sensors + settings @@ -36,9 +36,10 @@ async def main( print("Starting FlexMeasures HEMS") print("=" * 50) - # NOTE: Account and admin user creation must be done via FlexMeasures CLI first: + # NOTE: Create the account and account-admin user via FlexMeasures CLI first: # flexmeasures add account --name "MyCompany" - # flexmeasures add user --username admin --email admin@admin.com --account-id 2 --roles admin + # flexmeasures add user --username hems-admin --email hems-admin@example.com \ + # --account 2 --roles account-admin client = FlexMeasuresClient(email=usr, password=pwd, host=host) @@ -73,10 +74,11 @@ async def main( else: answer = input(f"Asset '{community_name}' already exists. Re-create?") if answer.lower() in ["y", "yes"]: - await cleanup_existing_assets( + await delete_hems_assets( client=client, account_id=account["id"], - site_names=[community_name], + community_name=community_name, + confirm_first=False, ) await create_community_asset( client, diff --git a/examples/HEMS/assets_setup.py b/examples/HEMS/assets_setup.py index 84d21650..69a61b32 100644 --- a/examples/HEMS/assets_setup.py +++ b/examples/HEMS/assets_setup.py @@ -17,20 +17,19 @@ from flexmeasures_client import FlexMeasuresClient -async def create_public_price_sensor(client: FlexMeasuresClient): - """Create a public price sensor (1h, EUR/kWh). +async def get_or_create_price_sensor(client: FlexMeasuresClient): + """Get or create an account-owned price sensor (1h, EUR/kWh). Returns the price sensor for use in flex-context. """ - print("Creating public price sensor...") + print("Getting or creating price sensor...") # Get the client account id account = await client.get_account() account_id = account["id"] print(f"Account ID: {account_id}") - # Create top-level market asset (not public, but still under the toy account) + # Create a top-level market asset in the current account. # Generic asset type 8 is typically used for market/price assets all_top_level_assets = await client.get_assets( - include_public=True, depth=0, fields=["id", "name", "account_id", "sensors"], ) @@ -57,22 +56,21 @@ async def create_public_price_sensor(client: FlexMeasuresClient): else: price_sensor = price_market_asset["sensors"][0] - print(f"Created public price sensor with ID: {price_sensor['id']}") + print(f"Price sensor ID: {price_sensor['id']}") return price_sensor -async def create_weather_station(client: FlexMeasuresClient): - """Create a public weather station with irradiation and cloud coverage sensors.""" - print("Creating weather station...") +async def get_or_create_weather_station(client: FlexMeasuresClient): + """Get or create an account-owned weather station and its sensors.""" + print("Getting or creating weather station...") # Get the client account id account = await client.get_account() account_id = account["id"] print(f"Account ID: {account_id}") - # Create top-level weather station asset (not public, but still under the toy account) + # Create a top-level weather station asset in the current account. # Generic asset type 7 (process) used for weather stations since no dedicated type exists # TODO: remove hard-coded ID, we should actually create a weather station type somehow all_top_level_assets = await client.get_assets( - include_public=True, depth=0, fields=["id", "name", "account_id", "sensors"], ) @@ -85,7 +83,7 @@ async def create_weather_station(client: FlexMeasuresClient): latitude=latitude, longitude=longitude, generic_asset_type_id=7, # Process asset type (for weather station) - account_id=account_id, # Public account ID + account_id=account_id, ) # Create irradiation sensor (1H, W/m²) @@ -934,12 +932,12 @@ async def create_community_asset( """Create an asset representing a community, which will serve as the parent asset for all sites in the community.""" # Get account id account_id = account["id"] - print("Creating price market asset and associated price sensor") - price_sensor = await create_public_price_sensor(client=client) + print("Getting or creating price market asset and associated price sensor") + price_sensor = await get_or_create_price_sensor(client=client) - print("Creating weather station with irradiation and cloud coverage sensors") + print("Getting or creating weather station and its sensors") weather_asset, irradiation_sensor, cloud_coverage_sensor = ( - await create_weather_station(client=client) + await get_or_create_weather_station(client=client) ) print(f"Weather station asset ID: {weather_asset['id']}") print(f"Irradiation sensor ID: {irradiation_sensor['id']}") diff --git a/examples/HEMS/const.py b/examples/HEMS/const.py index abeb4de2..97ba8314 100644 --- a/examples/HEMS/const.py +++ b/examples/HEMS/const.py @@ -4,8 +4,8 @@ Settings for the HEMS example script. """ # Connection details - UPDATE THESE FOR YOUR SETUP -usr = "admin@admin.com" # Admin user email -pwd = "admin" # Admin password +usr = "hems-admin@example.com" # Account-admin user email +pwd = "change-me" # Account-admin user password host = "127.0.0.1:5000" # FlexMeasures host # Asset and sensor names diff --git a/examples/HEMS/utils/asset_utils.py b/examples/HEMS/utils/asset_utils.py index d217545e..371ec63a 100644 --- a/examples/HEMS/utils/asset_utils.py +++ b/examples/HEMS/utils/asset_utils.py @@ -1,4 +1,3 @@ -import asyncio import os from pathlib import Path @@ -157,66 +156,55 @@ async def upload_data_for_first_two_weeks( return True -async def cleanup_existing_assets( - client: FlexMeasuresClient, account_id: int, site_names: list[str] -): - """Clean up existing HEMS assets to avoid naming conflicts.""" - print("Cleaning up existing assets...") - - for site_name in site_names: - # Asset names to clean up - asset_names_to_clean = [ - site_name, # Deleting this asset also deletes child assets (battery, PV, EVSEs) - weather_station_name, - price_market_name, - ] - - try: - # Get all existing assets - assets = await client.get_assets(parse_json_fields=True) - - # Find and delete assets that match our names - deleted_count = 0 - for asset in assets: - if asset["name"] in asset_names_to_clean: - print( - f"Deleting existing asset: {asset['name']} (ID: {asset['id']})" - ) - try: - if asset.get("account_id") != account_id: - print( - f"Warning: Asset {asset['name']} (ID: {asset['id']}) does not belong to the current account." - ) - raise - await client.delete_asset( - asset_id=asset["id"], confirm_first=False - ) - deleted_count += 1 - except Exception as delete_error: - # Check if it's a 404 error (asset not found) - if "404" in str(delete_error) or "NOT FOUND" in str( - delete_error - ): - print( - f"Asset {asset['name']} (ID: {asset['id']}) no longer exists, skipping..." - ) - else: - print( - f"Warning: Could not delete asset {asset['name']}: {delete_error}" - ) - # Continue with other assets - - if deleted_count > 0: - print(f"Cleaned up {deleted_count} existing assets") - else: - print("No existing assets to clean up") - - # Wait a moment for deletions to complete - await asyncio.sleep(1) +async def delete_hems_assets( + client: FlexMeasuresClient, + account_id: int, + community_name: str, + confirm_first: bool = True, +) -> int: + """Delete the top-level assets belonging to this HEMS example. - except Exception as e: - print(f"Warning: Error during cleanup: {e}") - print("Continuing with setup...") + Deleting the community asset also deletes all child assets, sensors, and data. + The price market and weather station are separate top-level assets, so they + are deleted explicitly. + """ + asset_names_to_delete = { + community_name, + weather_station_name, + price_market_name, + } + top_level_assets = await client.get_assets( + depth=0, + fields=["id", "name", "account_id"], + parse_json_fields=False, + ) + assets_to_delete = [ + asset + for asset in top_level_assets + if asset["name"] in asset_names_to_delete + and asset.get("account_id") == account_id + ] + + if not assets_to_delete: + print("No HEMS assets found in the current account.") + return 0 + + print("The following top-level HEMS assets will be deleted:") + for asset in assets_to_delete: + print(f"- {asset['name']} (ID: {asset['id']})") + print("Their child assets, sensors, and time-series data will also be deleted.") + + if confirm_first: + answer = input("Permanently delete these assets and all their data? [yN] ") + if answer.lower() not in ["y", "yes"]: + print("Aborting ...") + return 0 + + for asset in assets_to_delete: + await client.delete_asset(asset_id=asset["id"], confirm_first=False) + + print(f"Deleted {len(assets_to_delete)} top-level HEMS assets.") + return len(assets_to_delete) def load_and_align_csv_data( From 6ba7fcac16aa82a2e19c47b687702bede852fc52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Thu, 13 Aug 2026 14:24:40 +0200 Subject: [PATCH 02/21] improve docs for people running against docker-compose; add a section on circumventing rate limiting as well MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- docs/HEMS.rst | 30 +++++++++++++++++++++++++----- examples/HEMS/HEMS_setup.py | 5 ++++- examples/HEMS/const.py | 8 ++++---- 3 files changed, 33 insertions(+), 10 deletions(-) diff --git a/docs/HEMS.rst b/docs/HEMS.rst index 3cad002a..9327d2d7 100644 --- a/docs/HEMS.rst +++ b/docs/HEMS.rst @@ -76,18 +76,38 @@ In the second terminal, run a flexmeasures worker that listens to both the sched Note: you can run the same command in two terminals (2 workers), to speed up the computation! -In the third terminal, run the client script using the `/examples/HEMS` folder as the current working directory: +In the third terminal, go to the HEMS directory: .. code-block:: bash cd examples/HEMS - python3 HEMS_setup.py .. note:: - Report generation (see :ref:`hems-tutorial` note above) shells out to a ``flexmeasures`` CLI process, which by default is expected on ``PATH`` and configured against the same database as the server. If your FlexMeasures server runs elsewhere (e.g. inside a Docker Compose service), point report generation at it instead via two environment variables: + For the time being, report generation (see :ref:`hems-tutorial` note above) shells out to a ``flexmeasures`` CLI process, which by default is expected on ``PATH`` and configured against the same database as the server. If your FlexMeasures server runs elsewhere (e.g. inside a Docker Compose service), point report generation at it instead via two environment variables: + + - ``FLEXMEASURES_CLI_CMD``: the command used to invoke the CLI + - ``FLEXMEASURES_CLI_CONFIG_DIR``: the directory the CLI process sees the ``examples/HEMS/configs/`` files at, if different from their local path + + Here are steps if you use FlexMeasures' docker-compose: + - ``export FLEXMEASURES_CLI_CMD="docker compose -f full/path/to/docker-compose.yml exec -T server flexmeasures"``. + - Add this mount in docker-compose.yml under server.volumes, and restart it: ``- /full/path/to/flexmeasures-client/examples/HEMS/configs:/app/hems-configs:ro`` + - ``export FLEXMEASURES_CLI_CONFIG_DIR="/app/hems-configs"`` + +Another caveat is rate-limiting. Since v1.0, FlexMeasures only allows a limited number of schedule and forecasts per 5 minute interval. +Either give your account a generous plan (see the docs), or simply set ``FLEXMEASURES_MODE="play"`` and restart the server. +If you use docker-compose, you could do that like this: + +.. code-block:: bash - - ``FLEXMEASURES_CLI_CMD``: the command used to invoke the CLI, e.g. ``"docker compose exec -T server flexmeasures"``. - - ``FLEXMEASURES_CLI_CONFIG_DIR``: the directory the CLI process sees the ``examples/HEMS/configs/`` files at, if different from their local path (e.g. because that directory is bind-mounted into a container at a different path). + sudo chown -R "$(id -u):$(id -g)" /full/path/to/flexmeasures-instance + printf 'FLEXMEASURES_MODE = "play"\n' > /full/path/to/flexmeasures-instance/flexmeasures.cfg + docker compose restart name-of-flexmeasures-server-container + +Now run the client script using the `/examples/HEMS` folder as the current working directory: + +.. code-block:: bash + + python3 HEMS_setup.py Delete the tutorial assets and data diff --git a/examples/HEMS/HEMS_setup.py b/examples/HEMS/HEMS_setup.py index 13cd5489..e242ff51 100644 --- a/examples/HEMS/HEMS_setup.py +++ b/examples/HEMS/HEMS_setup.py @@ -72,7 +72,10 @@ async def main( client, account, community_name=community_name, site_names=site_names ) else: - answer = input(f"Asset '{community_name}' already exists. Re-create?") + answer = input( + f"Asset '{community_name}' already exists in account " + f"'{account['name']}' (ID: {account['id']}). Re-create it? [y/N] " + ) if answer.lower() in ["y", "yes"]: await delete_hems_assets( client=client, diff --git a/examples/HEMS/const.py b/examples/HEMS/const.py index 97ba8314..92a3a2d1 100644 --- a/examples/HEMS/const.py +++ b/examples/HEMS/const.py @@ -4,13 +4,13 @@ Settings for the HEMS example script. """ # Connection details - UPDATE THESE FOR YOUR SETUP -usr = "hems-admin@example.com" # Account-admin user email -pwd = "change-me" # Account-admin user password +usr = "toy-user@flexmeasures.io" # Account-admin user email +pwd = "toy-password" # Account-admin user password host = "127.0.0.1:5000" # FlexMeasures host # Asset and sensor names -COMMUNITY_NAME = "Community Site" -SITE_NAMES = ["My Home 1", "My Home 2"] +COMMUNITY_NAME = "Campus Site" +SITE_NAMES = ["Building A", "Building B"] pv_name = "Rooftop PV" battery_name = "Home Battery" From e339d3dcf2cf1955f2a24ca40a80ff14e1455276 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Thu, 13 Aug 2026 15:39:38 +0200 Subject: [PATCH 03/21] explain how to connect to https:// servers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- docs/HEMS.rst | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/HEMS.rst b/docs/HEMS.rst index 9327d2d7..a0ddd1cf 100644 --- a/docs/HEMS.rst +++ b/docs/HEMS.rst @@ -61,7 +61,16 @@ public assets, so a site-wide ``admin`` role is not required. Run the tutorial script ======================= -Before running the tutorial, make sure to update the connection details and other relevant settings (e.g., host, port, credentials) in examples/HEMS/const.py to match your local FlexMeasures setup. +Before running the tutorial, update the connection details and other relevant +settings in ``examples/HEMS/const.py``. Specify the host without an ``http://`` +or ``https://`` prefix, and set ``ssl = True`` when connecting over HTTPS. For +example: + +.. code-block:: python + + host = "ems.example.com" + ssl = True + Open three terminals. In the first terminal, run the server: .. code-block:: bash From 3ff87104dbf46aba97e36c31747ce7d7d461ff90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Thu, 13 Aug 2026 15:40:38 +0200 Subject: [PATCH 04/21] pass ssl parameter from const.py to Client MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- examples/HEMS/HEMS_cleanup.py | 4 ++-- examples/HEMS/HEMS_setup.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/HEMS/HEMS_cleanup.py b/examples/HEMS/HEMS_cleanup.py index bf831810..5543db7d 100644 --- a/examples/HEMS/HEMS_cleanup.py +++ b/examples/HEMS/HEMS_cleanup.py @@ -2,14 +2,14 @@ import asyncio -from const import COMMUNITY_NAME, host, pwd, usr +from const import COMMUNITY_NAME, host, pwd, ssl, usr from utils.asset_utils import delete_hems_assets from flexmeasures_client import FlexMeasuresClient async def main() -> None: - client = FlexMeasuresClient(email=usr, password=pwd, host=host) + client = FlexMeasuresClient(email=usr, password=pwd, host=host, ssl=ssl) try: account = await client.get_account() if not account: diff --git a/examples/HEMS/HEMS_setup.py b/examples/HEMS/HEMS_setup.py index e242ff51..96be1d39 100644 --- a/examples/HEMS/HEMS_setup.py +++ b/examples/HEMS/HEMS_setup.py @@ -8,7 +8,7 @@ from typing import Callable from assets_setup import create_community_asset -from const import COMMUNITY_NAME, SITE_NAMES, host, pwd, usr +from const import COMMUNITY_NAME, SITE_NAMES, host, pwd, ssl, usr from forecasting import generate_forecasts from reporters import create_reports from scheduling import just_continue, run_scheduling_simulation @@ -41,7 +41,7 @@ async def main( # flexmeasures add user --username hems-admin --email hems-admin@example.com \ # --account 2 --roles account-admin - client = FlexMeasuresClient(email=usr, password=pwd, host=host) + client = FlexMeasuresClient(email=usr, password=pwd, host=host, ssl=ssl) try: await client.ensure_minimum_server_version( From db37574009821c647882931c22f44e0d50b8c2a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Fri, 14 Aug 2026 12:44:17 +0200 Subject: [PATCH 05/21] add sensor data deletion, upload function now returns results - incl. tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- src/flexmeasures_client/client.py | 26 ++++++++++- tests/client/test_sensor.py | 75 ++++++++++++++++++++++++++++++- 2 files changed, 98 insertions(+), 3 deletions(-) diff --git a/src/flexmeasures_client/client.py b/src/flexmeasures_client/client.py index 4d552278..b679b50a 100644 --- a/src/flexmeasures_client/client.py +++ b/src/flexmeasures_client/client.py @@ -564,7 +564,7 @@ async def _post_sensor_data_json( if prior: json_payload["prior"] = pd.Timestamp(prior).isoformat() - _response, status = await self.request( + response, status = await self.request( uri=f"sensors/{sensor_id}/data", json_payload=json_payload, minimum_server_version="0.28.0", @@ -572,6 +572,7 @@ async def _post_sensor_data_json( ) check_for_status(status, 200) self.logger.info("Sensor data sent successfully via JSON.") + return response, status async def _post_sensor_data_file( self, @@ -1404,6 +1405,29 @@ async def delete_sensor(self, sensor_id: int, confirm_first: bool = True): _, status = await self.request(uri=uri, method="DELETE") check_for_status(status, 204) + async def delete_sensor_data( + self, sensor_id: int, confirm_first: bool = True + ) -> None: + """Delete all data from a sensor while preserving the sensor itself.""" + if confirm_first: + answer = input( + f"Permanently delete all data from sensor {sensor_id}? [y/N] " + ) + if answer.lower() not in ["y", "yes"]: + print("Aborting ...") + return + _, status = await self.request( + uri=f"sensors/{sensor_id}/data", + json_payload={}, + method="DELETE", + minimum_server_version="0.33.0", + minimum_server_version_msg=( + "Deleting sensor data without deleting the sensor requires " + "FlexMeasures server v0.33.0 or above." + ), + ) + check_for_status(status, 204) + async def trigger_schedule( self, start: str | datetime, diff --git a/tests/client/test_sensor.py b/tests/client/test_sensor.py index 14fbafc1..2d2e443d 100644 --- a/tests/client/test_sensor.py +++ b/tests/client/test_sensor.py @@ -2,7 +2,7 @@ import os import re -from unittest.mock import patch +from unittest.mock import AsyncMock, patch from urllib.parse import unquote import pandas as pd @@ -327,6 +327,48 @@ async def test_delete_sensor_confirm_no(): await client.close() +@pytest.mark.asyncio +async def test_delete_sensor_data_preserves_sensor(): + with aioresponses() as m: + client = FlexMeasuresClient(email="test@test.test", password="test") + client.access_token = "test-token" + client.server_version = "0.33.0" + m.delete( + "http://localhost:5000/api/v3_0/sensors/7/data", + status=204, + payload={}, + ) + + await client.delete_sensor_data(sensor_id=7, confirm_first=False) + + m.assert_called_once_with( + "http://localhost:5000/api/v3_0/sensors/7/data", + method="DELETE", + json={}, + headers={ + "Content-Type": "application/json", + "Authorization": "test-token", + }, + params=None, + ssl=False, + allow_redirects=False, + ) + await client.close() + + +@pytest.mark.asyncio +async def test_delete_sensor_data_confirmation_declined(): + client = FlexMeasuresClient(email="test@test.test", password="test") + client.access_token = "test-token" + with ( + patch("builtins.input", return_value="n"), + patch.object(client, "request", new_callable=AsyncMock) as request, + ): + await client.delete_sensor_data(sensor_id=7) + request.assert_not_awaited() + await client.close() + + @pytest.mark.asyncio async def test_post_sensor_data() -> None: with aioresponses() as m: @@ -347,13 +389,15 @@ async def test_post_sensor_data() -> None: values = "test" unit = "test" - await flexmeasures_client.post_sensor_data( + response, status = await flexmeasures_client.post_sensor_data( sensor_id=sensor_id, start=start, duration=duration, values=values, unit=unit, ) + assert response == {"test": "test"} + assert status == 200 m.assert_called_once_with( f"http://localhost:5000/api/v3_0/sensors/{sensor_id}/data", method="POST", @@ -371,6 +415,33 @@ async def test_post_sensor_data() -> None: await flexmeasures_client.close() +@pytest.mark.asyncio +async def test_post_sensor_data_json_accepted_returns_ingestion_job() -> None: + with aioresponses() as m: + client = FlexMeasuresClient(email="test@test.test", password="test") + client.access_token = "test-token" + m.post( + "http://localhost:5000/api/v3_0/sensors/5/data", + status=202, + payload={ + "job": "ingestion-job-id", + "status": "ACCEPTED", + }, + ) + + response, status = await client.post_sensor_data( + sensor_id=5, + start="2023-03-26T10:00+02:00", + duration="PT1H", + values=[1.0], + unit="kW", + ) + + assert response["job"] == "ingestion-job-id" + assert status == 202 + await client.close() + + @pytest.mark.asyncio async def test_post_sensor_data_no_params(): """No json params and no file_path raises ValueError.""" From cb39c622c26da1e9913a6f315732bfb8898b2e83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Fri, 14 Aug 2026 12:51:26 +0200 Subject: [PATCH 06/21] mark on asset if phases were completed; support wiping only data and not the strucure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- docs/HEMS.rst | 21 +++ examples/HEMS/HEMS_setup.py | 178 ++++++++++++++++++++---- examples/HEMS/assets_setup.py | 1 + examples/HEMS/const.py | 9 +- examples/HEMS/forecasting.py | 8 +- examples/HEMS/reporters.py | 8 +- examples/HEMS/scheduling.py | 62 +++++++-- examples/HEMS/utils/asset_utils.py | 99 ++++++++++++- examples/HEMS/utils/workflow_utils.py | 193 ++++++++++++++++++++++++++ 9 files changed, 522 insertions(+), 57 deletions(-) create mode 100644 examples/HEMS/utils/workflow_utils.py diff --git a/docs/HEMS.rst b/docs/HEMS.rst index a0ddd1cf..64c08c70 100644 --- a/docs/HEMS.rst +++ b/docs/HEMS.rst @@ -118,6 +118,27 @@ Now run the client script using the `/examples/HEMS` folder as the current worki python3 HEMS_setup.py +Rerunning or resuming the tutorial +================================== + +The setup script records completed phases in a namespaced attribute on the +community asset. If the community already exists, the script shows which phases +are complete and offers three choices: + +- ``y`` recreates the HEMS assets. This deletes their sensors, IDs, and data. +- ``w`` preserves the asset and sensor structure and IDs, but permanently + deletes all HEMS time-series data before restarting at data upload. This + includes uploads, forecasts, schedules, simulated measurements, and report + outputs. A second confirmation is required. +- ``n`` (the default) preserves everything and resumes at the first unfinished + phase. Completed phases are skipped. + +The workflow marker stores the exact sensor IDs created for the tutorial, so a +data wipe remains limited to that recorded set. If an existing setup predates +workflow markers, safe resume is unavailable because its completed phases are +unknown. Choose ``w`` to keep its IDs while refreshing its data, or ``y`` to +recreate it fully. + Delete the tutorial assets and data =================================== diff --git a/examples/HEMS/HEMS_setup.py b/examples/HEMS/HEMS_setup.py index 96be1d39..898c760a 100644 --- a/examples/HEMS/HEMS_setup.py +++ b/examples/HEMS/HEMS_setup.py @@ -13,10 +13,95 @@ from reporters import create_reports from scheduling import just_continue, run_scheduling_simulation from utils.asset_utils import delete_hems_assets, upload_data_for_first_two_weeks +from utils.workflow_utils import ( + DATA_UPLOAD_PHASE, + FORECASTING_PHASE, + PHASE_LABELS, + REPORTING_PHASE, + SCHEDULING_PHASE, + ensure_workflow_state, + initialize_workflow_state, + mark_phase_complete, + phase_is_complete, + wipe_hems_sensor_data, +) from flexmeasures_client import FlexMeasuresClient +def print_workflow_summary(state: dict) -> None: + """Show which phases resume mode will skip and run.""" + completed = set(state["completed-phases"]) + print("\nCompleted phases:") + for phase, label in PHASE_LABELS.items(): + if phase in completed: + print(f"- {label}") + print("Still to run:") + remaining = [ + label for phase, label in PHASE_LABELS.items() if phase not in completed + ] + if remaining: + for label in remaining: + print(f"- {label}") + else: + print("- Nothing; the tutorial is already complete.") + + +def prompt_for_existing_setup(account: dict, community_name: str, state: dict) -> str: + """Ask whether to recreate, wipe data, or resume an existing setup.""" + print( + f"Asset '{community_name}' already exists in account " + f"'{account['name']}' (ID: {account['id']})." + ) + print_workflow_summary(state) + while True: + answer = ( + input( + "\nChoose how to continue:\n" + " [y] Recreate assets — delete assets, sensors, IDs, and data.\n" + " [w] Wipe data — preserve the asset and sensor structure and IDs, " + "delete all HEMS time-series data, then restart at data upload.\n" + " [n] Resume — preserve everything and continue from the first " + "unfinished phase.\n" + "Choice [y/w/N]: " + ) + .strip() + .lower() + ) + if answer in {"y", "yes"}: + return "recreate" + if answer in {"w", "wipe"}: + return "wipe" + if answer in {"", "n", "no"}: + if state.get("status") == "wiping": + print( + "A previous data wipe was interrupted. Choose 'w' to resume " + "the wipe or 'y' to recreate the assets." + ) + continue + if state.get("status") == "untracked": + print( + "This setup predates HEMS phase markers, so its completed " + "phases cannot be determined safely. Choose 'w' for fresh " + "data with the same IDs, or 'y' to recreate everything." + ) + continue + return "resume" + print("Please choose 'y', 'w', or 'n'.") + + +def confirm_data_wipe(state: dict) -> bool: + """Require explicit confirmation before deleting HEMS sensor data.""" + sensor_count = len(state["sensor-ids"]) + answer = input( + f"This permanently deletes all time-series data from {sensor_count} " + "HEMS sensors, including uploads, forecasts, schedules, simulated " + "measurements, and report outputs. Asset and sensor IDs are preserved.\n" + "Type WIPE to continue: " + ) + return answer == "WIPE" + + async def main( community_name: str, site_names: list[str], callback: Callable = just_continue ): @@ -57,10 +142,10 @@ async def main( account_id = account["id"] print(f" Connected to account: {account['name']} (ID: {account_id})") - asset = None # Initialize asset variable + asset = None assets = await client.get_assets(parse_json_fields=True) for sst in assets: - if sst["name"] == community_name: + if sst["name"] == community_name and sst.get("account_id") == account_id: asset = sst break @@ -68,61 +153,94 @@ async def main( print( "Creating community Site asset with 2 building assets, each with PV and battery sensors, and weather station" ) - await create_community_asset( + asset = await create_community_asset( client, account, community_name=community_name, site_names=site_names ) + state = await initialize_workflow_state(client, asset, account_id) else: - answer = input( - f"Asset '{community_name}' already exists in account " - f"'{account['name']}' (ID: {account['id']}). Re-create it? [y/N] " - ) - if answer.lower() in ["y", "yes"]: + state = await ensure_workflow_state(client, asset, account_id) + action = prompt_for_existing_setup(account, community_name, state) + if action == "recreate": await delete_hems_assets( client=client, account_id=account["id"], community_name=community_name, confirm_first=False, ) - await create_community_asset( + asset = await create_community_asset( client, account, community_name=community_name, site_names=site_names, ) + state = await initialize_workflow_state(client, asset, account_id) + elif action == "wipe": + if not confirm_data_wipe(state): + print("Data wipe cancelled. No sensor data was deleted.") + return + state = await wipe_hems_sensor_data(client, asset["id"], state) else: - print("Assets already exist, skipping to data upload") + print("Resuming the existing HEMS setup.") # Part 2: Upload data for first two weeks print("\n" + "=" * 50) - print("PART 2: UPLOADING DATA") - await upload_data_for_first_two_weeks( - client, community_name=community_name, site_names=site_names - ) + if phase_is_complete(state, DATA_UPLOAD_PHASE): + print("PART 2: UPLOADING DATA (already complete; skipping)") + else: + print("PART 2: UPLOADING DATA") + await upload_data_for_first_two_weeks( + client, community_name=community_name, site_names=site_names + ) + state = await mark_phase_complete( + client, asset["id"], state, DATA_UPLOAD_PHASE + ) # Part 3: Generate PV forecasts for second week print("\n" + "=" * 50) - print("PART 3: GENERATING PV FORECASTS") - await generate_forecasts( - client, community_name=community_name, site_names=site_names - ) + if phase_is_complete(state, FORECASTING_PHASE): + print("PART 3: GENERATING PV FORECASTS (already complete; skipping)") + else: + print("PART 3: GENERATING PV FORECASTS") + await generate_forecasts( + client, community_name=community_name, site_names=site_names + ) + state = await mark_phase_complete( + client, asset["id"], state, FORECASTING_PHASE + ) # Part 4: Run scheduling simulation for third week print("\n" + "=" * 50) - print("PART 4: SCHEDULING SIMULATION") - await run_scheduling_simulation( - client, - community_name=community_name, - site_names=site_names, - callback=callback, - ) + if phase_is_complete(state, SCHEDULING_PHASE): + print("PART 4: SCHEDULING SIMULATION (already complete; skipping)") + else: + print("PART 4: SCHEDULING SIMULATION") + scheduling_succeeded = await run_scheduling_simulation( + client, + community_name=community_name, + site_names=site_names, + callback=callback, + ) + if not scheduling_succeeded: + raise RuntimeError("Scheduling simulation did not complete.") + state = await mark_phase_complete( + client, asset["id"], state, SCHEDULING_PHASE + ) # Part 5 : Create reports print("\n" + "=" * 50) - print("PART 5: CREATING REPORTS") - # todo B2: compute aggregate power flow for the community asset's power sensor - await create_reports( - client, community_name=community_name, site_names=site_names - ) + if phase_is_complete(state, REPORTING_PHASE): + print("PART 5: CREATING REPORTS (already complete; skipping)") + else: + print("PART 5: CREATING REPORTS") + # todo B2: compute aggregate power flow for the community asset's power sensor + reports_succeeded = await create_reports( + client, community_name=community_name, site_names=site_names + ) + if not reports_succeeded: + raise RuntimeError("Report generation did not complete.") + state = await mark_phase_complete( + client, asset["id"], state, REPORTING_PHASE + ) print("\n" + "=" * 50) print("HEMS Tutorial completed successfully!") diff --git a/examples/HEMS/assets_setup.py b/examples/HEMS/assets_setup.py index 69a61b32..37ed9b2f 100644 --- a/examples/HEMS/assets_setup.py +++ b/examples/HEMS/assets_setup.py @@ -992,3 +992,4 @@ async def create_community_asset( site_names=site_names, price_sensor=price_sensor, ) + return site_asset diff --git a/examples/HEMS/const.py b/examples/HEMS/const.py index 92a3a2d1..7d36b736 100644 --- a/examples/HEMS/const.py +++ b/examples/HEMS/const.py @@ -4,12 +4,13 @@ Settings for the HEMS example script. """ # Connection details - UPDATE THESE FOR YOUR SETUP -usr = "toy-user@flexmeasures.io" # Account-admin user email -pwd = "toy-password" # Account-admin user password -host = "127.0.0.1:5000" # FlexMeasures host +usr = "hems-admin@example.com" # Account-admin user email +pwd = "change-me" # Account-admin user password +host = "127.0.0.1:5000" # FlexMeasures host, without http:// or https:// +ssl = True # Use HTTPS (and port 443 unless host specifies another port) # Asset and sensor names -COMMUNITY_NAME = "Campus Site" +COMMUNITY_NAME = "Community Site" SITE_NAMES = ["Building A", "Building B"] pv_name = "Rooftop PV" diff --git a/examples/HEMS/forecasting.py b/examples/HEMS/forecasting.py index f00cc0ad..9f3eab2b 100644 --- a/examples/HEMS/forecasting.py +++ b/examples/HEMS/forecasting.py @@ -73,11 +73,9 @@ async def generate_sensor_forecasts( ) except Exception as exc: job_id = forecast_id if forecast_id is not None else "unknown" - print(f"Forecast job {job_id} failed for {sensor_name} on {asset_name}: {exc}") - print( - "Look up this job in the RQ dashboard for more details about the failure." - ) - return None + raise RuntimeError( + f"Forecast job {job_id} failed for {sensor_name} on " f"{asset_name}: {exc}" + ) from exc print(f"Forecast job completed for {sensor_name} on {asset_name}") diff --git a/examples/HEMS/reporters.py b/examples/HEMS/reporters.py index 958e387e..78a13cf5 100644 --- a/examples/HEMS/reporters.py +++ b/examples/HEMS/reporters.py @@ -34,6 +34,7 @@ async def create_reports( if check_result.returncode != 0: print("FlexMeasures CLI not found. Skipping report generation.") return False + all_reports_succeeded = True for i, site_name in enumerate(site_names, start=1): # Find all required sensors @@ -109,5 +110,10 @@ async def create_reports( start=SCHEDULING_START, end=SCHEDULING_END, ) + all_reports_succeeded = ( + self_consumption_result + and total_energy_costs_result + and all_reports_succeeded + ) - return self_consumption_result and total_energy_costs_result + return all_reports_succeeded diff --git a/examples/HEMS/scheduling.py b/examples/HEMS/scheduling.py index d77c5b86..4a3c1cb8 100644 --- a/examples/HEMS/scheduling.py +++ b/examples/HEMS/scheduling.py @@ -25,6 +25,8 @@ find_sensor_by_name_and_asset, find_top_level_asset_id, load_and_align_csv_data, + post_sensor_data_and_track_ingestion, + wait_for_ingestion_jobs, ) from utils.ev_utils import ( calculate_ev_soc_targets_and_constraints, @@ -179,6 +181,7 @@ async def run_scheduling_simulation( # Stop rescheduling break + pending_ingestion_jobs: list[str] = [] for index, site_name in enumerate(site_names, start=1): # Extract scheduled power for all devices for the next 4 hours # Update SoC for next step based on retrieved SoC schedules @@ -201,20 +204,29 @@ async def run_scheduling_simulation( heating_soc_schedule=heating_soc_schedules[index - 1], evse1_flex_model=evse1_flex_models[index - 1], evse2_flex_model=evse2_flex_models[index - 1], + pending_ingestion_jobs=pending_ingestion_jobs, ) next_current_soc_dict[site_name]["battery"] = battery_next_current_soc next_current_soc_dict[site_name]["evse1"] = evse1_next_current_soc next_current_soc_dict[site_name]["evse2"] = evse2_next_current_soc next_current_soc_dict[site_name]["heating"] = heating_next_current_soc + # Reporters read the measurements submitted above. Wait until the server + # has actually ingested them instead of treating HTTP 202 as completion. + await wait_for_ingestion_jobs(client, pending_ingestion_jobs) + # Run reporter to log community site aggregate power consumption each scheduling step - run_community_aggregate( + aggregate_reports_succeeded = run_community_aggregate( sensors=sensors, current_time=current_time, step_end_time=step_end_time, community_asset=community_asset, site_names=site_names, ) + if not aggregate_reports_succeeded: + raise RuntimeError( + f"Aggregate report generation failed for simulation step {step_num}." + ) # Move to next simulation step current_time = step_end_time @@ -482,6 +494,7 @@ async def compute_site_measurements( evse1_flex_model: dict, evse2_flex_model: dict, index: int, + pending_ingestion_jobs: list[str], ): # Initialize power schedules @@ -533,7 +546,9 @@ async def compute_site_measurements( # Upload battery power measurements battery_power_duration = timedelta(hours=SIMULATION_STEP_HOURS) - await client.post_sensor_data( + await post_sensor_data_and_track_ingestion( + client=client, + pending_ingestion_jobs=pending_ingestion_jobs, sensor_id=sensors[f"battery-power-{index}"]["id"], start=current_time, duration=battery_power_duration, @@ -559,7 +574,9 @@ async def compute_site_measurements( min(raw, scheduled) for raw, scheduled in zip(pv_raw_power, pv_scheduled_power) ] - await client.post_sensor_data( + await post_sensor_data_and_track_ingestion( + client=client, + pending_ingestion_jobs=pending_ingestion_jobs, sensor_id=sensors[f"pv-power-{index}"][ "id" ], # use power sensor to store realized data @@ -571,7 +588,9 @@ async def compute_site_measurements( ) # Upload EVSE 1 power measurements - await client.post_sensor_data( + await post_sensor_data_and_track_ingestion( + client=client, + pending_ingestion_jobs=pending_ingestion_jobs, sensor_id=sensors[f"evse1-power-{index}"]["id"], start=current_time, duration=battery_power_duration, @@ -581,7 +600,9 @@ async def compute_site_measurements( ) # Upload EVSE 2 power measurements - await client.post_sensor_data( + await post_sensor_data_and_track_ingestion( + client=client, + pending_ingestion_jobs=pending_ingestion_jobs, sensor_id=sensors[f"evse2-power-{index}"]["id"], start=current_time, duration=battery_power_duration, @@ -591,7 +612,9 @@ async def compute_site_measurements( ) # Upload heating power measurements - await client.post_sensor_data( + await post_sensor_data_and_track_ingestion( + client=client, + pending_ingestion_jobs=pending_ingestion_jobs, sensor_id=sensors[f"heating-power-{index}"]["id"], start=current_time, duration=battery_power_duration, @@ -614,7 +637,9 @@ async def compute_site_measurements( ) + pd.Timedelta(minutes=15) ) - await client.post_sensor_data( + await post_sensor_data_and_track_ingestion( + client=client, + pending_ingestion_jobs=pending_ingestion_jobs, sensor_id=sensors[f"building-consumption-{index}"]["id"], start=building_data_step["event_start"].iloc[0], duration=step_duration, @@ -688,7 +713,9 @@ async def compute_site_measurements( # Upload battery SoC measurements (FlexMeasures computed) if battery_soc_values: - await client.post_sensor_data( + await post_sensor_data_and_track_ingestion( + client=client, + pending_ingestion_jobs=pending_ingestion_jobs, sensor_id=sensors[f"battery-soc-{index}"]["id"], start=current_time, duration=pd.Timedelta(hours=SIMULATION_STEP_HOURS).isoformat(), @@ -702,7 +729,9 @@ async def compute_site_measurements( # Upload EVSE 1 SoC measurements (FlexMeasures computed) if evse1_soc_values: - await client.post_sensor_data( + await post_sensor_data_and_track_ingestion( + client=client, + pending_ingestion_jobs=pending_ingestion_jobs, sensor_id=sensors[f"evse1-soc-{index}"]["id"], start=current_time, duration=pd.Timedelta(hours=SIMULATION_STEP_HOURS).isoformat(), @@ -716,7 +745,9 @@ async def compute_site_measurements( # Upload EVSE 2 SoC measurements (FlexMeasures computed) if evse2_soc_values: - await client.post_sensor_data( + await post_sensor_data_and_track_ingestion( + client=client, + pending_ingestion_jobs=pending_ingestion_jobs, sensor_id=sensors[f"evse2-soc-{index}"]["id"], start=current_time, duration=pd.Timedelta(hours=SIMULATION_STEP_HOURS).isoformat(), @@ -729,7 +760,9 @@ async def compute_site_measurements( ) # Upload heating SoC measurements (FlexMeasures computed) if heating_soc_values: - await client.post_sensor_data( + await post_sensor_data_and_track_ingestion( + client=client, + pending_ingestion_jobs=pending_ingestion_jobs, sensor_id=sensors[f"heating-soc-{index}"]["id"], start=current_time, duration=pd.Timedelta(hours=SIMULATION_STEP_HOURS).isoformat(), @@ -919,6 +952,7 @@ def run_community_aggregate( community_power_sensor = x break # Run each site's aggregate reporter + all_reports_succeeded = True for index, site_name in enumerate(site_names, start=1): # Fill reporter parameters for each site fill_reporter_params( @@ -936,11 +970,12 @@ def run_community_aggregate( reporter_type="aggregate", ) # Run AggregatorReporter - run_report_cmd( + report_succeeded = run_report_cmd( reporter_map={"name": "aggregate", "reporter": "AggregatorReporter"}, start=current_time.isoformat(), end=step_end_time.isoformat(), ) + all_reports_succeeded = report_succeeded and all_reports_succeeded fill_reporter_params( input_sensors=[ @@ -953,8 +988,9 @@ def run_community_aggregate( reporter_type="aggregate", ) # Run AggregatorReporter - run_report_cmd( + community_report_succeeded = run_report_cmd( reporter_map={"name": "aggregate", "reporter": "AggregatorReporter"}, start=current_time.isoformat(), end=step_end_time.isoformat(), ) + return community_report_succeeded and all_reports_succeeded diff --git a/examples/HEMS/utils/asset_utils.py b/examples/HEMS/utils/asset_utils.py index 371ec63a..ac9a7113 100644 --- a/examples/HEMS/utils/asset_utils.py +++ b/examples/HEMS/utils/asset_utils.py @@ -1,5 +1,7 @@ +import asyncio import os from pathlib import Path +from typing import Any import pandas as pd from const import heating_name, price_market_name, pv_name, weather_station_name @@ -9,6 +11,86 @@ BASE_DIR = Path(__file__).parent.parent +async def post_sensor_data_and_track_ingestion( + client: FlexMeasuresClient, + pending_ingestion_jobs: list[str], + **kwargs: Any, +) -> None: + """Post sensor data and remember asynchronous ingestion jobs.""" + result = await client.post_sensor_data(**kwargs) + if not isinstance(result, tuple) or len(result) != 2: + raise RuntimeError( + "This HEMS example requires a FlexMeasures client whose " + "post_sensor_data() method returns the server response and status." + ) + response, status = result + + if status != 202: + return + + job_id = None + if isinstance(response, dict): + # ``job`` is the canonical field. ``job_id`` was used by older + # FlexMeasures servers and is retained here for compatibility. + job_id = response.get("job") or response.get("job_id") + if not job_id: + raise RuntimeError( + "The server accepted sensor data for asynchronous ingestion " + "but did not return a job ID." + ) + pending_ingestion_jobs.append(job_id) + + +async def wait_for_ingestion_jobs( + client: FlexMeasuresClient, pending_ingestion_jobs: list[str] +) -> None: + """Wait until all tracked sensor-data ingestion jobs have finished.""" + if not pending_ingestion_jobs: + return + + print(f"Waiting for {len(pending_ingestion_jobs)} ingestion job(s)...") + for job_id in pending_ingestion_jobs: + deadline = asyncio.get_running_loop().time() + client.polling_timeout + polling_step = 0 + + while True: + # FlexMeasures 0.33 returns HTTP 200 even while a job is in + # progress. Newer versions return 202, which client.request polls + # internally. Inspecting the status field supports both versions. + job, _ = await client.request( + uri=f"jobs/{job_id}", + method="GET", + ) + job_status = ( + str(job.get("status", "")).upper() if isinstance(job, dict) else "" + ) + if job_status == "FINISHED": + break + if job_status not in {"QUEUED", "STARTED", "DEFERRED", "SCHEDULED"}: + raise RuntimeError( + f"Ingestion job {job_id} did not finish successfully: {job}" + ) + + polling_step += 1 + if polling_step >= client.max_polling_steps: + raise ConnectionError( + f"Max polling steps reached while waiting for ingestion job " + f"{job_id}. Last status: {job_status}" + ) + + remaining = deadline - asyncio.get_running_loop().time() + if remaining <= 0: + raise ConnectionError( + f"Client polling timeout while waiting for ingestion job " + f"{job_id}. Last status: {job_status}" + ) + sleep_interval = min( + client.polling_interval * (2 ** (polling_step - 1)), remaining + ) + await asyncio.sleep(sleep_interval) + pending_ingestion_jobs.clear() + + async def find_sensor_by_name_and_asset( client: FlexMeasuresClient, sensor_name: str, @@ -44,20 +126,23 @@ async def upload_csv_file_to_sensor( sensor_id: int, file_path: str, belief_time_measured_instantly: bool, + pending_ingestion_jobs: list[str], ): - """Upload CSV file directly to a sensor using file upload.""" + """Upload a CSV file and track asynchronous ingestion.""" try: full_path = os.path.join(BASE_DIR, file_path) - await client.post_sensor_data( + await post_sensor_data_and_track_ingestion( + client=client, + pending_ingestion_jobs=pending_ingestion_jobs, sensor_id=sensor_id, file_path=full_path, belief_time_measured_instantly=belief_time_measured_instantly, # Set belief_time immediately after event ends ) - print(f"Uploaded {file_path} to sensor {sensor_id}") + print(f"Submitted {file_path} to sensor {sensor_id}") return True except Exception as e: print(f"Failed to upload {file_path} to sensor {sensor_id}: {e}") - return False + raise async def find_top_level_asset_id( @@ -101,6 +186,7 @@ async def upload_data_for_first_two_weeks( ): """Upload historical data for the first two weeks.""" print("Uploading data for first two weeks...") + pending_ingestion_jobs: list[str] = [] for i, site_name in enumerate(site_names, start=1): # Find all required sensors @@ -146,6 +232,7 @@ async def upload_data_for_first_two_weeks( sensor_id=sensors[sensor_key]["id"], file_path=file_path, belief_time_measured_instantly=belief_time_measured_instantly, + pending_ingestion_jobs=pending_ingestion_jobs, ) if success: @@ -153,6 +240,10 @@ async def upload_data_for_first_two_weeks( else: print(f"Failed to upload {sensor_key} data") + # File uploads may only have been accepted (HTTP 202), not processed yet. + # Forecasting must not start until all historical data is available. + await wait_for_ingestion_jobs(client, pending_ingestion_jobs) + return True diff --git a/examples/HEMS/utils/workflow_utils.py b/examples/HEMS/utils/workflow_utils.py new file mode 100644 index 00000000..56e7fd49 --- /dev/null +++ b/examples/HEMS/utils/workflow_utils.py @@ -0,0 +1,193 @@ +from __future__ import annotations + +import json + +from const import price_market_name, weather_station_name + +from flexmeasures_client import FlexMeasuresClient + +WORKFLOW_ATTRIBUTE = "hems_tutorial" +WORKFLOW_VERSION = 1 + +ASSET_SETUP_PHASE = "asset-setup" +DATA_UPLOAD_PHASE = "historical-data-upload" +FORECASTING_PHASE = "forecasting" +SCHEDULING_PHASE = "scheduling" +REPORTING_PHASE = "reporting" + +PHASE_LABELS = { + ASSET_SETUP_PHASE: "Asset setup", + DATA_UPLOAD_PHASE: "Historical data upload", + FORECASTING_PHASE: "Forecast generation", + SCHEDULING_PHASE: "Scheduling simulation", + REPORTING_PHASE: "Report generation", +} + + +def get_workflow_state(community_asset: dict) -> dict | None: + """Read a valid HEMS workflow marker from a community asset.""" + attributes = community_asset.get("attributes", {}) + if isinstance(attributes, str): + try: + attributes = json.loads(attributes) + except json.JSONDecodeError: + return None + if not isinstance(attributes, dict): + return None + + state = attributes.get(WORKFLOW_ATTRIBUTE) + if not isinstance(state, dict) or state.get("workflow-version") != WORKFLOW_VERSION: + return None + if not isinstance(state.get("completed-phases"), list): + return None + if not isinstance(state.get("sensor-ids"), list): + return None + return state + + +async def save_workflow_state( + client: FlexMeasuresClient, + community_asset_id: int, + state: dict, +) -> dict: + """Save workflow state without replacing unrelated asset attributes.""" + community_asset = await client.get_asset( + asset_id=community_asset_id, parse_json_fields=True + ) + attributes = community_asset.get("attributes", {}) + if not isinstance(attributes, dict): + attributes = {} + attributes[WORKFLOW_ATTRIBUTE] = state + await client.update_asset( + asset_id=community_asset_id, + updates={"attributes": attributes}, + parse_json_fields=False, + ) + return state + + +async def collect_hems_structure_ids( + client: FlexMeasuresClient, + community_asset: dict, + account_id: int, +) -> tuple[list[int], list[int]]: + """Collect the asset and sensor IDs that belong to this HEMS tutorial.""" + all_assets = await client.get_assets( + fields=["id", "name", "account_id", "parent_asset_id"], + parse_json_fields=False, + ) + top_level_asset_ids = [community_asset["id"]] + for asset_name in (price_market_name, weather_station_name): + matching_assets = [ + asset + for asset in all_assets + if asset.get("name") == asset_name + and asset.get("account_id") == account_id + and asset.get("parent_asset_id") is None + ] + if len(matching_assets) != 1: + raise LookupError( + f"Expected one top-level HEMS asset named '{asset_name}' in " + f"account {account_id}, found {len(matching_assets)}." + ) + top_level_asset_ids.append(matching_assets[0]["id"]) + + hems_asset_ids: set[int] = set() + for root_id in top_level_asset_ids: + hems_asset_ids.add(root_id) + descendants = await client.get_assets( + root=root_id, + fields=["id"], + parse_json_fields=False, + ) + hems_asset_ids.update(asset["id"] for asset in descendants) + + sensor_ids: set[int] = set() + for asset_id in sorted(hems_asset_ids): + sensors = await client.get_sensors( + asset_id=asset_id, + parse_json_fields=False, + ) + sensor_ids.update(sensor["id"] for sensor in sensors) + + return sorted(top_level_asset_ids), sorted(sensor_ids) + + +async def initialize_workflow_state( + client: FlexMeasuresClient, + community_asset: dict, + account_id: int, + status: str = "ready", +) -> dict: + """Create the workflow marker after the complete asset structure exists.""" + top_level_asset_ids, sensor_ids = await collect_hems_structure_ids( + client=client, + community_asset=community_asset, + account_id=account_id, + ) + state = { + "workflow-version": WORKFLOW_VERSION, + "status": status, + "completed-phases": [ASSET_SETUP_PHASE], + "top-level-asset-ids": top_level_asset_ids, + "sensor-ids": sensor_ids, + } + return await save_workflow_state(client, community_asset["id"], state) + + +async def ensure_workflow_state( + client: FlexMeasuresClient, + community_asset: dict, + account_id: int, +) -> dict: + """Return existing workflow state or initialize legacy HEMS assets.""" + state = get_workflow_state(community_asset) + if state is not None: + return state + print( + "No compatible HEMS phase marker exists yet. Treating the asset " + "structure as complete; data phases are not assumed to be complete." + ) + return await initialize_workflow_state( + client, community_asset, account_id, status="untracked" + ) + + +def phase_is_complete(state: dict, phase: str) -> bool: + return phase in state["completed-phases"] + + +async def mark_phase_complete( + client: FlexMeasuresClient, + community_asset_id: int, + state: dict, + phase: str, +) -> dict: + """Mark one successfully finished phase as complete.""" + completed_phases = list(state["completed-phases"]) + if phase not in completed_phases: + completed_phases.append(phase) + state = {**state, "status": "ready", "completed-phases": completed_phases} + return await save_workflow_state(client, community_asset_id, state) + + +async def wipe_hems_sensor_data( + client: FlexMeasuresClient, + community_asset_id: int, + state: dict, +) -> dict: + """Delete HEMS time-series data and reset all data phase markers.""" + state = { + **state, + "status": "wiping", + "completed-phases": [ASSET_SETUP_PHASE], + } + await save_workflow_state(client, community_asset_id, state) + + sensor_ids = state["sensor-ids"] + print(f"Deleting time-series data from {len(sensor_ids)} HEMS sensors...") + for sensor_id in sensor_ids: + await client.delete_sensor_data(sensor_id, confirm_first=False) + + state = {**state, "status": "ready"} + return await save_workflow_state(client, community_asset_id, state) From eedab46ed6268d68c1c03cec2cae4c14d1c41e84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Fri, 14 Aug 2026 13:21:51 +0200 Subject: [PATCH 07/21] fix whitespace MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- docs/HEMS.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/HEMS.rst b/docs/HEMS.rst index 64c08c70..20ac7c3c 100644 --- a/docs/HEMS.rst +++ b/docs/HEMS.rst @@ -96,14 +96,14 @@ In the third terminal, go to the HEMS directory: - ``FLEXMEASURES_CLI_CMD``: the command used to invoke the CLI - ``FLEXMEASURES_CLI_CONFIG_DIR``: the directory the CLI process sees the ``examples/HEMS/configs/`` files at, if different from their local path - + Here are steps if you use FlexMeasures' docker-compose: - ``export FLEXMEASURES_CLI_CMD="docker compose -f full/path/to/docker-compose.yml exec -T server flexmeasures"``. - Add this mount in docker-compose.yml under server.volumes, and restart it: ``- /full/path/to/flexmeasures-client/examples/HEMS/configs:/app/hems-configs:ro`` - ``export FLEXMEASURES_CLI_CONFIG_DIR="/app/hems-configs"`` Another caveat is rate-limiting. Since v1.0, FlexMeasures only allows a limited number of schedule and forecasts per 5 minute interval. -Either give your account a generous plan (see the docs), or simply set ``FLEXMEASURES_MODE="play"`` and restart the server. +Either give your account a generous plan (see the docs), or simply set ``FLEXMEASURES_MODE="play"`` and restart the server. If you use docker-compose, you could do that like this: .. code-block:: bash @@ -115,7 +115,7 @@ If you use docker-compose, you could do that like this: Now run the client script using the `/examples/HEMS` folder as the current working directory: .. code-block:: bash - + python3 HEMS_setup.py Rerunning or resuming the tutorial From 638b639442bf955d73b023f82c8058fdd11e6961 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Fri, 14 Aug 2026 20:35:53 +0200 Subject: [PATCH 08/21] no ssl by default, and schedule two days MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- examples/HEMS/const.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/HEMS/const.py b/examples/HEMS/const.py index 7d36b736..3d24fece 100644 --- a/examples/HEMS/const.py +++ b/examples/HEMS/const.py @@ -7,7 +7,7 @@ usr = "hems-admin@example.com" # Account-admin user email pwd = "change-me" # Account-admin user password host = "127.0.0.1:5000" # FlexMeasures host, without http:// or https:// -ssl = True # Use HTTPS (and port 443 unless host specifies another port) +ssl = False # Use HTTPS (and port 443 unless host specifies another port) # Asset and sensor names COMMUNITY_NAME = "Community Site" @@ -29,7 +29,7 @@ TUTORIAL_START_DATE = "2030-01-01T00:00:00+01:00" FORECASTING_START = "2030-01-15T00:00:00+01:00" SCHEDULING_START = "2030-01-15T00:00:00+01:00" -SCHEDULING_END = "2030-01-16T00:00:00+01:00" +SCHEDULING_END = "2030-01-17T00:00:00+01:00" SIMULATION_STEP_HOURS = 4 FORECAST_HORIZON_HOURS = 24 MAX_RESCHEDULING_ITERATIONS = 2 From 887fc05c06a6b5c5eb1f497c21fae0d79a82ba03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Fri, 14 Aug 2026 20:36:56 +0200 Subject: [PATCH 09/21] Increase prompt interaction for re-creating assets, similar to wiping; add some helpful prints in the startup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- docs/HEMS.rst | 6 ++++-- examples/HEMS/HEMS_setup.py | 28 +++++++++++++++++++++++++--- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/docs/HEMS.rst b/docs/HEMS.rst index 20ac7c3c..e1824fa7 100644 --- a/docs/HEMS.rst +++ b/docs/HEMS.rst @@ -125,11 +125,13 @@ The setup script records completed phases in a namespaced attribute on the community asset. If the community already exists, the script shows which phases are complete and offers three choices: -- ``y`` recreates the HEMS assets. This deletes their sensors, IDs, and data. +- ``y`` recreates the HEMS assets. This deletes their sensors, IDs, and data, + including the HEMS energy market and weather station, before creating + replacements with new IDs. You must confirm this by typing ``RECREATE``. - ``w`` preserves the asset and sensor structure and IDs, but permanently deletes all HEMS time-series data before restarting at data upload. This includes uploads, forecasts, schedules, simulated measurements, and report - outputs. A second confirmation is required. + outputs. You must confirm this by typing ``WIPE``. - ``n`` (the default) preserves everything and resumes at the first unfinished phase. Completed phases are skipped. diff --git a/examples/HEMS/HEMS_setup.py b/examples/HEMS/HEMS_setup.py index 898c760a..58d79ab1 100644 --- a/examples/HEMS/HEMS_setup.py +++ b/examples/HEMS/HEMS_setup.py @@ -58,9 +58,11 @@ def prompt_for_existing_setup(account: dict, community_name: str, state: dict) - answer = ( input( "\nChoose how to continue:\n" - " [y] Recreate assets — delete assets, sensors, IDs, and data.\n" + " [y] Recreate assets — delete assets, sensors, IDs, and data " + "(requires typing RECREATE).\n" " [w] Wipe data — preserve the asset and sensor structure and IDs, " - "delete all HEMS time-series data, then restart at data upload.\n" + "delete all HEMS time-series data, then restart at data upload " + "(requires typing WIPE).\n" " [n] Resume — preserve everything and continue from the first " "unfinished phase.\n" "Choice [y/w/N]: " @@ -90,6 +92,19 @@ def prompt_for_existing_setup(account: dict, community_name: str, state: dict) - print("Please choose 'y', 'w', or 'n'.") +def confirm_recreation(account: dict, community_name: str) -> bool: + """Require explicit confirmation before replacing the HEMS structure.""" + answer = input( + f"This permanently deletes the HEMS setup '{community_name}' from account " + f"'{account['name']}' (ID: {account['id']}), including its assets, sensors, " + "IDs, and all time-series data. The replacement assets and sensors will " + "receive new IDs. The HEMS energy market and weather station in this " + "account will also be replaced.\n" + "Type RECREATE to continue: " + ) + return answer == "RECREATE" + + def confirm_data_wipe(state: dict) -> bool: """Require explicit confirmation before deleting HEMS sensor data.""" sensor_count = len(state["sensor-ids"]) @@ -129,18 +144,22 @@ async def main( client = FlexMeasuresClient(email=usr, password=pwd, host=host, ssl=ssl) try: + print( + f"Checking server is up and on supported version ... connecting to {host} (ssl: {ssl})" + ) await client.ensure_minimum_server_version( "0.31.0", "The HEMS example requires a FlexMeasures server of v0.31.0 or above.", ) # Get user account information + print(f"Logging in as {usr} ...") account = await client.get_account() if not account: raise Exception("No account found. Please create an account first.") account_id = account["id"] - print(f" Connected to account: {account['name']} (ID: {account_id})") + print(f"Connected to account: {account['name']} (ID: {account_id})") asset = None assets = await client.get_assets(parse_json_fields=True) @@ -161,6 +180,9 @@ async def main( state = await ensure_workflow_state(client, asset, account_id) action = prompt_for_existing_setup(account, community_name, state) if action == "recreate": + if not confirm_recreation(account, community_name): + print("Recreation cancelled. No assets or data were deleted.") + return await delete_hems_assets( client=client, account_id=account["id"], From 4f27b619a9091c86a5a72c55a80b4da43e1158d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Fri, 14 Aug 2026 21:21:46 +0200 Subject: [PATCH 10/21] apply the 15 minute factor 0.25 to energy costs - they were too high MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- .../HEMS/configs/total-energy-costs_reporter_config.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/examples/HEMS/configs/total-energy-costs_reporter_config.json b/examples/HEMS/configs/total-energy-costs_reporter_config.json index 52c5c98a..1dcadd7f 100644 --- a/examples/HEMS/configs/total-energy-costs_reporter_config.json +++ b/examples/HEMS/configs/total-energy-costs_reporter_config.json @@ -46,6 +46,12 @@ "args": ["@feed_in_revenue"], "df_output": "total-energy-costs" }, + { + "df_input": "total-energy-costs", + "method": "multiply", + "args": [0.25], + "df_output": "total-energy-costs" + }, { "df_input": "total-energy-costs", "method": "resample", From 24be50e12fb622eac8635ba17d8d52493f3127f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Fri, 14 Aug 2026 21:26:02 +0200 Subject: [PATCH 11/21] define EV usage batter: soc-usage in 30 min intervals, show their SoC in graph MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- examples/HEMS/assets_setup.py | 12 +- examples/HEMS/const.py | 11 +- examples/HEMS/scheduling.py | 64 +++---- examples/HEMS/utils/ev_utils.py | 211 +++++++++++++----------- examples/HEMS/utils/scheduling_utils.py | 10 +- 5 files changed, 162 insertions(+), 146 deletions(-) diff --git a/examples/HEMS/assets_setup.py b/examples/HEMS/assets_setup.py index 37ed9b2f..38a28d57 100644 --- a/examples/HEMS/assets_setup.py +++ b/examples/HEMS/assets_setup.py @@ -425,6 +425,7 @@ async def create_evse_asset( capacity = EV_CONFIG["default_capacity_kwh"] attributes_flex_model = { "soc_at_start": capacity * EV_CONFIG["min_soc_percent"], # Start at minimum SoC + "capacity_kwh": capacity, } flex_model = { @@ -676,7 +677,9 @@ async def configure_site_dashboard( battery_power_sensor, battery_soc_sensor, evse1_power_sensor, + evse1_soc_sensor, evse2_power_sensor, + evse2_soc_sensor, heating_power_sensor, heating_soc_sensor, aggregate_sensor, @@ -726,7 +729,12 @@ async def configure_site_dashboard( }, { "title": "Storages SoC", - "sensors": [battery_soc_sensor["id"], heating_soc_sensor["id"]], + "sensors": [ + battery_soc_sensor["id"], + evse1_soc_sensor["id"], + evse2_soc_sensor["id"], + heating_soc_sensor["id"], + ], }, { "title": "Site capacity", @@ -909,7 +917,9 @@ async def create_sites_assets_and_sensors( battery_power_sensor=battery_power_sensor, battery_soc_sensor=battery_soc_sensor, evse1_power_sensor=evse1_power_sensor, + evse1_soc_sensor=evse1_soc_sensor, evse2_power_sensor=evse2_power_sensor, + evse2_soc_sensor=evse2_soc_sensor, heating_power_sensor=heating_power_sensor, heating_soc_sensor=heating_soc_sensor, aggregate_sensor=aggregate_sensor, diff --git a/examples/HEMS/const.py b/examples/HEMS/const.py index 3d24fece..a87c88b6 100644 --- a/examples/HEMS/const.py +++ b/examples/HEMS/const.py @@ -38,13 +38,14 @@ # Configuration constants EV_CONFIG = { - "default_capacity_kwh": 60.0, + "default_capacity_kwh": 40.0, "default_power_capacity_kw": 11.0, "min_soc_percent": 0.20, # 20% minimum SoC "roundtrip_efficiency": 0.85, # 85% efficiency "random_trip_probability": 0.10, # 10% chance per step "random_trip_consumption_range": (0.10, 0.20), # 10-20% consumption "driving_consumption_kwh_per_hour": 7.5, # 15 kWh/100km at 50 km/h average + "one_way_commute_duration_hours": 0.5, } BATTERY_CONFIG = { @@ -60,10 +61,10 @@ # Each entry represents: (needs_charging_overnight, departure_time, return_time, target_soc_percent) # Index 0 = Monday, 1 = Tuesday, ..., 6 = Sunday EV_WEEKLY_PATTERNS = [ - (False, None, None, 40), # Monday - Free day, keep at moderate charge - (True, "07:00", "13:00", 80), # Tuesday - Work day, need 80% by 7am - (True, "08:00", "13:00", 80), # Wednesday - Work day, need 80% by 8am - (True, "07:00", "13:00", 80), # Thursday - Work day, need 80% by 7am + (True, "07:00", "13:00", 60), # Monday - Work day, need 60% by 7am + (True, "07:00", "13:00", 60), # Tuesday - Work day, need 60% by 7am + (True, "08:00", "13:00", 60), # Wednesday - Work day, need 60% by 8am + (True, "07:00", "13:00", 60), # Thursday - Work day, need 60% by 7am (False, None, None, 60), # Friday - Free day, charge to 60% for weekend (False, None, None, 40), # Saturday - Free day (False, None, None, 40), # Sunday - Free day diff --git a/examples/HEMS/scheduling.py b/examples/HEMS/scheduling.py index 4a3c1cb8..433d6d79 100644 --- a/examples/HEMS/scheduling.py +++ b/examples/HEMS/scheduling.py @@ -302,33 +302,29 @@ async def compute_site_schedules( evse2_constraints = calculate_ev_soc_targets_and_constraints( current_time_ts, evse2_capacity, evse2_has_trip ) - if not evse1_constraints.get("unavailable"): - - # Create flex models for EVSE 1 - if evse1_next_current_soc is None: - # Use initial SoC for first step - evse1_current_soc = evse1_flex_model.get("soc_at_start", 12.0) - else: - evse1_current_soc = evse1_next_current_soc - # Create dynamic flex model for EVSE 1 (Current SoC updated each step) - evse1_scheduling_dynamic_flex_model = create_dynamic_storage_flex_model( - current_soc=evse1_current_soc, - constraints=evse1_constraints, + # Keep EVs in the model while they are away. Their zero consumption-capacity + # prevents charging, while soc-usage continues to account for driving. + if evse1_next_current_soc is None: + evse1_current_soc = evse1_flex_model.get( + "soc_at_start", EV_CONFIG["min_soc_percent"] * evse1_capacity ) + else: + evse1_current_soc = evse1_next_current_soc + evse1_scheduling_dynamic_flex_model = create_dynamic_storage_flex_model( + current_soc=evse1_current_soc, + constraints=evse1_constraints, + ) - if not evse2_constraints.get("unavailable"): - - # Create flex models for EVSE 2 (similar pattern, could be different car) - if evse2_next_current_soc is None: - # Use initial SoC for first step - evse2_current_soc = evse2_flex_model.get("soc_at_start", 12.0) - else: - evse2_current_soc = evse2_next_current_soc - # Create dynamic flex model for EVSE 2 (Current SoC updated each step) - evse2_scheduling_dynamic_flex_model = create_dynamic_storage_flex_model( - current_soc=evse2_current_soc, - constraints=evse2_constraints, + if evse2_next_current_soc is None: + evse2_current_soc = evse2_flex_model.get( + "soc_at_start", EV_CONFIG["min_soc_percent"] * evse2_capacity ) + else: + evse2_current_soc = evse2_next_current_soc + evse2_scheduling_dynamic_flex_model = create_dynamic_storage_flex_model( + current_soc=evse2_current_soc, + constraints=evse2_constraints, + ) if heating_next_current_soc is None: # Use initial SoC for first step @@ -365,26 +361,18 @@ async def compute_site_schedules( }, ] - # Conditionally add EVSE flex models if they are not on a trip - if not evse1_constraints.get("unavailable"): - final_flex_models.append( + final_flex_models.extend( + [ { "sensor": sensors[f"evse1-power-{index}"]["id"], **evse1_scheduling_dynamic_flex_model, - } - ) - else: - print("EVSE 1 is on a trip, skipping scheduling.") - - if not evse2_constraints.get("unavailable"): - final_flex_models.append( + }, { "sensor": sensors[f"evse2-power-{index}"]["id"], **evse2_scheduling_dynamic_flex_model, - } - ) - else: - print("EVSE 2 is on a trip, skipping scheduling.") + }, + ] + ) print("[FLEX-MODEL-DEBUG] === FLEX MODELS SENT TO SCHEDULER ===") for i, model in enumerate(final_flex_models): diff --git a/examples/HEMS/utils/ev_utils.py b/examples/HEMS/utils/ev_utils.py index 6fd42431..6cfab8c9 100644 --- a/examples/HEMS/utils/ev_utils.py +++ b/examples/HEMS/utils/ev_utils.py @@ -32,122 +32,125 @@ def calculate_ev_soc_targets_and_constraints( """ Calculate dynamic SoC targets and availability constraints for EV charging. - Returns a dict with: - - soc_targets: List of target SoC values with datetimes - - soc_minima: List of minimum SoC constraints during unavailable periods - - consumption_capacity: Availability windows (0 during unavailable periods) + Build SoC and availability constraints for the next 24 hours. + + Driving is represented by two explicit SoC-usage periods: one after departure + and one before returning home. Keeping these separate from the SoC minima makes + rolling rescheduling preserve the remaining part of a trip. """ if capacity_kwh is None: capacity_kwh = EV_CONFIG["default_capacity_kwh"] print( - f"[EV-CALC] Calculating EV constraints for {current_time.strftime('%Y-%m-%d %H:%M')}" + "[EV-CALC] Calculating EV constraints for " + f"{current_time.strftime('%Y-%m-%d %H:%M')}" ) print(f" [CAPACITY] Battery capacity: {capacity_kwh} kWh") - needs_charging, departure_time_str, return_time_str, target_soc_percent = ( - get_day_pattern(current_time) - ) - - target_soc_kwh = (target_soc_percent / 100.0) * capacity_kwh min_soc_kwh = EV_CONFIG["min_soc_percent"] * capacity_kwh - - print(f" [TARGET] Target SoC: {target_soc_percent}% = {target_soc_kwh:.1f} kWh") print( - f" [MINIMUM] Minimum SoC: {EV_CONFIG['min_soc_percent']*100:.0f}% = {min_soc_kwh:.1f} kWh" + f" [MINIMUM] Minimum SoC: {EV_CONFIG['min_soc_percent']*100:.0f}% " + f"= {min_soc_kwh:.1f} kWh" ) constraints = { - "soc_targets": [], "soc_minima": [], + "soc_usage": [], "consumption_capacity": [], } + usage_segments = [] + planning_end = current_time + pd.Timedelta(hours=24) + commute_duration = pd.Timedelta(hours=EV_CONFIG["one_way_commute_duration_hours"]) + + def add_usage_segment(start: pd.Timestamp, end: pd.Timestamp) -> None: + """Add the part of a driving period that remains in the planning window.""" + start = max(start, current_time) + end = min(end, planning_end) + if start < end: + usage_segments.append( + { + "start": start.isoformat(), + "end": end.isoformat(), + "value": f'{EV_CONFIG["driving_consumption_kwh_per_hour"]} kW', + } + ) - if needs_charging and departure_time_str and return_time_str: - # Work day - need to be charged by departure time - print( - f" [WORK-DAY] Departure at {departure_time_str}, return at {return_time_str}" - ) - departure_hour, departure_minute = map(int, departure_time_str.split(":")) - return_hour, return_minute = map(int, return_time_str.split(":")) - - # Target: charged to 80% by departure time - departure_datetime = current_time.replace( - hour=departure_hour, minute=departure_minute, second=0, microsecond=0 + # Include both the remainder of today's pattern and tomorrow's pattern. This + # matters when replanning while an EV is away or shortly before tomorrow's + # departure. + for day_offset in (0, 1): + day = current_time.normalize() + pd.Timedelta(days=day_offset) + needs_charging, departure_time_str, return_time_str, target_soc_percent = ( + get_day_pattern(day) ) - - # If departure is already past today, target tomorrow - if departure_datetime <= current_time: - departure_datetime += pd.Timedelta(days=1) - print( - f" [SCHEDULE] Departure time adjusted to next day: {departure_datetime.strftime('%Y-%m-%d %H:%M')}" + target_soc_kwh = target_soc_percent / 100 * capacity_kwh + + if needs_charging and departure_time_str and return_time_str: + departure_hour, departure_minute = map(int, departure_time_str.split(":")) + return_hour, return_minute = map(int, return_time_str.split(":")) + departure_datetime = day.replace( + hour=departure_hour, + minute=departure_minute, + second=0, + microsecond=0, ) - else: - print( - f" [SCHEDULE] Departure time: {departure_datetime.strftime('%Y-%m-%d %H:%M')}" + return_datetime = day.replace( + hour=return_hour, + minute=return_minute, + second=0, + microsecond=0, ) - constraints["soc_minima"] = [ - { - "datetime": departure_datetime.isoformat(), - "value": f"{target_soc_kwh} kWh", - } - ] - print( - f" [MINIMUM-SET] SoC minimum set: {target_soc_kwh:.1f} kWh by {departure_datetime.strftime('%H:%M')}" - ) - - # Unavailable period: departure time to return time (same day as departure) - return_datetime = departure_datetime.replace( - hour=return_hour, minute=return_minute - ) - unavailable_duration = return_datetime - departure_datetime - - # Check if we are currently in the unavailable period - if current_time >= departure_datetime and current_time <= return_datetime: - print(" [UNAVAILABLE] Currently in unavailable period") - return_datetime += pd.Timedelta(days=1) print( - f" [UNAVAILABLE] Period: {departure_datetime.strftime('%H:%M')} - {return_datetime.strftime('%H:%M')} ({unavailable_duration})" + f" [WORK-DAY] {day.date()}: {target_soc_percent}% " + f"({target_soc_kwh:.1f} kWh) by {departure_time_str}; " + f"return at {return_time_str}" ) - constraints["unavailable"] = True + if current_time <= departure_datetime <= planning_end: + constraints["soc_minima"].append( + { + "datetime": departure_datetime.isoformat(), + "value": f"{target_soc_kwh} kWh", + } + ) + + unavailable_start = max(departure_datetime, current_time) + unavailable_end = min(return_datetime, planning_end) + if unavailable_start < unavailable_end: + constraints["consumption_capacity"].append( + { + "start": unavailable_start.isoformat(), + "end": unavailable_end.isoformat(), + "value": "0 kW", + } + ) + constraints["soc_minima"].append( + { + "start": unavailable_start.isoformat(), + "end": unavailable_end.isoformat(), + "value": f"{min_soc_kwh} kWh", + } + ) + + add_usage_segment(departure_datetime, departure_datetime + commute_duration) + add_usage_segment(return_datetime - commute_duration, return_datetime) else: + end_of_day = day + pd.Timedelta(days=1) + if current_time <= end_of_day <= planning_end: + constraints["soc_minima"].append( + { + "datetime": end_of_day.isoformat(), + "value": f"{target_soc_kwh} kWh", + } + ) print( - f" [UNAVAILABLE] Period: {departure_datetime.strftime('%H:%M')} - {return_datetime.strftime('%H:%M')} ({unavailable_duration})" + f" [FLEXIBLE-DAY] {day.date()}: maintain " + f"{target_soc_percent}% ({target_soc_kwh:.1f} kWh)" ) - # Disable charging during unavailable period by setting consumption capacity to 0 - constraints["consumption_capacity"] = [ - { - "start": departure_datetime.isoformat(), - "end": return_datetime.isoformat(), - "value": "0 kW", - } - ] - - # Extend minimum SoC constraint during unavailable period - constraints["soc_minima"].append( - { - "start": departure_datetime.isoformat(), - "end": return_datetime.isoformat(), - "value": f"{min_soc_kwh} kWh", - } - ) - print(" [DISABLED] Charging disabled during unavailable period (0 kW)") - print( - f" [MIN-SOC] Minimum SoC maintained: {min_soc_kwh:.1f} kWh during unavailable period" - ) - else: - # Free day - just maintain minimum SoC by end of planning horizon - print(f" [FREE-DAY] Flexible charging to {target_soc_percent}%") - end_of_day = current_time.replace(hour=23, minute=59, second=59, microsecond=0) - constraints["soc_minima"] = [ - {"datetime": end_of_day.isoformat(), "value": f"{target_soc_kwh} kWh"} - ] - print( - f" [FLEXIBLE] Minimum: {target_soc_kwh:.1f} kWh by end of day ({end_of_day.strftime('%H:%M')})" - ) - print(" [AVAILABLE] No availability restrictions - can charge anytime") + if usage_segments: + # soc-usage is a list of components; this component is a time series. + constraints["soc_usage"] = [usage_segments] # Handle random trips - reduce SoC randomly to simulate unplanned usage if has_random_trip: @@ -158,11 +161,15 @@ def calculate_ev_soc_targets_and_constraints( trip_consumption_kwh = trip_consumption_percent * capacity_kwh print( - f" [CONSUMPTION] Trip consumption: {trip_consumption_percent*100:.1f}% = {trip_consumption_kwh:.1f} kWh" + " [CONSUMPTION] Trip consumption: " + f"{trip_consumption_percent*100:.1f}% = " + f"{trip_consumption_kwh:.1f} kWh" ) - # Adjust minima to account for trip consumption + # Adjust point-in-time targets to account for the unexpected trip. for minimum in constraints["soc_minima"]: + if "datetime" not in minimum: + continue original_minimum_kwh = float(minimum["value"].split()[0]) # Ensure we charge enough to cover the trip consumption adjusted_minimum = min( @@ -170,7 +177,9 @@ def calculate_ev_soc_targets_and_constraints( ) minimum["value"] = f"{adjusted_minimum} kWh" print( - f" [ADJUSTED] Minimum: {original_minimum_kwh:.1f} kWh -> {adjusted_minimum:.1f} kWh (+{trip_consumption_kwh:.1f} kWh for trip)" + f" [ADJUSTED] Minimum: {original_minimum_kwh:.1f} kWh " + f"-> {adjusted_minimum:.1f} kWh " + f"(+{trip_consumption_kwh:.1f} kWh for trip)" ) print(" [SUMMARY] Final constraints:") @@ -187,15 +196,29 @@ def calculate_ev_soc_targets_and_constraints( start_dt = pd.to_datetime(minima["start"]) end_dt = pd.to_datetime(minima["end"]) print( - f" [MINIMUM] {minima['value']} from {start_dt.strftime('%H:%M')} to {end_dt.strftime('%H:%M')}" + f" [MINIMUM] {minima['value']} from " + f"{start_dt.strftime('%H:%M')} to {end_dt.strftime('%H:%M')}" ) if constraints["consumption_capacity"]: for capacity in constraints["consumption_capacity"]: start_dt = pd.to_datetime(capacity["start"]) end_dt = pd.to_datetime(capacity["end"]) print( - f" [DISABLED] Charging: {start_dt.strftime('%H:%M')} to {end_dt.strftime('%H:%M')} ({capacity['value']})" + f" [DISABLED] Charging: {start_dt.strftime('%H:%M')} " + f"to {end_dt.strftime('%H:%M')} ({capacity['value']})" ) + if usage_segments: + total_driving_hours = sum( + ( + pd.Timestamp(segment["end"]) - pd.Timestamp(segment["start"]) + ).total_seconds() + / 3600 + for segment in usage_segments + ) + print( + f" [DRIVING] {total_driving_hours:.1f} h at " + f'{EV_CONFIG["driving_consumption_kwh_per_hour"]:.1f} kW' + ) print() diff --git a/examples/HEMS/utils/scheduling_utils.py b/examples/HEMS/utils/scheduling_utils.py index 4b5fcef3..bec46b64 100644 --- a/examples/HEMS/utils/scheduling_utils.py +++ b/examples/HEMS/utils/scheduling_utils.py @@ -1,7 +1,5 @@ from typing import Any -from const import EV_CONFIG - def create_dynamic_storage_flex_model( current_soc: float, @@ -23,13 +21,9 @@ def create_dynamic_storage_flex_model( # Add dynamic constraints if provided if constraints: if constraints.get("soc_minima"): - # todo: here we remove the last soc_minima constraint and set it up as a soc_usage component instead - # this is a workaround; we should define the SoC drop during the trip as a soc_usage component straightaway - soc_usage = constraints["soc_minima"].pop(-1) flex_model["soc-minima"] = constraints["soc_minima"] - soc_usage["value"] = f'{EV_CONFIG["driving_consumption_kwh_per_hour"]} kW' - # add soc_usage as a component (soc-usage supports a list of usage components) - flex_model["soc-usage"] = [[soc_usage]] + if constraints.get("soc_usage"): + flex_model["soc-usage"] = constraints["soc_usage"] if constraints.get("consumption_capacity"): flex_model["consumption-capacity"] = constraints["consumption_capacity"] From 49654f9a179947762e39cde6d3fbbf962ae4bd0d Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Mon, 17 Aug 2026 23:40:30 +0100 Subject: [PATCH 12/21] fix(client): honor asset depth and sensor data filters HEMS cleanup relies on depth zero to select only top-level assets, but the client discarded that value as falsy. This could broaden cleanup queries and make nested name collisions unsafe. Preserve zero-valued asset query parameters and expose the server's source and event-time filters for sensor data deletion. Add regression coverage for both request payloads. Signed-off-by: Mohamed Belhsan Hmida --- src/flexmeasures_client/client.py | 28 +++++++++++++++++----- tests/client/test_asset.py | 19 +++++++++++++++ tests/client/test_sensor.py | 39 +++++++++++++++++++++++++++++++ 3 files changed, 80 insertions(+), 6 deletions(-) diff --git a/src/flexmeasures_client/client.py b/src/flexmeasures_client/client.py index b679b50a..ec5cb3cd 100644 --- a/src/flexmeasures_client/client.py +++ b/src/flexmeasures_client/client.py @@ -885,7 +885,7 @@ async def get_assets( if account_id and isinstance(account_id, int): uri += f"&account_id={account_id}" - if root or depth or fields: + if root is not None or depth is not None or fields: if self.server_version is not None and Version( self.server_version ) < Version("0.31.0"): @@ -893,9 +893,9 @@ async def get_assets( "get_assets(): The 'root', 'depth' and 'fields' parameters require FlexMeasures server version 0.31.0 or above. " f"These parameters will be ignored for server version {self.server_version}." ) - if root and isinstance(root, int): + if root is not None and isinstance(root, int): uri += f"&root={root}" - if depth and isinstance(depth, int): + if depth is not None and isinstance(depth, int): uri += f"&depth={depth}" if fields and isinstance(fields, list): fields_str = "|".join(fields) @@ -1406,9 +1406,17 @@ async def delete_sensor(self, sensor_id: int, confirm_first: bool = True): check_for_status(status, 204) async def delete_sensor_data( - self, sensor_id: int, confirm_first: bool = True + self, + sensor_id: int, + confirm_first: bool = True, + source: int | None = None, + start: str | datetime | None = None, + until: str | datetime | None = None, ) -> None: - """Delete all data from a sensor while preserving the sensor itself.""" + """Delete sensor data while preserving the sensor itself. + + Optionally limit deletion to one source and/or an event-time range. + """ if confirm_first: answer = input( f"Permanently delete all data from sensor {sensor_id}? [y/N] " @@ -1416,9 +1424,17 @@ async def delete_sensor_data( if answer.lower() not in ["y", "yes"]: print("Aborting ...") return + json_payload = {} + if source is not None: + json_payload["source"] = source + if start is not None: + json_payload["start"] = pd.Timestamp(start).isoformat() + if until is not None: + json_payload["until"] = pd.Timestamp(until).isoformat() + _, status = await self.request( uri=f"sensors/{sensor_id}/data", - json_payload={}, + json_payload=json_payload, method="DELETE", minimum_server_version="0.33.0", minimum_server_version_msg=( diff --git a/tests/client/test_asset.py b/tests/client/test_asset.py index edb52ea4..c5ae7b9f 100644 --- a/tests/client/test_asset.py +++ b/tests/client/test_asset.py @@ -204,6 +204,25 @@ async def test_get_assets_root_depth_fields_new_server(): await client.close() +@pytest.mark.asyncio +async def test_get_assets_includes_zero_depth(): + """depth=0 is meaningful and must be included in the URL.""" + with aioresponses() as m: + client = FlexMeasuresClient(email="test@test.test", password="test") + client.access_token = "test-token" + client.server_version = "0.31.0" + m.get( + "http://localhost:5000/api/v3_0/assets?all_accessible=False&sort_by=id&sort_dir=asc&include_public=False&depth=0", + status=200, + payload=[{"id": 1, "name": "top-level"}], + ) + + assets = await client.get_assets(depth=0, parse_json_fields=False) + + assert assets == [{"id": 1, "name": "top-level"}] + await client.close() + + @pytest.mark.asyncio async def test_get_assets_root_old_server_warning(caplog): """root param on server < 0.31.0 emits warning.""" diff --git a/tests/client/test_sensor.py b/tests/client/test_sensor.py index 2d2e443d..828c2333 100644 --- a/tests/client/test_sensor.py +++ b/tests/client/test_sensor.py @@ -369,6 +369,45 @@ async def test_delete_sensor_data_confirmation_declined(): await client.close() +@pytest.mark.asyncio +async def test_delete_sensor_data_with_filters(): + with aioresponses() as m: + client = FlexMeasuresClient(email="test@test.test", password="test") + client.access_token = "test-token" + client.server_version = "0.33.0" + m.delete( + "http://localhost:5000/api/v3_0/sensors/7/data", + status=204, + payload={}, + ) + + await client.delete_sensor_data( + sensor_id=7, + confirm_first=False, + source=3, + start="2030-01-01T00:00:00+00:00", + until="2030-01-02T00:00:00+00:00", + ) + + m.assert_called_once_with( + "http://localhost:5000/api/v3_0/sensors/7/data", + method="DELETE", + json={ + "source": 3, + "start": "2030-01-01T00:00:00+00:00", + "until": "2030-01-02T00:00:00+00:00", + }, + headers={ + "Content-Type": "application/json", + "Authorization": "test-token", + }, + params=None, + ssl=False, + allow_redirects=False, + ) + await client.close() + + @pytest.mark.asyncio async def test_post_sensor_data() -> None: with aioresponses() as m: From f6258e6c642fcbfb564b7bcc69b8aa0bc77a35b5 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Mon, 17 Aug 2026 23:40:37 +0100 Subject: [PATCH 13/21] fix(hems): make interrupted recovery explicit Interrupted asset creation, partial data wipes, and legacy site names could leave users with unsafe or destructive recovery paths. Global name lookups could also select assets outside the intended community or account. Make structure repair idempotent, preserve existing IDs, and offer explicit keep, rename, recreate, continue-wipe, and exit choices where applicable. Validate stored workflow state, scope asset and sensor resolution, retain canonical ingestion jobs, and document the recovery behavior. Signed-off-by: Mohamed Belhsan Hmida --- docs/HEMS.rst | 30 ++- examples/HEMS/HEMS_setup.py | 263 ++++++++++++++++---- examples/HEMS/assets_setup.py | 338 +++++++++++++++----------- examples/HEMS/const.py | 2 +- examples/HEMS/scheduling.py | 31 ++- examples/HEMS/utils/asset_utils.py | 115 +++++---- examples/HEMS/utils/workflow_utils.py | 73 ++++-- tests/examples/test_hems_workflow.py | 325 +++++++++++++++++++++++++ 8 files changed, 893 insertions(+), 284 deletions(-) create mode 100644 tests/examples/test_hems_workflow.py diff --git a/docs/HEMS.rst b/docs/HEMS.rst index e1824fa7..699c7a78 100644 --- a/docs/HEMS.rst +++ b/docs/HEMS.rst @@ -43,6 +43,8 @@ Or, alternatively, to install released versions into a fresh project: Next steps: - Follow instructions to set up flexmeasures (fresh database, etc). +- Use FlexMeasures 0.33.0 or newer. The tutorial's data-preserving wipe uses + the sensor-data deletion endpoint introduced in that version. - Create an organisation account and a user with the ``account-admin`` role: .. code-block:: bash @@ -68,8 +70,10 @@ example: .. code-block:: python - host = "ems.example.com" - ssl = True + host = "127.0.0.1:5000" + ssl = False + +For an HTTPS deployment, use its host name and set ``ssl = True``. Open three terminals. In the first terminal, run the server: @@ -77,11 +81,12 @@ Open three terminals. In the first terminal, run the server: flexmeasures run -In the second terminal, run a flexmeasures worker that listens to both the scheduling and forecasting queues: +In the second terminal, run a flexmeasures worker that listens to the +forecasting, scheduling, and ingestion queues: .. code-block:: bash - flexmeasures jobs run-worker --queue "forecasting|scheduling" + flexmeasures jobs run-worker --queue "forecasting|scheduling|ingestion" Note: you can run the same command in two terminals (2 workers), to speed up the computation! @@ -106,10 +111,12 @@ Another caveat is rate-limiting. Since v1.0, FlexMeasures only allows a limited Either give your account a generous plan (see the docs), or simply set ``FLEXMEASURES_MODE="play"`` and restart the server. If you use docker-compose, you could do that like this: +Add ``FLEXMEASURES_MODE = "play"`` to the existing +``/full/path/to/flexmeasures-instance/flexmeasures.cfg`` file without replacing +its other settings, then restart the server container: + .. code-block:: bash - sudo chown -R "$(id -u):$(id -g)" /full/path/to/flexmeasures-instance - printf 'FLEXMEASURES_MODE = "play"\n' > /full/path/to/flexmeasures-instance/flexmeasures.cfg docker compose restart name-of-flexmeasures-server-container Now run the client script using the `/examples/HEMS` folder as the current working directory: @@ -134,6 +141,17 @@ are complete and offers three choices: outputs. You must confirm this by typing ``WIPE``. - ``n`` (the default) preserves everything and resumes at the first unfinished phase. Completed phases are skipped. +- ``q`` exits without changing the setup. + +If an earlier data wipe was interrupted, normal resume is disabled because +some sensors may already be empty while others still contain old data. The +script instead offers to continue the wipe, recreate the setup, or exit. + +If asset creation was interrupted before the setup marker was saved, the +script offers to complete missing assets and sensors while preserving existing +IDs, recreate the setup, or exit. For an older setup with different site names, +it also offers to keep those names or rename the sites to the names configured +in ``const.py``. The workflow marker stores the exact sensor IDs created for the tutorial, so a data wipe remains limited to that recorded set. If an existing setup predates diff --git a/examples/HEMS/HEMS_setup.py b/examples/HEMS/HEMS_setup.py index 58d79ab1..1b31256f 100644 --- a/examples/HEMS/HEMS_setup.py +++ b/examples/HEMS/HEMS_setup.py @@ -19,10 +19,12 @@ PHASE_LABELS, REPORTING_PHASE, SCHEDULING_PHASE, - ensure_workflow_state, + get_site_assets, + get_workflow_state, initialize_workflow_state, mark_phase_complete, phase_is_complete, + rename_site_assets, wipe_hems_sensor_data, ) @@ -65,7 +67,8 @@ def prompt_for_existing_setup(account: dict, community_name: str, state: dict) - "(requires typing WIPE).\n" " [n] Resume — preserve everything and continue from the first " "unfinished phase.\n" - "Choice [y/w/N]: " + " [q] Exit without making changes.\n" + "Choice [y/w/N/q]: " ) .strip() .lower() @@ -74,22 +77,100 @@ def prompt_for_existing_setup(account: dict, community_name: str, state: dict) - return "recreate" if answer in {"w", "wipe"}: return "wipe" + if answer in {"q", "quit", "exit"}: + return "exit" if answer in {"", "n", "no"}: - if state.get("status") == "wiping": - print( - "A previous data wipe was interrupted. Choose 'w' to resume " - "the wipe or 'y' to recreate the assets." - ) - continue - if state.get("status") == "untracked": - print( - "This setup predates HEMS phase markers, so its completed " - "phases cannot be determined safely. Choose 'w' for fresh " - "data with the same IDs, or 'y' to recreate everything." - ) - continue return "resume" - print("Please choose 'y', 'w', or 'n'.") + print("Please choose 'y', 'w', 'n', or 'q'.") + + +def prompt_for_interrupted_wipe() -> str: + """Require an explicit recovery choice after a partial data wipe.""" + print( + "A previous data wipe was interrupted. Some sensor data may already " + "be deleted, so normal resume is not safe." + ) + while True: + answer = ( + input( + "\nChoose how to recover:\n" + " [c] Continue the interrupted wipe and preserve all IDs.\n" + " [y] Recreate assets, sensors, IDs, and data " + "(requires typing RECREATE).\n" + " [q] Exit without deleting anything else.\n" + "Choice [c/y/Q]: " + ) + .strip() + .lower() + ) + if answer in {"c", "continue"}: + return "continue-wipe" + if answer in {"y", "yes", "recreate"}: + return "recreate" + if answer in {"", "q", "quit", "exit", "n", "no"}: + return "exit" + print("Please choose 'c', 'y', or 'q'.") + + +def prompt_for_untracked_setup( + existing_site_names: list[str], configured_site_names: list[str] +) -> str: + """Choose how to recover a setup whose asset phase was not recorded.""" + legacy_names = any( + name not in configured_site_names for name in existing_site_names + ) + if legacy_names: + print( + "This setup uses different site names than the current tutorial:\n" + f"- Existing: {existing_site_names}\n" + f"- Configured: {configured_site_names}" + ) + while True: + answer = ( + input( + "\nChoose how to recover the asset structure:\n" + " [k] Keep the existing site names and complete missing items.\n" + " [m] Rename existing sites to the configured names and " + "complete missing items.\n" + " [y] Recreate the complete setup with new IDs " + "(requires typing RECREATE).\n" + " [q] Exit without making changes.\n" + "Choice [k/m/y/Q]: " + ) + .strip() + .lower() + ) + if answer in {"k", "keep"}: + return "keep-names" + if answer in {"m", "migrate", "rename"}: + return "rename-sites" + if answer in {"y", "yes", "recreate"}: + return "recreate" + if answer in {"", "q", "quit", "exit"}: + return "exit" + print("Please choose 'k', 'm', 'y', or 'q'.") + + while True: + answer = ( + input( + "The existing setup has no completed asset-setup marker and may " + "be incomplete.\n" + " [c] Complete missing assets and sensors, preserving existing IDs.\n" + " [y] Recreate the complete setup with new IDs " + "(requires typing RECREATE).\n" + " [q] Exit without making changes.\n" + "Choice [c/y/Q]: " + ) + .strip() + .lower() + ) + if answer in {"c", "continue", "complete", "repair"}: + return "repair" + if answer in {"y", "yes", "recreate"}: + return "recreate" + if answer in {"", "q", "quit", "exit"}: + return "exit" + print("Please choose 'c', 'y', or 'q'.") def confirm_recreation(account: dict, community_name: str) -> bool: @@ -148,8 +229,8 @@ async def main( f"Checking server is up and on supported version ... connecting to {host} (ssl: {ssl})" ) await client.ensure_minimum_server_version( - "0.31.0", - "The HEMS example requires a FlexMeasures server of v0.31.0 or above.", + "0.33.0", + "The HEMS example requires a FlexMeasures server of v0.33.0 or above.", ) # Get user account information @@ -161,48 +242,126 @@ async def main( account_id = account["id"] print(f"Connected to account: {account['name']} (ID: {account_id})") - asset = None - assets = await client.get_assets(parse_json_fields=True) - for sst in assets: - if sst["name"] == community_name and sst.get("account_id") == account_id: - asset = sst - break + active_site_names = list(site_names) + top_level_assets = await client.get_assets( + account_id=account_id, + depth=0, + fields=["id", "name", "account_id", "parent_asset_id", "attributes"], + parse_json_fields=True, + ) + matching_communities = [ + candidate + for candidate in top_level_assets + if candidate.get("name") == community_name + and candidate.get("account_id") == account_id + ] + if len(matching_communities) > 1: + raise LookupError( + f"Expected at most one top-level asset named '{community_name}', " + f"found {len(matching_communities)}." + ) + asset = matching_communities[0] if matching_communities else None if not asset: print( "Creating community Site asset with 2 building assets, each with PV and battery sensors, and weather station" ) asset = await create_community_asset( - client, account, community_name=community_name, site_names=site_names + client, + account, + community_name=community_name, + site_names=active_site_names, + ) + state = await initialize_workflow_state( + client, asset, account_id, active_site_names ) - state = await initialize_workflow_state(client, asset, account_id) else: - state = await ensure_workflow_state(client, asset, account_id) - action = prompt_for_existing_setup(account, community_name, state) - if action == "recreate": - if not confirm_recreation(account, community_name): - print("Recreation cancelled. No assets or data were deleted.") + existing_site_assets = await get_site_assets( + client, asset["id"], account_id + ) + existing_site_names = [site["name"] for site in existing_site_assets] + state = get_workflow_state(asset) + + if state is None or state.get("status") == "untracked": + action = prompt_for_untracked_setup(existing_site_names, site_names) + if action == "exit": + print("Exiting without making changes.") return - await delete_hems_assets( - client=client, - account_id=account["id"], - community_name=community_name, - confirm_first=False, + if action == "recreate": + if not confirm_recreation(account, community_name): + print("Recreation cancelled. No assets or data were deleted.") + return + await delete_hems_assets( + client=client, + account_id=account["id"], + community_name=community_name, + confirm_first=False, + ) + asset = await create_community_asset( + client, + account, + community_name=community_name, + site_names=active_site_names, + ) + else: + if action == "keep-names": + active_site_names = existing_site_names + elif action == "rename-sites": + await rename_site_assets( + client, existing_site_assets, active_site_names + ) + asset = await create_community_asset( + client, + account, + community_name=community_name, + site_names=active_site_names, + community_asset=asset, + ) + state = await initialize_workflow_state( + client, asset, account_id, active_site_names ) - asset = await create_community_asset( - client, - account, - community_name=community_name, - site_names=site_names, + else: + active_site_names = list( + state.get("site-names") or existing_site_names or site_names ) - state = await initialize_workflow_state(client, asset, account_id) - elif action == "wipe": - if not confirm_data_wipe(state): - print("Data wipe cancelled. No sensor data was deleted.") + if "site-names" not in state: + state = {**state, "site-names": active_site_names} + + if state.get("status") == "wiping": + action = prompt_for_interrupted_wipe() + else: + action = prompt_for_existing_setup(account, community_name, state) + + if action == "exit": + print("Exiting without making changes.") return - state = await wipe_hems_sensor_data(client, asset["id"], state) - else: - print("Resuming the existing HEMS setup.") + if action == "recreate": + if not confirm_recreation(account, community_name): + print("Recreation cancelled. No assets or data were deleted.") + return + active_site_names = list(site_names) + await delete_hems_assets( + client=client, + account_id=account["id"], + community_name=community_name, + confirm_first=False, + ) + asset = await create_community_asset( + client, + account, + community_name=community_name, + site_names=active_site_names, + ) + state = await initialize_workflow_state( + client, asset, account_id, active_site_names + ) + elif action in {"wipe", "continue-wipe"}: + if not confirm_data_wipe(state): + print("Data wipe cancelled. No additional data was deleted.") + return + state = await wipe_hems_sensor_data(client, asset["id"], state) + else: + print("Resuming the existing HEMS setup.") # Part 2: Upload data for first two weeks print("\n" + "=" * 50) @@ -211,7 +370,7 @@ async def main( else: print("PART 2: UPLOADING DATA") await upload_data_for_first_two_weeks( - client, community_name=community_name, site_names=site_names + client, community_name=community_name, site_names=active_site_names ) state = await mark_phase_complete( client, asset["id"], state, DATA_UPLOAD_PHASE @@ -224,7 +383,7 @@ async def main( else: print("PART 3: GENERATING PV FORECASTS") await generate_forecasts( - client, community_name=community_name, site_names=site_names + client, community_name=community_name, site_names=active_site_names ) state = await mark_phase_complete( client, asset["id"], state, FORECASTING_PHASE @@ -239,7 +398,7 @@ async def main( scheduling_succeeded = await run_scheduling_simulation( client, community_name=community_name, - site_names=site_names, + site_names=active_site_names, callback=callback, ) if not scheduling_succeeded: @@ -256,7 +415,7 @@ async def main( print("PART 5: CREATING REPORTS") # todo B2: compute aggregate power flow for the community asset's power sensor reports_succeeded = await create_reports( - client, community_name=community_name, site_names=site_names + client, community_name=community_name, site_names=active_site_names ) if not reports_succeeded: raise RuntimeError("Report generation did not complete.") diff --git a/examples/HEMS/assets_setup.py b/examples/HEMS/assets_setup.py index 37ed9b2f..a9868475 100644 --- a/examples/HEMS/assets_setup.py +++ b/examples/HEMS/assets_setup.py @@ -12,11 +12,81 @@ pv_name, weather_station_name, ) -from utils.asset_utils import get_first_asset_by_name - from flexmeasures_client import FlexMeasuresClient +async def get_or_create_asset( + client: FlexMeasuresClient, + *, + name: str, + account_id: int, + generic_asset_type_id: int, + parent_asset_id: int | None = None, +) -> dict: + """Return one exact asset or create it in the requested hierarchy position.""" + assets = await client.get_assets( + account_id=account_id, + fields=["id", "name", "account_id", "parent_asset_id"], + parse_json_fields=False, + ) + matches = [ + asset + for asset in assets + if asset.get("name") == name + and asset.get("account_id") == account_id + and asset.get("parent_asset_id") == parent_asset_id + ] + if len(matches) > 1: + raise LookupError( + f"Expected at most one asset named '{name}' under parent " + f"{parent_asset_id}, found {len(matches)}." + ) + if matches: + print(f"Reusing asset '{name}' with ID {matches[0]['id']}") + return matches[0] + return await client.add_asset( + name=name, + latitude=latitude, + longitude=longitude, + generic_asset_type_id=generic_asset_type_id, + account_id=account_id, + parent_asset_id=parent_asset_id, + ) + + +async def get_or_create_sensor( + client: FlexMeasuresClient, + *, + name: str, + event_resolution: str, + unit: str, + generic_asset_id: int, + timezone: str | None = "Europe/Amsterdam", + attributes: dict | None = None, +) -> dict: + """Return one exact sensor on an asset or create the missing sensor.""" + sensors = await client.get_sensors( + asset_id=generic_asset_id, parse_json_fields=False + ) + matches = [sensor for sensor in sensors if sensor.get("name") == name] + if len(matches) > 1: + raise LookupError( + f"Expected at most one sensor named '{name}' on asset " + f"{generic_asset_id}, found {len(matches)}." + ) + if matches: + print(f"Reusing sensor '{name}' with ID {matches[0]['id']}") + return matches[0] + return await client.add_sensor( + name=name, + event_resolution=event_resolution, + unit=unit, + generic_asset_id=generic_asset_id, + timezone=timezone, + attributes=attributes, + ) + + async def get_or_create_price_sensor(client: FlexMeasuresClient): """Get or create an account-owned price sensor (1h, EUR/kWh). @@ -27,34 +97,19 @@ async def get_or_create_price_sensor(client: FlexMeasuresClient): account = await client.get_account() account_id = account["id"] print(f"Account ID: {account_id}") - # Create a top-level market asset in the current account. - # Generic asset type 8 is typically used for market/price assets - all_top_level_assets = await client.get_assets( - depth=0, - fields=["id", "name", "account_id", "sensors"], - ) - price_market_asset = get_first_asset_by_name( - assets=all_top_level_assets, name=price_market_name, account_id=account_id - ) - if price_market_asset is None: - price_market_asset = await client.add_asset( - name=price_market_name, - latitude=latitude, - longitude=longitude, - generic_asset_type_id=8, # Transmission zone A grid regulated & balanced as a whole, usually a national grid. - account_id=account_id, - ) - - # Create price sensor with 1-hour resolution - price_sensor = await client.add_sensor( - name="electricity-price", - event_resolution="PT1H", - unit="EUR/kWh", - generic_asset_id=price_market_asset["id"], - timezone="Europe/Amsterdam", - ) - else: - price_sensor = price_market_asset["sensors"][0] + price_market_asset = await get_or_create_asset( + client, + name=price_market_name, + account_id=account_id, + generic_asset_type_id=8, + ) + price_sensor = await get_or_create_sensor( + client, + name="electricity-price", + event_resolution="PT1H", + unit="EUR/kWh", + generic_asset_id=price_market_asset["id"], + ) print(f"Price sensor ID: {price_sensor['id']}") return price_sensor @@ -67,55 +122,26 @@ async def get_or_create_weather_station(client: FlexMeasuresClient): account = await client.get_account() account_id = account["id"] print(f"Account ID: {account_id}") - # Create a top-level weather station asset in the current account. - # Generic asset type 7 (process) used for weather stations since no dedicated type exists - # TODO: remove hard-coded ID, we should actually create a weather station type somehow - all_top_level_assets = await client.get_assets( - depth=0, - fields=["id", "name", "account_id", "sensors"], - ) - weather_asset = get_first_asset_by_name( - assets=all_top_level_assets, name=weather_station_name, account_id=account_id - ) - if weather_asset is None: - weather_asset = await client.add_asset( - name=weather_station_name, - latitude=latitude, - longitude=longitude, - generic_asset_type_id=7, # Process asset type (for weather station) - account_id=account_id, - ) - - # Create irradiation sensor (1H, W/m²) - irradiation_sensor = await client.add_sensor( - name="irradiation", - event_resolution="PT1H", - unit="W/m²", - generic_asset_id=weather_asset["id"], - timezone="Europe/Amsterdam", - ) - - # Create cloud coverage sensor (1H, %) - cloud_coverage_sensor = await client.add_sensor( - name="cloud-coverage", - event_resolution="PT1H", - unit="%", - generic_asset_id=weather_asset["id"], - timezone="Europe/Amsterdam", - ) - else: - sensors = weather_asset["sensors"] - cloud_coverage_sensor = None - irradiation_sensor = None - for sensor in sensors: - if sensor["name"] == "cloud-coverage": - cloud_coverage_sensor = sensor - elif sensor["name"] == "irradiation": - irradiation_sensor = sensor - if cloud_coverage_sensor is None or irradiation_sensor is None: - raise ValueError( - "Could not identify cloud_coverage_sensor or irradiation_sensor. Maybe a name changed?" - ) + weather_asset = await get_or_create_asset( + client, + name=weather_station_name, + account_id=account_id, + generic_asset_type_id=7, + ) + irradiation_sensor = await get_or_create_sensor( + client, + name="irradiation", + event_resolution="PT1H", + unit="W/m²", + generic_asset_id=weather_asset["id"], + ) + cloud_coverage_sensor = await get_or_create_sensor( + client, + name="cloud-coverage", + event_resolution="PT1H", + unit="%", + generic_asset_id=weather_asset["id"], + ) print(f"Created weather station with ID: {weather_asset['id']}") return weather_asset, irradiation_sensor, cloud_coverage_sensor @@ -132,17 +158,17 @@ async def create_site_asset( print("Creating Site asset...") # Create site asset (generic_asset_type_id=6 for building) - site_asset = await client.add_asset( + site_asset = await get_or_create_asset( + client, name=site_name, - latitude=latitude, - longitude=longitude, parent_asset_id=site_asset_id, - generic_asset_type_id=6, # Building asset type + generic_asset_type_id=6, account_id=account_id, ) # Create general consumption sensor (15min resolution, kW) - consumption_sensor = await client.add_sensor( + consumption_sensor = await get_or_create_sensor( + client, name="electricity-consumption", event_resolution="PT15M", unit="kW", @@ -152,7 +178,8 @@ async def create_site_asset( ) # Create energy costs KPI sensor (1D resolution, EUR) - energy_costs_sensor = await client.add_sensor( + energy_costs_sensor = await get_or_create_sensor( + client, name="energy-costs-kpi", event_resolution="P1D", unit="EUR", @@ -161,7 +188,8 @@ async def create_site_asset( ) # Create aggregate power sensor for the site - aggregate_sensor = await client.add_sensor( + aggregate_sensor = await get_or_create_sensor( + client, name="electricity-aggregate", event_resolution="PT15M", unit="kW", @@ -171,7 +199,8 @@ async def create_site_asset( ) # Create max production capacity sensor for the site - max_production_sensor = await client.add_sensor( + max_production_sensor = await get_or_create_sensor( + client, name="max-production-capacity", event_resolution="PT1H", unit="kW", @@ -181,7 +210,8 @@ async def create_site_asset( ) # Create max consumption capacity sensor for the site - max_consumption_sensor = await client.add_sensor( + max_consumption_sensor = await get_or_create_sensor( + client, name="max-consumption-capacity", event_resolution="PT1H", unit="kW", @@ -191,7 +221,8 @@ async def create_site_asset( ) # Create site-peak-consumption-price sensor (15min resolution, EUR/kW) - site_peak_consumption_price_sensor = await client.add_sensor( # noqa: F841 + site_peak_consumption_price_sensor = await get_or_create_sensor( + client, name="site-peak-consumption-price", event_resolution="PT15M", unit="EUR/kW", @@ -200,7 +231,8 @@ async def create_site_asset( ) # Create site-peak-production-price sensor (15min resolution, EUR/kW) - site_peak_production_price_sensor = await client.add_sensor( # noqa: F841 + site_peak_production_price_sensor = await get_or_create_sensor( + client, name="site-peak-production-price", event_resolution="PT15M", unit="EUR/kW", @@ -209,7 +241,8 @@ async def create_site_asset( ) # Create self-consumption sensor for the site - self_consumption_sensor = await client.add_sensor( + self_consumption_sensor = await get_or_create_sensor( + client, name="self-consumption", event_resolution="PT15M", unit="kW", @@ -219,7 +252,8 @@ async def create_site_asset( ) # Create total energy costs sensor for the site - total_energy_costs_sensor = await client.add_sensor( + total_energy_costs_sensor = await get_or_create_sensor( + client, name="total-energy-costs", event_resolution="PT15M", unit="EUR", @@ -228,7 +262,8 @@ async def create_site_asset( ) # Create daily total energy costs sensor for the site - daily_total_energy_costs_sensor = await client.add_sensor( + daily_total_energy_costs_sensor = await get_or_create_sensor( + client, name="daily-total-energy-costs", event_resolution="P1D", unit="EUR", @@ -237,7 +272,8 @@ async def create_site_asset( ) # Create daily share of self-consumption sensor for the site - daily_share_of_self_consumption_sensor = await client.add_sensor( + daily_share_of_self_consumption_sensor = await get_or_create_sensor( + client, name="daily-share-of-self-consumption", event_resolution="P1D", unit="%", @@ -269,17 +305,17 @@ async def create_pv_asset( print("Creating PV asset...") # Create PV asset (generic_asset_type_id=1 for solar/PV) - pv_asset = await client.add_asset( + pv_asset = await get_or_create_asset( + client, name=pv_name, - latitude=latitude, - longitude=longitude, - generic_asset_type_id=1, # Solar/PV asset type + generic_asset_type_id=1, account_id=account_id, - parent_asset_id=site_asset_id, # Child of site + parent_asset_id=site_asset_id, ) # Create production sensor (15min, kW) - pv_production_sensor = await client.add_sensor( # to store raw generation values + pv_production_sensor = await get_or_create_sensor( + client, name="electricity-production", event_resolution="PT15M", unit="kW", @@ -288,7 +324,8 @@ async def create_pv_asset( ) # Create power sensor (15min, kW) - pv_power_sensor = await client.add_sensor( # to store realized generation values + pv_power_sensor = await get_or_create_sensor( + client, name="electricity-power", event_resolution="PT15M", unit="kW", @@ -310,17 +347,17 @@ async def create_battery_asset( print("Creating battery asset...") # Create battery asset (generic_asset_type_id=5 for battery) - battery_asset = await client.add_asset( + battery_asset = await get_or_create_asset( + client, name=battery_name, - latitude=latitude, - longitude=longitude, - generic_asset_type_id=5, # Battery asset type + generic_asset_type_id=5, account_id=account_id, - parent_asset_id=site_asset_id, # Child of site + parent_asset_id=site_asset_id, ) # Create power sensor (15min, kW) - battery_power_sensor = await client.add_sensor( + battery_power_sensor = await get_or_create_sensor( + client, name="electricity-power", event_resolution="PT15M", unit="kW", @@ -330,7 +367,8 @@ async def create_battery_asset( ) # Create state-of-charge sensor (0min, kWh) - battery_soc_sensor = await client.add_sensor( + battery_soc_sensor = await get_or_create_sensor( + client, name="state-of-charge", event_resolution="PT0M", unit="kWh", @@ -374,17 +412,17 @@ async def create_evse_asset( # Create EVSE asset - using generic type 4 for one-way EVSE based on the codebase search # Note: We'll use a basic asset type since one-way_evse might not be available by default - evse_asset = await client.add_asset( + evse_asset = await get_or_create_asset( + client, name=evse_name, - latitude=latitude, - longitude=longitude, - generic_asset_type_id=4, # Using a generic type, could be EVSE specific if available + generic_asset_type_id=4, account_id=account_id, - parent_asset_id=site_asset_id, # Child of site + parent_asset_id=site_asset_id, ) # Create power sensor (15min, kW) - evse_power_sensor = await client.add_sensor( + evse_power_sensor = await get_or_create_sensor( + client, name="electricity-power", event_resolution="PT15M", unit="kW", @@ -394,7 +432,8 @@ async def create_evse_asset( ) # Create state-of-charge sensor (instantaneous, kWh) - evse_soc_sensor = await client.add_sensor( + evse_soc_sensor = await get_or_create_sensor( + client, name="state-of-charge", event_resolution="PT0M", unit="kWh", @@ -403,7 +442,8 @@ async def create_evse_asset( ) # Create soc-min sensor (15min, kWh) - evse_soc_min_sensor = await client.add_sensor( + evse_soc_min_sensor = await get_or_create_sensor( + client, name="soc-min", event_resolution="PT15M", unit="kWh", @@ -412,7 +452,8 @@ async def create_evse_asset( ) # Create soc-max sensor (15min, kWh) - evse_soc_max_sensor = await client.add_sensor( + evse_soc_max_sensor = await get_or_create_sensor( + client, name="soc-max", event_resolution="PT15M", unit="kWh", @@ -488,17 +529,17 @@ async def create_heating_asset( print(f"Creating heating asset: {heating_name}...") # Create heating asset (generic asset type id = 5 if heating not defined in DB) - heating_asset = await client.add_asset( + heating_asset = await get_or_create_asset( + client, name=heating_name, - latitude=latitude, - longitude=longitude, - generic_asset_type_id=5, # Using battery type as placeholder for heating asset + generic_asset_type_id=5, account_id=account_id, parent_asset_id=site_asset_id, ) # Power sensors (15min, kW) - heating_power_sensor = await client.add_sensor( + heating_power_sensor = await get_or_create_sensor( + client, name="power", event_resolution="PT15M", unit="kW", @@ -508,7 +549,8 @@ async def create_heating_asset( ) # Soc usage sensor (15min, kW) - heating_soc_usage_sensor = await client.add_sensor( + heating_soc_usage_sensor = await get_or_create_sensor( + client, name="soc-usage", event_resolution="PT15M", unit="kW", @@ -518,21 +560,24 @@ async def create_heating_asset( ) # State of Charge sensors (15min, kWh) - heating_soc_sensor = await client.add_sensor( + heating_soc_sensor = await get_or_create_sensor( + client, name="state of charge", event_resolution="PT0M", unit="kWh", generic_asset_id=heating_asset["id"], timezone="Europe/Amsterdam", ) - heating_min_soc_sensor = await client.add_sensor( + heating_min_soc_sensor = await get_or_create_sensor( + client, name="min SoC", event_resolution="PT15M", unit="kWh", generic_asset_id=heating_asset["id"], timezone="Europe/Amsterdam", ) - heating_max_soc_sensor = await client.add_sensor( + heating_max_soc_sensor = await get_or_create_sensor( + client, name="max SoC", event_resolution="PT15M", unit="kWh", @@ -541,7 +586,8 @@ async def create_heating_asset( ) # COP (Coefficient of Performance) - heating_COP = await client.add_sensor( + heating_COP = await get_or_create_sensor( + client, name="COP", event_resolution="PT15M", unit="%", @@ -928,8 +974,9 @@ async def create_community_asset( account: dict, community_name: str, site_names: list[str], + community_asset: dict | None = None, ): - """Create an asset representing a community, which will serve as the parent asset for all sites in the community.""" + """Create or complete the HEMS asset structure without replacing existing IDs.""" # Get account id account_id = account["id"] print("Getting or creating price market asset and associated price sensor") @@ -943,32 +990,35 @@ async def create_community_asset( print(f"Irradiation sensor ID: {irradiation_sensor['id']}") print(f"Cloud coverage sensor ID: {cloud_coverage_sensor['id']}") print("Creating community asset...") - # Create Site asset (generic_asset_type_id=6 for building) - site_asset = await client.add_asset( - name=community_name, - latitude=latitude, - longitude=longitude, - generic_asset_type_id=6, # Building asset type - account_id=account_id, - ) + if community_asset is None: + community_asset = await get_or_create_asset( + client, + name=community_name, + generic_asset_type_id=6, + account_id=account_id, + ) + elif community_asset.get("parent_asset_id") is not None: + raise ValueError("The HEMS community asset must be a top-level asset.") # Create site power capacity sensor (15min resolution, kW) - site_power_capacity_sensor = await client.add_sensor( + site_power_capacity_sensor = await get_or_create_sensor( + client, name="site-power-capacity", event_resolution="PT15M", unit="kW", - generic_asset_id=site_asset["id"], + generic_asset_id=community_asset["id"], timezone="Europe/Amsterdam", attributes=dict(consumption_is_positive=True), ) # Create site power sensor (15min resolution, kW) # this is used to store aggregate assets power measurements - site_power_sensor = await client.add_sensor( # noqa: F841 + site_power_sensor = await get_or_create_sensor( # noqa: F841 + client, name="power", event_resolution="PT15M", unit="kW", - generic_asset_id=site_asset["id"], + generic_asset_id=community_asset["id"], timezone="Europe/Amsterdam", attributes=dict(consumption_is_positive=True), ) @@ -977,19 +1027,19 @@ async def create_community_asset( flex_context = { "site-power-capacity": {"sensor": site_power_capacity_sensor["id"]}, } - print(f"Created site asset with ID: {site_asset['id']}") + print(f"Community asset ID: {community_asset['id']}") # Update site asset with flex-context await client.update_asset( - asset_id=site_asset["id"], updates={"flex_context": flex_context} + asset_id=community_asset["id"], updates={"flex_context": flex_context} ) for i in range(len(site_names)): await create_sites_assets_and_sensors( client=client, account=account, - community_asset_id=site_asset["id"], + community_asset_id=community_asset["id"], site_index=i + 1, site_names=site_names, price_sensor=price_sensor, ) - return site_asset + return community_asset diff --git a/examples/HEMS/const.py b/examples/HEMS/const.py index 3d24fece..0e439abb 100644 --- a/examples/HEMS/const.py +++ b/examples/HEMS/const.py @@ -7,7 +7,7 @@ usr = "hems-admin@example.com" # Account-admin user email pwd = "change-me" # Account-admin user password host = "127.0.0.1:5000" # FlexMeasures host, without http:// or https:// -ssl = False # Use HTTPS (and port 443 unless host specifies another port) +ssl = False # Local development server uses HTTP; set True for HTTPS deployments # Asset and sensor names COMMUNITY_NAME = "Community Site" diff --git a/examples/HEMS/scheduling.py b/examples/HEMS/scheduling.py index 4a3c1cb8..b205c2f1 100644 --- a/examples/HEMS/scheduling.py +++ b/examples/HEMS/scheduling.py @@ -860,12 +860,22 @@ async def get_site_assets( index: int, ): """Get all assets in a site's child building.""" + community_asset_id = await find_top_level_asset_id(client, community_name) + community_asset = await client.get_asset(community_asset_id, parse_json_fields=True) assets = await client.get_assets( - fields=["id", "name", "attributes", "sensors"], parse_json_fields=True - ) - assets_by_name = {a["name"]: a for a in assets} + root=community_asset_id, + fields=["id", "name", "attributes", "sensors", "parent_asset_id"], + parse_json_fields=True, + ) + assets_by_name: dict[str, dict] = {community_name: community_asset} + for asset in assets: + if asset["name"] in assets_by_name: + raise LookupError( + f"Asset name '{asset['name']}' is ambiguous in community " + f"'{community_name}'." + ) + assets_by_name[asset["name"]] = asset - community_asset = assets_by_name.get(community_name) site_asset = assets_by_name.get(site_name) battery_asset = assets_by_name.get(f"{battery_name} {index}") evse1_asset = assets_by_name.get(f"{evse1_name} {index}") @@ -928,14 +938,13 @@ async def map_site_sensors( for sensor_key, asset_name, sensor_name in sensor_mappings: sensor = await find_sensor_by_name_and_asset( - client, sensor_name, asset_name, top_level_asset_id=top_level_asset_id + client, + sensor_name, + asset_name, + top_level_asset_id=top_level_asset_id, + allow_top_level_asset=asset_name == price_market_name, ) - if sensor: - sensors[sensor_key] = sensor - else: - raise LookupError( - f"Could not find sensor '{sensor_name}' in asset '{asset_name}'" - ) + sensors[sensor_key] = sensor return sensors diff --git a/examples/HEMS/utils/asset_utils.py b/examples/HEMS/utils/asset_utils.py index ac9a7113..cb512019 100644 --- a/examples/HEMS/utils/asset_utils.py +++ b/examples/HEMS/utils/asset_utils.py @@ -17,22 +17,12 @@ async def post_sensor_data_and_track_ingestion( **kwargs: Any, ) -> None: """Post sensor data and remember asynchronous ingestion jobs.""" - result = await client.post_sensor_data(**kwargs) - if not isinstance(result, tuple) or len(result) != 2: - raise RuntimeError( - "This HEMS example requires a FlexMeasures client whose " - "post_sensor_data() method returns the server response and status." - ) - response, status = result + response, status = await client.post_sensor_data(**kwargs) if status != 202: return - job_id = None - if isinstance(response, dict): - # ``job`` is the canonical field. ``job_id`` was used by older - # FlexMeasures servers and is retained here for compatibility. - job_id = response.get("job") or response.get("job_id") + job_id = response.get("job") if isinstance(response, dict) else None if not job_id: raise RuntimeError( "The server accepted sensor data for asynchronous ingestion " @@ -96,29 +86,51 @@ async def find_sensor_by_name_and_asset( sensor_name: str, asset_name: str, top_level_asset_id: int | None = None, + allow_top_level_asset: bool = False, ): - """Find a sensor by name within a specific asset.""" - assets = await client.get_assets( - root=top_level_asset_id - ) # first list those that are part of the community - assets += await client.get_assets( - parse_json_fields=True - ) # then list all accessible assets - target_asset = None - for asset in assets: - if asset["name"] == asset_name: - target_asset = asset - break + """Find one sensor in the community tree or an explicitly allowed root.""" + if top_level_asset_id is None: + raise ValueError("top_level_asset_id is required for scoped sensor lookup") + + community_asset = await client.get_asset(top_level_asset_id, parse_json_fields=True) + account_id = community_asset.get("account_id") + if not isinstance(account_id, int): + account = await client.get_account() + account_id = account["id"] + assets = [community_asset] + assets.extend( + await client.get_assets(root=top_level_asset_id, parse_json_fields=True) + ) + if allow_top_level_asset: + assets.extend( + await client.get_assets( + account_id=account_id, depth=0, parse_json_fields=True + ) + ) - if not target_asset: + assets_by_id = {asset["id"]: asset for asset in assets} + matches = [ + asset for asset in assets_by_id.values() if asset.get("name") == asset_name + ] + if not matches: raise LookupError(f"Asset '{asset_name}' not found") + if len(matches) > 1: + raise LookupError( + f"Asset name '{asset_name}' is ambiguous in the allowed HEMS scope." + ) + target_asset = matches[0] - sensors = await client.get_sensors(asset_id=target_asset["id"]) - for sensor in sensors: - if sensor["name"] == sensor_name: - return sensor - - raise LookupError(f"Sensor '{sensor_name}' not found in asset '{asset_name}'") + sensors = await client.get_sensors( + asset_id=target_asset["id"], parse_json_fields=True + ) + matches = [sensor for sensor in sensors if sensor.get("name") == sensor_name] + if not matches: + raise LookupError(f"Sensor '{sensor_name}' not found in asset '{asset_name}'") + if len(matches) > 1: + raise LookupError( + f"Sensor name '{sensor_name}' is ambiguous on asset '{asset_name}'." + ) + return matches[0] async def upload_csv_file_to_sensor( @@ -139,7 +151,6 @@ async def upload_csv_file_to_sensor( belief_time_measured_instantly=belief_time_measured_instantly, # Set belief_time immediately after event ends ) print(f"Submitted {file_path} to sensor {sensor_id}") - return True except Exception as e: print(f"Failed to upload {file_path} to sensor {sensor_id}: {e}") raise @@ -149,12 +160,19 @@ async def find_top_level_asset_id( client: FlexMeasuresClient, name: str, ) -> int: + account = await client.get_account() top_level_assets = await client.get_assets( - depth=0, fields=["id", "name"], parse_json_fields=True + account_id=account["id"], + depth=0, + fields=["id", "name"], + parse_json_fields=True, ) - for asset in top_level_assets: - if asset["name"] == name: - return asset["id"] + matches = [asset for asset in top_level_assets if asset["name"] == name] + if len(matches) != 1: + raise LookupError( + f"Expected one top-level asset named '{name}', found {len(matches)}." + ) + return matches[0]["id"] async def find_sensors_by_asset( @@ -170,14 +188,14 @@ async def find_sensors_by_asset( sensors = {} for key, sensor_name, asset_name in sensor_mappings: sensor = await find_sensor_by_name_and_asset( - client, sensor_name, asset_name, top_level_asset_id + client, + sensor_name, + asset_name, + top_level_asset_id, + allow_top_level_asset=asset_name + in {price_market_name, weather_station_name}, ) - if sensor: - sensors[key] = sensor - else: - raise LookupError( - f"Could not find sensor '{sensor_name}' in asset '{asset_name}'" - ) + sensors[key] = sensor return sensors @@ -220,14 +238,10 @@ async def upload_data_for_first_two_weeks( 2: ] # Remove site power capacity and price datafiles to not fill them more than once for file_path, sensor_key, belief_time_measured_instantly in data_files: - if sensor_key not in sensors: - print(f"Skipping {file_path} - sensor not found") - continue - print(f"Processing {file_path}...") # Upload CSV file directly - success = await upload_csv_file_to_sensor( + await upload_csv_file_to_sensor( client=client, sensor_id=sensors[sensor_key]["id"], file_path=file_path, @@ -235,10 +249,7 @@ async def upload_data_for_first_two_weeks( pending_ingestion_jobs=pending_ingestion_jobs, ) - if success: - print(f"Successfully uploaded {sensor_key} data") - else: - print(f"Failed to upload {sensor_key} data") + print(f"Submitted {sensor_key} data for ingestion") # File uploads may only have been accepted (HTTP 202), not processed yet. # Forecasting must not start until all historical data is available. diff --git a/examples/HEMS/utils/workflow_utils.py b/examples/HEMS/utils/workflow_utils.py index 56e7fd49..20aa682c 100644 --- a/examples/HEMS/utils/workflow_utils.py +++ b/examples/HEMS/utils/workflow_utils.py @@ -42,9 +42,62 @@ def get_workflow_state(community_asset: dict) -> dict | None: return None if not isinstance(state.get("sensor-ids"), list): return None + if not all(isinstance(sensor_id, int) for sensor_id in state["sensor-ids"]): + return None + if not isinstance(state.get("top-level-asset-ids"), list): + return None + if not all(isinstance(asset_id, int) for asset_id in state["top-level-asset-ids"]): + return None + if "site-names" in state and not ( + isinstance(state["site-names"], list) + and all(isinstance(name, str) for name in state["site-names"]) + ): + return None return state +async def get_site_assets( + client: FlexMeasuresClient, + community_asset_id: int, + account_id: int, +) -> list[dict]: + """Return the community's direct child sites in stable creation order.""" + assets = await client.get_assets( + account_id=account_id, + fields=["id", "name", "account_id", "parent_asset_id"], + parse_json_fields=False, + ) + return sorted( + ( + asset + for asset in assets + if asset.get("account_id") == account_id + and asset.get("parent_asset_id") == community_asset_id + ), + key=lambda asset: asset["id"], + ) + + +async def rename_site_assets( + client: FlexMeasuresClient, + site_assets: list[dict], + site_names: list[str], +) -> None: + """Rename existing sites in stable order while preserving their IDs.""" + if len(site_assets) > len(site_names): + raise ValueError( + f"Cannot map {len(site_assets)} existing sites to " + f"{len(site_names)} configured site names." + ) + for site_asset, site_name in zip(site_assets, site_names): + if site_asset["name"] != site_name: + await client.update_asset( + asset_id=site_asset["id"], + updates={"name": site_name}, + parse_json_fields=False, + ) + + async def save_workflow_state( client: FlexMeasuresClient, community_asset_id: int, @@ -117,6 +170,7 @@ async def initialize_workflow_state( client: FlexMeasuresClient, community_asset: dict, account_id: int, + site_names: list[str], status: str = "ready", ) -> dict: """Create the workflow marker after the complete asset structure exists.""" @@ -131,28 +185,11 @@ async def initialize_workflow_state( "completed-phases": [ASSET_SETUP_PHASE], "top-level-asset-ids": top_level_asset_ids, "sensor-ids": sensor_ids, + "site-names": list(site_names), } return await save_workflow_state(client, community_asset["id"], state) -async def ensure_workflow_state( - client: FlexMeasuresClient, - community_asset: dict, - account_id: int, -) -> dict: - """Return existing workflow state or initialize legacy HEMS assets.""" - state = get_workflow_state(community_asset) - if state is not None: - return state - print( - "No compatible HEMS phase marker exists yet. Treating the asset " - "structure as complete; data phases are not assumed to be complete." - ) - return await initialize_workflow_state( - client, community_asset, account_id, status="untracked" - ) - - def phase_is_complete(state: dict, phase: str) -> bool: return phase in state["completed-phases"] diff --git a/tests/examples/test_hems_workflow.py b/tests/examples/test_hems_workflow.py new file mode 100644 index 00000000..cc26ffc3 --- /dev/null +++ b/tests/examples/test_hems_workflow.py @@ -0,0 +1,325 @@ +from __future__ import annotations + +import sys +from pathlib import Path +from unittest.mock import AsyncMock, call, patch + +import pytest + +HEMS_DIR = Path(__file__).parents[2] / "examples" / "HEMS" +sys.path.insert(0, str(HEMS_DIR)) + +from HEMS_setup import ( # noqa: E402 + prompt_for_interrupted_wipe, + prompt_for_untracked_setup, +) +from assets_setup import ( # noqa: E402 + create_community_asset, + get_or_create_asset, + get_or_create_sensor, +) +from utils.asset_utils import find_sensor_by_name_and_asset # noqa: E402 +from utils.workflow_utils import ( # noqa: E402 + ASSET_SETUP_PHASE, + get_site_assets, + get_workflow_state, + rename_site_assets, + wipe_hems_sensor_data, +) + + +class InMemoryClient: + """Small API-shaped store for exercising idempotent HEMS asset repair.""" + + def __init__(self): + self.assets = [ + { + "id": 1, + "name": "Community Site", + "account_id": 9, + "parent_asset_id": None, + }, + { + "id": 2, + "name": "Building A", + "account_id": 9, + "parent_asset_id": 1, + }, + ] + self.sensors = [ + { + "id": 100, + "name": "electricity-consumption", + "generic_asset_id": 2, + } + ] + self.next_asset_id = 3 + self.next_sensor_id = 101 + + async def get_account(self): + return {"id": 9, "name": "test"} + + async def get_assets(self, **kwargs): + return [dict(asset) for asset in self.assets] + + async def get_sensors(self, asset_id, **kwargs): + return [ + dict(sensor) + for sensor in self.sensors + if sensor["generic_asset_id"] == asset_id + ] + + async def add_asset(self, **asset): + asset = { + **asset, + "id": self.next_asset_id, + "parent_asset_id": asset.get("parent_asset_id"), + } + self.next_asset_id += 1 + self.assets.append(asset) + return dict(asset) + + async def add_sensor(self, **sensor): + sensor = {**sensor, "id": self.next_sensor_id} + self.next_sensor_id += 1 + self.sensors.append(sensor) + return dict(sensor) + + async def update_asset(self, asset_id, updates, **kwargs): + asset = next(asset for asset in self.assets if asset["id"] == asset_id) + asset.update(updates) + return dict(asset) + + +def test_interrupted_wipe_requires_explicit_continue(): + with patch("builtins.input", return_value="n"): + assert prompt_for_interrupted_wipe() == "exit" + with patch("builtins.input", return_value="c"): + assert prompt_for_interrupted_wipe() == "continue-wipe" + + +def test_untracked_setup_offers_repair(): + with patch("builtins.input", return_value="c"): + assert ( + prompt_for_untracked_setup(["Building A"], ["Building A", "Building B"]) + == "repair" + ) + + +@pytest.mark.parametrize( + ("answer", "expected"), + [("k", "keep-names"), ("m", "rename-sites"), ("y", "recreate")], +) +def test_legacy_names_offer_user_choice(answer: str, expected: str): + with patch("builtins.input", return_value=answer): + assert ( + prompt_for_untracked_setup( + ["My Home 1", "My Home 2"], ["Building A", "Building B"] + ) + == expected + ) + + +def test_workflow_state_validates_recorded_ids_and_site_names(): + valid_state = { + "workflow-version": 1, + "status": "ready", + "completed-phases": [ASSET_SETUP_PHASE], + "top-level-asset-ids": [1, 2, 3], + "sensor-ids": [10, 11], + "site-names": ["Building A", "Building B"], + } + assert ( + get_workflow_state({"attributes": {"hems_tutorial": valid_state}}) + == valid_state + ) + assert ( + get_workflow_state( + { + "attributes": { + "hems_tutorial": {**valid_state, "top-level-asset-ids": ["1"]} + } + } + ) + is None + ) + + +@pytest.mark.asyncio +async def test_get_or_create_asset_reuses_exact_hierarchy_position(): + client = AsyncMock() + client.get_assets.return_value = [ + {"id": 1, "name": "Building A", "account_id": 9, "parent_asset_id": 4}, + {"id": 2, "name": "Building A", "account_id": 9, "parent_asset_id": 8}, + ] + + asset = await get_or_create_asset( + client, + name="Building A", + account_id=9, + parent_asset_id=4, + generic_asset_type_id=6, + ) + + assert asset["id"] == 1 + client.add_asset.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_get_or_create_sensor_only_creates_missing_sensor(): + client = AsyncMock() + client.get_sensors.return_value = [{"id": 10, "name": "existing"}] + client.add_sensor.return_value = {"id": 11, "name": "missing"} + + sensor = await get_or_create_sensor( + client, + name="missing", + event_resolution="PT15M", + unit="kW", + generic_asset_id=1, + ) + + assert sensor["id"] == 11 + client.add_sensor.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_repair_completes_structure_without_replacing_existing_ids(): + client = InMemoryClient() + existing_community = client.assets[0] + + community = await create_community_asset( + client, + account={"id": 9}, + community_name="Community Site", + site_names=["Building A", "Building B"], + community_asset=existing_community, + ) + first_asset_count = len(client.assets) + first_sensor_count = len(client.sensors) + + await create_community_asset( + client, + account={"id": 9}, + community_name="Community Site", + site_names=["Building A", "Building B"], + community_asset=community, + ) + + assert community["id"] == 1 + assert ( + next(asset for asset in client.assets if asset["name"] == "Building A")["id"] + == 2 + ) + assert ( + next( + sensor + for sensor in client.sensors + if sensor["name"] == "electricity-consumption" + and sensor["generic_asset_id"] == 2 + )["id"] + == 100 + ) + assert len(client.assets) == first_asset_count + assert len(client.sensors) == first_sensor_count + + +@pytest.mark.asyncio +async def test_sensor_lookup_stays_inside_community_by_default(): + client = AsyncMock() + client.get_asset.return_value = { + "id": 1, + "name": "Community Site", + "account_id": 9, + } + client.get_assets.return_value = [{"id": 2, "name": "Building A"}] + client.get_sensors.return_value = [{"id": 20, "name": "power"}] + + sensor = await find_sensor_by_name_and_asset( + client, + sensor_name="power", + asset_name="Building A", + top_level_asset_id=1, + ) + + assert sensor["id"] == 20 + client.get_assets.assert_awaited_once_with(root=1, parse_json_fields=True) + + +@pytest.mark.asyncio +async def test_explicit_top_level_sensor_lookup_stays_in_community_account(): + client = AsyncMock() + client.get_asset.return_value = { + "id": 1, + "name": "Community Site", + "account_id": 9, + } + client.get_assets.side_effect = [ + [{"id": 2, "name": "Building A", "account_id": 9}], + [{"id": 3, "name": "Price Market", "account_id": 9}], + ] + client.get_sensors.return_value = [{"id": 30, "name": "electricity-price"}] + + sensor = await find_sensor_by_name_and_asset( + client, + sensor_name="electricity-price", + asset_name="Price Market", + top_level_asset_id=1, + allow_top_level_asset=True, + ) + + assert sensor["id"] == 30 + assert client.get_assets.await_args_list == [ + call(root=1, parse_json_fields=True), + call(account_id=9, depth=0, parse_json_fields=True), + ] + + +@pytest.mark.asyncio +async def test_get_site_assets_only_returns_direct_children(): + client = AsyncMock() + client.get_assets.return_value = [ + {"id": 3, "name": "Battery", "account_id": 9, "parent_asset_id": 2}, + {"id": 2, "name": "Building A", "account_id": 9, "parent_asset_id": 1}, + {"id": 4, "name": "Building B", "account_id": 9, "parent_asset_id": 1}, + ] + + sites = await get_site_assets(client, community_asset_id=1, account_id=9) + + assert [site["id"] for site in sites] == [2, 4] + + +@pytest.mark.asyncio +async def test_rename_site_assets_preserves_ids(): + client = AsyncMock() + sites = [{"id": 20, "name": "My Home 1"}, {"id": 21, "name": "My Home 2"}] + + await rename_site_assets(client, sites, ["Building A", "Building B"]) + + assert client.update_asset.await_args_list == [ + call(asset_id=20, updates={"name": "Building A"}, parse_json_fields=False), + call(asset_id=21, updates={"name": "Building B"}, parse_json_fields=False), + ] + + +@pytest.mark.asyncio +async def test_wipe_can_be_repeated_and_finishes_ready(): + client = AsyncMock() + client.get_asset.return_value = {"id": 1, "attributes": {}} + state = { + "workflow-version": 1, + "status": "wiping", + "completed-phases": [ASSET_SETUP_PHASE], + "top-level-asset-ids": [1, 2, 3], + "sensor-ids": [10, 11], + "site-names": ["Building A", "Building B"], + } + + result = await wipe_hems_sensor_data(client, 1, state) + + assert result["status"] == "ready" + assert client.delete_sensor_data.await_args_list == [ + call(10, confirm_first=False), + call(11, confirm_first=False), + ] + assert client.update_asset.await_count == 2 From 3837e29685d66524e0dfb9e00063511c66be932f Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Tue, 18 Aug 2026 01:25:29 +0100 Subject: [PATCH 14/21] fix(hems): support 0.33 response scopes FlexMeasures 0.33 returns ingestion IDs under job_id and expands asset and sensor queries to include roots or descendants. The HEMS recovery flow treated these valid responses as missing or ambiguous, preventing clean runs and incomplete-structure repair. Accept both job field names, deduplicate repeated roots by ID, and constrain sensor matches to exact owners while retaining genuine ambiguity checks. Align deletion confirmation and recovery documentation, remove the obsolete global lookup helper, and add regression coverage. Signed-off-by: Mohamed Belhsan Hmida --- docs/HEMS.rst | 14 ++-- examples/HEMS/assets_setup.py | 8 ++- examples/HEMS/scheduling.py | 7 +- examples/HEMS/utils/asset_utils.py | 29 +++----- src/flexmeasures_client/client.py | 8 ++- tests/client/test_sensor.py | 19 +++++- tests/examples/test_hems_workflow.py | 98 +++++++++++++++++++++++++++- 7 files changed, 148 insertions(+), 35 deletions(-) diff --git a/docs/HEMS.rst b/docs/HEMS.rst index 699c7a78..3cf4e1e9 100644 --- a/docs/HEMS.rst +++ b/docs/HEMS.rst @@ -129,8 +129,8 @@ Rerunning or resuming the tutorial ================================== The setup script records completed phases in a namespaced attribute on the -community asset. If the community already exists, the script shows which phases -are complete and offers three choices: +community asset. If a tracked community already exists, the script shows which +phases are complete and offers four choices: - ``y`` recreates the HEMS assets. This deletes their sensors, IDs, and data, including the HEMS energy market and weather station, before creating @@ -153,11 +153,11 @@ IDs, recreate the setup, or exit. For an older setup with different site names, it also offers to keep those names or rename the sites to the names configured in ``const.py``. -The workflow marker stores the exact sensor IDs created for the tutorial, so a -data wipe remains limited to that recorded set. If an existing setup predates -workflow markers, safe resume is unavailable because its completed phases are -unknown. Choose ``w`` to keep its IDs while refreshing its data, or ``y`` to -recreate it fully. +The workflow marker stores the sensor IDs in the HEMS structure when the marker +is created, so a data wipe remains limited to that recorded set. If an existing +setup predates workflow markers, safe resume is unavailable because its +completed phases are unknown. Choose the offered repair option to preserve IDs +and complete missing structure, or choose recreation to replace the setup. Delete the tutorial assets and data diff --git a/examples/HEMS/assets_setup.py b/examples/HEMS/assets_setup.py index a9868475..bbd991a3 100644 --- a/examples/HEMS/assets_setup.py +++ b/examples/HEMS/assets_setup.py @@ -68,7 +68,13 @@ async def get_or_create_sensor( sensors = await client.get_sensors( asset_id=generic_asset_id, parse_json_fields=False ) - matches = [sensor for sensor in sensors if sensor.get("name") == name] + # FlexMeasures 0.33 may include sensors on descendant assets here. + matches = [ + sensor + for sensor in sensors + if sensor.get("name") == name + and sensor.get("generic_asset_id") == generic_asset_id + ] if len(matches) > 1: raise LookupError( f"Expected at most one sensor named '{name}' on asset " diff --git a/examples/HEMS/scheduling.py b/examples/HEMS/scheduling.py index b205c2f1..7ec563dc 100644 --- a/examples/HEMS/scheduling.py +++ b/examples/HEMS/scheduling.py @@ -867,9 +867,10 @@ async def get_site_assets( fields=["id", "name", "attributes", "sensors", "parent_asset_id"], parse_json_fields=True, ) - assets_by_name: dict[str, dict] = {community_name: community_asset} - for asset in assets: - if asset["name"] in assets_by_name: + assets_by_name: dict[str, dict] = {} + for asset in [community_asset, *assets]: + existing_asset = assets_by_name.get(asset["name"]) + if existing_asset is not None and existing_asset["id"] != asset["id"]: raise LookupError( f"Asset name '{asset['name']}' is ambiguous in community " f"'{community_name}'." diff --git a/examples/HEMS/utils/asset_utils.py b/examples/HEMS/utils/asset_utils.py index cb512019..728876e9 100644 --- a/examples/HEMS/utils/asset_utils.py +++ b/examples/HEMS/utils/asset_utils.py @@ -22,7 +22,10 @@ async def post_sensor_data_and_track_ingestion( if status != 202: return - job_id = response.get("job") if isinstance(response, dict) else None + # FlexMeasures 0.33 calls this field ``job_id``; newer servers use ``job``. + job_id = None + if isinstance(response, dict): + job_id = response.get("job") or response.get("job_id") if not job_id: raise RuntimeError( "The server accepted sensor data for asynchronous ingestion " @@ -123,7 +126,12 @@ async def find_sensor_by_name_and_asset( sensors = await client.get_sensors( asset_id=target_asset["id"], parse_json_fields=True ) - matches = [sensor for sensor in sensors if sensor.get("name") == sensor_name] + matches = [ + sensor + for sensor in sensors + if sensor.get("name") == sensor_name + and sensor.get("generic_asset_id") == target_asset["id"] + ] if not matches: raise LookupError(f"Sensor '{sensor_name}' not found in asset '{asset_name}'") if len(matches) > 1: @@ -328,20 +336,3 @@ def load_and_align_csv_data( print(f"Aligned {len(df)} records from {file_path}") return aligned_df - - -def get_first_asset_by_name( - assets: list[dict], name: str, account_id: int | None = 0 -) -> dict | None: - """ - :param assets: List of dictionaries describing assets, each with at least a "name". - :param name: The asset name to find the first occurrence for. - :param account_id: Optionally, filter by account_id (a positive integer, or None for a public account). - To use this filter, each dictionary in `assets` should contain the "account_id", too. - NB the 0 default is used to signal the argument is missing (real IDs are strictly positive). - """ - for asset in assets: - if asset["name"] == name: - if account_id != 0 and asset["account_id"] != account_id: - continue - return asset diff --git a/src/flexmeasures_client/client.py b/src/flexmeasures_client/client.py index ec5cb3cd..ffa55f2f 100644 --- a/src/flexmeasures_client/client.py +++ b/src/flexmeasures_client/client.py @@ -1418,8 +1418,14 @@ async def delete_sensor_data( Optionally limit deletion to one source and/or an event-time range. """ if confirm_first: + deletion_scope = ( + "all data" + if source is None and start is None and until is None + else "matching data" + ) answer = input( - f"Permanently delete all data from sensor {sensor_id}? [y/N] " + f"Permanently delete {deletion_scope} from sensor " + f"{sensor_id}? [y/N] " ) if answer.lower() not in ["y", "yes"]: print("Aborting ...") diff --git a/tests/client/test_sensor.py b/tests/client/test_sensor.py index 828c2333..1fd39fc9 100644 --- a/tests/client/test_sensor.py +++ b/tests/client/test_sensor.py @@ -361,10 +361,27 @@ async def test_delete_sensor_data_confirmation_declined(): client = FlexMeasuresClient(email="test@test.test", password="test") client.access_token = "test-token" with ( - patch("builtins.input", return_value="n"), + patch("builtins.input", return_value="n") as prompt, patch.object(client, "request", new_callable=AsyncMock) as request, ): await client.delete_sensor_data(sensor_id=7) + prompt.assert_called_once_with("Permanently delete all data from sensor 7? [y/N] ") + request.assert_not_awaited() + await client.close() + + +@pytest.mark.asyncio +async def test_delete_filtered_sensor_data_confirmation_is_scoped(): + client = FlexMeasuresClient(email="test@test.test", password="test") + client.access_token = "test-token" + with ( + patch("builtins.input", return_value="n") as prompt, + patch.object(client, "request", new_callable=AsyncMock) as request, + ): + await client.delete_sensor_data(sensor_id=7, source=3) + prompt.assert_called_once_with( + "Permanently delete matching data from sensor 7? [y/N] " + ) request.assert_not_awaited() await client.close() diff --git a/tests/examples/test_hems_workflow.py b/tests/examples/test_hems_workflow.py index cc26ffc3..70369ed9 100644 --- a/tests/examples/test_hems_workflow.py +++ b/tests/examples/test_hems_workflow.py @@ -18,7 +18,11 @@ get_or_create_asset, get_or_create_sensor, ) -from utils.asset_utils import find_sensor_by_name_and_asset # noqa: E402 +from scheduling import get_site_assets as get_scheduling_site_assets # noqa: E402 +from utils.asset_utils import ( # noqa: E402 + find_sensor_by_name_and_asset, + post_sensor_data_and_track_ingestion, +) from utils.workflow_utils import ( # noqa: E402 ASSET_SETUP_PHASE, get_site_assets, @@ -183,6 +187,56 @@ async def test_get_or_create_sensor_only_creates_missing_sensor(): client.add_sensor.assert_awaited_once() +@pytest.mark.asyncio +async def test_get_or_create_sensor_ignores_same_name_on_descendant_assets(): + client = AsyncMock() + client.get_sensors.return_value = [ + {"id": 10, "name": "power", "generic_asset_id": 2}, + {"id": 11, "name": "power", "generic_asset_id": 1}, + ] + + sensor = await get_or_create_sensor( + client, + name="power", + event_resolution="PT15M", + unit="kW", + generic_asset_id=1, + ) + + assert sensor["id"] == 11 + client.add_sensor.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_get_or_create_sensor_rejects_duplicates_on_same_asset(): + client = AsyncMock() + client.get_sensors.return_value = [ + {"id": 10, "name": "power", "generic_asset_id": 1}, + {"id": 11, "name": "power", "generic_asset_id": 1}, + ] + + with pytest.raises(LookupError, match="found 2"): + await get_or_create_sensor( + client, + name="power", + event_resolution="PT15M", + unit="kW", + generic_asset_id=1, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("job_field", ["job", "job_id"]) +async def test_async_sensor_data_ingestion_tracks_supported_job_fields(job_field): + client = AsyncMock() + client.post_sensor_data.return_value = ({job_field: "job-123"}, 202) + pending_jobs = [] + + await post_sensor_data_and_track_ingestion(client, pending_jobs, sensor_id=1) + + assert pending_jobs == ["job-123"] + + @pytest.mark.asyncio async def test_repair_completes_structure_without_replacing_existing_ids(): client = InMemoryClient() @@ -233,7 +287,10 @@ async def test_sensor_lookup_stays_inside_community_by_default(): "account_id": 9, } client.get_assets.return_value = [{"id": 2, "name": "Building A"}] - client.get_sensors.return_value = [{"id": 20, "name": "power"}] + client.get_sensors.return_value = [ + {"id": 19, "name": "power", "generic_asset_id": 3}, + {"id": 20, "name": "power", "generic_asset_id": 2}, + ] sensor = await find_sensor_by_name_and_asset( client, @@ -258,7 +315,9 @@ async def test_explicit_top_level_sensor_lookup_stays_in_community_account(): [{"id": 2, "name": "Building A", "account_id": 9}], [{"id": 3, "name": "Price Market", "account_id": 9}], ] - client.get_sensors.return_value = [{"id": 30, "name": "electricity-price"}] + client.get_sensors.return_value = [ + {"id": 30, "name": "electricity-price", "generic_asset_id": 3} + ] sensor = await find_sensor_by_name_and_asset( client, @@ -289,6 +348,39 @@ async def test_get_site_assets_only_returns_direct_children(): assert [site["id"] for site in sites] == [2, 4] +@pytest.mark.asyncio +async def test_scheduling_assets_accept_root_repeated_by_server(): + client = AsyncMock() + community = {"id": 1, "name": "Community Site"} + client.get_asset.return_value = community + client.get_assets.return_value = [ + community, + {"id": 2, "name": "Building A"}, + {"id": 3, "name": "Home Battery 1"}, + {"id": 4, "name": "EV Connector 1 1"}, + {"id": 5, "name": "EV Connector 2 1"}, + {"id": 6, "name": "Heat Pump 1"}, + ] + + with patch("scheduling.find_top_level_asset_id", AsyncMock(return_value=1)): + assets = await get_scheduling_site_assets( + client, "Building A", "Community Site", 1 + ) + + assert [asset["id"] for asset in assets] == [1, 2, 3, 4, 5, 6] + + +@pytest.mark.asyncio +async def test_scheduling_assets_reject_different_assets_with_same_name(): + client = AsyncMock() + client.get_asset.return_value = {"id": 1, "name": "Community Site"} + client.get_assets.return_value = [{"id": 99, "name": "Community Site"}] + + with patch("scheduling.find_top_level_asset_id", AsyncMock(return_value=1)): + with pytest.raises(LookupError, match="ambiguous"): + await get_scheduling_site_assets(client, "Building A", "Community Site", 1) + + @pytest.mark.asyncio async def test_rename_site_assets_preserves_ids(): client = AsyncMock() From 2f402e0895c71d28bde128f3e556c99353f5d93e Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Tue, 18 Aug 2026 01:49:04 +0100 Subject: [PATCH 15/21] fix(style): add missing newline for better readability Signed-off-by: Mohamed Belhsan Hmida --- examples/HEMS/assets_setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/HEMS/assets_setup.py b/examples/HEMS/assets_setup.py index bbd991a3..845e7482 100644 --- a/examples/HEMS/assets_setup.py +++ b/examples/HEMS/assets_setup.py @@ -12,6 +12,7 @@ pv_name, weather_station_name, ) + from flexmeasures_client import FlexMeasuresClient From b3d398037c409a6675ea7adb3edf802de6950333 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Tue, 18 Aug 2026 01:53:36 +0100 Subject: [PATCH 16/21] style(tests): apply isort ordering The pre-commit workflow reordered the HEMS workflow test imports, causing CI to fail because the generated formatting diff was not committed. Apply the deterministic import order so the repository remains unchanged when the full pre-commit suite runs. Signed-off-by: Mohamed Belhsan Hmida --- tests/examples/test_hems_workflow.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/examples/test_hems_workflow.py b/tests/examples/test_hems_workflow.py index 70369ed9..d257c61f 100644 --- a/tests/examples/test_hems_workflow.py +++ b/tests/examples/test_hems_workflow.py @@ -9,15 +9,15 @@ HEMS_DIR = Path(__file__).parents[2] / "examples" / "HEMS" sys.path.insert(0, str(HEMS_DIR)) -from HEMS_setup import ( # noqa: E402 - prompt_for_interrupted_wipe, - prompt_for_untracked_setup, -) from assets_setup import ( # noqa: E402 create_community_asset, get_or_create_asset, get_or_create_sensor, ) +from HEMS_setup import ( # noqa: E402 + prompt_for_interrupted_wipe, + prompt_for_untracked_setup, +) from scheduling import get_site_assets as get_scheduling_site_assets # noqa: E402 from utils.asset_utils import ( # noqa: E402 find_sensor_by_name_and_asset, From 041c0d56323d595b138deabd822f73273f4e9825 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Wed, 19 Aug 2026 16:13:09 +0200 Subject: [PATCH 17/21] PV Improvements: The self-consumption KPI now divides self-consumed solar by delivered PV instead of available PV, so curtailed energy no longer incorrectly lowers the KPI. Feed-in and curtailment are also reported separately. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- docs/HEMS.rst | 16 +++++ examples/HEMS/HEMS_setup.py | 15 ++++- examples/HEMS/assets_setup.py | 56 ++++++++++++++-- .../self-consumption_reporter_config.json | 24 +++++-- examples/HEMS/const.py | 9 ++- examples/HEMS/reporters.py | 4 ++ examples/HEMS/scheduling.py | 65 +++++++++++++++---- 7 files changed, 159 insertions(+), 30 deletions(-) diff --git a/docs/HEMS.rst b/docs/HEMS.rst index e1824fa7..60fa06fb 100644 --- a/docs/HEMS.rst +++ b/docs/HEMS.rst @@ -22,6 +22,7 @@ Set up your environment ======================== To run the HEMS example (``HEMS_setup.py``), you'll need an environment in which both ``flexmeasures`` (the server) and ``flexmeasures-client`` is installed. +The example requires FlexMeasures 1.0 or newer. We use `uv `_ to manage dependencies. First, `install uv `_. @@ -71,6 +72,21 @@ example: host = "ems.example.com" ssl = True +PV is inflexible by default: all available production is delivered and any +surplus is treated as grid feed-in. Set ``PV_MODE = "curtailable"`` when the PV +gateway can reduce production, for example at a site whose grid-production +capacity is zero. In that mode the simulated gateway treats the PV schedule as +a maximum setpoint; it can reduce available production but cannot increase it. +Recreate an existing tutorial structure after changing this setting so its flex +context and PV sensors match the selected mode. + +The PV chart distinguishes available production, delivered production, +self-consumption, grid feed-in, and curtailment. The reporter calculates the +latter two after realization as ``max(delivered PV - local load, 0)`` and +``max(available PV - delivered PV, 0)``. Consequently, the daily +self-consumption percentage uses delivered rather than merely available PV as +its denominator. + Open three terminals. In the first terminal, run the server: .. code-block:: bash diff --git a/examples/HEMS/HEMS_setup.py b/examples/HEMS/HEMS_setup.py index 58d79ab1..b7fa9b35 100644 --- a/examples/HEMS/HEMS_setup.py +++ b/examples/HEMS/HEMS_setup.py @@ -8,7 +8,7 @@ from typing import Callable from assets_setup import create_community_asset -from const import COMMUNITY_NAME, SITE_NAMES, host, pwd, ssl, usr +from const import COMMUNITY_NAME, PV_MODE, SITE_NAMES, host, pwd, ssl, usr from forecasting import generate_forecasts from reporters import create_reports from scheduling import just_continue, run_scheduling_simulation @@ -136,6 +136,11 @@ async def main( print("Starting FlexMeasures HEMS") print("=" * 50) + if PV_MODE not in {"inflexible", "curtailable"}: + raise ValueError( + f"Unsupported PV_MODE {PV_MODE!r}; choose 'inflexible' or 'curtailable'." + ) + # NOTE: Create the account and account-admin user via FlexMeasures CLI first: # flexmeasures add account --name "MyCompany" # flexmeasures add user --username hems-admin --email hems-admin@example.com \ @@ -147,9 +152,13 @@ async def main( print( f"Checking server is up and on supported version ... connecting to {host} (ssl: {ssl})" ) + # The sign-explicit ``inflexible-consumption`` and + # ``inflexible-production`` flex-context fields were introduced for + # FlexMeasures 1.0. Accept its development releases for testing, too. await client.ensure_minimum_server_version( - "0.31.0", - "The HEMS example requires a FlexMeasures server of v0.31.0 or above.", + "1.0.0.dev0", + "The HEMS example requires a FlexMeasures server from the v1.0 " + "series or above.", ) # Get user account information diff --git a/examples/HEMS/assets_setup.py b/examples/HEMS/assets_setup.py index 38a28d57..eac431b0 100644 --- a/examples/HEMS/assets_setup.py +++ b/examples/HEMS/assets_setup.py @@ -2,6 +2,7 @@ BATTERY_CONFIG, EV_CONFIG, HEATING_CONFIG, + PV_MODE, battery_name, evse1_name, evse2_name, @@ -285,6 +286,7 @@ async def create_pv_asset( unit="kW", generic_asset_id=pv_asset["id"], timezone="Europe/Amsterdam", + attributes=dict(consumption_is_positive=False), ) # Create power sensor (15min, kW) @@ -294,10 +296,35 @@ async def create_pv_asset( unit="kW", generic_asset_id=pv_asset["id"], timezone="Europe/Amsterdam", + attributes=dict(consumption_is_positive=False), + ) + + pv_feed_in_sensor = await client.add_sensor( + name="solar-feed-in", + event_resolution="PT15M", + unit="kW", + generic_asset_id=pv_asset["id"], + timezone="Europe/Amsterdam", + attributes=dict(consumption_is_positive=False), + ) + + pv_curtailment_sensor = await client.add_sensor( + name="solar-curtailment", + event_resolution="PT15M", + unit="kW", + generic_asset_id=pv_asset["id"], + timezone="Europe/Amsterdam", + attributes=dict(consumption_is_positive=False), ) print(f"Created PV asset with ID: {pv_asset['id']}") - return pv_asset, pv_production_sensor, pv_power_sensor + return ( + pv_asset, + pv_production_sensor, + pv_power_sensor, + pv_feed_in_sensor, + pv_curtailment_sensor, + ) async def create_battery_asset( @@ -653,12 +680,15 @@ async def configure_site_flex_context( # "site-production-breach-price": "10000000 EUR/MW", # "consumption-breach-price": "1000 EUR/MW", # "production-breach-price": "1000 EUR/MW", - # Add inflexible devices as requested - "inflexible-device-sensors": [ - consumption_sensor["id"], # General consumption - ], + "inflexible-consumption": [{"sensor": consumption_sensor["id"]}], "aggregate-power": {"sensor": aggregate_sensor["id"]}, } + if PV_MODE == "inflexible": + flex_context["inflexible-production"] = [{"sensor": pv_production_sensor["id"]}] + elif PV_MODE != "curtailable": + raise ValueError( + f"Unsupported PV_MODE {PV_MODE!r}; choose 'inflexible' or 'curtailable'." + ) # Update site asset with flex-context await client.update_asset( @@ -674,6 +704,8 @@ async def configure_site_dashboard( consumption_sensor, pv_production_sensor, pv_power_sensor, + pv_feed_in_sensor, + pv_curtailment_sensor, battery_power_sensor, battery_soc_sensor, evse1_power_sensor, @@ -708,11 +740,13 @@ async def configure_site_dashboard( ], }, { - "title": "Solar self-consumption", + "title": "PV production and use", "sensors": [ self_consumption_sensor["id"], pv_production_sensor["id"], pv_power_sensor["id"], + pv_feed_in_sensor["id"], + pv_curtailment_sensor["id"], ], }, { @@ -814,7 +848,13 @@ async def create_sites_assets_and_sensors( print(f"Max consumption sensor ID: {max_consumption_sensor['id']}") print(f"Self-consumption sensor ID: {self_consumption_sensor['id']}") print("Creating PV asset with production sensor") - pv_asset, pv_production_sensor, pv_power_sensor = await create_pv_asset( + ( + pv_asset, + pv_production_sensor, + pv_power_sensor, + pv_feed_in_sensor, + pv_curtailment_sensor, + ) = await create_pv_asset( client, account_id, site_asset["id"], pv_name=f"{pv_name} {site_index}" ) print(f"PV asset ID: {pv_asset['id']}") @@ -914,6 +954,8 @@ async def create_sites_assets_and_sensors( consumption_sensor=consumption_sensor, pv_production_sensor=pv_production_sensor, pv_power_sensor=pv_power_sensor, + pv_feed_in_sensor=pv_feed_in_sensor, + pv_curtailment_sensor=pv_curtailment_sensor, battery_power_sensor=battery_power_sensor, battery_soc_sensor=battery_soc_sensor, evse1_power_sensor=evse1_power_sensor, diff --git a/examples/HEMS/configs/self-consumption_reporter_config.json b/examples/HEMS/configs/self-consumption_reporter_config.json index 0b24709a..46e58c27 100644 --- a/examples/HEMS/configs/self-consumption_reporter_config.json +++ b/examples/HEMS/configs/self-consumption_reporter_config.json @@ -1,6 +1,6 @@ { "required_input" :[{"name":"production","unit": "kW"}, {"name":"pv-power","unit": "kW"}, {"name":"heating-power","unit": "kW"}, {"name":"evse1-consumption","unit": "kW"}, {"name":"evse2-consumption","unit": "kW"}, {"name":"building-consumption","unit": "kW"}, {"name":"battery-power","unit": "kW"}], - "required_output" :[{"name":"self-consumption","unit": "kW"}, {"name":"daily-share-of-self-consumption", "unit": "%"}], + "required_output" :[{"name":"self-consumption","unit": "kW"}, {"name":"solar-feed-in","unit": "kW"}, {"name":"solar-curtailment","unit": "kW"}, {"name":"daily-share-of-self-consumption", "unit": "%"}], "droplevels": true, "transformations" : [ { @@ -39,7 +39,7 @@ }, { "df_input": "excess-pv", - "df_output": "excess-pv", + "df_output": "solar-feed-in", "method": "clip", "kwargs": {"lower": 0} }, @@ -47,7 +47,19 @@ "df_input": "pv-power", "df_output": "self-consumption", "method": "sub", - "args": ["@excess-pv"] + "args": ["@solar-feed-in"] + }, + { + "df_input": "production", + "df_output": "solar-curtailment", + "method": "sub", + "args": ["@pv-power"] + }, + { + "df_input": "solar-curtailment", + "df_output": "solar-curtailment", + "method": "clip", + "kwargs": {"lower": 0} }, { "df_input": "self-consumption", @@ -59,8 +71,8 @@ "method": "sum" }, { - "df_input": "production", - "df_output": "daily solar production", + "df_input": "pv-power", + "df_output": "daily delivered solar production", "method": "resample", "args": ["1D"] }, @@ -71,7 +83,7 @@ "df_input": "daily self-consumption", "df_output": "daily-share-of-self-consumption", "method": "divide", - "args": ["@daily solar production"] + "args": ["@daily delivered solar production"] }, { "method": "multiply", diff --git a/examples/HEMS/const.py b/examples/HEMS/const.py index a87c88b6..6a504324 100644 --- a/examples/HEMS/const.py +++ b/examples/HEMS/const.py @@ -4,11 +4,16 @@ Settings for the HEMS example script. """ # Connection details - UPDATE THESE FOR YOUR SETUP -usr = "hems-admin@example.com" # Account-admin user email -pwd = "change-me" # Account-admin user password +usr = "toy-user@flexmeasures.io" # Account-admin user email +pwd = "toy-password" # Account-admin user password host = "127.0.0.1:5000" # FlexMeasures host, without http:// or https:// ssl = False # Use HTTPS (and port 443 unless host specifies another port) +# PV operation mode: +# - "inflexible": all available PV is delivered; surplus production is fed in. +# - "curtailable": FlexMeasures may schedule PV below its available production. +PV_MODE = "inflexible" + # Asset and sensor names COMMUNITY_NAME = "Community Site" SITE_NAMES = ["Building A", "Building B"] diff --git a/examples/HEMS/reporters.py b/examples/HEMS/reporters.py index 78a13cf5..99e4431a 100644 --- a/examples/HEMS/reporters.py +++ b/examples/HEMS/reporters.py @@ -41,6 +41,8 @@ async def create_reports( sensor_mappings = [ ("electricity-production", "electricity-production", f"{pv_name} {i}"), ("pv-power", "electricity-power", f"{pv_name} {i}"), + ("solar-feed-in", "solar-feed-in", f"{pv_name} {i}"), + ("solar-curtailment", "solar-curtailment", f"{pv_name} {i}"), ("electricity-consumption", "electricity-consumption", site_name), ("electricity-power", "electricity-power", f"{battery_name} {i}"), ("evse1-power", "electricity-power", f"{evse1_name} {i}"), @@ -74,6 +76,8 @@ async def create_reports( ], output_sensors=[ sensors["self-consumption"], + sensors["solar-feed-in"], + sensors["solar-curtailment"], sensors["daily-share-of-self-consumption"], ], start=SCHEDULING_START, diff --git a/examples/HEMS/scheduling.py b/examples/HEMS/scheduling.py index 433d6d79..e9d85886 100644 --- a/examples/HEMS/scheduling.py +++ b/examples/HEMS/scheduling.py @@ -10,6 +10,7 @@ FORECAST_HORIZON_HOURS, HEATING_CONFIG, MAX_RESCHEDULING_ITERATIONS, + PV_MODE, SCHEDULING_END, SCHEDULING_START, SIMULATION_STEP_HOURS, @@ -44,6 +45,28 @@ async def just_continue(*args, **kwargs): return True +def realize_pv_power( + available_power: list[float], + scheduled_power: list[float], + pv_mode: str, +) -> list[float]: + """Return delivered PV power for the configured operating mode.""" + if pv_mode == "inflexible": + return list(available_power) + if pv_mode != "curtailable": + raise ValueError( + f"Unsupported PV_MODE {pv_mode!r}; choose 'inflexible' or 'curtailable'." + ) + if len(available_power) != len(scheduled_power): + raise ValueError( + "Available and scheduled PV power must contain the same number of values." + ) + return [ + max(min(available, scheduled), 0) + for available, scheduled in zip(available_power, scheduled_power) + ] + + async def run_scheduling_simulation( client: FlexMeasuresClient, community_name: str, @@ -338,7 +361,10 @@ async def compute_site_schedules( current_soc=heating_current_soc, ) - # Start with the battery and PV flex models + # Curtailable PV is modeled as production which the scheduler may reduce, + # but never increase above the available-production forecast. Be careful + # when using its realized schedule in reports: a forecast underestimate can + # look like deliberate curtailment if the schedule is treated as a hard cap. curtailable_pv_flex_model = { "power-capacity": "12 kW", "consumption-capacity": "0 kW", @@ -349,17 +375,23 @@ async def compute_site_schedules( "sensor": sensors[f"battery-power-{index}"]["id"], **battery_scheduling_dynamic_flex_model, }, - { - "sensor": sensors[f"pv-power-{index}"][ - "id" - ], # use power sensor to store realized data - **curtailable_pv_flex_model, - }, { "sensor": sensors[f"heating-power-{index}"]["id"], **heating_scheduling_dynamic_flex_model, }, ] + if PV_MODE == "curtailable": + final_flex_models.insert( + 1, + { + "sensor": sensors[f"pv-power-{index}"]["id"], + **curtailable_pv_flex_model, + }, + ) + elif PV_MODE != "inflexible": + raise ValueError( + f"Unsupported PV_MODE {PV_MODE!r}; choose 'inflexible' or 'curtailable'." + ) final_flex_models.extend( [ @@ -375,8 +407,15 @@ async def compute_site_schedules( ) print("[FLEX-MODEL-DEBUG] === FLEX MODELS SENT TO SCHEDULER ===") - for i, model in enumerate(final_flex_models): - device_name = ["Battery", "PV", "Heating", "EVSE-1", "EVSE-2"][i] + device_names_by_sensor_id = { + sensors[f"battery-power-{index}"]["id"]: "Battery", + sensors[f"pv-power-{index}"]["id"]: "PV", + sensors[f"heating-power-{index}"]["id"]: "Heating", + sensors[f"evse1-power-{index}"]["id"]: "EVSE-1", + sensors[f"evse2-power-{index}"]["id"]: "EVSE-2", + } + for model in final_flex_models: + device_name = device_names_by_sensor_id[model["sensor"]] print(f"[FLEX-MODEL] {device_name}: {model}") print() @@ -558,9 +597,11 @@ async def compute_site_measurements( f"Failed to fetch PV raw power from sensor {sensors[f'pv-production-{index}']['id']}" ) - pv_realized_power = [ - min(raw, scheduled) for raw, scheduled in zip(pv_raw_power, pv_scheduled_power) - ] + # In curtailable mode this emulates a gateway which can lower PV output to + # the scheduled setpoint. It cannot produce more power than is available. + pv_realized_power = realize_pv_power( + pv_raw_power, pv_scheduled_power, pv_mode=PV_MODE + ) await post_sensor_data_and_track_ingestion( client=client, From eb31a4a29d6f16c764228cd406579bd55e2437ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Wed, 19 Aug 2026 16:33:08 +0200 Subject: [PATCH 18/21] fix: use get_or_create_sensor() for new sensor creation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- examples/HEMS/assets_setup.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/HEMS/assets_setup.py b/examples/HEMS/assets_setup.py index 1057f66f..64fb00bc 100644 --- a/examples/HEMS/assets_setup.py +++ b/examples/HEMS/assets_setup.py @@ -343,7 +343,8 @@ async def create_pv_asset( attributes=dict(consumption_is_positive=False), ) - pv_feed_in_sensor = await client.add_sensor( + pv_feed_in_sensor = await get_or_create_sensor( + client, name="solar-feed-in", event_resolution="PT15M", unit="kW", @@ -352,7 +353,8 @@ async def create_pv_asset( attributes=dict(consumption_is_positive=False), ) - pv_curtailment_sensor = await client.add_sensor( + pv_curtailment_sensor = await get_or_create_sensor( + client, name="solar-curtailment", event_resolution="PT15M", unit="kW", From b30b2c0e3e6ed0d5f293de4d136965961734838f Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Wed, 19 Aug 2026 22:57:15 +0100 Subject: [PATCH 19/21] fix(hems): upgrade older setups in place instead of stranding them WORKFLOW_VERSION stayed at 1 while the PV commits changed the asset structure: they added the solar-feed-in and solar-curtailment sensors, switched the site flex-context to inflexible-consumption/production, and added dashboard panels. get_workflow_state() accepts any marker whose version equals WORKFLOW_VERSION, so a setup created before those commits still validated as current. That left every existing setup unusable. Asset setup only runs from the create, recreate and untracked-repair paths; resume and wipe both skip it, so the new sensors were never created. Report generation then looks them up unconditionally and find_sensor_by_name_and_asset() raises rather than skipping, so create_reports() failed with LookupError: Sensor 'solar-feed-in' not found in asset 'PV 1' which propagates out of main(). Every rerun hit the same wall. The wipe path was worst: it reset the phase markers, re-ran the whole scheduling simulation, and only then failed. The sole escape was recreation, which deletes every asset, sensor, ID and all time-series data. Bump WORKFLOW_VERSION to 2 and accept any marker at or below it, so an older setup is recognised as upgradable rather than current. After the existing prompts have been resolved, run the idempotent asset setup again and re-record the marker. Ordering matters: the upgrade runs last, so an interrupted wipe is still recovered through its own prompt first, and a recreation has already rebuilt the structure at the current version. The upgrade is deliberately cheap. Asset and sensor IDs are preserved because get_or_create_* reuse whatever exists. Sensor IDs are re-collected from the server afterwards, so a later wipe also covers sensors the upgrade added. The recorded site names and the status are kept, so an interrupted wipe stays recoverable. Only report generation is re-run, since the reports write the sensors an upgrade tends to add, while the uploaded data, forecasts and schedules stay valid. Markers from a newer version are still rejected, since this script cannot know what they describe. Signed-off-by: Mohamed Belhsan Hmida --- examples/HEMS/HEMS_setup.py | 57 ++++++++++++ examples/HEMS/utils/workflow_utils.py | 58 +++++++++++- tests/examples/test_hems_workflow.py | 126 ++++++++++++++++++++++++++ 3 files changed, 239 insertions(+), 2 deletions(-) diff --git a/examples/HEMS/HEMS_setup.py b/examples/HEMS/HEMS_setup.py index e6f5e4de..81d0c61b 100644 --- a/examples/HEMS/HEMS_setup.py +++ b/examples/HEMS/HEMS_setup.py @@ -19,12 +19,15 @@ PHASE_LABELS, REPORTING_PHASE, SCHEDULING_PHASE, + WORKFLOW_VERSION, get_site_assets, get_workflow_state, initialize_workflow_state, mark_phase_complete, phase_is_complete, rename_site_assets, + state_needs_upgrade, + upgrade_workflow_state, wipe_hems_sensor_data, ) @@ -198,6 +201,48 @@ def confirm_data_wipe(state: dict) -> bool: return answer == "WIPE" +async def upgrade_existing_setup( + client: FlexMeasuresClient, + account: dict, + community_asset: dict, + community_name: str, + site_names: list[str], + state: dict, +) -> dict: + """Bring an older setup up to the current tutorial version, in place. + + Asset setup is idempotent, so re-running it adds whatever the newer tutorial + version introduced (sensors, flex-context fields, dashboard panels) while + every existing asset and sensor keeps its ID and its data. + + Only report generation is re-run afterwards: the reports are what write the + sensors an upgrade is most likely to have added, while the uploaded data, + forecasts and schedules remain valid. + """ + print( + f"Upgrading this setup from tutorial version {state['workflow-version']} " + f"to {WORKFLOW_VERSION}." + ) + print( + "Existing assets, sensors and data are preserved; missing structure is " + "added and reports are regenerated." + ) + community_asset = await create_community_asset( + client, + account, + community_name=community_name, + site_names=site_names, + community_asset=community_asset, + ) + return await upgrade_workflow_state( + client=client, + community_asset=community_asset, + account_id=account["id"], + state=state, + phases_to_rerun=(REPORTING_PHASE,), + ) + + async def main( community_name: str, site_names: list[str], callback: Callable = just_continue ): @@ -372,6 +417,18 @@ async def main( else: print("Resuming the existing HEMS setup.") + # Run last, so that an interrupted wipe is recovered first and + # a recreation has already rebuilt the structure from scratch. + if state_needs_upgrade(state): + state = await upgrade_existing_setup( + client=client, + account=account, + community_asset=asset, + community_name=community_name, + site_names=active_site_names, + state=state, + ) + # Part 2: Upload data for first two weeks print("\n" + "=" * 50) if phase_is_complete(state, DATA_UPLOAD_PHASE): diff --git a/examples/HEMS/utils/workflow_utils.py b/examples/HEMS/utils/workflow_utils.py index 20aa682c..a863f9c7 100644 --- a/examples/HEMS/utils/workflow_utils.py +++ b/examples/HEMS/utils/workflow_utils.py @@ -7,7 +7,10 @@ from flexmeasures_client import FlexMeasuresClient WORKFLOW_ATTRIBUTE = "hems_tutorial" -WORKFLOW_VERSION = 1 +#: Bump whenever the tutorial changes the asset/sensor structure, the site +#: flex-context or the dashboard, so that setups created by an older version +#: are upgraded in place instead of silently keeping the old structure. +WORKFLOW_VERSION = 2 ASSET_SETUP_PHASE = "asset-setup" DATA_UPLOAD_PHASE = "historical-data-upload" @@ -36,7 +39,15 @@ def get_workflow_state(community_asset: dict) -> dict | None: return None state = attributes.get(WORKFLOW_ATTRIBUTE) - if not isinstance(state, dict) or state.get("workflow-version") != WORKFLOW_VERSION: + if not isinstance(state, dict): + return None + # Older markers stay valid: they are upgraded in place, which preserves both + # the existing asset and sensor IDs and any interrupted-wipe status. Newer + # markers are rejected, since this script cannot know what they describe. + version = state.get("workflow-version") + if isinstance(version, bool) or not isinstance(version, int): + return None + if not 1 <= version <= WORKFLOW_VERSION: return None if not isinstance(state.get("completed-phases"), list): return None @@ -190,6 +201,49 @@ async def initialize_workflow_state( return await save_workflow_state(client, community_asset["id"], state) +def state_needs_upgrade(state: dict) -> bool: + """Tell whether a valid marker was written by an older tutorial version.""" + return state["workflow-version"] < WORKFLOW_VERSION + + +async def upgrade_workflow_state( + client: FlexMeasuresClient, + community_asset: dict, + account_id: int, + state: dict, + phases_to_rerun: tuple[str, ...] = (), +) -> dict: + """Re-record an upgraded setup without discarding unaffected progress. + + Call this only after the idempotent asset setup has run again, so that the + structure on the server already matches the current tutorial version. The + asset and sensor IDs are re-collected as they are now, which is what makes a + later data wipe cover sensors that the upgrade added. + + Everything the upgrade does not invalidate is preserved: the recorded site + names, the ``status`` (so an interrupted wipe stays recoverable) and every + completed phase except those the caller asks to re-run. + """ + top_level_asset_ids, sensor_ids = await collect_hems_structure_ids( + client=client, + community_asset=community_asset, + account_id=account_id, + ) + completed_phases = [ + phase for phase in state["completed-phases"] if phase not in phases_to_rerun + ] + if ASSET_SETUP_PHASE not in completed_phases: + completed_phases.insert(0, ASSET_SETUP_PHASE) + state = { + **state, + "workflow-version": WORKFLOW_VERSION, + "completed-phases": completed_phases, + "top-level-asset-ids": top_level_asset_ids, + "sensor-ids": sensor_ids, + } + return await save_workflow_state(client, community_asset["id"], state) + + def phase_is_complete(state: dict, phase: str) -> bool: return phase in state["completed-phases"] diff --git a/tests/examples/test_hems_workflow.py b/tests/examples/test_hems_workflow.py index d257c61f..8974bc72 100644 --- a/tests/examples/test_hems_workflow.py +++ b/tests/examples/test_hems_workflow.py @@ -25,9 +25,16 @@ ) from utils.workflow_utils import ( # noqa: E402 ASSET_SETUP_PHASE, + DATA_UPLOAD_PHASE, + FORECASTING_PHASE, + REPORTING_PHASE, + SCHEDULING_PHASE, + WORKFLOW_VERSION, get_site_assets, get_workflow_state, rename_site_assets, + state_needs_upgrade, + upgrade_workflow_state, wipe_hems_sensor_data, ) @@ -415,3 +422,122 @@ async def test_wipe_can_be_repeated_and_finishes_ready(): call(11, confirm_first=False), ] assert client.update_asset.await_count == 2 + + +def _state(**overrides) -> dict: + """A valid workflow marker, by default written by the current version.""" + return { + "workflow-version": WORKFLOW_VERSION, + "status": "ready", + "completed-phases": [ASSET_SETUP_PHASE], + "top-level-asset-ids": [1, 2, 3], + "sensor-ids": [10, 11], + "site-names": ["Building A", "Building B"], + **overrides, + } + + +def test_older_workflow_state_stays_valid_so_it_can_be_upgraded(): + """A setup from an older tutorial version must not be discarded. + + Rejecting it would leave resume and wipe unable to add newly introduced + sensors, and recreation (losing every ID and all data) the only way out. + """ + older = _state(**{"workflow-version": 1}) + + assert get_workflow_state({"attributes": {"hems_tutorial": older}}) == older + assert state_needs_upgrade(older) is True + assert state_needs_upgrade(_state()) is False + + +def test_workflow_state_from_a_newer_version_is_rejected(): + newer = _state(**{"workflow-version": WORKFLOW_VERSION + 1}) + + assert get_workflow_state({"attributes": {"hems_tutorial": newer}}) is None + + +def test_workflow_state_rejects_a_non_integer_version(): + for version in ("1", 1.5, True, None): + state = _state(**{"workflow-version": version}) + assert get_workflow_state({"attributes": {"hems_tutorial": state}}) is None + + +@pytest.mark.asyncio +async def test_upgrade_preserves_ids_status_and_unaffected_phases(): + """Upgrading must cost the user only the phases it actually invalidates.""" + client = AsyncMock() + client.get_asset.return_value = {"id": 1, "attributes": {}} + client.get_assets.return_value = [ + {"id": 1, "name": "Community Site", "account_id": 9, "parent_asset_id": None}, + {"id": 2, "name": "Energy Market", "account_id": 9, "parent_asset_id": None}, + { + "id": 3, + "name": "Local Weather Station", + "account_id": 9, + "parent_asset_id": None, + }, + ] + # sensor 12 is new: added by the asset setup that the upgrade re-ran + client.get_sensors.return_value = [{"id": 10}, {"id": 11}, {"id": 12}] + state = _state( + **{ + "workflow-version": 1, + "status": "wiping", + "completed-phases": [ + ASSET_SETUP_PHASE, + DATA_UPLOAD_PHASE, + FORECASTING_PHASE, + SCHEDULING_PHASE, + REPORTING_PHASE, + ], + } + ) + + upgraded = await upgrade_workflow_state( + client=client, + community_asset={"id": 1}, + account_id=9, + state=state, + phases_to_rerun=(REPORTING_PHASE,), + ) + + assert upgraded["workflow-version"] == WORKFLOW_VERSION + # only reporting is dropped; the slow phases are not re-run + assert upgraded["completed-phases"] == [ + ASSET_SETUP_PHASE, + DATA_UPLOAD_PHASE, + FORECASTING_PHASE, + SCHEDULING_PHASE, + ] + # a newly added sensor joins the list, so a later wipe also covers it + assert upgraded["sensor-ids"] == [10, 11, 12] + # an interrupted wipe stays recoverable, and site names survive + assert upgraded["status"] == "wiping" + assert upgraded["site-names"] == ["Building A", "Building B"] + + +@pytest.mark.asyncio +async def test_upgrade_always_records_asset_setup_as_complete(): + client = AsyncMock() + client.get_asset.return_value = {"id": 1, "attributes": {}} + client.get_assets.return_value = [ + {"id": 1, "name": "Community Site", "account_id": 9, "parent_asset_id": None}, + {"id": 2, "name": "Energy Market", "account_id": 9, "parent_asset_id": None}, + { + "id": 3, + "name": "Local Weather Station", + "account_id": 9, + "parent_asset_id": None, + }, + ] + client.get_sensors.return_value = [{"id": 10}] + + upgraded = await upgrade_workflow_state( + client=client, + community_asset={"id": 1}, + account_id=9, + state=_state(**{"workflow-version": 1, "completed-phases": []}), + phases_to_rerun=(REPORTING_PHASE,), + ) + + assert upgraded["completed-phases"] == [ASSET_SETUP_PHASE] From 63a6bd29619a5662895983c6adeb66d223a42b51 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Wed, 19 Aug 2026 22:59:00 +0100 Subject: [PATCH 20/21] docs(hems): drop the contradictory minimum server version docs/HEMS.rst stated two different requirements. The environment section said the example requires FlexMeasures 1.0 or newer, while the setup steps below still said 0.33.0 or newer was enough for the data-preserving wipe. 1.0 is the requirement that matches the code. HEMS_setup.py gates on "1.0.0.dev0", and the floor is substantive rather than cosmetic: the site flex-context is written with inflexible-consumption and inflexible-production, which only exist from the 1.0 series. A reader who followed the 0.33 line would set up a server that rejects the flex-context on the first schedule. Remove the stale 0.33 bullet. The 1.0 statement already sits above, next to the install instructions, and the rate-limiting note further down also refers to v1.0. Signed-off-by: Mohamed Belhsan Hmida --- docs/HEMS.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/HEMS.rst b/docs/HEMS.rst index 8b1f0079..13ce7f2b 100644 --- a/docs/HEMS.rst +++ b/docs/HEMS.rst @@ -44,8 +44,6 @@ Or, alternatively, to install released versions into a fresh project: Next steps: - Follow instructions to set up flexmeasures (fresh database, etc). -- Use FlexMeasures 0.33.0 or newer. The tutorial's data-preserving wipe uses - the sensor-data deletion endpoint introduced in that version. - Create an organisation account and a user with the ``account-admin`` role: .. code-block:: bash From 58ad78c34ab78befa4041cf56725717a3ea7fb6f Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Thu, 20 Aug 2026 15:47:48 +0100 Subject: [PATCH 21/21] docs(hems): document the toy account the script now defaults to const.py was changed to default to toy-user@flexmeasures.io / toy-password, the account created by `flexmeasures add toy-account`. The setup instructions were not changed with it: docs/HEMS.rst and the NOTE comment in HEMS_setup.py still told the reader to create an account named "HEMS tutorial" with a hems-admin@example.com user. Following those instructions produced a server with an account the script never tries to log into. The script then failed at startup with "User with email 'hems-admin@example.com' does not exist", and nothing pointed at the shipped defaults as the cause. Align the instructions with the code rather than the other way around. The toy account is a single command instead of three, and it already grants the account-admin role the tutorial needs, so the quick path is now the documented one and needs no edit to const.py. Keep the custom-account route for readers who want their own account, now stated as the alternative it has become, and say explicitly that const.py has to be updated to match in that case. Also mention that the toy account brings its own demo assets (toy-building and its children). The tutorial neither uses nor deletes them: delete_hems_assets() only removes assets it knows by name, and collect_hems_structure_ids() only walks the community, market and weather-station trees. Verified by running the tutorial end to end against a toy account. Signed-off-by: Mohamed Belhsan Hmida --- docs/HEMS.rst | 29 ++++++++++++++++++++++++----- examples/HEMS/HEMS_setup.py | 5 ++++- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/docs/HEMS.rst b/docs/HEMS.rst index 13ce7f2b..dc590519 100644 --- a/docs/HEMS.rst +++ b/docs/HEMS.rst @@ -44,7 +44,26 @@ Or, alternatively, to install released versions into a fresh project: Next steps: - Follow instructions to set up flexmeasures (fresh database, etc). -- Create an organisation account and a user with the ``account-admin`` role: +- Create an account and a user with the ``account-admin`` role. The quickest way + is FlexMeasures' toy account, which creates both in one step: + +.. code-block:: bash + + flexmeasures add toy-account + +This is what ``examples/HEMS/const.py`` expects out of the box, so you can run +the tutorial without editing it: + +.. code-block:: python + + usr = "toy-user@flexmeasures.io" + pwd = "toy-password" + +The toy account also adds a few unrelated demo assets (``toy-building`` and its +children). The tutorial ignores them and never deletes them. + +To use your own account instead, create it and give its user the +``account-admin`` role: .. code-block:: bash @@ -52,11 +71,11 @@ Next steps: flexmeasures add user --username hems-admin --email hems-admin@example.com \ --account 2 --roles account-admin -Replace ``2`` with the account ID printed by the first command. The tutorial -creates all assets and sensors in this organisation account. It does not create -public assets, so a site-wide ``admin`` role is not required. +Replace ``2`` with the account ID printed by the first command, and update +``usr`` and ``pwd`` in ``examples/HEMS/const.py`` to match. -- Update the credentials in the ``examples/HEMS/const.py`` script accordingly. +Either way, the tutorial creates all assets and sensors in that one account. It +does not create public assets, so a site-wide ``admin`` role is not required. Run the tutorial script diff --git a/examples/HEMS/HEMS_setup.py b/examples/HEMS/HEMS_setup.py index 81d0c61b..906bd6af 100644 --- a/examples/HEMS/HEMS_setup.py +++ b/examples/HEMS/HEMS_setup.py @@ -267,7 +267,10 @@ async def main( f"Unsupported PV_MODE {PV_MODE!r}; choose 'inflexible' or 'curtailable'." ) - # NOTE: Create the account and account-admin user via FlexMeasures CLI first: + # NOTE: Create an account and an account-admin user via the FlexMeasures CLI + # first. The credentials in const.py default to FlexMeasures' toy account: + # flexmeasures add toy-account + # To use your own account instead, create it and update const.py to match: # flexmeasures add account --name "MyCompany" # flexmeasures add user --username hems-admin --email hems-admin@example.com \ # --account 2 --roles account-admin