diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst index 541079bb5f..2b89cb8deb 100644 --- a/documentation/api/change_log.rst +++ b/documentation/api/change_log.rst @@ -19,6 +19,7 @@ v3.0-33 | September 1, 2026 - Added ``GET /api/v3_0/assets//automations`` and ``GET /api/v3_0/assets//automations/`` for listing and inspecting forecast automations, including the sensors an automation reads from and writes to. Each automation shows the IANA ``timezone`` in which its cron expression is interpreted, and a ``cursor``: the offset-aware UTC time of the most recent run it committed to. The cursor advances just before queueing, so it does not indicate that queueing or the forecast itself succeeded. Asset job entries now include ``created_via`` provenance; automation identity is included only when the caller may read that automation. - Added ``GET /api/v3_0/sources/`` to show the full record of one data source, including the attributes in which data generators store their configuration. - ``GET /api/v3_0/sensors//stats`` now reports an ``All sources`` entry summarising every data source, whenever more than one recorded. Its mean divides by the values that were summed, not by ``Number of values``, which also counts rows holding NaN. +- Automation list entries now include recent ``job_stats`` counts, collected in one batched cache pass. If Redis is unavailable, the list remains available with empty counts and a ``redis_connection_err`` message. v3.0-32 | August 11, 2026 """"""""""""""""""""""""" diff --git a/documentation/changelog.rst b/documentation/changelog.rst index e5b096d1f4..a3a4f94e36 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -92,6 +92,7 @@ New features ------------- * Reports can be computed on a recurring basis by automations (``flexmeasures add automation --type reporting``), with a rolling report window expressed as Pandas offsets, or defaulting to the period since the automation's last covered window [see `PR #2297 `_] +* An asset's *Automations* page now shows each automation's recent job counts with the listing itself, counted in one pass over the job cache, rather than asking for them once per automation, and loads an automation's full details only when they are opened [see `PR #2299 `_] * ``flexmeasures show data-sources`` now shows which organisation a data source belongs to, and can list the sensors holding data recorded by a given source [see `PR #2401 `_] * The flex-context can now define multiple commodities, each specifying their own prices and grid capacities [see `PR #1946 `_, `PR #2172 `_, `PR #2235 `_, `PR #2271 `_, `PR #2355 `_ and `PR #2380 `_] * Support multiple feeders to a shared storage [see `PR #2001 `_, `PR #2321 `_, `PR #2322 `_, `PR #2325 `_ and `PR #2431 `_] diff --git a/flexmeasures/api/v3_0/assets.py b/flexmeasures/api/v3_0/assets.py index 09da4f2b38..3ade972a65 100644 --- a/flexmeasures/api/v3_0/assets.py +++ b/flexmeasures/api/v3_0/assets.py @@ -13,6 +13,7 @@ from flask_sqlalchemy.pagination import SelectPagination from marshmallow import fields, post_load, ValidationError, Schema, validate +from redis.exceptions import RedisError from webargs.flaskparser import use_kwargs, use_args from sqlalchemy import select, func, or_ @@ -58,6 +59,7 @@ create_automation, delete_automation as remove_automation, describe_cronstr, + get_asset_automations_job_stats, get_automation_job_stats, resolve_automation_sensors, run_automation, @@ -1404,10 +1406,11 @@ def get_automations(self, id: int, asset: GenericAsset): get: summary: Get all automations defined on an asset. description: | - The response will be a list of automations: recurring forecasting or scheduling tasks + The response will be a list of automations: recurring forecasting, scheduling or reporting tasks defined on the asset. Each entry shows the automation's ID, when it was created, - its type, name, activation status, and its recurrence, both as a cron string - and described in natural language. Each entry also shows the IANA timezone in which its cron expression is interpreted, and its cursor. + its type, name, activation status, recurrence, IANA timezone, cursor, + and counts of recently created jobs per job status. Jobs in Redis have a limited TTL, + so not all past jobs are counted. security: - ApiKeyAuth: [] parameters: @@ -1437,6 +1440,9 @@ def get_automations(self, id: int, asset: GenericAsset): cursor: "2026-07-11T04:00:00+00:00" recurrence_description: "At 06:00" active: true + job_stats: + finished: 3 + redis_connection_err: null 400: description: INVALID_REQUEST, REQUIRED_INFO_MISSING, UNEXPECTED_PARAMS 401: @@ -1448,14 +1454,33 @@ def get_automations(self, id: int, asset: GenericAsset): tags: - Assets """ + redis_connection_err = None + try: + job_stats = get_asset_automations_job_stats(asset) + except NoRedisConfigured as e: + job_stats = {} + redis_connection_err = e.args[0] + except RedisError: + current_app.logger.warning( + "Could not load automation job statistics because Redis is unavailable.", + exc_info=True, + ) + job_stats = {} + redis_connection_err = ( + "Redis is unavailable; job statistics could not be loaded." + ) automations_data = [] for automation in asset.automations: automation_data = automation_schema.dump(automation) automation_data["recurrence_description"] = describe_cronstr( automation.cronstr ) + automation_data["job_stats"] = job_stats.get(automation.id, {}) automations_data.append(automation_data) - return {"automations": automations_data}, 200 + return { + "automations": automations_data, + "redis_connection_err": redis_connection_err, + }, 200 @route("//automations/", methods=["GET"]) @use_kwargs( @@ -1588,6 +1613,15 @@ def get_automation(self, id: int, automation_id: int, asset: GenericAsset): except NoRedisConfigured as e: automation_data["job_stats"] = {} redis_connection_err = e.args[0] + except RedisError: + current_app.logger.warning( + "Could not load automation job statistics because Redis is unavailable.", + exc_info=True, + ) + automation_data["job_stats"] = {} + redis_connection_err = ( + "Redis is unavailable; job statistics could not be loaded." + ) automation_data["redis_connection_err"] = redis_connection_err return automation_data, 200 diff --git a/flexmeasures/api/v3_0/tests/test_automations_api.py b/flexmeasures/api/v3_0/tests/test_automations_api.py index 2c1d11f1d6..44e3df067e 100644 --- a/flexmeasures/api/v3_0/tests/test_automations_api.py +++ b/flexmeasures/api/v3_0/tests/test_automations_api.py @@ -6,6 +6,7 @@ import pytest from flask import url_for +from redis.exceptions import TimeoutError as RedisTimeoutError from sqlalchemy import select from flexmeasures.data.models.automations import Automation @@ -99,12 +100,48 @@ def test_get_automations( assert day_ahead["recurrence_description"] == "At 06:00" assert day_ahead["active"] is True assert day_ahead["created_at"] is not None + assert day_ahead["job_stats"] == {} # this automation has not queued any jobs # generator and parameters are not listed assert "generator_id" not in day_ahead assert "generator" not in day_ahead assert "parameters" not in day_ahead +@pytest.mark.parametrize( + "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True +) +def test_get_automations_when_redis_times_out( + app, + add_battery_assets_fresh_db, + add_automations, + requesting_user, + monkeypatch, +): + """The automation list remains available when Redis times out.""" + battery = add_battery_assets_fresh_db["Test battery"] + + def raise_redis_timeout(asset): + raise RedisTimeoutError("Redis timed out at private-host.example") + + monkeypatch.setattr( + "flexmeasures.api.v3_0.assets.get_asset_automations_job_stats", + raise_redis_timeout, + ) + + with app.test_client() as client: + response = client.get(url_for("AssetAPI:get_automations", id=battery.id)) + + assert response.status_code == 200 + assert len(response.json["automations"]) == 2 + assert all( + automation["job_stats"] == {} for automation in response.json["automations"] + ) + assert response.json["redis_connection_err"] == ( + "Redis is unavailable; job statistics could not be loaded." + ) + assert "private-host.example" not in response.text + + @pytest.mark.parametrize( "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True ) diff --git a/flexmeasures/data/services/automations.py b/flexmeasures/data/services/automations.py index df6c959e02..9dbc098a16 100644 --- a/flexmeasures/data/services/automations.py +++ b/flexmeasures/data/services/automations.py @@ -35,6 +35,7 @@ from flexmeasures.data.queries.generic_assets import ( asset_and_ancestor_ids, asset_is_in_subtree, + descendants_cte, ) from flexmeasures.utils.time_utils import apply_offset_chain, get_timezone, server_now @@ -804,30 +805,34 @@ def get_automations_involving_sensor(sensor: Sensor) -> list[Automation]: return involved -def get_automation_job_stats(automation: Automation) -> dict[str, int]: - """Count the jobs created by this automation, per job status. +def _asset_subtree_sensor_ids(asset_id: int) -> set[int]: + """Return all sensor IDs on an asset and its descendants.""" + tree = descendants_cte(root_asset_id=asset_id, max_depth=None) + return set( + db.session.scalars( + select(Sensor.id).where(Sensor.generic_asset_id.in_(select(tree.c.id))) + ).all() + ) + - Note that jobs in Redis have a limited TTL, so this only counts fairly recent jobs. +def _job_cache_refs( + automation: Automation, schedule_sensor_ids: set[int] | None = None +) -> set[tuple[int, str, str]]: + """The job-cache entries in which an automation's jobs may live. + + Forecasting and reporting jobs are cached under their target/output sensor(s), + which may belong to a different asset than the automation's own asset. """ - # Determine the job cache entries to scan. Forecasting and reporting jobs are cached under their target/output sensor(s), - # which may belong to a different asset than the automation's own asset. parameters = automation.parameters or {} if automation.type == "scheduling": - # Scheduling jobs are cached under the asset (multi-device wrap-up jobs) # and under individual device sensors (per-device jobs), which may belong # to child assets rather than the automation's own (site) asset. - sensor_ids = _relevant_sensor_ids( - automation, - [ - entry.get("sensor") - for entry in parameters.get("flex-model", []) or [] - if isinstance(entry, dict) - ], - ) - cache_refs = [(automation.asset_id, "scheduling", "asset")] + [ - (sensor_id, "scheduling", "sensor") for sensor_id in sensor_ids - ] + if schedule_sensor_ids is None: + schedule_sensor_ids = _asset_subtree_sensor_ids(automation.asset_id) + return {(automation.asset_id, "scheduling", "asset")} | { + (sensor_id, "scheduling", "sensor") for sensor_id in schedule_sensor_ids + } elif automation.type == "reporting": sensor_ids = _relevant_sensor_ids( automation, @@ -837,27 +842,60 @@ def get_automation_job_stats(automation: Automation) -> dict[str, int]: if isinstance(output, dict) ], ) - cache_refs = [(sensor_id, "reporting", "sensor") for sensor_id in sensor_ids] + return {(sensor_id, "reporting", "sensor") for sensor_id in sensor_ids} else: sensor_ids = _relevant_sensor_ids( automation, [parameters.get("sensor"), parameters.get("sensor-to-save")], ) - cache_refs = [(sensor_id, "forecasting", "sensor") for sensor_id in sensor_ids] + return {(sensor_id, "forecasting", "sensor") for sensor_id in sensor_ids} - counts: dict[str, int] = {} + +def _count_automation_jobs( + cache_refs: set[tuple[int, str, str]], automation_ids: set[int] +) -> dict[int, dict[str, int]]: + """Count jobs per automation and status in one pass over the cache entries.""" + counts: dict[int, dict[str, int]] = { + automation_id: {} for automation_id in automation_ids + } seen_job_ids: set[str] = set() for entity_id, queue, asset_or_sensor_type in cache_refs: for job in current_app.job_cache.get(entity_id, queue, asset_or_sensor_type): if job.id in seen_job_ids: continue seen_job_ids.add(job.id) - if job.meta.get("trigger", {}).get("automation_id") == automation.id: + automation_id = job.meta.get("trigger", {}).get("automation_id") + if automation_id in counts: status = str(job.get_status().value) - counts[status] = counts.get(status, 0) + 1 + counts[automation_id][status] = counts[automation_id].get(status, 0) + 1 return counts +def get_automation_job_stats(automation: Automation) -> dict[str, int]: + """Count the recent jobs created by this automation, per job status.""" + return _count_automation_jobs(_job_cache_refs(automation), {automation.id})[ + automation.id + ] + + +def get_asset_automations_job_stats(asset) -> dict[int, dict[str, int]]: + """Count recent jobs for all of an asset's automations in one cache pass.""" + automations = asset.automations + if not automations: + return {} + schedule_sensor_ids = ( + _asset_subtree_sensor_ids(asset.id) + if any(automation.type == "scheduling" for automation in automations) + else None + ) + cache_refs: set[tuple[int, str, str]] = set() + for automation in automations: + cache_refs |= _job_cache_refs(automation, schedule_sensor_ids) + return _count_automation_jobs( + cache_refs, {automation.id for automation in automations} + ) + + def _prepare_forecast_automation( asset, parameters: dict, generator_class: str | None, config: dict | None, source ) -> tuple[Forecaster, dict, list[str]]: diff --git a/flexmeasures/data/tests/test_automations_fresh_db.py b/flexmeasures/data/tests/test_automations_fresh_db.py index 6038d25df9..d9af4645ec 100644 --- a/flexmeasures/data/tests/test_automations_fresh_db.py +++ b/flexmeasures/data/tests/test_automations_fresh_db.py @@ -341,6 +341,17 @@ def test_schedule_automation_stats_include_descendant_jobs_once( app.job_cache.add(root.id, job.id, "scheduling", "asset") app.job_cache.add(child_sensor.id, job.id, "scheduling", "sensor") + child_job = Job.create( + "flexmeasures.utils.time_utils.server_now", connection=queue.connection + ) + child_job.meta["trigger"] = { + "origin": "automation", + "automation_id": schedule_automation.id, + } + child_job.save_meta() + queue.enqueue_job(child_job) + app.job_cache.add(child_sensor.id, child_job.id, "scheduling", "sensor") + other_job = Job.create( "flexmeasures.utils.time_utils.server_now", connection=queue.connection ) @@ -352,7 +363,7 @@ def test_schedule_automation_stats_include_descendant_jobs_once( queue.enqueue_job(other_job) app.job_cache.add(child_sensor.id, other_job.id, "scheduling", "sensor") - assert get_automation_job_stats(schedule_automation) == {"queued": 1} + assert get_automation_job_stats(schedule_automation) == {"queued": 2} def test_automation_has_valid_timezone_and_aware_cursor(automation_with_generator): diff --git a/flexmeasures/ui/static/openapi-specs.json b/flexmeasures/ui/static/openapi-specs.json index 428da1f417..0b0710b135 100644 --- a/flexmeasures/ui/static/openapi-specs.json +++ b/flexmeasures/ui/static/openapi-specs.json @@ -3599,7 +3599,7 @@ "/api/v3_0/assets/{id}/automations": { "get": { "summary": "Get all automations defined on an asset.", - "description": "The response will be a list of automations: recurring forecasting or scheduling tasks\ndefined on the asset. Each entry shows the automation's ID, when it was created,\nits type, name, activation status, and its recurrence, both as a cron string\nand described in natural language. Each entry also shows the IANA timezone in which its cron expression is interpreted, and its cursor.\n", + "description": "The response will be a list of automations: recurring forecasting, scheduling or reporting tasks\ndefined on the asset. Each entry shows the automation's ID, when it was created,\nits type, name, activation status, recurrence, IANA timezone, cursor,\nand counts of recently created jobs per job status. Jobs in Redis have a limited TTL,\nso not all past jobs are counted.\n", "security": [ { "ApiKeyAuth": [] @@ -3636,9 +3636,13 @@ "timezone": "Europe/Amsterdam", "cursor": "2026-07-11T04:00:00+00:00", "recurrence_description": "At 06:00", - "active": true + "active": true, + "job_stats": { + "finished": 3 + } } - ] + ], + "redis_connection_err": null } } } diff --git a/flexmeasures/ui/templates/assets/asset_automations.html b/flexmeasures/ui/templates/assets/asset_automations.html index cf94fd7dc0..fcdd5302b9 100644 --- a/flexmeasures/ui/templates/assets/asset_automations.html +++ b/flexmeasures/ui/templates/assets/asset_automations.html @@ -213,6 +213,12 @@
Edit automation
$("#automations_err").removeClass("d-none").text(message); } + function jobStatsText(jobStats) { + return Object.keys(jobStats || {}).length + ? Object.entries(jobStats).map(([status, count]) => `${status}: ${count}`).join(", ") + : "none"; + } + function showAutomationsMessage(message) { $("#automations_msg").removeClass("d-none").text(message); } @@ -245,9 +251,9 @@
Edit automation
`${esc(automation.recurrence_description)}` ), timezone: esc(automation.timezone), - jobs: `…`, + jobs: `${esc(jobStatsText(automation.job_stats))}`, details: `${runButton(automation)} - Details + Details