From 0649bbe083e9b4c2eede12a5ce9184e68316dbee Mon Sep 17 00:00:00 2001 From: Ernesto Ruge Date: Mon, 14 Apr 2025 20:20:11 +0200 Subject: [PATCH] fix giro-e --- requirements.txt | 6 +- webapp/cli/cli.py | 2 - webapp/cli/giroe_cli.py | 110 ----------------- .../generic_import_heartbeat_tasks.py | 2 +- .../import_services/giroe/giroe_mapper.py | 1 + .../import_services/giroe/giroe_service.py | 113 ++++++++++++++---- .../import_services/giroe/giroe_validator.py | 2 +- 7 files changed, 95 insertions(+), 141 deletions(-) delete mode 100644 webapp/cli/giroe_cli.py diff --git a/requirements.txt b/requirements.txt index 4cf19bcf..93679904 100644 --- a/requirements.txt +++ b/requirements.txt @@ -31,6 +31,6 @@ urllib3~=2.4.0 # binary butterfly shared libraries (https://git.binary-butterfly.de/public_libraries) --extra-index-url https://git.binary-butterfly.de/api/v4/groups/262/-/packages/pypi/simple -flask-openapi~=2.1.3 -validataclass-search-queries~=0.5.0 -butterfly_pubsub~=2.0.0 +flask-openapi~=2.1.4 +validataclass-search-queries~=0.5.1 +butterfly_pubsub~=2.1.1 diff --git a/webapp/cli/cli.py b/webapp/cli/cli.py index 6f96402d..7adbed80 100644 --- a/webapp/cli/cli.py +++ b/webapp/cli/cli.py @@ -20,7 +20,6 @@ from .bnetza_cli import bnetza_cli from .chargeit_cli import chargeit_cli -from .giroe_cli import giroe_cli from .import_cli import import_cli from .match_cli import match_cli from .source_cli import source_cli @@ -31,7 +30,6 @@ def register_cli_to_app(app: Flask): app.cli.add_command(bnetza_cli) app.cli.add_command(chargeit_cli) - app.cli.add_command(giroe_cli) app.cli.add_command(match_cli) app.cli.add_command(import_cli) app.cli.add_command(stadtnavi_cli) diff --git a/webapp/cli/giroe_cli.py b/webapp/cli/giroe_cli.py deleted file mode 100644 index e7746b22..00000000 --- a/webapp/cli/giroe_cli.py +++ /dev/null @@ -1,110 +0,0 @@ -""" -Open ChargePoint DataBase OCPDB -Copyright (C) 2021 binary butterfly GmbH - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see . -""" - -from datetime import datetime, timedelta -from typing import Optional - -import click -from flask.cli import AppGroup - -from webapp.common.error_handling import catch_exception -from webapp.dependencies import dependencies -from webapp.models.connector import ConnectorStatus -from webapp.services.import_services.giroe.pub_sub_services import PubSubService - -giroe_cli = AppGroup('giroe') - - -@giroe_cli.command('import', help='Giro-e: downloads and saves chargepoint updates') -@click.option( - '-p', - '--preset', - 'preset', - type=click.Choice(['daily', 'weekly']), - help='preset for daily (last 36 hours) or weekly (last 10 days) sync', -) -@click.option( - '-cf', - '--created-since', - 'created_since', - type=click.DateTime(formats=['%Y-%m-%d', '%Y-%m-%dT%H:%M:%S']), - help='created since', -) -@click.option( - '-ct', - '--created-until', - 'created_until', - type=click.DateTime(formats=['%Y-%m-%d', '%Y-%m-%dT%H:%M:%S']), - help='created until', -) -@click.option( - '-cf', - '--modified-since', - 'modified_since', - type=click.DateTime(formats=['%Y-%m-%d', '%Y-%m-%dT%H:%M:%S']), - help='modified since', -) -@click.option( - '-ct', - '--modified-until', - 'modified_until', - type=click.DateTime(formats=['%Y-%m-%d', '%Y-%m-%dT%H:%M:%S']), - help='modified until', -) -@catch_exception -def cli_download_and_save( - preset: Optional[str] = None, - created_since: Optional[datetime] = None, - created_until: Optional[datetime] = None, - modified_since: Optional[datetime] = None, - modified_until: Optional[datetime] = None, -): - if preset and (created_since or created_until or modified_since or modified_until): - raise Exception('cannot use preset together with created / modified parameters') - - if preset == 'daily': - modified_since = datetime.utcnow() - timedelta(hours=36) - elif preset == 'weekly': - modified_since = datetime.utcnow() - timedelta(days=10) - - dependencies.get_import_services().giroe_import_service.download_and_save( - created_since=created_since, - created_until=created_until, - modified_since=modified_since, - modified_until=modified_until, - ) - - -@giroe_cli.command('set-connector-status', help='set connector status') -@click.argument('connector_uid', type=str) -@click.argument('connector_status', type=click.Choice([item.value for item in ConnectorStatus])) -def set_connector_status(connector_uid: str, connector_status: str): - pubsub_client = dependencies.get_pubsub_client() - pubsub_client.pub(f'CONNECTOR.{connector_uid.upper()}.STATUS', connector_status) - - -@giroe_cli.command('subscribe', help='subscribe to Giro-e redis pubsub') -def subscribe_connectors(): - pub_sub_connector_service = PubSubService( - pubsub_client=dependencies.get_pubsub_client(), - source_repository=dependencies.get_source_repository(), - evse_repository=dependencies.get_evse_repository(), - **dependencies.get_base_service_dependencies(), - ) - pub_sub_connector_service.register() - pub_sub_connector_service.listen_for_updates() diff --git a/webapp/services/import_services/generic_import_heartbeat_tasks.py b/webapp/services/import_services/generic_import_heartbeat_tasks.py index 1384c7dc..47775d5f 100644 --- a/webapp/services/import_services/generic_import_heartbeat_tasks.py +++ b/webapp/services/import_services/generic_import_heartbeat_tasks.py @@ -40,5 +40,5 @@ def realtime_import_task(source: str): def image_import_task(): from webapp.dependencies import dependencies - image_import_services: ImageImportService = dependencies.get_image_import_services() + image_import_services: ImageImportService = dependencies.get_image_import_service() image_import_services.fetch_images() diff --git a/webapp/services/import_services/giroe/giroe_mapper.py b/webapp/services/import_services/giroe/giroe_mapper.py index 8bc2dcee..ac175ba2 100644 --- a/webapp/services/import_services/giroe/giroe_mapper.py +++ b/webapp/services/import_services/giroe/giroe_mapper.py @@ -72,6 +72,7 @@ def map_location_input_to_update(self, location_data: LocationInput) -> Location def map_station_connector_to_evse_connector(self, station_input: StationInput, connector_input: ConnectorInput): return EvseUpdate( uid=connector_input.uid, + evse_id=connector_input.uid, status=self.map_charge_connector_status_to_evse_status(connector_input.status), capabilities=[ Capability.UNLOCK_CAPABLE, diff --git a/webapp/services/import_services/giroe/giroe_service.py b/webapp/services/import_services/giroe/giroe_service.py index 9fa1c37a..ff7a56eb 100644 --- a/webapp/services/import_services/giroe/giroe_service.py +++ b/webapp/services/import_services/giroe/giroe_service.py @@ -16,8 +16,7 @@ along with this program. If not, see . """ -from datetime import datetime -from typing import List, Optional +from datetime import datetime, timezone from validataclass.exceptions import ValidationError from validataclass.validators import DataclassValidator @@ -25,16 +24,17 @@ from webapp.common.remote_helper import RemoteException, RemoteServerType from webapp.models.source import SourceStatus from webapp.services.import_services.base_import_service import BaseImportService, SourceInfo -from webapp.services.import_services.models import LocationUpdate +from webapp.services.import_services.models import EvseUpdate, LocationUpdate from .giroe_mapper import GiroeMapper -from .giroe_validator import LocationInput, LocationListInput +from .giroe_validator import ConnectorInput, ItemListInput, LocationInput class GiroeImportService(BaseImportService): giroe_mapper: GiroeMapper - location_list_validator = DataclassValidator(LocationListInput) + item_list_validator = DataclassValidator(ItemListInput) location_validator = DataclassValidator(LocationInput) + connector_validator = DataclassValidator(ConnectorInput) source_info = SourceInfo( uid='giroe', @@ -47,16 +47,11 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.giroe_mapper = GiroeMapper(config_helper=self.config_helper) - def download_and_save( - self, - created_since: Optional[datetime] = None, - created_until: Optional[datetime] = None, - modified_since: Optional[datetime] = None, - modified_until: Optional[datetime] = None, - ): + def fetch_static_data(self): source = self.get_source() - location_updates: List[LocationUpdate] = [] + location_updates: list[LocationUpdate] = [] static_error_count = 0 + static_data_updated_at = datetime.now(timezone.utc) try: location_list_data = self.remote_helper.get( @@ -64,19 +59,16 @@ def download_and_save( path='/api/server/v1/charge-locations', params={ 'technical_backend': 'tcc', - **({} if created_since is None else {'created_since': created_since}), - **({} if created_until is None else {'created_until': created_until}), - **({} if modified_since is None else {'modified_since': modified_since}), - **({} if modified_until is None else {'modified_until': modified_until}), + 'public': True, }, ) - location_list_input: LocationListInput = self.location_list_validator.validate(location_list_data) + location_list_input: ItemListInput = self.item_list_validator.validate(location_list_data) except (ValidationError, RemoteException) as e: self.logger.info('import-giro-e', f'giro-e static data has error: {e.to_dict()}') self.update_source(source, static_status=SourceStatus.FAILED) return - location_dicts: List[dict] = location_list_input.items + location_dicts: list[dict] = location_list_input.items while location_list_input.next_path: try: @@ -84,7 +76,7 @@ def download_and_save( remote_server_type=RemoteServerType.GIROE, path=location_list_input.next_path, ) - location_list_input: LocationListInput = self.location_list_validator.validate(location_list_data) + location_list_input: ItemListInput = self.item_list_validator.validate(location_list_data) location_dicts += location_list_input.items except (ValidationError, RemoteException) as e: self.logger.info('import-giro-e', f'giro-e static data has error: {e.to_dict()}') @@ -102,11 +94,84 @@ def download_and_save( static_error_count += 1 continue - if not location_input.public: - continue - location_updates.append(self.giroe_mapper.map_location_input_to_update(location_input)) self.save_location_updates(location_updates) - self.update_source(source=source, static_error_count=static_error_count) + self.update_source( + source=source, + static_status=SourceStatus.ACTIVE, + static_error_count=static_error_count, + static_data_updated_at=static_data_updated_at, + ) + + def fetch_realtime_data(self): + source = self.get_source() + # Don't fetch realtime updates if there is no static data + if source.static_status != SourceStatus.ACTIVE: + return + + evse_updates: list[EvseUpdate] = [] + realtime_error_count = 0 + realtime_data_updated_at = datetime.now(timezone.utc) + + params = { + 'technical_backend': 'tcc', + 'public': True, + } + if source.realtime_data_updated_at: + params['modified_since'] = source.realtime_data_updated_at.isoformat() + try: + connector_list_data = self.remote_helper.get( + remote_server_type=RemoteServerType.GIROE, + path='/api/server/v1/charge-connectors', + params=params, + ) + connector_list_input: ItemListInput = self.item_list_validator.validate(connector_list_data) + except (ValidationError, RemoteException) as e: + self.logger.info('import-giro-e', f'giro-e realtime data has error: {e.to_dict()}') + self.update_source(source, realtime_status=SourceStatus.FAILED) + return + + connector_dicts: list[dict] = connector_list_input.items + + while connector_list_input.next_path: + try: + connector_list_data = self.remote_helper.get( + remote_server_type=RemoteServerType.GIROE, + path=connector_list_input.next_path, + ) + connector_list_input: ItemListInput = self.item_list_validator.validate(connector_list_data) + connector_dicts += connector_list_input.items + except (ValidationError, RemoteException) as e: + self.logger.info('import-giro-e', f'giro-e realtime data has error: {e.to_dict()}') + self.update_source(source, realtime_status=SourceStatus.FAILED) + return + + for connector_dict in connector_dicts: + try: + connector_input: ConnectorInput = self.connector_validator.validate(connector_dict) + except ValidationError as e: + self.logger.info( + 'import-giro-e', + f'connector {connector_dict} has validation error: {e.to_dict()}', + ) + realtime_error_count += 1 + continue + + evse_updates.append( + EvseUpdate( + uid=connector_input.uid, + evse_id=connector_input.uid, + last_updated=connector_input.modified, + ), + ) + + self.save_evse_updates(evse_updates) + + self.update_source( + source=source, + realtime_status=SourceStatus.ACTIVE, + realtime_error_count=realtime_error_count, + realtime_data_updated_at=realtime_data_updated_at, + ) diff --git a/webapp/services/import_services/giroe/giroe_validator.py b/webapp/services/import_services/giroe/giroe_validator.py index 325d42c2..e1ad03f1 100644 --- a/webapp/services/import_services/giroe/giroe_validator.py +++ b/webapp/services/import_services/giroe/giroe_validator.py @@ -97,6 +97,6 @@ class LocationInput: @validataclass -class LocationListInput: +class ItemListInput: items: List[dict] = ListValidator(UnvalidatedDictValidator()) next_path: Optional[str] = StringValidator(), Default(None)