diff --git a/requirements.txt b/requirements.txt
index 4cf19bcf..93679904 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -31,6 +31,6 @@ urllib3~=2.4.0
# binary butterfly shared libraries (https://git.binary-butterfly.de/public_libraries)
--extra-index-url https://git.binary-butterfly.de/api/v4/groups/262/-/packages/pypi/simple
-flask-openapi~=2.1.3
-validataclass-search-queries~=0.5.0
-butterfly_pubsub~=2.0.0
+flask-openapi~=2.1.4
+validataclass-search-queries~=0.5.1
+butterfly_pubsub~=2.1.1
diff --git a/webapp/cli/cli.py b/webapp/cli/cli.py
index 6f96402d..7adbed80 100644
--- a/webapp/cli/cli.py
+++ b/webapp/cli/cli.py
@@ -20,7 +20,6 @@
from .bnetza_cli import bnetza_cli
from .chargeit_cli import chargeit_cli
-from .giroe_cli import giroe_cli
from .import_cli import import_cli
from .match_cli import match_cli
from .source_cli import source_cli
@@ -31,7 +30,6 @@
def register_cli_to_app(app: Flask):
app.cli.add_command(bnetza_cli)
app.cli.add_command(chargeit_cli)
- app.cli.add_command(giroe_cli)
app.cli.add_command(match_cli)
app.cli.add_command(import_cli)
app.cli.add_command(stadtnavi_cli)
diff --git a/webapp/cli/giroe_cli.py b/webapp/cli/giroe_cli.py
deleted file mode 100644
index e7746b22..00000000
--- a/webapp/cli/giroe_cli.py
+++ /dev/null
@@ -1,110 +0,0 @@
-"""
-Open ChargePoint DataBase OCPDB
-Copyright (C) 2021 binary butterfly GmbH
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program. If not, see .
-"""
-
-from datetime import datetime, timedelta
-from typing import Optional
-
-import click
-from flask.cli import AppGroup
-
-from webapp.common.error_handling import catch_exception
-from webapp.dependencies import dependencies
-from webapp.models.connector import ConnectorStatus
-from webapp.services.import_services.giroe.pub_sub_services import PubSubService
-
-giroe_cli = AppGroup('giroe')
-
-
-@giroe_cli.command('import', help='Giro-e: downloads and saves chargepoint updates')
-@click.option(
- '-p',
- '--preset',
- 'preset',
- type=click.Choice(['daily', 'weekly']),
- help='preset for daily (last 36 hours) or weekly (last 10 days) sync',
-)
-@click.option(
- '-cf',
- '--created-since',
- 'created_since',
- type=click.DateTime(formats=['%Y-%m-%d', '%Y-%m-%dT%H:%M:%S']),
- help='created since',
-)
-@click.option(
- '-ct',
- '--created-until',
- 'created_until',
- type=click.DateTime(formats=['%Y-%m-%d', '%Y-%m-%dT%H:%M:%S']),
- help='created until',
-)
-@click.option(
- '-cf',
- '--modified-since',
- 'modified_since',
- type=click.DateTime(formats=['%Y-%m-%d', '%Y-%m-%dT%H:%M:%S']),
- help='modified since',
-)
-@click.option(
- '-ct',
- '--modified-until',
- 'modified_until',
- type=click.DateTime(formats=['%Y-%m-%d', '%Y-%m-%dT%H:%M:%S']),
- help='modified until',
-)
-@catch_exception
-def cli_download_and_save(
- preset: Optional[str] = None,
- created_since: Optional[datetime] = None,
- created_until: Optional[datetime] = None,
- modified_since: Optional[datetime] = None,
- modified_until: Optional[datetime] = None,
-):
- if preset and (created_since or created_until or modified_since or modified_until):
- raise Exception('cannot use preset together with created / modified parameters')
-
- if preset == 'daily':
- modified_since = datetime.utcnow() - timedelta(hours=36)
- elif preset == 'weekly':
- modified_since = datetime.utcnow() - timedelta(days=10)
-
- dependencies.get_import_services().giroe_import_service.download_and_save(
- created_since=created_since,
- created_until=created_until,
- modified_since=modified_since,
- modified_until=modified_until,
- )
-
-
-@giroe_cli.command('set-connector-status', help='set connector status')
-@click.argument('connector_uid', type=str)
-@click.argument('connector_status', type=click.Choice([item.value for item in ConnectorStatus]))
-def set_connector_status(connector_uid: str, connector_status: str):
- pubsub_client = dependencies.get_pubsub_client()
- pubsub_client.pub(f'CONNECTOR.{connector_uid.upper()}.STATUS', connector_status)
-
-
-@giroe_cli.command('subscribe', help='subscribe to Giro-e redis pubsub')
-def subscribe_connectors():
- pub_sub_connector_service = PubSubService(
- pubsub_client=dependencies.get_pubsub_client(),
- source_repository=dependencies.get_source_repository(),
- evse_repository=dependencies.get_evse_repository(),
- **dependencies.get_base_service_dependencies(),
- )
- pub_sub_connector_service.register()
- pub_sub_connector_service.listen_for_updates()
diff --git a/webapp/services/import_services/generic_import_heartbeat_tasks.py b/webapp/services/import_services/generic_import_heartbeat_tasks.py
index 1384c7dc..47775d5f 100644
--- a/webapp/services/import_services/generic_import_heartbeat_tasks.py
+++ b/webapp/services/import_services/generic_import_heartbeat_tasks.py
@@ -40,5 +40,5 @@ def realtime_import_task(source: str):
def image_import_task():
from webapp.dependencies import dependencies
- image_import_services: ImageImportService = dependencies.get_image_import_services()
+ image_import_services: ImageImportService = dependencies.get_image_import_service()
image_import_services.fetch_images()
diff --git a/webapp/services/import_services/giroe/giroe_mapper.py b/webapp/services/import_services/giroe/giroe_mapper.py
index 8bc2dcee..ac175ba2 100644
--- a/webapp/services/import_services/giroe/giroe_mapper.py
+++ b/webapp/services/import_services/giroe/giroe_mapper.py
@@ -72,6 +72,7 @@ def map_location_input_to_update(self, location_data: LocationInput) -> Location
def map_station_connector_to_evse_connector(self, station_input: StationInput, connector_input: ConnectorInput):
return EvseUpdate(
uid=connector_input.uid,
+ evse_id=connector_input.uid,
status=self.map_charge_connector_status_to_evse_status(connector_input.status),
capabilities=[
Capability.UNLOCK_CAPABLE,
diff --git a/webapp/services/import_services/giroe/giroe_service.py b/webapp/services/import_services/giroe/giroe_service.py
index 9fa1c37a..ff7a56eb 100644
--- a/webapp/services/import_services/giroe/giroe_service.py
+++ b/webapp/services/import_services/giroe/giroe_service.py
@@ -16,8 +16,7 @@
along with this program. If not, see .
"""
-from datetime import datetime
-from typing import List, Optional
+from datetime import datetime, timezone
from validataclass.exceptions import ValidationError
from validataclass.validators import DataclassValidator
@@ -25,16 +24,17 @@
from webapp.common.remote_helper import RemoteException, RemoteServerType
from webapp.models.source import SourceStatus
from webapp.services.import_services.base_import_service import BaseImportService, SourceInfo
-from webapp.services.import_services.models import LocationUpdate
+from webapp.services.import_services.models import EvseUpdate, LocationUpdate
from .giroe_mapper import GiroeMapper
-from .giroe_validator import LocationInput, LocationListInput
+from .giroe_validator import ConnectorInput, ItemListInput, LocationInput
class GiroeImportService(BaseImportService):
giroe_mapper: GiroeMapper
- location_list_validator = DataclassValidator(LocationListInput)
+ item_list_validator = DataclassValidator(ItemListInput)
location_validator = DataclassValidator(LocationInput)
+ connector_validator = DataclassValidator(ConnectorInput)
source_info = SourceInfo(
uid='giroe',
@@ -47,16 +47,11 @@ def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.giroe_mapper = GiroeMapper(config_helper=self.config_helper)
- def download_and_save(
- self,
- created_since: Optional[datetime] = None,
- created_until: Optional[datetime] = None,
- modified_since: Optional[datetime] = None,
- modified_until: Optional[datetime] = None,
- ):
+ def fetch_static_data(self):
source = self.get_source()
- location_updates: List[LocationUpdate] = []
+ location_updates: list[LocationUpdate] = []
static_error_count = 0
+ static_data_updated_at = datetime.now(timezone.utc)
try:
location_list_data = self.remote_helper.get(
@@ -64,19 +59,16 @@ def download_and_save(
path='/api/server/v1/charge-locations',
params={
'technical_backend': 'tcc',
- **({} if created_since is None else {'created_since': created_since}),
- **({} if created_until is None else {'created_until': created_until}),
- **({} if modified_since is None else {'modified_since': modified_since}),
- **({} if modified_until is None else {'modified_until': modified_until}),
+ 'public': True,
},
)
- location_list_input: LocationListInput = self.location_list_validator.validate(location_list_data)
+ location_list_input: ItemListInput = self.item_list_validator.validate(location_list_data)
except (ValidationError, RemoteException) as e:
self.logger.info('import-giro-e', f'giro-e static data has error: {e.to_dict()}')
self.update_source(source, static_status=SourceStatus.FAILED)
return
- location_dicts: List[dict] = location_list_input.items
+ location_dicts: list[dict] = location_list_input.items
while location_list_input.next_path:
try:
@@ -84,7 +76,7 @@ def download_and_save(
remote_server_type=RemoteServerType.GIROE,
path=location_list_input.next_path,
)
- location_list_input: LocationListInput = self.location_list_validator.validate(location_list_data)
+ location_list_input: ItemListInput = self.item_list_validator.validate(location_list_data)
location_dicts += location_list_input.items
except (ValidationError, RemoteException) as e:
self.logger.info('import-giro-e', f'giro-e static data has error: {e.to_dict()}')
@@ -102,11 +94,84 @@ def download_and_save(
static_error_count += 1
continue
- if not location_input.public:
- continue
-
location_updates.append(self.giroe_mapper.map_location_input_to_update(location_input))
self.save_location_updates(location_updates)
- self.update_source(source=source, static_error_count=static_error_count)
+ self.update_source(
+ source=source,
+ static_status=SourceStatus.ACTIVE,
+ static_error_count=static_error_count,
+ static_data_updated_at=static_data_updated_at,
+ )
+
+ def fetch_realtime_data(self):
+ source = self.get_source()
+ # Don't fetch realtime updates if there is no static data
+ if source.static_status != SourceStatus.ACTIVE:
+ return
+
+ evse_updates: list[EvseUpdate] = []
+ realtime_error_count = 0
+ realtime_data_updated_at = datetime.now(timezone.utc)
+
+ params = {
+ 'technical_backend': 'tcc',
+ 'public': True,
+ }
+ if source.realtime_data_updated_at:
+ params['modified_since'] = source.realtime_data_updated_at.isoformat()
+ try:
+ connector_list_data = self.remote_helper.get(
+ remote_server_type=RemoteServerType.GIROE,
+ path='/api/server/v1/charge-connectors',
+ params=params,
+ )
+ connector_list_input: ItemListInput = self.item_list_validator.validate(connector_list_data)
+ except (ValidationError, RemoteException) as e:
+ self.logger.info('import-giro-e', f'giro-e realtime data has error: {e.to_dict()}')
+ self.update_source(source, realtime_status=SourceStatus.FAILED)
+ return
+
+ connector_dicts: list[dict] = connector_list_input.items
+
+ while connector_list_input.next_path:
+ try:
+ connector_list_data = self.remote_helper.get(
+ remote_server_type=RemoteServerType.GIROE,
+ path=connector_list_input.next_path,
+ )
+ connector_list_input: ItemListInput = self.item_list_validator.validate(connector_list_data)
+ connector_dicts += connector_list_input.items
+ except (ValidationError, RemoteException) as e:
+ self.logger.info('import-giro-e', f'giro-e realtime data has error: {e.to_dict()}')
+ self.update_source(source, realtime_status=SourceStatus.FAILED)
+ return
+
+ for connector_dict in connector_dicts:
+ try:
+ connector_input: ConnectorInput = self.connector_validator.validate(connector_dict)
+ except ValidationError as e:
+ self.logger.info(
+ 'import-giro-e',
+ f'connector {connector_dict} has validation error: {e.to_dict()}',
+ )
+ realtime_error_count += 1
+ continue
+
+ evse_updates.append(
+ EvseUpdate(
+ uid=connector_input.uid,
+ evse_id=connector_input.uid,
+ last_updated=connector_input.modified,
+ ),
+ )
+
+ self.save_evse_updates(evse_updates)
+
+ self.update_source(
+ source=source,
+ realtime_status=SourceStatus.ACTIVE,
+ realtime_error_count=realtime_error_count,
+ realtime_data_updated_at=realtime_data_updated_at,
+ )
diff --git a/webapp/services/import_services/giroe/giroe_validator.py b/webapp/services/import_services/giroe/giroe_validator.py
index 325d42c2..e1ad03f1 100644
--- a/webapp/services/import_services/giroe/giroe_validator.py
+++ b/webapp/services/import_services/giroe/giroe_validator.py
@@ -97,6 +97,6 @@ class LocationInput:
@validataclass
-class LocationListInput:
+class ItemListInput:
items: List[dict] = ListValidator(UnvalidatedDictValidator())
next_path: Optional[str] = StringValidator(), Default(None)