OpenTelemetry logging

.pre-commit-config.yaml

@@ -1,8 +1,24 @@
 repos:
-  - repo: local
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
     hooks:
-      - id: format-and-lint
-        name: Check formatting and linting
-        entry: mobidata-bw-proxy-proxy:latest ruff ./app
-        language: docker_image
-        types: [ python ]
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-added-large-files
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.9.10
+    hooks:
+      - id: ruff-format
+      - id: ruff
+        args: [ --fix, --exit-non-zero-on-fix ]
+
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.15.0
+    hooks:
+    - id: mypy
+      exclude: ^(tests|dev)/
+      additional_dependencies: [
+        'types-PyYAML',
+      ]

addons.py

@@ -1,7 +1,19 @@
 """
 MobiData BW Proxy
 Copyright (c) 2023, binary butterfly GmbH
-All rights reserved.
+
+Licensed under the EUPL, Version 1.2 or – as soon they will be approved by
+the European Commission - subsequent versions of the EUPL (the "Licence");
+You may not use this work except in compliance with the Licence.
+You may obtain a copy of the Licence at:
+
+https://joinup.ec.europa.eu/software/page/eupl
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the Licence is distributed on an "AS IS" basis,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the Licence for the specific language governing permissions and
+limitations under the Licence.
 """
 
 from app import App

app/__init__.py

@@ -1,7 +1,19 @@
 """
 MobiData BW Proxy
 Copyright (c) 2023, binary butterfly GmbH
-All rights reserved.
+
+Licensed under the EUPL, Version 1.2 or – as soon they will be approved by
+the European Commission - subsequent versions of the EUPL (the "Licence");
+You may not use this work except in compliance with the Licence.
+You may obtain a copy of the Licence at:
+
+https://joinup.ec.europa.eu/software/page/eupl
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the Licence is distributed on an "AS IS" basis,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the Licence for the specific language governing permissions and
+limitations under the Licence.
 """
 
 from .app import App

app/app.py

@@ -1,31 +1,55 @@
 """
 MobiData BW Proxy
 Copyright (c) 2023, binary butterfly GmbH
-All rights reserved.
+
+Licensed under the EUPL, Version 1.2 or – as soon they will be approved by
+the European Commission - subsequent versions of the EUPL (the "Licence");
+You may not use this work except in compliance with the Licence.
+You may obtain a copy of the Licence at:
+
+https://joinup.ec.europa.eu/software/page/eupl
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the Licence is distributed on an "AS IS" basis,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the Licence for the specific language governing permissions and
+limitations under the Licence.
 """
 
 import json
 import logging
+import traceback
 from importlib import import_module
 from inspect import isclass
 from json import JSONDecodeError
+from logging.config import dictConfig
 from pathlib import Path
 from pkgutil import iter_modules
 from typing import Dict, List
 
-from mitmproxy.http import HTTPFlow
+from mitmproxy.http import HTTPFlow, Response
 
 from app.base_converter import BaseConverter
 from app.config_helper import ConfigHelper
+from app.utils import ContextHelper
+from app.utils.context_helper import context_helper
+from app.utils.default_json_encoder import DefaultJSONEncoder
 
-logger = logging.getLogger('converters.requests')
+logger = logging.getLogger(__name__)
 
 
 class App:
+    config_helper: ConfigHelper
+    context_helper: ContextHelper
+
     json_converters: Dict[str, List[BaseConverter]]
 
     def __init__(self):
         self.config_helper = ConfigHelper()
+        self.context_helper = context_helper
+
+        # configure logging
+        dictConfig(self.config_helper.get('LOGGING'))
 
         self.json_converters = {}
 
@@ -48,29 +72,60 @@ def __init__(self):
                         self.json_converters[hostname].append(obj)
 
     def request(self, flow: HTTPFlow):
+        self.context_helper.initialize_context()
+
         if flow.request.host in self.config_helper.get('HTTP_TO_HTTPS_HOSTS', []):
             flow.request.scheme = 'https'
             flow.request.port = 443
 
+        self.context_helper.set_attribute('url.host', flow.request.host)
+        self.context_helper.set_attribute('url.scheme', flow.request.scheme)
+        self.context_helper.set_attribute('url.port', flow.request.port)
+        self.context_helper.set_attribute('url.path', flow.request.path)
+
     def response(self, flow: HTTPFlow):
         # Log requests
-        logger.info(f'{flow.request.method} {flow.request.url}: HTTP {"-" if flow.response is None else flow.response.status_code}')
+        logger.debug(f'{flow.request.method} {flow.request.url}: HTTP {"-" if flow.response is None else flow.response.status_code}')
 
         # if there is no converter for the requested host, don't do anything
         if flow.request.host not in self.json_converters:
+            logger.warning('No JSON converter for request.')
             return
 
         # try to load the response. If there is any error, return.
-        if not flow.response or not flow.response.text:
+        if not flow.response:
+            logger.warning('No response for request.')
             return
+
+        response: Response = flow.response
+
+        if not response.text:
+            logger.warning('Empty response for request.')
+            return
+
         try:
-            json_data = json.loads(flow.response.text)
+            json_data = json.loads(response.text)
         except (JSONDecodeError, TypeError):
+            logger.warning(f'Invalid JSON in request: {response.text}.')
             return
 
         # iterate all converters and apply them
         for json_converter in self.json_converters[flow.request.host]:
-            json_data = json_converter.convert(data=json_data, path=flow.request.path)
+            self.context_helper.set_attribute('converter', json_converter.__class__.__name__)
+            try:
+                json_data = json_converter.convert(data=json_data, path=flow.request.path)
+            except Exception as e:
+                logger.error(
+                    f'Converter {json_converter.__class__.__name__} threw an exception {e.__class__.__name__}: {e}',
+                    extra={
+                        'attributes': {
+                            # This needs to be json_data, as json_data is maybe already transformed
+                            'data': json.dumps(json_data),
+                            'traceback': traceback.format_exc(),
+                        },
+                    },
+                )
+                return
 
         # set the returning json content
-        flow.response.text = json.dumps(json_data)
+        flow.response.text = json.dumps(json_data, cls=DefaultJSONEncoder)

app/base_converter.py

@@ -1,7 +1,19 @@
 """
 MobiData BW Proxy
 Copyright (c) 2023, binary butterfly GmbH
-All rights reserved.
+
+Licensed under the EUPL, Version 1.2 or – as soon they will be approved by
+the European Commission - subsequent versions of the EUPL (the "Licence");
+You may not use this work except in compliance with the Licence.
+You may obtain a copy of the Licence at:
+
+https://joinup.ec.europa.eu/software/page/eupl
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the Licence is distributed on an "AS IS" basis,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the Licence for the specific language governing permissions and
+limitations under the Licence.
 """
 
 from abc import ABC, abstractmethod

app/config_helper.py

@@ -1,7 +1,19 @@
 """
 MobiData BW Proxy
 Copyright (c) 2023, binary butterfly GmbH
-All rights reserved.
+
+Licensed under the EUPL, Version 1.2 or – as soon they will be approved by
+the European Commission - subsequent versions of the EUPL (the "Licence");
+You may not use this work except in compliance with the Licence.
+You may obtain a copy of the Licence at:
+
+https://joinup.ec.europa.eu/software/page/eupl
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the Licence is distributed on an "AS IS" basis,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the Licence for the specific language governing permissions and
+limitations under the Licence.
 """
 
 from pathlib import Path
@@ -22,5 +34,5 @@ def __init__(self):
         else:
             self._config = {}
 
-    def get(self, key: str, default: Any) -> Any:
+    def get(self, key: str, default: Any = None) -> Any:
         return self._config.get(key, default)

app/converters/donkey_free_bike_status_fix.py

@@ -1,7 +1,19 @@
 """
 MobiData BW Proxy
 Copyright (c) 2023, systect Holger Bruch
-All rights reserved.
+
+Licensed under the EUPL, Version 1.2 or – as soon they will be approved by
+the European Commission - subsequent versions of the EUPL (the "Licence");
+You may not use this work except in compliance with the Licence.
+You may obtain a copy of the Licence at:
+
+https://joinup.ec.europa.eu/software/page/eupl
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the Licence is distributed on an "AS IS" basis,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the Licence for the specific language governing permissions and
+limitations under the Licence.
 """
 
 import logging
@@ -10,7 +22,7 @@
 
 from app.base_converter import BaseConverter
 
-logger = logging.getLogger('converters.DonkeyFreeBikeStatusConverter')
+logger = logging.getLogger(__name__)
 
 
 class DonkeyFreeBikeStatusConverter(BaseConverter):

app/utils/__init__.py

@@ -0,0 +1,20 @@
+"""
+MobiData BW Proxy
+Copyright (c) 2023, binary butterfly GmbH
+
+Licensed under the EUPL, Version 1.2 or – as soon they will be approved by
+the European Commission - subsequent versions of the EUPL (the "Licence");
+You may not use this work except in compliance with the Licence.
+You may obtain a copy of the Licence at:
+
+https://joinup.ec.europa.eu/software/page/eupl
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the Licence is distributed on an "AS IS" basis,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the Licence for the specific language governing permissions and
+limitations under the Licence.
+"""
+
+from .context_helper import Context, ContextHelper
+from .gbfs_util import update_stations_availability_status

app/utils/context_helper.py

@@ -0,0 +1,69 @@
+"""
+MobiData BW Proxy
+Copyright (c) 2025, binary butterfly GmbH
+
+Licensed under the EUPL, Version 1.2 or – as soon they will be approved by
+the European Commission - subsequent versions of the EUPL (the "Licence");
+You may not use this work except in compliance with the Licence.
+You may obtain a copy of the Licence at:
+
+https://joinup.ec.europa.eu/software/page/eupl
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the Licence is distributed on an "AS IS" basis,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the Licence for the specific language governing permissions and
+limitations under the Licence.
+"""
+
+import secrets
+from dataclasses import dataclass, field
+
+
+@dataclass
+class Context:
+    trace_id: str
+    span_id: str
+    attributes: dict[str, str | int | float] = field(default_factory=dict)
+
+
+class InitializationRequiredException(Exception): ...
+
+
+class ContextHelper:
+    _current_context: Context | None
+
+    def initialize_context(self, trace_id: str | None = None, span_id: str | None = None):
+        if trace_id is None:
+            trace_id = secrets.token_hex(16)
+        if span_id is None:
+            span_id = secrets.token_hex(8)
+        self._current_context = Context(trace_id=trace_id, span_id=span_id)
+
+    def set_attribute(self, key: str, value: str | int | float):
+        if self._current_context is None:
+            raise InitializationRequiredException()
+
+        self._current_context.attributes[key] = value
+
+    def get_current_attributes(self) -> dict[str, str | int | float]:
+        if self._current_context is None:
+            raise InitializationRequiredException()
+
+        return self._current_context.attributes
+
+    def get_current_trace_id(self) -> str:
+        if self._current_context is None:
+            raise InitializationRequiredException()
+
+        return self._current_context.trace_id
+
+    def get_current_span_id(self) -> str:
+        if self._current_context is None:
+            raise InitializationRequiredException()
+
+        return self._current_context.span_id
+
+
+# The ContextHelper is initialized globally for logging system access
+context_helper = ContextHelper()