diff --git a/CHANGELOG.md b/CHANGELOG.md index fe01248..12a6209 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ The changelog lists relevant feature changes between each release. Search GitHub issues and pull requests for smaller issues. +## [Unrelease] +- add support for GBFSv3 +- for yoio, fix timestamp isoformat (truncate microseconds and set UTC timezone) +- chore(deps): bump dependencies + ## 2025-08-20 - add converter for `share_birrer_ch` feed: correct form_factor and propulsion_type diff --git a/app/converters/donkey_free_bike_status_fix.py b/app/converters/donkey_free_bike_status_fix.py index 6be4fb9..2613fbc 100644 --- a/app/converters/donkey_free_bike_status_fix.py +++ b/app/converters/donkey_free_bike_status_fix.py @@ -33,10 +33,10 @@ def _convert_str_to_float(bike: dict[str, Any], property: str) -> None: bike[property] = float(value) def convert(self, data: dict | list, path: str) -> dict | list: - if not isinstance(data, dict) or not path.endswith('/free_bike_status.json'): + if not isinstance(data, dict) or not path.endswith(('/free_bike_status.json', '/vehicle_status.json')): return data - for bike in data['data'].get('bikes', []): + for bike in data['data'].get('bikes', data['data'].get('vehicles', [])): last_reported = bike.get('last_reported') if isinstance(last_reported, str): try: diff --git a/app/converters/gbfs_bird_remove_stations_or_vehicles.py b/app/converters/gbfs_bird_remove_stations_or_vehicles.py index c95eb3a..85855f1 100644 --- a/app/converters/gbfs_bird_remove_stations_or_vehicles.py +++ b/app/converters/gbfs_bird_remove_stations_or_vehicles.py @@ -25,7 +25,7 @@ def convert(self, data: Union[dict, list], path: str) -> Union[dict, list]: continue if system_id in ['basel', 'biel', 'kloten', 'zurich'] and feed.get('name') in ['station_information', 'station_status']: continue - if system_id in ['sarreguemines'] and feed.get('name') in ['free_bike_status']: + if system_id in ['sarreguemines'] and feed.get('name') in ['free_bike_status', 'vehicle_status']: continue new_feeds.append(feed) diff --git a/app/converters/gbfs_fix_gbfv3_timestamps.py b/app/converters/gbfs_fix_gbfv3_timestamps.py new file mode 100644 index 0000000..a6aef3e --- /dev/null +++ b/app/converters/gbfs_fix_gbfv3_timestamps.py @@ -0,0 +1,36 @@ +import logging +from datetime import UTC, datetime +from typing import Union + +from app.base_converter import BaseConverter + +logger = logging.getLogger(__name__) + + +class GbfsFixGbfsV3Timestamps(BaseConverter): + hostnames = [ + 'yoio.rideatom.com', + ] + + def convert(self, data: Union[dict, list], path: str) -> Union[dict, list]: + def clean_timestamp(isotimestring: str) -> str: + try: + parsed_date = datetime.fromisoformat(isotimestring).replace(microsecond=0) + + if parsed_date.tzinfo is None: + parsed_date = parsed_date.replace(tzinfo=UTC) + + return parsed_date.isoformat() + except ValueError: + logger.error('Could not parse %s as date for %s', isotimestring, path) + return isotimestring + + if not isinstance(data, dict): + return data + + if 'last_updated' not in data or not isinstance(data['last_updated'], str): + return data + + data['last_updated'] = clean_timestamp(data['last_updated']) + + return data diff --git a/app/converters/gbfs_https_to_http.py b/app/converters/gbfs_https_to_http.py index 4589ddd..a42e1e9 100644 --- a/app/converters/gbfs_https_to_http.py +++ b/app/converters/gbfs_https_to_http.py @@ -10,6 +10,15 @@ class GbfsHttpsToHttpConverter(BaseConverter): + """ + This Converter rewrites the https-protocol of + all feeds' gbfs files to http, so that subsequent + requests of these will be handled by the proxy as well, + which responds to http requests but not https. + + It supports GBFSv2 and GBFSv3 format. + """ + hostnames = [ 'gbfs.nextbike.net', 'apis.deutschebahn.com', @@ -27,6 +36,17 @@ class GbfsHttpsToHttpConverter(BaseConverter): ] def convert(self, data: Union[dict, list], path: str) -> Union[dict, list]: + def rewrite_https_to_http_for_all_feeds(feeds: list[dict]) -> None: + for feed in feeds: + if ( + not isinstance(feed, dict) + or 'url' not in feed + or not isinstance(feed['url'], str) + or not feed['url'].startswith('https') + ): + continue + feed['url'] = f'http{feed["url"][5:]}' + if not isinstance(data, dict): return data if not (path.endswith(('/gbfs.json', '/gbfs'))): @@ -35,12 +55,13 @@ def convert(self, data: Union[dict, list], path: str) -> Union[dict, list]: if not isinstance(data, dict) or 'data' not in data or not isinstance(data['data'], dict): return data - for language in data['data']: - if 'feeds' not in data['data'][language] or not isinstance(data['data'][language]['feeds'], list): - continue - for feed in data['data'][language]['feeds']: - if not isinstance(feed, dict) or 'url' not in feed or not isinstance(feed['url'], str): + if isinstance(data['data'].get('feeds'), list): + # this gbfs file is >= v3 + rewrite_https_to_http_for_all_feeds(data['data']['feeds']) + else: + for language in data['data']: + if 'feeds' not in data['data'][language] or not isinstance(data['data'][language]['feeds'], list): continue - feed['url'] = f'http{feed["url"][5:]}' + rewrite_https_to_http_for_all_feeds(data['data'][language]['feeds']) return data diff --git a/app/converters/gbfs_nextbike_vehicle_availabilities.py b/app/converters/gbfs_nextbike_vehicle_availabilities.py index 5336daf..c95dd3b 100644 --- a/app/converters/gbfs_nextbike_vehicle_availabilities.py +++ b/app/converters/gbfs_nextbike_vehicle_availabilities.py @@ -49,7 +49,7 @@ def _get_system_id_from_path(path: str) -> str: def _convert_free_vehicle_status(self, system_id: str, data: dict, path: str) -> dict: # cache vehicles per feed - vehicles = data.get('data', {}).get('bikes', []) + vehicles = data.get('data', {}).get('bikes', data.get('vehicles', [])) if isinstance(vehicles, list): self.free_vehicles_cache_per_system[system_id] = vehicles return data diff --git a/app/converters/gbfs_pickebike_basel_change_pricing_plan_id.py b/app/converters/gbfs_pickebike_basel_change_pricing_plan_id.py index 0cc31ae..5d7aeda 100644 --- a/app/converters/gbfs_pickebike_basel_change_pricing_plan_id.py +++ b/app/converters/gbfs_pickebike_basel_change_pricing_plan_id.py @@ -16,7 +16,7 @@ def convert(self, data: Union[dict, list], path: str) -> Union[dict, list]: fields = data.get('data', {}) if not isinstance(fields, dict): return data - vehicles = fields.get('bikes', []) + vehicles = fields.get('bikes', fields.get('vehicles', [])) if not isinstance(vehicles, list): return data for vehicle in vehicles: diff --git a/app/converters/gbfs_set_ttl.py b/app/converters/gbfs_set_ttl.py index dbb6935..199dba5 100644 --- a/app/converters/gbfs_set_ttl.py +++ b/app/converters/gbfs_set_ttl.py @@ -22,7 +22,16 @@ def convert(self, data: Union[dict, list], path: str) -> Union[dict, list]: if not isinstance(data, dict): return data - if not path.endswith(('/station_status', '/station_status.json', '/free_bike_status', '/free_bike_status.json')): + if not path.endswith( + ( + '/station_status', + '/station_status.json', + '/free_bike_status', + '/free_bike_status.json', + '/vehicle_status', + '/vehicle_status.json', + ) + ): if 'ttl' not in data or data['ttl'] == 0: data['ttl'] = SECONDS_PER_HOUR return data diff --git a/requirements-dev.txt b/requirements-dev.txt index 191f980..10b1128 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,3 @@ -ruff~=0.9.6 +ruff~=0.11.9 mypy~=1.15.0 -types-pyyaml~=6.0.12.20241230 +types-pyyaml~=6.0.12.20250915 diff --git a/requirements.txt b/requirements.txt index ec43220..3270aaf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ -mitmproxy~=11.1.2 -pyyaml~=6.0.2 +mitmproxy~=12.2.1 +pyyaml~=6.0.3