diff --git a/src/lamp_py/ingestion/config_rt_trip.py b/src/lamp_py/ingestion/config_rt_trip.py index 2cf53f07..43272c90 100644 --- a/src/lamp_py/ingestion/config_rt_trip.py +++ b/src/lamp_py/ingestion/config_rt_trip.py @@ -2,11 +2,7 @@ import pyarrow from lamp_py.ingestion.gtfs_rt_detail import GTFSRTDetail -from lamp_py.ingestion.gtfs_rt_structs import ( - trip_descriptor, - vehicle_descriptor, - stop_time_event, -) +from lamp_py.ingestion.gtfs_rt_structs import trip_descriptor, vehicle_descriptor, stop_time_event, trip_properties from lamp_py.ingestion.utils import explode_table_column, flatten_table_schema @@ -17,7 +13,7 @@ class RtTripDetail(GTFSRTDetail): """ def transform_for_write(self, table: pyarrow.table) -> pyarrow.table: - """modify table schema before write to parquet""" + """Modify table schema before write to parquet""" return flatten_table_schema(explode_table_column(flatten_table_schema(table), "trip_update.stop_time_update")) @property @@ -56,6 +52,7 @@ def import_schema(self) -> pyarrow.schema: ) ), ), + ("trip_properties", trip_properties.pyarrow_dtype), ("timestamp", pyarrow.uint64()), ("delay", pyarrow.int32()), ] diff --git a/src/lamp_py/ingestion/gtfs_rt_structs.py b/src/lamp_py/ingestion/gtfs_rt_structs.py index 7ae77e00..0e7b8135 100644 --- a/src/lamp_py/ingestion/gtfs_rt_structs.py +++ b/src/lamp_py/ingestion/gtfs_rt_structs.py @@ -66,3 +66,5 @@ "uncertainty": dy.Int32(nullable=True), }, ) + +trip_properties = dy.Struct(inner={"trip_headsign": dy.String(nullable=True)}) diff --git a/tests/test_files/ingestion_GTFS-RT_TU.parquet b/tests/test_files/ingestion_GTFS-RT_TU.parquet index 938a7dff..50eec8ae 100644 Binary files a/tests/test_files/ingestion_GTFS-RT_TU.parquet and b/tests/test_files/ingestion_GTFS-RT_TU.parquet differ diff --git a/tests/test_files/ingestion_GTFS-RT_TU_OLD.parquet b/tests/test_files/ingestion_GTFS-RT_TU_OLD.parquet index b015784b..7f3a87f3 100644 Binary files a/tests/test_files/ingestion_GTFS-RT_TU_OLD.parquet and b/tests/test_files/ingestion_GTFS-RT_TU_OLD.parquet differ