|
39 | 39 | _expression_to_complementary_pyarrow, |
40 | 40 | _HasIds, |
41 | 41 | _NullNaNUnmentionedTermsCollector, |
| 42 | + _primitive_to_physical, |
42 | 43 | _pyarrow_schema_ensure_large_types, |
43 | 44 | _pyarrow_schema_ensure_small_types, |
44 | 45 | pyarrow_to_schema, |
|
61 | 62 | LongType, |
62 | 63 | MapType, |
63 | 64 | NestedField, |
| 65 | + PrimitiveType, |
64 | 66 | StringType, |
65 | 67 | StructType, |
66 | 68 | TimestampNanoType, |
67 | 69 | TimestampType, |
68 | 70 | TimestamptzType, |
69 | 71 | TimeType, |
| 72 | + UUIDType, |
70 | 73 | ) |
71 | 74 |
|
72 | 75 |
|
@@ -259,6 +262,34 @@ def test_pyarrow_variable_binary_to_iceberg(pyarrow_type: pa.DataType) -> None: |
259 | 262 | assert visit(converted_iceberg_type, _ConvertToArrowSchema()) == pa.large_binary() |
260 | 263 |
|
261 | 264 |
|
| 265 | +@pytest.mark.parametrize( |
| 266 | + ("iceberg_type", "pyarrow_type", "parquet_physical_type"), |
| 267 | + [ |
| 268 | + (BooleanType(), pa.bool_(), "BOOLEAN"), |
| 269 | + (IntegerType(), pa.int32(), "INT32"), |
| 270 | + (LongType(), pa.int64(), "INT64"), |
| 271 | + (FloatType(), pa.float32(), "FLOAT"), |
| 272 | + (DoubleType(), pa.float64(), "DOUBLE"), |
| 273 | + (DateType(), pa.date32(), "INT32"), |
| 274 | + (TimeType(), pa.time64("us"), "INT64"), |
| 275 | + (TimestampType(), pa.timestamp("us"), "INT64"), |
| 276 | + (TimestamptzType(), pa.timestamp("us", tz="UTC"), "INT64"), |
| 277 | + (StringType(), pa.large_string(), "BYTE_ARRAY"), |
| 278 | + (UUIDType(), pa.uuid(), "FIXED_LEN_BYTE_ARRAY"), |
| 279 | + (BinaryType(), pa.large_binary(), "BYTE_ARRAY"), |
| 280 | + (FixedType(16), pa.binary(16), "FIXED_LEN_BYTE_ARRAY"), |
| 281 | + (DecimalType(9, 2), pa.decimal128(9, 2), "INT32"), |
| 282 | + (DecimalType(10, 2), pa.decimal128(10, 2), "INT64"), |
| 283 | + (DecimalType(19, 2), pa.decimal128(19, 2), "FIXED_LEN_BYTE_ARRAY"), |
| 284 | + ], |
| 285 | +) |
| 286 | +def test_iceberg_primitive_types_map_to_pyarrow_and_parquet_physical_types( |
| 287 | + iceberg_type: PrimitiveType, pyarrow_type: pa.DataType, parquet_physical_type: str |
| 288 | +) -> None: |
| 289 | + assert schema_to_pyarrow(iceberg_type) == pyarrow_type |
| 290 | + assert _primitive_to_physical(iceberg_type) == parquet_physical_type |
| 291 | + |
| 292 | + |
262 | 293 | def test_pyarrow_struct_to_iceberg() -> None: |
263 | 294 | pyarrow_struct = pa.struct( |
264 | 295 | [ |
|
0 commit comments