From d8780135799bb51b7b7219f130503472c0986c5f Mon Sep 17 00:00:00 2001 From: Tyler Schauer Date: Sun, 20 Sep 2026 18:03:31 -0400 Subject: [PATCH 1/2] Add TODO: nested nullability accuracy, slow-path bytes/duration panics Co-Authored-By: Claude Opus 5 (1M context) --- TODO.md | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 TODO.md diff --git a/TODO.md b/TODO.md new file mode 100644 index 0000000..e71a386 --- /dev/null +++ b/TODO.md @@ -0,0 +1,49 @@ +# TODO + +## Accurate nullability for fields nested under a nullable struct + +Today every descendant of a nullable Avro record (`["null", {"type":"record",...}]`) is +reported as nullable in the Arrow schema, and `list` items are unconditionally nullable — +which then makes the item struct's fields nullable too. So the same record `S` produces +`struct` as a direct field but `list>` inside an +array. Consumers that enforce schemas (Parquet/Delta writers, Polars) lose that information. + +Why it's this way (commit `2c3e45d`): when a struct is null, both decoders push a **null** +into every child builder (`RecordDecoder::append_null` in `fast_decode.rs`, +`StructContainer::add_val` in `complex.rs`). arrow-rs's `StructArray::try_new` only allows +nulls in a non-nullable child when they're masked by the *immediate* parent's null buffer, +so a non-nullable record `U` nested inside nullable `S` breaks: `U` has no null buffer, and +`U.z`'s null is only masked by the grandparent → `Found unmasked nulls for non-nullable +StructArray field "z"`. Propagating `nullable` down in `schema/translate.rs` is the +workaround. + +Fix (builders, not schema): +- On a null parent, append a **placeholder** (`0`, `""`, empty list/map, …) to non-nullable + children instead of a null. The Arrow spec says child slots under a null parent are + undefined, so this is legal. Nullable children keep getting a null. +- Touch points: `RecordDecoder::append_null` and each `FieldDecoder` variant's + `append_null` in `ruhvro/src/fast_decode.rs`; `StructContainer::add_val` (the + `Value::Null` arm) in `ruhvro/src/complex.rs`. +- Then in `ruhvro/src/schema/translate.rs` pass `false` for record children and list + items and let each child's own `["null", T]` union set nullability. +- Verify the serialize side (`serialization_containers.rs`, `fast_encode.rs`) skips + children under a null parent rather than reading the placeholder. The byte-exact + round-trip on nested null records already passes, so this is likely fine, but confirm. +- Regression test: nullable `S` containing non-nullable `U { z: long }`, with a null `S` + row, on both the fast path and `per_datum_deserialize_baseline`; assert the Arrow schema + reports `z` as `not null` and `pyarrow` `validate(full=True)` passes. + +## Slow path panics on `bytes` and `duration` + +`add_data_to_array_builder` in `ruhvro/src/complex.rs` has `unimplemented!()` for +`DataType::Binary` (Avro `bytes`) and hits another `unimplemented!()` for `duration`. Any +schema containing these falls off the fast path (`fast_decode::is_supported` returns +`false`) and then panics in the slow path, which surfaces in Python as a +`pyo3_runtime.PanicException` rather than a `ValueError`. + +- Implement `Binary` (`BinaryBuilder`) and `Duration` in `add_data_to_array_builder`, or +- at minimum replace the `unimplemented!()` arms with `Err(anyhow!(...))` so unsupported + types raise a proper error. The same applies to `DataType::Map` / `RunEndEncoded` / + the catch-all in `default_field_name` in `ruhvro/src/schema/translate.rs`. +- Longer term, add `bytes` to the fast path's supported set so common schemas don't + hit the slow path at all. From e782706f426339adcb2f53c5b095f0c1bb0980a3 Mon Sep 17 00:00:00 2001 From: Tyler Schauer Date: Sun, 20 Sep 2026 18:04:21 -0400 Subject: [PATCH 2/2] Release v0.4.0: schema module, avro_to_arrow_schema, nullable map fixes Co-Authored-By: Claude Opus 5 (1M context) --- Cargo.lock | 4 ++-- Cargo.toml | 4 ++-- ruhvro/Cargo.toml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dfec695..d566785 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1194,7 +1194,7 @@ dependencies = [ [[package]] name = "pyruhvro" -version = "0.3.1" +version = "0.4.0" dependencies = [ "apache-avro", "arrow", @@ -1303,7 +1303,7 @@ checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" [[package]] name = "ruhvro" -version = "0.3.1" +version = "0.4.0" dependencies = [ "anyhow", "apache-avro", diff --git a/Cargo.toml b/Cargo.toml index 07b6b7b..279847b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,7 +16,7 @@ members = [ [package] name = "pyruhvro" -version = "0.3.1" +version = "0.4.0" edition = "2021" license = "MIT" keywords = ["avro", "arrow", "kafka"] @@ -31,7 +31,7 @@ crate-type = ["cdylib"] [dependencies] pyo3 = "0.29" -ruhvro = {path = "ruhvro", version = "0.3.1"} +ruhvro = {path = "ruhvro", version = "0.4.0"} apache-avro = "0.22" arrow = {version = "59", features = ["pyarrow"]} diff --git a/ruhvro/Cargo.toml b/ruhvro/Cargo.toml index be62f2e..5ecd1d8 100644 --- a/ruhvro/Cargo.toml +++ b/ruhvro/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ruhvro" -version = "0.3.1" +version = "0.4.0" edition = "2021" license = "MIT" keywords = ["avro", "arrow", "kafka"]