From c93ec2e25dcd034a3239f1cf96e4d73e9c2029ce Mon Sep 17 00:00:00 2001 From: Abigale Kim Date: Wed, 5 Aug 2026 21:39:21 +0000 Subject: [PATCH 01/23] first commit of status opt in feature --- cpp/include/cudf/io/experimental/variant.hpp | 39 +- .../cudf/io/experimental/variant_spec.hpp | 17 + .../parquet/experimental/variant_extract.cu | 676 ++++++++++++++++-- .../io/experimental/variant_extract_test.cpp | 550 ++++++++++++++ 4 files changed, 1229 insertions(+), 53 deletions(-) diff --git a/cpp/include/cudf/io/experimental/variant.hpp b/cpp/include/cudf/io/experimental/variant.hpp index 7c53a89e4e7f..f0baa16604e4 100644 --- a/cpp/include/cudf/io/experimental/variant.hpp +++ b/cpp/include/cudf/io/experimental/variant.hpp @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -50,9 +51,17 @@ namespace io::parquet::experimental { * @param path JSONPath-like path string identifying the target field * @param stream CUDA stream * @param mr Device memory resource + * @param status_out If non-null, receives a `UINT8` column of `variant_operation_status` values + * (one per row, aligned with the returned value column). SQL-null input rows produce a null + * status entry. All other rows receive a valid status: `success`, `missing_path`, + * `variant_null`, or `malformed_variant`. Missing-path and malformed rows produce a SQL-null + * output regardless; `success` and `variant_null` rows produce non-null output with the + * resolved bytes. When `nullptr` (the default), no status column is produced. * @return `list` column with the extracted value's encoded bytes. A row is null when the - * input row is null, a name is absent, an index is out of bounds, or a step does not match - * the current value. + * input row is null, a name is absent, an index is out of bounds, a step does not match + * the current value, or bytes are malformed. Encoded VARIANT-null terminal values are + * always returned as the raw VARIANT-null bytes (non-null output); callers can detect them + * via the status column or by inspecting the returned bytes. * * @throws std::invalid_argument on empty path or malformed syntax (`[*]` wildcards, negative * indices, out-of-range indices, and quoted names inside `[...]` are not supported) @@ -60,8 +69,9 @@ namespace io::parquet::experimental { [[nodiscard]] std::unique_ptr get_variant_field( column_view const& variant_column, std::string_view path, - rmm::cuda_stream_view stream = cudf::get_default_stream(), - rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()); + rmm::cuda_stream_view stream = cudf::get_default_stream(), + rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref(), + std::unique_ptr* status_out = nullptr); /** * @brief Decode a VARIANT value column's blobs into a typed cuDF column. @@ -74,6 +84,13 @@ namespace io::parquet::experimental { * `FLOAT32`/`FLOAT64`, or `BOOL8`) * @param stream CUDA stream * @param mr Device memory resource + * @param incoming_status Optional status column from a prior `get_variant_field` call. When + * non-null, rows whose status is null remain null in both the output and the new status + * column, and rows with a non-`success` status are propagated unchanged (output is null, + * status is preserved). Only `success` rows are decoded. + * @param status_out If non-null, receives a `UINT8` column of `variant_operation_status` values + * aligned with the output column. SQL-null input rows (or rows whose incoming status is + * null) produce null status entries. * @return Typed column decoded from the VARIANT value blobs * * @throws std::invalid_argument if `values` is not a `list` column, or if `desired_type` @@ -83,8 +100,10 @@ namespace io::parquet::experimental { [[nodiscard]] std::unique_ptr cast_variant( column_view const& values, data_type desired_type, - rmm::cuda_stream_view stream = cudf::get_default_stream(), - rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()); + rmm::cuda_stream_view stream = cudf::get_default_stream(), + rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref(), + column_view const* incoming_status = nullptr, + std::unique_ptr* status_out = nullptr); /** * @brief Convenience wrapper: extract a nested object value by path and decode into a typed column. @@ -98,6 +117,9 @@ namespace io::parquet::experimental { * `FLOAT32`/`FLOAT64`, or `BOOL8` * @param stream CUDA stream * @param mr Device memory resource + * @param status_out If non-null, receives a `UINT8` column of `variant_operation_status` values + * aligned with the output column, combining extraction and decode outcomes. SQL-null input + * rows produce null status entries. * @return Column of `desired_type` * * @throws std::invalid_argument on empty path or malformed syntax @@ -106,8 +128,9 @@ namespace io::parquet::experimental { column_view const& variant_column, std::string_view path, data_type desired_type, - rmm::cuda_stream_view stream = cudf::get_default_stream(), - rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()); + rmm::cuda_stream_view stream = cudf::get_default_stream(), + rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref(), + std::unique_ptr* status_out = nullptr); /** @} */ } // namespace io::parquet::experimental diff --git a/cpp/include/cudf/io/experimental/variant_spec.hpp b/cpp/include/cudf/io/experimental/variant_spec.hpp index 6b71dc57385c..263e4da53d3b 100644 --- a/cpp/include/cudf/io/experimental/variant_spec.hpp +++ b/cpp/include/cudf/io/experimental/variant_spec.hpp @@ -46,4 +46,21 @@ enum class variant_primitive_type : uint8_t { UUID = 20, }; +/** + * @brief Per-row outcome of a VARIANT extraction or conversion operation. + * + * A SQL-null input row produces a null status (the status column entry is null). + * Every other row receives one of these valid status values. + */ +enum class variant_operation_status : uint8_t { + success, ///< The requested output was produced. + missing_path, ///< Path resolution failed: key absent, index out of range, + ///< or a non-container/null value before the final step. + variant_null, ///< The resolved value is an encoded VARIANT null. + type_mismatch, ///< Source type is not accepted for the requested operation. + malformed_variant, ///< Bytes needed by the requested operation are invalid or truncated. + overflow, ///< Conversion is outside the target range or precision. + invalid_conversion, ///< Conversion failed for another value-dependent reason. +}; + } // namespace cudf::io::parquet::experimental diff --git a/cpp/src/io/parquet/experimental/variant_extract.cu b/cpp/src/io/parquet/experimental/variant_extract.cu index f463230489b5..83ce891f7ccd 100644 --- a/cpp/src/io/parquet/experimental/variant_extract.cu +++ b/cpp/src/io/parquet/experimental/variant_extract.cu @@ -408,6 +408,166 @@ __device__ device_span locate_array_element(device_span enc) +{ + if (enc.empty()) { return false; } + auto const vm = enc[0]; + return decode_basic_type(vm) == basic_type::PRIMITIVE && + variant_value_header(vm) == static_cast(primitive_type::NULLVAL); +} + +// Status-aware version of find_key_in_metadata. Returns (id, status) where status is either +// missing_path (key absent) or malformed_variant (blob is malformed). On success, id has a value. +__device__ cuda::std::pair, op_status> +find_key_in_metadata_with_status(device_span meta, cudf::string_view key) +{ + auto const meta_len = static_cast(meta.size()); + if (meta_len < 1) { return {cuda::std::nullopt, op_status::malformed_variant}; } + + auto const header = meta[0]; + int const version = header & 0x0F; + if (version != variant_version_v1) { return {cuda::std::nullopt, op_status::malformed_variant}; } + int const offset_size = ((header >> 6) & 0x03) + 1; + + size_type pos = 1; + auto const num_entries = narrow_cast(read_uint64(meta, pos, offset_size)); + if (!num_entries.has_value()) { return {cuda::std::nullopt, op_status::malformed_variant}; } + pos += offset_size; + + auto const offsets_start = pos; + auto const offsets_bytes = (static_cast(num_entries.value()) + 1) * offset_size; + if (cuda::std::cmp_greater(offsets_bytes, meta_len - offsets_start)) { + return {cuda::std::nullopt, op_status::malformed_variant}; + } + + auto start_off = read_uint64(meta, offsets_start, offset_size); + if (!start_off.has_value()) { return {cuda::std::nullopt, op_status::malformed_variant}; } + auto const strings_base = offsets_start + static_cast(offsets_bytes); + auto const strings_extent = meta_len - strings_base; + for (size_type i = 0; i < num_entries.value(); ++i) { + auto const end_off = read_uint64(meta, offsets_start + (i + 1) * offset_size, offset_size); + if (!end_off.has_value()) { return {cuda::std::nullopt, op_status::malformed_variant}; } + if (end_off.value() < start_off.value() || end_off.value() > strings_extent) { + return {cuda::std::nullopt, op_status::malformed_variant}; + } + cudf::string_view const entry{ + reinterpret_cast(meta.data() + strings_base + start_off.value()), + static_cast(end_off.value() - start_off.value())}; + if (entry == key) { return {i, op_status::success}; } + start_off = end_off; + } + return {cuda::std::nullopt, op_status::missing_path}; +} + +// Status-aware version of locate_object_field. Returns (span, status). +// Status is missing_path when the value is not an object or the field is absent; +// malformed_variant when the bytes are truncated or otherwise invalid; success otherwise. +__device__ cuda::std::pair, op_status> locate_object_field_with_status( + device_span val, int id) +{ + auto const val_len = static_cast(val.size()); + if (val_len < 1) { return {{}, op_status::malformed_variant}; } + auto const value_metadata = val[0]; + // Not an object: treat as missing path (non-container, can't descend) + if (decode_basic_type(value_metadata) != basic_type::OBJECT) { + return {{}, op_status::missing_path}; + } + + auto const [offset_size, id_size, num_elements_size] = + decode_object_array_header(variant_value_header(value_metadata), true); + + size_type pos = 1; + auto const num_fields = narrow_cast(read_uint64(val, pos, num_elements_size)); + if (!num_fields.has_value()) { return {{}, op_status::malformed_variant}; } + pos += num_elements_size; + + auto const ids_start = pos; + auto const ids_bytes = static_cast(num_fields.value()) * id_size; + if (ids_bytes > val_len - ids_start) { return {{}, op_status::malformed_variant}; } + + auto const offsets_start = ids_start + static_cast(ids_bytes); + auto const offsets_bytes = (static_cast(num_fields.value()) + 1) * offset_size; + if (offsets_bytes > val_len - offsets_start) { return {{}, op_status::malformed_variant}; } + + auto const values_base = offsets_start + static_cast(offsets_bytes); + auto const values_extent = val_len - values_base; + + bool found = false; + uint64_t match_start = 0; + for (size_type i = 0; i < num_fields.value(); ++i) { + auto const current_id = read_uint64(val, ids_start + i * id_size, id_size); + if (!current_id.has_value()) { return {{}, op_status::malformed_variant}; } + if (cuda::std::cmp_not_equal(current_id.value(), id)) { continue; } + auto const match_offset = read_uint64(val, offsets_start + i * offset_size, offset_size); + if (!match_offset.has_value()) { return {{}, op_status::malformed_variant}; } + if (match_offset.value() > values_extent) { return {{}, op_status::malformed_variant}; } + match_start = match_offset.value(); + found = true; + break; + } + if (!found) { return {{}, op_status::missing_path}; } + + auto const value = val.subspan(values_base + match_start); + auto const value_len = variant_value_length(value); + if (!value_len.has_value()) { return {{}, op_status::malformed_variant}; } + auto const match_end = match_start + value_len.value(); + if (match_end > values_extent) { return {{}, op_status::malformed_variant}; } + return {val.subspan(values_base + match_start, value_len.value()), op_status::success}; +} + +// Status-aware version of locate_array_element. Returns (span, status). +__device__ cuda::std::pair, op_status> locate_array_element_with_status( + device_span value, size_type index) +{ + if (index < 0) { return {{}, op_status::missing_path}; } + auto const value_size = static_cast(value.size()); + if (value_size < 1) { return {{}, op_status::malformed_variant}; } + uint8_t const value_metadata = value[0]; + if (decode_basic_type(value_metadata) != basic_type::ARRAY) { + return {{}, op_status::missing_path}; + } + + int const value_header = variant_value_header(value_metadata); + [[maybe_unused]] auto const [offset_size, _, num_elements_size] = + decode_object_array_header(value_header, false); + + size_type position = 1; + auto const num_elements_value = narrow_cast(read_uint64(value, position, num_elements_size)); + if (!num_elements_value.has_value()) { return {{}, op_status::malformed_variant}; } + auto const num_elements = num_elements_value.value(); + if (index >= num_elements) { return {{}, op_status::missing_path}; } + position += num_elements_size; + + size_type const offsets_start = position; + auto const offsets_bytes = (static_cast(num_elements) + 1) * offset_size; + if (cuda::std::cmp_greater(offsets_bytes, value_size - offsets_start)) { + return {{}, op_status::malformed_variant}; + } + size_type const values_base = offsets_start + static_cast(offsets_bytes); + auto const values_extent = value_size - values_base; + + auto const start_offset_pos = offsets_start + static_cast(index) * offset_size; + auto const end_offset_pos = offsets_start + (static_cast(index) + 1) * offset_size; + if (cuda::std::cmp_greater(end_offset_pos + offset_size, value_size)) { + return {{}, op_status::malformed_variant}; + } + auto const start_offset = read_uint64(value, start_offset_pos, offset_size); + auto const end_offset = read_uint64(value, end_offset_pos, offset_size); + if (!start_offset.has_value() || !end_offset.has_value()) { + return {{}, op_status::malformed_variant}; + } + auto const element_start = *start_offset; + auto const element_end = *end_offset; + if (element_end < element_start || cuda::std::cmp_greater(element_end, values_extent)) { + return {{}, op_status::malformed_variant}; + } + return {value.subspan(values_base + element_start, element_end - element_start), + op_status::success}; +} + // The fixed-width signed integers a VARIANT value can be cast to: INT{8,16,32,64}. Matches the // exact width types (not e.g. __int128) since those are the only variant primitive int headers. template @@ -537,6 +697,40 @@ __device__ device_span resolve_path(device_span me return sub_val; } +// Walk a path of object-key or array-index steps, returning (final_span, status). +// Status distinguishes success, missing_path, variant_null, and malformed_variant. +// On variant_null, span points to the VARIANT null bytes (not empty). +__device__ cuda::std::pair, op_status> resolve_path_with_status( + device_span meta, device_span val, column_device_view path) +{ + device_span sub_val = val; + for (size_type i = 0; i < path.size(); ++i) { + auto const step = path.element(i); + + if (step.size_bytes() >= 1 && step.data()[0] == '[') { + auto const index = parse_index_step(step); + if (!index.has_value()) { return {{}, op_status::missing_path}; } + auto const [span, st] = locate_array_element_with_status(sub_val, index.value()); + if (st != op_status::success) { return {{}, st}; } + sub_val = span; + } else { + auto const [field_id, meta_st] = find_key_in_metadata_with_status(meta, step); + if (meta_st == op_status::malformed_variant) { return {{}, op_status::malformed_variant}; } + if (!field_id.has_value()) { return {{}, op_status::missing_path}; } + auto const [span, st] = locate_object_field_with_status(sub_val, field_id.value()); + if (st != op_status::success) { return {{}, st}; } + sub_val = span; + } + + // VARIANT null before the end of the path is missing_path per spec. + if (i + 1 < path.size() && is_variant_null(sub_val)) { return {{}, op_status::missing_path}; } + } + + // Terminal VARIANT null: return the bytes with variant_null status. + if (is_variant_null(sub_val)) { return {sub_val, op_status::variant_null}; } + return {sub_val, op_status::success}; +} + __device__ cuda::std::optional> decode_string( device_span enc) { @@ -625,6 +819,57 @@ CUDF_KERNEL __launch_bounds__(block_size) void locate_variant_fields_kernel( } } +/** + * @brief Status-tracking version of locate_variant_fields_kernel. + * + * Fills `d_status[row]` with the per-row `variant_operation_status`. SQL-null rows get status null + * (their null bit in `d_status_null_mask` is cleared). VARIANT-null terminal values are preserved + * in the output (null bit NOT cleared) and receive `variant_null` status. + */ +CUDF_KERNEL __launch_bounds__(block_size) void locate_variant_fields_with_status_kernel( + cudf::lists_column_device_view metadata, + cudf::lists_column_device_view values, + column_device_view path, + device_span d_sizes, + device_span d_src_offsets, + bitmask_type* d_null_mask, + device_span d_status, + bitmask_type* d_status_null_mask) +{ + auto const num_rows = static_cast(d_sizes.size()); + auto const tid = cudf::detail::grid_1d::global_thread_id(); + auto const stride = cudf::detail::grid_1d::grid_stride(); + + for (auto row = tid; row < num_rows; row += stride) { + if (!cudf::bit_is_set(d_null_mask, row)) { + // SQL-null input: null output, null status + d_sizes[row] = 0; + d_src_offsets[row] = 0; + cudf::clear_bit(d_status_null_mask, row); + continue; + } + + auto const [meta, val] = metadata_and_value_at(metadata, values, row); + auto const [field, st] = resolve_path_with_status(meta, val, path); + + d_status[row] = st; + + if (st == op_status::success) { + d_sizes[row] = static_cast(field.size()); + d_src_offsets[row] = static_cast(field.data() - val.data()); + } else if (st == op_status::variant_null) { + // Keep the VARIANT null bytes in the output; do NOT clear the null bit. + d_sizes[row] = static_cast(field.size()); + d_src_offsets[row] = static_cast(field.data() - val.data()); + } else { + // missing_path or malformed_variant: SQL null output + d_sizes[row] = 0; + d_src_offsets[row] = 0; + cudf::clear_bit(d_null_mask, row); + } + } +} + /** * @brief Per-row kernel: decode each VARIANT value blob into a fixed-width primitive of type `T`. * @@ -660,6 +905,142 @@ CUDF_KERNEL __launch_bounds__(block_size) void cast_variant_primitive_kernel( } } +// Compute the cast status for a single non-null VARIANT value blob targeting a fixed-width +// primitive type T, given that the row's null bit is already set (i.e. the row is valid). +template + requires(is_variant_numerical) +__device__ op_status cast_status_for_primitive(device_span val) +{ + if (val.empty()) { return op_status::malformed_variant; } + if (is_variant_null(val)) { return op_status::variant_null; } + if (decode_primitive(val).has_value()) { return op_status::success; } + // Has a value but not the right type + if (decode_basic_type(val[0]) != basic_type::PRIMITIVE) { return op_status::type_mismatch; } + auto const vhdr = variant_value_header(val[0]); + // Check if it's a valid primitive type at all (otherwise malformed) + switch (static_cast(vhdr)) { + case primitive_type::NULLVAL: + case primitive_type::BOOLEAN_TRUE: + case primitive_type::BOOLEAN_FALSE: + case primitive_type::INT8: + case primitive_type::INT16: + case primitive_type::INT32: + case primitive_type::INT64: + case primitive_type::FLOAT32: + case primitive_type::FLOAT64: + case primitive_type::LONG_STRING: + case primitive_type::BINARY: return op_status::type_mismatch; + default: return op_status::malformed_variant; + } +} + +__device__ op_status cast_status_for_bool(device_span val) +{ + if (val.empty()) { return op_status::malformed_variant; } + if (is_variant_null(val)) { return op_status::variant_null; } + if (decode_bool(val).has_value()) { return op_status::success; } + if (decode_basic_type(val[0]) != basic_type::PRIMITIVE) { return op_status::type_mismatch; } + auto const vhdr = variant_value_header(val[0]); + switch (static_cast(vhdr)) { + case primitive_type::INT8: + case primitive_type::INT16: + case primitive_type::INT32: + case primitive_type::INT64: + case primitive_type::FLOAT32: + case primitive_type::FLOAT64: + case primitive_type::LONG_STRING: return op_status::type_mismatch; + default: return op_status::malformed_variant; + } +} + +__device__ op_status cast_status_for_string(device_span val) +{ + if (val.empty()) { return op_status::malformed_variant; } + if (is_variant_null(val)) { return op_status::variant_null; } + if (decode_string(val).has_value()) { return op_status::success; } + // long_string with truncated payload is malformed; other types are type_mismatch + auto const btype = decode_basic_type(val[0]); + if (btype == basic_type::PRIMITIVE && + variant_value_header(val[0]) == static_cast(primitive_type::LONG_STRING)) { + return op_status::malformed_variant; + } + return op_status::type_mismatch; +} + +// Determine the effective mask and status for a row when an incoming_status column is present. +// Returns (should_decode, row_status_or_nullopt). +// - If incoming status is null → should_decode=false, status is null (nullopt means "write null"). +// - If incoming status is non-success → should_decode=false, propagate that status. +// - If incoming status is success → should_decode=true, status determined by decode result. +__device__ cuda::std::pair> apply_incoming_status( + column_device_view incoming, size_type row) +{ + if (incoming.is_null(row)) { return {false, cuda::std::nullopt}; } + auto const s = incoming.element(row); + if (s != static_cast(op_status::success)) { return {false, static_cast(s)}; } + return {true, cuda::std::nullopt}; // status determined after decode +} + +// Status-tracking variant of cast_variant_primitive_kernel. +// When incoming_status is present, only rows with success status are decoded. +template +CUDF_KERNEL __launch_bounds__(block_size) void cast_variant_primitive_status_kernel( + cudf::lists_column_device_view values, + device_span d_output, + bitmask_type* d_null_mask, + column_device_view incoming_status, + bool has_incoming, + device_span d_status, + bitmask_type* d_status_null_mask) +{ + auto const num_rows = static_cast(d_output.size()); + auto const tid = cudf::detail::grid_1d::global_thread_id(); + auto const stride = cudf::detail::grid_1d::grid_stride(); + + for (auto row = tid; row < num_rows; row += stride) { + if (has_incoming) { + // When an upstream status is present, it is the sole authority: use it to gate decoding. + // The value-column null mask may have been pre-cleared by get_variant_field for non-success + // rows, so we must not rely on it to distinguish SQL-null from missing/malformed here. + auto const [decode, propagated] = apply_incoming_status(incoming_status, row); + if (!decode) { + d_output[row] = T{}; + if (!cudf::bit_is_set(d_null_mask, row)) { + // Already null in the value column (get_variant_field cleared it for non-success rows). + } else { + cudf::clear_bit(d_null_mask, row); + } + if (propagated.has_value()) { + d_status[row] = *propagated; + } else { + // Incoming status was null → this was a SQL-null input row + cudf::clear_bit(d_status_null_mask, row); + } + continue; + } + // Incoming status == success: fall through to decode using the value bytes. + } else { + if (!cudf::bit_is_set(d_null_mask, row)) { + // SQL-null input (no incoming status) → null output, null status + d_output[row] = T{}; + cudf::clear_bit(d_status_null_mask, row); + continue; + } + } + + auto const val = list_row_span(values, row); + auto const decoded = decode_primitive(val); + if (decoded.has_value()) { + d_output[row] = *decoded; + d_status[row] = op_status::success; + } else { + d_output[row] = T{}; + cudf::clear_bit(d_null_mask, row); + d_status[row] = cast_status_for_primitive(val); + } + } +} + /** * @brief Strings-children functor: decode each VARIANT value blob into a string. * @@ -674,28 +1055,63 @@ struct cast_variant_string_fn { size_type* d_sizes; char* d_chars; cudf::detail::input_offsetalator d_offsets; + // Status tracking (optional: both must be non-null to enable) + op_status* d_status{nullptr}; + bitmask_type* d_status_null_mask{nullptr}; + column_device_view incoming_status; + bool has_incoming{false}; __device__ void operator()(size_type row) { - if (!cudf::bit_is_set(d_null_mask, row)) { - if (!d_chars) { d_sizes[row] = 0; } - return; + // Status and the status null mask are only written on the sizing pass (d_chars == nullptr). + // On the writing pass the null mask may already be cleared from the sizing pass, so we must + // not re-inspect it to write status (that would misidentify a decode-failed row as SQL null). + bool const sizing = (d_chars == nullptr); + + if (has_incoming) { + // Incoming status takes precedence over the value-column null mask. + auto const [decode, propagated] = apply_incoming_status(incoming_status, row); + if (!decode) { + if (sizing) { d_sizes[row] = 0; } + if (!cudf::bit_is_set(d_null_mask, row)) { + // already null from get_variant_field + } else { + cudf::clear_bit(d_null_mask, row); + } + if (sizing && d_status) { + if (propagated.has_value()) { + d_status[row] = *propagated; + } else { + cudf::clear_bit(d_status_null_mask, row); + } + } + return; + } + // incoming success: fall through to decode + } else { + if (!cudf::bit_is_set(d_null_mask, row)) { + if (sizing) { d_sizes[row] = 0; } + if (sizing && d_status) { cudf::clear_bit(d_status_null_mask, row); } + return; + } } auto const val = list_row_span(d_values, row); auto const str = decode_string(val); if (!str) { - if (!d_chars) { d_sizes[row] = 0; } + if (sizing) { d_sizes[row] = 0; } cudf::clear_bit(d_null_mask, row); + if (sizing && d_status) { d_status[row] = cast_status_for_string(val); } return; } - if (!d_chars) { + if (sizing) { d_sizes[row] = str->size(); } else { cuda::std::memcpy(d_chars + d_offsets[row], str->data(), str->size()); } + if (sizing && d_status) { d_status[row] = op_status::success; } } }; @@ -709,6 +1125,23 @@ void validate_variant_child(column_view const& child) std::invalid_argument); } +// Build the status column from d_status and d_status_null_mask buffers. +std::unique_ptr make_status_column(rmm::device_buffer status_data, + rmm::device_buffer status_null_mask, + size_type num_rows, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) +{ + auto const null_count = cudf::detail::count_unset_bits( + static_cast(status_null_mask.data()), 0, num_rows, stream); + return std::make_unique( + data_type{type_id::UINT8}, + num_rows, + std::move(status_data), + null_count > 0 ? std::move(status_null_mask) : rmm::device_buffer{}, + null_count); +} + struct cast_variant_fn { cudf::lists_column_device_view values; size_type num_rows; @@ -717,17 +1150,37 @@ struct cast_variant_fn { rmm::device_buffer null_mask; rmm::cuda_stream_view stream; rmm::device_async_resource_ref mr; + // Optional status tracking + column_device_view incoming_status_view; + bool has_incoming{false}; + std::unique_ptr* status_out{nullptr}; template std::unique_ptr operator()() requires(is_variant_numerical) { rmm::device_buffer data{num_rows * sizeof(T), stream, mr}; - - auto grid = cudf::detail::grid_1d{num_rows, block_size}; - cast_variant_primitive_kernel<<>>( - values, {static_cast(data.data()), static_cast(num_rows)}, d_null_mask); - CUDF_CUDA_TRY(cudaGetLastError()); + auto const grid = cudf::detail::grid_1d{num_rows, block_size}; + + if (status_out != nullptr) { + rmm::device_buffer status_data{num_rows * sizeof(op_status), stream, mr}; + auto status_null_mask = cudf::create_null_mask(num_rows, mask_state::ALL_VALID, stream, mr); + cast_variant_primitive_status_kernel<<>>( + values, + {static_cast(data.data()), static_cast(num_rows)}, + d_null_mask, + incoming_status_view, + has_incoming, + {static_cast(status_data.data()), static_cast(num_rows)}, + static_cast(status_null_mask.data())); + CUDF_CUDA_TRY(cudaGetLastError()); + *status_out = make_status_column( + std::move(status_data), std::move(status_null_mask), num_rows, stream, mr); + } else { + cast_variant_primitive_kernel<<>>( + values, {static_cast(data.data()), static_cast(num_rows)}, d_null_mask); + CUDF_CUDA_TRY(cudaGetLastError()); + } auto const null_count = num_rows - cudf::detail::count_set_bits(d_null_mask, 0, num_rows, stream); @@ -744,19 +1197,74 @@ struct cast_variant_fn { { rmm::device_buffer data{num_rows * sizeof(bool), stream, mr}; - thrust::transform( - rmm::exec_policy_nosync(stream, cudf::get_current_device_resource_ref()), - cuda::counting_iterator(0), - cuda::counting_iterator(num_rows), - static_cast(data.data()), - [values = this->values, d_null_mask = this->d_null_mask] __device__(size_type row) -> bool { - if (!cudf::bit_is_set(d_null_mask, row)) { return false; } - auto const val = list_row_span(values, row); - auto const decoded = decode_bool(val); - if (decoded.has_value()) { return *decoded; } - cudf::clear_bit(d_null_mask, row); - return false; - }); + if (status_out != nullptr) { + rmm::device_buffer status_data{num_rows * sizeof(op_status), stream, mr}; + auto status_null_mask = cudf::create_null_mask(num_rows, mask_state::ALL_VALID, stream, mr); + auto* d_status = static_cast(status_data.data()); + auto* d_status_null_mask = static_cast(status_null_mask.data()); + auto const inc_view = incoming_status_view; + auto const hi = has_incoming; + thrust::for_each(rmm::exec_policy_nosync(stream, cudf::get_current_device_resource_ref()), + cuda::counting_iterator(0), + cuda::counting_iterator(num_rows), + [vals = this->values, + d_out = static_cast(data.data()), + d_null_mask = this->d_null_mask, + d_status, + d_status_null_mask, + inc_view, + hi] __device__(size_type row) { + if (hi) { + auto const [decode, propagated] = apply_incoming_status(inc_view, row); + if (!decode) { + d_out[row] = false; + if (!cudf::bit_is_set(d_null_mask, row)) { + // already null from get_variant_field + } else { + cudf::clear_bit(d_null_mask, row); + } + if (propagated.has_value()) { + d_status[row] = *propagated; + } else { + cudf::clear_bit(d_status_null_mask, row); + } + return; + } + } else { + if (!cudf::bit_is_set(d_null_mask, row)) { + d_out[row] = false; + cudf::clear_bit(d_status_null_mask, row); + return; + } + } + auto const val = list_row_span(vals, row); + auto const decoded = decode_bool(val); + if (decoded.has_value()) { + d_out[row] = *decoded; + d_status[row] = op_status::success; + } else { + d_out[row] = false; + cudf::clear_bit(d_null_mask, row); + d_status[row] = cast_status_for_bool(val); + } + }); + *status_out = make_status_column( + std::move(status_data), std::move(status_null_mask), num_rows, stream, mr); + } else { + thrust::transform( + rmm::exec_policy_nosync(stream, cudf::get_current_device_resource_ref()), + cuda::counting_iterator(0), + cuda::counting_iterator(num_rows), + static_cast(data.data()), + [vals = this->values, d_null_mask = this->d_null_mask] __device__(size_type row) -> bool { + if (!cudf::bit_is_set(d_null_mask, row)) { return false; } + auto const val = list_row_span(vals, row); + auto const decoded = decode_bool(val); + if (decoded.has_value()) { return *decoded; } + cudf::clear_bit(d_null_mask, row); + return false; + }); + } auto const null_count = num_rows - cudf::detail::count_set_bits(d_null_mask, 0, num_rows, stream); @@ -771,10 +1279,35 @@ struct cast_variant_fn { std::unique_ptr operator()() requires(cuda::std::is_same_v) { - cast_variant_string_fn fn{values, d_null_mask, nullptr, nullptr, {}}; + rmm::device_buffer status_data; + rmm::device_buffer status_null_mask; + op_status* d_status_ptr{nullptr}; + bitmask_type* d_status_null_mask_ptr{nullptr}; + + if (status_out != nullptr) { + status_data = rmm::device_buffer{num_rows * sizeof(op_status), stream, mr}; + status_null_mask = cudf::create_null_mask(num_rows, mask_state::ALL_VALID, stream, mr); + d_status_ptr = static_cast(status_data.data()); + d_status_null_mask_ptr = static_cast(status_null_mask.data()); + } + + cast_variant_string_fn fn{values, + d_null_mask, + nullptr, + nullptr, + {}, + d_status_ptr, + d_status_null_mask_ptr, + incoming_status_view, + has_incoming}; auto [offsets_column, chars] = cudf::strings::detail::make_strings_children(fn, num_rows, stream, mr); + if (status_out != nullptr) { + *status_out = make_status_column( + std::move(status_data), std::move(status_null_mask), num_rows, stream, mr); + } + auto const null_count = num_rows - cudf::detail::count_set_bits(d_null_mask, 0, num_rows, stream); return make_strings_column(num_rows, @@ -826,7 +1359,8 @@ namespace detail { std::unique_ptr get_variant_field(column_view const& variant_column, std::string_view path, rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) + rmm::device_async_resource_ref mr, + std::unique_ptr* status_out) { // Validate the variant column CUDF_EXPECTS(variant_column.type().id() == type_id::STRUCT, @@ -843,6 +1377,7 @@ std::unique_ptr get_variant_field(column_view const& variant_column, auto const num_rows = variant_column.size(); if (num_rows == 0) { + if (status_out != nullptr) { *status_out = make_empty_column(data_type{type_id::UINT8}); } return cudf::make_lists_column( 0, make_empty_column(type_id::INT32), make_empty_column(type_id::UINT8), 0, {}); } @@ -871,16 +1406,35 @@ std::unique_ptr get_variant_field(column_view const& variant_column, : cudf::create_null_mask(variant_column.size(), mask_state::ALL_VALID, stream, mr); auto* d_null_mask = static_cast(null_mask.data()); - // Parse the path per row and compute the output sizes auto grid = cudf::detail::grid_1d{num_rows, block_size}; - locate_variant_fields_kernel<<>>( - meta_lists_device_view, - val_lists_device_view, - *path_device_view, - d_sizes, - d_src_offsets, - d_null_mask); - CUDF_CUDA_TRY(cudaGetLastError()); + + if (status_out != nullptr) { + rmm::device_buffer status_data{num_rows * sizeof(op_status), stream, mr}; + auto status_null_mask = cudf::create_null_mask(num_rows, mask_state::ALL_VALID, stream, mr); + + locate_variant_fields_with_status_kernel<<>>( + meta_lists_device_view, + val_lists_device_view, + *path_device_view, + d_sizes, + d_src_offsets, + d_null_mask, + {static_cast(status_data.data()), static_cast(num_rows)}, + static_cast(status_null_mask.data())); + CUDF_CUDA_TRY(cudaGetLastError()); + + *status_out = + make_status_column(std::move(status_data), std::move(status_null_mask), num_rows, stream, mr); + } else { + locate_variant_fields_kernel<<>>( + meta_lists_device_view, + val_lists_device_view, + *path_device_view, + d_sizes, + d_src_offsets, + d_null_mask); + CUDF_CUDA_TRY(cudaGetLastError()); + } // Convert sizes to offsets auto [offsets_column, total_bytes] = @@ -923,11 +1477,16 @@ std::unique_ptr get_variant_field(column_view const& variant_column, std::unique_ptr cast_variant(column_view const& values, data_type desired_type, rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) + rmm::device_async_resource_ref mr, + column_view const* incoming_status, + std::unique_ptr* status_out) { validate_variant_child(values); size_type const num_rows = values.size(); - if (num_rows == 0) { return make_empty_column(desired_type); } + if (num_rows == 0) { + if (status_out != nullptr) { *status_out = make_empty_column(data_type{type_id::UINT8}); } + return make_empty_column(desired_type); + } auto val_device_view = column_device_view::create(values, stream); cudf::lists_column_device_view val_lists_device_view(*val_device_view); @@ -938,6 +1497,15 @@ std::unique_ptr cast_variant(column_view const& values, : cudf::create_null_mask(num_rows, mask_state::ALL_VALID, stream, mr); auto* d_null_mask = static_cast(null_mask.data()); + // Build device view for incoming status if provided; keep a placeholder when absent so that + // cast_variant_fn always holds a valid column_device_view (kernel ignores it when !has_incoming). + auto placeholder_col = make_empty_column(data_type{type_id::UINT8}); + auto placeholder_dev_view = column_device_view::create(*placeholder_col, stream); + auto incoming_dev_view = (incoming_status != nullptr) + ? column_device_view::create(*incoming_status, stream) + : column_device_view::create(*placeholder_col, stream); + bool const has_incoming = (incoming_status != nullptr); + return cudf::type_dispatcher(desired_type, cast_variant_fn{val_lists_device_view, num_rows, @@ -945,7 +1513,10 @@ std::unique_ptr cast_variant(column_view const& values, d_null_mask, std::move(null_mask), stream, - mr}); + mr, + *incoming_dev_view, + has_incoming, + status_out}); } } // namespace detail @@ -953,31 +1524,46 @@ std::unique_ptr cast_variant(column_view const& values, std::unique_ptr get_variant_field(column_view const& variant_column, std::string_view path, rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) + rmm::device_async_resource_ref mr, + std::unique_ptr* status_out) { CUDF_FUNC_RANGE(); - return detail::get_variant_field(variant_column, path, stream, mr); + return detail::get_variant_field(variant_column, path, stream, mr, status_out); } std::unique_ptr cast_variant(column_view const& values, data_type desired_type, rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) + rmm::device_async_resource_ref mr, + column_view const* incoming_status, + std::unique_ptr* status_out) { CUDF_FUNC_RANGE(); - return detail::cast_variant(values, desired_type, stream, mr); + return detail::cast_variant(values, desired_type, stream, mr, incoming_status, status_out); } std::unique_ptr extract_variant_field(column_view const& variant_column, std::string_view path, data_type desired_type, rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) + rmm::device_async_resource_ref mr, + std::unique_ptr* status_out) { CUDF_FUNC_RANGE(); - auto value = detail::get_variant_field( - variant_column, path, stream, cudf::get_current_device_resource_ref()); - return detail::cast_variant(value->view(), desired_type, stream, mr); + auto const temp_mr = cudf::get_current_device_resource_ref(); + + if (status_out != nullptr) { + // Get the raw field bytes, capturing intermediate extraction status + std::unique_ptr extract_status; + auto value = detail::get_variant_field(variant_column, path, stream, temp_mr, &extract_status); + // Cast using the extraction status as incoming; cast populates the final status + auto const extract_status_view = extract_status->view(); + return detail::cast_variant( + value->view(), desired_type, stream, mr, &extract_status_view, status_out); + } + + auto value = detail::get_variant_field(variant_column, path, stream, temp_mr, nullptr); + return detail::cast_variant(value->view(), desired_type, stream, mr, nullptr, nullptr); } } // namespace io::parquet::experimental diff --git a/cpp/tests/io/experimental/variant_extract_test.cpp b/cpp/tests/io/experimental/variant_extract_test.cpp index e810bbe142a0..84991bca374e 100644 --- a/cpp/tests/io/experimental/variant_extract_test.cpp +++ b/cpp/tests/io/experimental/variant_extract_test.cpp @@ -1381,3 +1381,553 @@ TEST_F(InvalidInputShapeTest, CastVariantRejectsMalformedInput) std::invalid_argument); } } + +// --------------------------------------------------------------------------- +// Status column tests +// --------------------------------------------------------------------------- +using op_status = cudf::io::parquet::experimental::variant_operation_status; +namespace expns = cudf::io::parquet::experimental; +auto const& cmr = cudf::get_current_device_resource_ref; + +// Helper using fixed_width_column_wrapper comparison for the common case where the status +// column has no nulls. +static void expect_status_values(cudf::column_view const& status, + std::vector const& expected) +{ + cudf::test::fixed_width_column_wrapper exp(expected.begin(), expected.end()); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(status, exp); +} + +constexpr uint8_t ST_SUCCESS = static_cast(op_status::success); +constexpr uint8_t ST_MISSING = static_cast(op_status::missing_path); +constexpr uint8_t ST_VNULL = static_cast(op_status::variant_null); +constexpr uint8_t ST_MISMATCH = static_cast(op_status::type_mismatch); +constexpr uint8_t ST_MALFORMED = static_cast(op_status::malformed_variant); + +// --------------------------------------------------------------------------- +// GetVariantField status tests +// --------------------------------------------------------------------------- + +struct GetVariantFieldStatusTest : public cudf::test::BaseFixture {}; + +// SQL-null input row → null output + null status +TEST_F(GetVariantFieldStatusTest, SqlNullInputProducesNullStatus) +{ + cudf::test::lists_column_wrapper meta{{0x01, 0x01, 0x00, 0x01, 'x'}}; + cudf::test::lists_column_wrapper val{{0x14, 0x07, 0x00, 0x00, 0x00}}; + cudf::test::structs_column_wrapper col{{meta, val}, std::vector{false}}; + + auto stream = cudf::test::get_default_stream(); + std::unique_ptr status; + auto got = cudf::io::parquet::experimental::get_variant_field(col, "x", stream, cmr(), &status); + + ASSERT_NE(status, nullptr); + ASSERT_EQ(status->size(), 1); + ASSERT_EQ(status->null_count(), 1); + ASSERT_EQ(got->null_count(), 1); +} + +// Successful extraction → success status +TEST_F(GetVariantFieldStatusTest, SuccessStatus) +{ + auto col = make_xyz_three_row_variant(); + auto stream = cudf::test::get_default_stream(); + + std::unique_ptr status; + auto got = cudf::io::parquet::experimental::get_variant_field(col, "x", stream, cmr(), &status); + + // Row 0: x=INT32(7) → success; Row 1: x=INT32(42) → success; Row 2: no x → missing_path + ASSERT_NE(status, nullptr); + expect_status_values(*status, {ST_SUCCESS, ST_SUCCESS, ST_MISSING}); + // Output rows 0,1 valid; row 2 null + EXPECT_EQ(got->null_count(), 1); +} + +// Missing key → missing_path status +TEST_F(GetVariantFieldStatusTest, MissingKeyProducesMissingPathStatus) +{ + auto col = make_apache_variant(avf::object_primitive); + auto stream = cudf::test::get_default_stream(); + + std::unique_ptr status; + auto got = cudf::io::parquet::experimental::get_variant_field( + col, "no_such_field", stream, cmr(), &status); + + ASSERT_NE(status, nullptr); + ASSERT_EQ(status->size(), 1); + expect_status_values(*status, {ST_MISSING}); + EXPECT_EQ(got->null_count(), 1); +} + +// VARIANT null terminal value → variant_null status, preserved bytes (non-null output) +TEST_F(GetVariantFieldStatusTest, VariantNullPreservedWithStatus) +{ + // Build a single-row VARIANT: object {null_field: VARIANT_NULL} + // metadata: {null_field}, value: object wrapping NULLVAL primitive + auto const m = build_metadata({"null_field"}); + auto const v = build_single_field_object(/*fid=*/0, enc_null()); + auto col = wrap_single_variant(m, v); + auto stream = cudf::test::get_default_stream(); + + std::unique_ptr status; + auto got = + cudf::io::parquet::experimental::get_variant_field(col, "null_field", stream, cmr(), &status); + + ASSERT_NE(status, nullptr); + ASSERT_EQ(status->size(), 1); + expect_status_values(*status, {ST_VNULL}); + // With status requested, the VARIANT null bytes are preserved (output is NOT SQL null) + EXPECT_EQ(got->null_count(), 0); +} + +// Without status_out, VARIANT null is returned as bytes (non-null list row), same as with status. +// Only cast_variant turns a VARIANT null blob into a SQL null. +TEST_F(GetVariantFieldStatusTest, VariantNullReturnedAsBytesWithoutStatus) +{ + auto const m = build_metadata({"null_field"}); + auto const v = build_single_field_object(/*fid=*/0, enc_null()); + auto col = wrap_single_variant(m, v); + auto stream = cudf::test::get_default_stream(); + + // No status_out: get_variant_field returns the VARIANT null bytes as a non-null list row. + auto got = cudf::io::parquet::experimental::get_variant_field(col, "null_field", stream); + EXPECT_EQ(got->null_count(), 0); + EXPECT_EQ(got->size(), 1); +} + +// Malformed metadata → malformed_variant status +TEST_F(GetVariantFieldStatusTest, MalformedMetadataProducesMalformedStatus) +{ + std::vector const bad_meta = {0x02}; // too short / version ≠ 1 + std::vector const val = {0x14, 0x07, 0x00, 0x00, 0x00}; + auto col = wrap_single_variant(bad_meta, val); + auto stream = cudf::test::get_default_stream(); + + std::unique_ptr status; + auto got = cudf::io::parquet::experimental::get_variant_field(col, "x", stream, cmr(), &status); + + ASSERT_NE(status, nullptr); + expect_status_values(*status, {ST_MALFORMED}); + EXPECT_EQ(got->null_count(), 1); +} + +// VARIANT null before end of a nested path → missing_path +TEST_F(GetVariantFieldStatusTest, VariantNullBeforeEndIsMissingPath) +{ + // Object {a: VARIANT_NULL}; path "$.a.b" should be missing_path (null intermediate) + auto const m = build_metadata({"a"}); + auto const v = build_single_field_object(/*fid=*/0, enc_null()); + auto col = wrap_single_variant(m, v); + auto stream = cudf::test::get_default_stream(); + + std::unique_ptr status; + auto got = + cudf::io::parquet::experimental::get_variant_field(col, "$.a.b", stream, cmr(), &status); + + ASSERT_NE(status, nullptr); + expect_status_values(*status, {ST_MISSING}); + EXPECT_EQ(got->null_count(), 1); +} + +// Mixed rows: success / missing / variant_null / malformed / SQL null +TEST_F(GetVariantFieldStatusTest, MixedRows) +{ + auto stream = cudf::test::get_default_stream(); + + auto const dict = build_metadata({"x"}); + + // Row 0: {x: INT32(5)} → success + auto const v0 = build_single_field_object(/*fid=*/0, enc_int32(5)); + // Row 1: {x: NULLVAL} → variant_null + auto const v1 = build_single_field_object(/*fid=*/0, enc_null()); + // Row 2: {} (no x key) → missing_path + auto const m2 = build_metadata({}); + auto const v2 = build_single_field_object(/*fid=*/0, enc_int32(0)); // fid 0 but dict empty + // Row 3: SQL null → null status + auto const v3 = enc_int32(0); + + cudf::test::lists_column_wrapper meta{{dict.begin(), dict.end()}, + {dict.begin(), dict.end()}, + {m2.begin(), m2.end()}, + {dict.begin(), dict.end()}}; + cudf::test::lists_column_wrapper val{ + {v0.begin(), v0.end()}, {v1.begin(), v1.end()}, {v2.begin(), v2.end()}, {v3.begin(), v3.end()}}; + // Row 3 is SQL null + cudf::test::structs_column_wrapper col{{meta, val}, std::vector{true, true, true, false}}; + + std::unique_ptr status; + auto got = cudf::io::parquet::experimental::get_variant_field(col, "x", stream, cmr(), &status); + + ASSERT_NE(status, nullptr); + ASSERT_EQ(status->size(), 4); + // Row 3 (SQL null input) → null status; rows 0-2 → valid status + ASSERT_EQ(status->null_count(), 1); + // Check per-row status values via expected column (rows 0-2 valid, row 3 null) + std::vector const exp_sv{ST_SUCCESS, ST_VNULL, ST_MISSING, 0}; + std::vector const exp_sv_valid{true, true, true, false}; + cudf::test::fixed_width_column_wrapper expected_status( + exp_sv.begin(), exp_sv.end(), exp_sv_valid.begin()); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*status, expected_status); + + // Row 0: valid (INT32 bytes), Row 1: valid (VARIANT null bytes preserved), Row 2+3: null + EXPECT_EQ(got->null_count(), 2); +} + +// Empty input → empty status column +TEST_F(GetVariantFieldStatusTest, EmptyInput) +{ + auto const stream = cudf::test::get_default_stream(); + auto const variant = cudf::empty_like(make_xyz_three_row_variant()); + + std::unique_ptr status; + auto got = + cudf::io::parquet::experimental::get_variant_field(*variant, "x", stream, cmr(), &status); + + ASSERT_NE(status, nullptr); + EXPECT_EQ(status->size(), 0); + EXPECT_EQ(got->size(), 0); +} + +// --------------------------------------------------------------------------- +// CastVariant status tests +// --------------------------------------------------------------------------- + +struct CastVariantStatusTest : public cudf::test::BaseFixture {}; + +namespace { + +inline cudf::test::lists_column_wrapper make_value_col(std::vector const& bytes) +{ + return cudf::test::lists_column_wrapper(bytes.begin(), bytes.end()); +} + +} // namespace + +// Success → success status +TEST_F(CastVariantStatusTest, SuccessProducesSuccessStatus) +{ + auto stream = cudf::test::get_default_stream(); + auto values = make_value_col(enc_int32(42)); + std::unique_ptr status; + auto got = cudf::io::parquet::experimental::cast_variant( + values, cudf::data_type{cudf::type_id::INT32}, stream, cmr(), nullptr, &status); + + ASSERT_NE(status, nullptr); + expect_status_values(*status, {ST_SUCCESS}); + cudf::test::fixed_width_column_wrapper expected{42}; + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); +} + +// VARIANT null → variant_null status +TEST_F(CastVariantStatusTest, VariantNullProducesVariantNullStatus) +{ + auto stream = cudf::test::get_default_stream(); + auto values = make_value_col(enc_null()); + std::unique_ptr status; + auto got = cudf::io::parquet::experimental::cast_variant( + values, cudf::data_type{cudf::type_id::INT32}, stream, cmr(), nullptr, &status); + + ASSERT_NE(status, nullptr); + expect_status_values(*status, {ST_VNULL}); + EXPECT_EQ(got->null_count(), 1); +} + +// Type mismatch → type_mismatch status +TEST_F(CastVariantStatusTest, TypeMismatchStatus) +{ + auto stream = cudf::test::get_default_stream(); + auto values = make_value_col(enc_int8(5)); // INT8 cast to INT32 target → mismatch + std::unique_ptr status; + auto got = cudf::io::parquet::experimental::cast_variant( + values, cudf::data_type{cudf::type_id::INT32}, stream, cmr(), nullptr, &status); + + ASSERT_NE(status, nullptr); + expect_status_values(*status, {ST_MISMATCH}); + EXPECT_EQ(got->null_count(), 1); +} + +// SQL-null input (null list row) → null status +TEST_F(CastVariantStatusTest, SqlNullInputProducesNullStatus) +{ + auto stream = cudf::test::get_default_stream(); + + // Build the values list column directly (two rows), then mask row 1 null. + auto b0 = enc_int32(42); + auto b1 = enc_int32(0); + // offsets: 0, b0.size(), b0.size()+b1.size() + std::vector offsets{ + 0, static_cast(b0.size()), static_cast(b0.size() + b1.size())}; + std::vector flat; + flat.insert(flat.end(), b0.begin(), b0.end()); + flat.insert(flat.end(), b1.begin(), b1.end()); + auto offs_col = + cudf::test::fixed_width_column_wrapper(offsets.begin(), offsets.end()).release(); + auto data_col = + cudf::test::fixed_width_column_wrapper(flat.begin(), flat.end()).release(); + auto values_col = cudf::make_lists_column(2, std::move(offs_col), std::move(data_col), 0, {}); + // Mask row 1 SQL null + auto null_mask = cudf::create_null_mask(2, cudf::mask_state::ALL_VALID, stream, cmr()); + cudf::set_null_mask(static_cast(null_mask.data()), 1, 2, false); + stream.synchronize(); + values_col->set_null_mask(std::move(null_mask), 1); + + std::unique_ptr status; + auto got = cudf::io::parquet::experimental::cast_variant( + values_col->view(), cudf::data_type{cudf::type_id::INT32}, stream, cmr(), nullptr, &status); + + ASSERT_NE(status, nullptr); + // Row 0: success; row 1: null status + ASSERT_EQ(status->null_count(), 1); + EXPECT_EQ(cudf::test::to_host(*status).first[0], ST_SUCCESS); + EXPECT_EQ(got->null_count(), 1); +} + +// Incoming status propagation: non-success upstream → propagated status +TEST_F(CastVariantStatusTest, IncomingStatusPropagation) +{ + auto stream = cudf::test::get_default_stream(); + + // 3 rows: success, missing_path, variant_null (from a prior get_variant_field) + // The values column: row 0 = INT32(7), rows 1+2 = anything (won't be decoded for non-success) + std::vector> const val_rows{enc_int32(7), enc_int32(0), enc_null()}; + auto col = + wrap_multi_row_variant(std::vector>(3, build_metadata({})), val_rows); + auto const values = cudf::structs_column_view{col}.get_sliced_child(1, stream); + + // Build incoming_status column: {success, missing_path, variant_null} + cudf::test::fixed_width_column_wrapper incoming_status_w( + {ST_SUCCESS, ST_MISSING, ST_VNULL}); + auto incoming_status_col = incoming_status_w.release(); + auto const incoming_view1 = incoming_status_col->view(); + + std::unique_ptr status; + auto got = cudf::io::parquet::experimental::cast_variant( + values, cudf::data_type{cudf::type_id::INT32}, stream, cmr(), &incoming_view1, &status); + + ASSERT_NE(status, nullptr); + // Row 0: success (decoded), Row 1: missing_path (propagated), Row 2: variant_null (propagated) + expect_status_values(*status, {ST_SUCCESS, ST_MISSING, ST_VNULL}); + cudf::test::fixed_width_column_wrapper expected({7, 0, 0}, {true, false, false}); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); +} + +// Incoming status with null entry → null output and null status for that row +TEST_F(CastVariantStatusTest, IncomingNullStatusProducesNullStatus) +{ + auto stream = cudf::test::get_default_stream(); + + std::vector> const val_rows{enc_int32(7), enc_int32(1)}; + auto col = + wrap_multi_row_variant(std::vector>(2, build_metadata({})), val_rows); + auto const values = cudf::structs_column_view{col}.get_sliced_child(1, stream); + + // Row 0: success, Row 1: null incoming status + cudf::test::fixed_width_column_wrapper incoming_status_w2({ST_SUCCESS, ST_SUCCESS}, + {true, false}); + auto incoming_status_col2 = incoming_status_w2.release(); + auto const incoming_view2 = incoming_status_col2->view(); + + std::unique_ptr status; + auto got = cudf::io::parquet::experimental::cast_variant( + values, cudf::data_type{cudf::type_id::INT32}, stream, cmr(), &incoming_view2, &status); + + ASSERT_NE(status, nullptr); + ASSERT_EQ(status->null_count(), 1); + // Row 0: success; Row 1: null status + std::vector const exp_s2{ST_SUCCESS, 0}; + std::vector const exp_s2_valid{true, false}; + cudf::test::fixed_width_column_wrapper expected_status2( + exp_s2.begin(), exp_s2.end(), exp_s2_valid.begin()); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*status, expected_status2); + + cudf::test::fixed_width_column_wrapper expected({7, 0}, {true, false}); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); +} + +// Status for bool target +TEST_F(CastVariantStatusTest, BoolStatusTracking) +{ + auto stream = cudf::test::get_default_stream(); + + // 3 rows: bool_true (success), null (variant_null), int32 (type_mismatch) + std::vector> const val_rows{enc_bool(true), enc_null(), enc_int32(1)}; + auto col = + wrap_multi_row_variant(std::vector>(3, build_metadata({})), val_rows); + auto values = cudf::structs_column_view{col}.get_sliced_child(1, stream); + + std::unique_ptr status; + auto got = cudf::io::parquet::experimental::cast_variant( + values, cudf::data_type{cudf::type_id::BOOL8}, stream, cmr(), nullptr, &status); + + ASSERT_NE(status, nullptr); + expect_status_values(*status, {ST_SUCCESS, ST_VNULL, ST_MISMATCH}); + cudf::test::fixed_width_column_wrapper expected({true, false, false}, {true, false, false}); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); +} + +// Status for string target: short_string, variant_null, type_mismatch, malformed long_string +TEST_F(CastVariantStatusTest, StringStatusTracking) +{ + auto stream = cudf::test::get_default_stream(); + + std::vector> const val_rows{ + enc_short_string("hi"), // success + enc_null(), // variant_null + enc_int32(5), // type_mismatch + // malformed long_string: header + declares 10 bytes but only 2 present + {make_variant_primitive(cudf::io::parquet::experimental::variant_primitive_type::LONG_STRING), + 0x0A, + 0x00, + 0x00, + 0x00, + 'a', + 'b'}, + }; + auto col = + wrap_multi_row_variant(std::vector>(4, build_metadata({})), val_rows); + auto values = cudf::structs_column_view{col}.get_sliced_child(1, stream); + + std::unique_ptr status; + auto got = cudf::io::parquet::experimental::cast_variant( + values, cudf::data_type{cudf::type_id::STRING}, stream, cmr(), nullptr, &status); + + ASSERT_NE(status, nullptr); + expect_status_values(*status, {ST_SUCCESS, ST_VNULL, ST_MISMATCH, ST_MALFORMED}); + EXPECT_EQ(got->null_count(), 3); // all but row 0 are null +} + +// Empty input → empty status column +TEST_F(CastVariantStatusTest, EmptyInput) +{ + auto const stream = cudf::test::get_default_stream(); + auto const values = + cudf::empty_like(cudf::structs_column_view{make_xyz_three_row_variant()}.child(1)); + std::unique_ptr status; + auto got = cudf::io::parquet::experimental::cast_variant( + *values, cudf::data_type{cudf::type_id::INT32}, stream, cmr(), nullptr, &status); + + ASSERT_NE(status, nullptr); + EXPECT_EQ(status->size(), 0); + EXPECT_EQ(got->size(), 0); +} + +// --------------------------------------------------------------------------- +// ExtractVariantField status tests (end-to-end: extraction + decode) +// --------------------------------------------------------------------------- + +struct ExtractVariantFieldStatusTest : public cudf::test::BaseFixture {}; + +// Success path: object {x: INT32(7)} extracted as INT32 +TEST_F(ExtractVariantFieldStatusTest, SuccessStatus) +{ + auto col = make_xyz_three_row_variant(); + auto stream = cudf::test::get_default_stream(); + + std::unique_ptr status; + auto got = cudf::io::parquet::experimental::extract_variant_field( + col, "x", cudf::data_type{cudf::type_id::INT32}, stream, cmr(), &status); + + ASSERT_NE(status, nullptr); + // Rows 0,1 have x as INT32 → success; row 2 has no x → missing_path + expect_status_values(*status, {ST_SUCCESS, ST_SUCCESS, ST_MISSING}); + cudf::test::fixed_width_column_wrapper expected({7, 42, 0}, {true, true, false}); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); +} + +// SQL null input → null status +TEST_F(ExtractVariantFieldStatusTest, SqlNullInputProducesNullStatus) +{ + cudf::test::lists_column_wrapper meta{{0x01, 0x01, 0x00, 0x01, 'x'}}; + cudf::test::lists_column_wrapper val{{0x14, 0x07, 0x00, 0x00, 0x00}}; + cudf::test::structs_column_wrapper col{{meta, val}, std::vector{false}}; + + auto stream = cudf::test::get_default_stream(); + std::unique_ptr status; + auto got = cudf::io::parquet::experimental::extract_variant_field( + col, "x", cudf::data_type{cudf::type_id::INT32}, stream, cmr(), &status); + + ASSERT_NE(status, nullptr); + EXPECT_EQ(status->null_count(), 1); + EXPECT_EQ(got->null_count(), 1); +} + +// VARIANT null → variant_null status (from extraction phase) +TEST_F(ExtractVariantFieldStatusTest, VariantNullStatus) +{ + auto const m = build_metadata({"f"}); + auto const v = build_single_field_object(/*fid=*/0, enc_null()); + auto col = wrap_single_variant(m, v); + auto stream = cudf::test::get_default_stream(); + + std::unique_ptr status; + auto got = cudf::io::parquet::experimental::extract_variant_field( + col, "f", cudf::data_type{cudf::type_id::INT32}, stream, cmr(), &status); + + ASSERT_NE(status, nullptr); + expect_status_values(*status, {ST_VNULL}); + EXPECT_EQ(got->null_count(), 1); +} + +// Type mismatch: field exists but is a string, requested as INT32 +TEST_F(ExtractVariantFieldStatusTest, TypeMismatchStatus) +{ + auto const m = build_metadata({"s"}); + auto const v = build_single_field_object(/*fid=*/0, enc_short_string("hello")); + auto col = wrap_single_variant(m, v); + auto stream = cudf::test::get_default_stream(); + + std::unique_ptr status; + auto got = cudf::io::parquet::experimental::extract_variant_field( + col, "s", cudf::data_type{cudf::type_id::INT32}, stream, cmr(), &status); + + ASSERT_NE(status, nullptr); + expect_status_values(*status, {ST_MISMATCH}); + EXPECT_EQ(got->null_count(), 1); +} + +// Missing path for a multi-step path +TEST_F(ExtractVariantFieldStatusTest, MissingNestedPathStatus) +{ + auto col = make_apache_variant(avf::object_nested); + auto stream = cudf::test::get_default_stream(); + + std::unique_ptr status; + auto got = cudf::io::parquet::experimental::extract_variant_field( + col, "$.species.nope", cudf::data_type{cudf::type_id::STRING}, stream, cmr(), &status); + + ASSERT_NE(status, nullptr); + expect_status_values(*status, {ST_MISSING}); + EXPECT_EQ(got->null_count(), 1); +} + +// All-null column → all-null status column +TEST_F(ExtractVariantFieldStatusTest, AllNullInput) +{ + // Single-row variant column with the row marked SQL null + cudf::test::lists_column_wrapper meta{{0x01, 0x01, 0x00, 0x01, 'x'}}; + cudf::test::lists_column_wrapper val{{0x14, 0x07, 0x00, 0x00, 0x00}}; + cudf::test::structs_column_wrapper all_null{{meta, val}, std::vector{false}}; + + auto stream = cudf::test::get_default_stream(); + std::unique_ptr status; + auto got = cudf::io::parquet::experimental::extract_variant_field( + all_null, "x", cudf::data_type{cudf::type_id::INT32}, stream, cmr(), &status); + + ASSERT_NE(status, nullptr); + EXPECT_EQ(status->null_count(), 1); + EXPECT_EQ(got->null_count(), 1); +} + +// Existing tests work without status_out (nullopt semantics) +TEST_F(ExtractVariantFieldStatusTest, ExistingCallsUnchanged) +{ + auto col = make_xyz_three_row_variant(); + auto stream = cudf::test::get_default_stream(); + + // No status_out — existing behavior + auto got = cudf::io::parquet::experimental::extract_variant_field( + col, "x", cudf::data_type{cudf::type_id::INT32}, stream); + + cudf::test::fixed_width_column_wrapper expected({7, 42, 0}, {true, true, false}); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); +} From 78bbb90be23b423f336c4d049db22f8cbe691a68 Mon Sep 17 00:00:00 2001 From: Abigale Kim Date: Wed, 5 Aug 2026 22:02:06 +0000 Subject: [PATCH 02/23] refactor --- .../parquet/experimental/variant_extract.cu | 711 ++++++------------ 1 file changed, 224 insertions(+), 487 deletions(-) diff --git a/cpp/src/io/parquet/experimental/variant_extract.cu b/cpp/src/io/parquet/experimental/variant_extract.cu index 83ce891f7ccd..03b235584835 100644 --- a/cpp/src/io/parquet/experimental/variant_extract.cu +++ b/cpp/src/io/parquet/experimental/variant_extract.cu @@ -62,6 +62,8 @@ using basic_type = variant_basic_type; // For a primitive value, the value_header is the physical type id of the payload. using primitive_type = variant_primitive_type; +using op_status = variant_operation_status; + __device__ cuda::std::optional read_uint64(device_span data, size_type pos, int width) @@ -227,48 +229,48 @@ __device__ cuda::std::optional variant_value_length(device_span find_key_in_metadata(device_span meta, - cudf::string_view key) +__device__ cuda::std::pair, op_status> find_key_in_metadata( + device_span meta, cudf::string_view key) { auto const meta_len = static_cast(meta.size()); - if (meta_len < 1) { return cuda::std::nullopt; } + if (meta_len < 1) { return {cuda::std::nullopt, op_status::malformed_variant}; } auto const header = meta[0]; int const version = header & 0x0F; - if (version != variant_version_v1) { return cuda::std::nullopt; } + if (version != variant_version_v1) { return {cuda::std::nullopt, op_status::malformed_variant}; } int const offset_size = ((header >> 6) & 0x03) + 1; size_type pos = 1; auto const num_entries = narrow_cast(read_uint64(meta, pos, offset_size)); - if (!num_entries.has_value()) { return cuda::std::nullopt; } + if (!num_entries.has_value()) { return {cuda::std::nullopt, op_status::malformed_variant}; } pos += offset_size; auto const offsets_start = pos; auto const offsets_bytes = (static_cast(num_entries.value()) + 1) * offset_size; if (cuda::std::cmp_greater(offsets_bytes, meta_len - offsets_start)) { - return cuda::std::nullopt; + return {cuda::std::nullopt, op_status::malformed_variant}; } auto start_off = read_uint64(meta, offsets_start, offset_size); - if (!start_off.has_value()) { return cuda::std::nullopt; } - auto const strings_base = offsets_start + static_cast(offsets_bytes); - // Bytes available for dictionary string payloads + if (!start_off.has_value()) { return {cuda::std::nullopt, op_status::malformed_variant}; } + auto const strings_base = offsets_start + static_cast(offsets_bytes); auto const strings_extent = meta_len - strings_base; for (size_type i = 0; i < num_entries.value(); ++i) { auto const end_off = read_uint64(meta, offsets_start + (i + 1) * offset_size, offset_size); - if (!end_off.has_value()) { return cuda::std::nullopt; } + if (!end_off.has_value()) { return {cuda::std::nullopt, op_status::malformed_variant}; } if (end_off.value() < start_off.value() || end_off.value() > strings_extent) { - return cuda::std::nullopt; + return {cuda::std::nullopt, op_status::malformed_variant}; } cudf::string_view const entry{ reinterpret_cast(meta.data() + strings_base + start_off.value()), static_cast(end_off.value() - start_off.value())}; - if (entry == key) { return i; } + if (entry == key) { return {i, op_status::success}; } start_off = end_off; } - return cuda::std::nullopt; + return {cuda::std::nullopt, op_status::missing_path}; } /** @@ -294,184 +296,15 @@ __device__ cuda::std::optional find_key_in_metadata(device_span locate_object_field(device_span val, int id) -{ - auto const val_len = static_cast(val.size()); - if (val_len < 1) { return {}; } - auto const value_metadata = val[0]; - if (decode_basic_type(value_metadata) != basic_type::OBJECT) { return {}; } - - auto const [offset_size, id_size, num_elements_size] = - decode_object_array_header(variant_value_header(value_metadata), true); - - size_type pos = 1; - auto const num_fields = narrow_cast(read_uint64(val, pos, num_elements_size)); - if (!num_fields.has_value()) { return {}; } - pos += num_elements_size; - - auto const ids_start = pos; - auto const ids_bytes = static_cast(num_fields.value()) * id_size; - if (ids_bytes > val_len - ids_start) { return {}; } - - auto const offsets_start = ids_start + static_cast(ids_bytes); - auto const offsets_bytes = (static_cast(num_fields.value()) + 1) * offset_size; - if (offsets_bytes > val_len - offsets_start) { return {}; } - - auto const values_base = offsets_start + static_cast(offsets_bytes); - // Maximum legitimate field-offset value: bytes available after values_base - auto const values_extent = val_len - values_base; - - // Find the matching field ID and its start offset - bool found = false; - uint64_t match_start = 0; - for (size_type i = 0; i < num_fields.value(); ++i) { - auto const current_id = read_uint64(val, ids_start + i * id_size, id_size); - if (!current_id.has_value()) { return {}; } - if (cuda::std::cmp_not_equal(current_id.value(), id)) { continue; } - - auto const match_offset = read_uint64(val, offsets_start + i * offset_size, offset_size); - if (!match_offset.has_value()) { return {}; } - if (match_offset.value() > values_extent) { return {}; } - match_start = match_offset.value(); - found = true; - break; - } - if (!found) { return {}; } - - // Derive field's value length from its header - auto const value = val.subspan(values_base + match_start); - auto const value_len = variant_value_length(value); - if (!value_len.has_value()) { return {}; } - auto const match_end = match_start + value_len.value(); - if (match_end > values_extent) { return {}; } - return val.subspan(values_base + match_start, value_len.value()); -} - -// Parse an array value header and return the sub-span of the element at `index` (0-based) within -// `val`. Returns an empty span if `val` is not an array (`basic_type != array`), if `index` is out -// of bounds, or if the encoded data is truncated. -// -// Array layout per the Variant spec: -// byte 0: header (basic_type=array in low 2 bits; value_header in high 6 bits) -// value_header bits: (offset_size - 1) in bits 0-1, is_large in bit 2, bits 3-5 unused -// num_elements: 1 byte if !is_large else 4 bytes (little-endian) -// offsets: (num_elements + 1) entries, each `offset_size` bytes, relative to the end of -// offsets -// values: concatenated element blobs -// -// Array element offsets are monotonically increasing, so the element length is taken directly from -// the offset delta (o1 - o0) rather than from the element's own header. -__device__ device_span locate_array_element(device_span value, - size_type index) -{ - if (index < 0) { return {}; } - - auto const value_size = static_cast(value.size()); - if (value_size < 1) { return {}; } - uint8_t const value_metadata = value[0]; - if (decode_basic_type(value_metadata) != basic_type::ARRAY) { return {}; } - - int const value_header = variant_value_header(value_metadata); - [[maybe_unused]] auto const [offset_size, _, num_elements_size] = - decode_object_array_header(value_header, false); - - size_type position = 1; - auto const num_elements_value = narrow_cast(read_uint64(value, position, num_elements_size)); - if (!num_elements_value.has_value()) { return {}; } - auto const num_elements = num_elements_value.value(); - if (index >= num_elements) { return {}; } - position += num_elements_size; - - size_type const offsets_start = position; - // Computed in 64-bit because (num_elements + 1) * offset_size can exceed the signed `size_type` - // range (which would be UB); the check below then rejects any array that overruns the value blob. - auto const offsets_bytes = (static_cast(num_elements) + 1) * offset_size; - if (cuda::std::cmp_greater(offsets_bytes, value_size - offsets_start)) { return {}; } - size_type const values_base = offsets_start + static_cast(offsets_bytes); - auto const values_extent = value_size - values_base; - - auto const start_offset_pos = offsets_start + static_cast(index) * offset_size; - auto const end_offset_pos = offsets_start + (static_cast(index) + 1) * offset_size; - if (cuda::std::cmp_greater(end_offset_pos + offset_size, value_size)) { return {}; } - - auto const start_offset = read_uint64(value, start_offset_pos, offset_size); - auto const end_offset = read_uint64(value, end_offset_pos, offset_size); - if (!start_offset.has_value() || !end_offset.has_value()) { return {}; } - auto const element_start = *start_offset; - auto const element_end = *end_offset; - if (element_end < element_start || cuda::std::cmp_greater(element_end, values_extent)) { - return {}; - } - return value.subspan(values_base + element_start, element_end - element_start); -} - -using op_status = variant_operation_status; - -// True when the value blob begins with the VARIANT null primitive header. -__device__ bool is_variant_null(device_span enc) -{ - if (enc.empty()) { return false; } - auto const vm = enc[0]; - return decode_basic_type(vm) == basic_type::PRIMITIVE && - variant_value_header(vm) == static_cast(primitive_type::NULLVAL); -} - -// Status-aware version of find_key_in_metadata. Returns (id, status) where status is either -// missing_path (key absent) or malformed_variant (blob is malformed). On success, id has a value. -__device__ cuda::std::pair, op_status> -find_key_in_metadata_with_status(device_span meta, cudf::string_view key) -{ - auto const meta_len = static_cast(meta.size()); - if (meta_len < 1) { return {cuda::std::nullopt, op_status::malformed_variant}; } - - auto const header = meta[0]; - int const version = header & 0x0F; - if (version != variant_version_v1) { return {cuda::std::nullopt, op_status::malformed_variant}; } - int const offset_size = ((header >> 6) & 0x03) + 1; - - size_type pos = 1; - auto const num_entries = narrow_cast(read_uint64(meta, pos, offset_size)); - if (!num_entries.has_value()) { return {cuda::std::nullopt, op_status::malformed_variant}; } - pos += offset_size; - - auto const offsets_start = pos; - auto const offsets_bytes = (static_cast(num_entries.value()) + 1) * offset_size; - if (cuda::std::cmp_greater(offsets_bytes, meta_len - offsets_start)) { - return {cuda::std::nullopt, op_status::malformed_variant}; - } - - auto start_off = read_uint64(meta, offsets_start, offset_size); - if (!start_off.has_value()) { return {cuda::std::nullopt, op_status::malformed_variant}; } - auto const strings_base = offsets_start + static_cast(offsets_bytes); - auto const strings_extent = meta_len - strings_base; - for (size_type i = 0; i < num_entries.value(); ++i) { - auto const end_off = read_uint64(meta, offsets_start + (i + 1) * offset_size, offset_size); - if (!end_off.has_value()) { return {cuda::std::nullopt, op_status::malformed_variant}; } - if (end_off.value() < start_off.value() || end_off.value() > strings_extent) { - return {cuda::std::nullopt, op_status::malformed_variant}; - } - cudf::string_view const entry{ - reinterpret_cast(meta.data() + strings_base + start_off.value()), - static_cast(end_off.value() - start_off.value())}; - if (entry == key) { return {i, op_status::success}; } - start_off = end_off; - } - return {cuda::std::nullopt, op_status::missing_path}; -} - -// Status-aware version of locate_object_field. Returns (span, status). -// Status is missing_path when the value is not an object or the field is absent; -// malformed_variant when the bytes are truncated or otherwise invalid; success otherwise. -__device__ cuda::std::pair, op_status> locate_object_field_with_status( +__device__ cuda::std::pair, op_status> locate_object_field( device_span val, int id) { auto const val_len = static_cast(val.size()); if (val_len < 1) { return {{}, op_status::malformed_variant}; } auto const value_metadata = val[0]; - // Not an object: treat as missing path (non-container, can't descend) if (decode_basic_type(value_metadata) != basic_type::OBJECT) { return {{}, op_status::missing_path}; } @@ -501,6 +334,7 @@ __device__ cuda::std::pair, op_status> locate_object_ auto const current_id = read_uint64(val, ids_start + i * id_size, id_size); if (!current_id.has_value()) { return {{}, op_status::malformed_variant}; } if (cuda::std::cmp_not_equal(current_id.value(), id)) { continue; } + auto const match_offset = read_uint64(val, offsets_start + i * offset_size, offset_size); if (!match_offset.has_value()) { return {{}, op_status::malformed_variant}; } if (match_offset.value() > values_extent) { return {{}, op_status::malformed_variant}; } @@ -518,11 +352,27 @@ __device__ cuda::std::pair, op_status> locate_object_ return {val.subspan(values_base + match_start, value_len.value()), op_status::success}; } -// Status-aware version of locate_array_element. Returns (span, status). -__device__ cuda::std::pair, op_status> locate_array_element_with_status( +// Parse an array value header and return the sub-span of the element at `index` (0-based) within +// `val`. Returns an empty span if `val` is not an array (`basic_type != array`), if `index` is out +// of bounds, or if the encoded data is truncated. +// +// Array layout per the Variant spec: +// byte 0: header (basic_type=array in low 2 bits; value_header in high 6 bits) +// value_header bits: (offset_size - 1) in bits 0-1, is_large in bit 2, bits 3-5 unused +// num_elements: 1 byte if !is_large else 4 bytes (little-endian) +// offsets: (num_elements + 1) entries, each `offset_size` bytes, relative to the end of +// offsets +// values: concatenated element blobs +// +// Array element offsets are monotonically increasing, so the element length is taken directly from +// the offset delta (o1 - o0) rather than from the element's own header. +// Returns `(span, success)` on success, `(empty, missing_path)` for out-of-bounds or non-array, +// and `(empty, malformed_variant)` for truncated data. +__device__ cuda::std::pair, op_status> locate_array_element( device_span value, size_type index) { if (index < 0) { return {{}, op_status::missing_path}; } + auto const value_size = static_cast(value.size()); if (value_size < 1) { return {{}, op_status::malformed_variant}; } uint8_t const value_metadata = value[0]; @@ -568,6 +418,15 @@ __device__ cuda::std::pair, op_status> locate_array_e op_status::success}; } +// True when the value blob begins with the VARIANT null primitive header. +__device__ bool is_variant_null(device_span enc) +{ + if (enc.empty()) { return false; } + auto const vm = enc[0]; + return decode_basic_type(vm) == basic_type::PRIMITIVE && + variant_value_header(vm) == static_cast(primitive_type::NULLVAL); +} + // The fixed-width signed integers a VARIANT value can be cast to: INT{8,16,32,64}. Matches the // exact width types (not e.g. __int128) since those are the only variant primitive int headers. template @@ -668,39 +527,10 @@ __device__ cuda::std::optional parse_index_step(cudf::string_view ste return index; } -// Walk a path of object-key or array-index steps level by level starting at `val` and return -// the span of the final value (subspan of `val`). Returns an empty span on failure. -// -// Each path step is encoded in the `path` strings column as either: -// - "" -> descend into an object by dictionary key, or -// - "[]" -> descend into an array by zero-based integer index. -// The step kind is inferred from the first byte (`'['` means index). -__device__ device_span resolve_path(device_span meta, - device_span val, - column_device_view path) -{ - device_span sub_val = val; - for (size_type i = 0; i < path.size(); ++i) { - auto const step = path.element(i); - - if (step.size_bytes() >= 1 && step.data()[0] == '[') { - auto const index = parse_index_step(step); - if (!index.has_value()) { return {}; } - sub_val = locate_array_element(sub_val, index.value()); - } else { - auto const field_id = find_key_in_metadata(meta, step); - if (!field_id.has_value()) { return {}; } - sub_val = locate_object_field(sub_val, field_id.value()); - } - if (sub_val.empty()) { return {}; } - } - return sub_val; -} - -// Walk a path of object-key or array-index steps, returning (final_span, status). -// Status distinguishes success, missing_path, variant_null, and malformed_variant. -// On variant_null, span points to the VARIANT null bytes (not empty). -__device__ cuda::std::pair, op_status> resolve_path_with_status( +// Walk a path of object-key or array-index steps. Returns `(span, status)`. +// On success the span is non-empty and status is `success` or `variant_null` (terminal null). +// On failure the span is empty and status is `missing_path` or `malformed_variant`. +__device__ cuda::std::pair, op_status> resolve_path( device_span meta, device_span val, column_device_view path) { device_span sub_val = val; @@ -710,14 +540,14 @@ __device__ cuda::std::pair, op_status> resolve_path_w if (step.size_bytes() >= 1 && step.data()[0] == '[') { auto const index = parse_index_step(step); if (!index.has_value()) { return {{}, op_status::missing_path}; } - auto const [span, st] = locate_array_element_with_status(sub_val, index.value()); + auto const [span, st] = locate_array_element(sub_val, index.value()); if (st != op_status::success) { return {{}, st}; } sub_val = span; } else { - auto const [field_id, meta_st] = find_key_in_metadata_with_status(meta, step); + auto const [field_id, meta_st] = find_key_in_metadata(meta, step); if (meta_st == op_status::malformed_variant) { return {{}, op_status::malformed_variant}; } if (!field_id.has_value()) { return {{}, op_status::missing_path}; } - auto const [span, st] = locate_object_field_with_status(sub_val, field_id.value()); + auto const [span, st] = locate_object_field(sub_val, field_id.value()); if (st != op_status::success) { return {{}, st}; } sub_val = span; } @@ -726,7 +556,6 @@ __device__ cuda::std::pair, op_status> resolve_path_w if (i + 1 < path.size() && is_variant_null(sub_val)) { return {{}, op_status::missing_path}; } } - // Terminal VARIANT null: return the bytes with variant_null status. if (is_variant_null(sub_val)) { return {sub_val, op_status::variant_null}; } return {sub_val, op_status::success}; } @@ -785,13 +614,19 @@ constexpr int block_size = 256; * `d_sizes[row]` and its offset within the row's value blob to `d_src_offsets[row]`. Rows that are * null, or whose path does not resolve, are marked null in `d_null_mask` with a size of 0. */ +// `HasStatus=false`: existing value-only behavior, no status output. +// `HasStatus=true`: also fills `d_status`/`d_status_null_mask`; SQL-null rows get null status; +// VARIANT-null terminal values are preserved in the output with `variant_null` status. +template CUDF_KERNEL __launch_bounds__(block_size) void locate_variant_fields_kernel( cudf::lists_column_device_view metadata, cudf::lists_column_device_view values, column_device_view path, device_span d_sizes, device_span d_src_offsets, - bitmask_type* d_null_mask) + bitmask_type* d_null_mask, + device_span d_status, // only read/written when HasStatus + bitmask_type* d_status_null_mask) // only read/written when HasStatus { auto const num_rows = static_cast(d_sizes.size()); auto const tid = cudf::detail::grid_1d::global_thread_id(); @@ -801,112 +636,27 @@ CUDF_KERNEL __launch_bounds__(block_size) void locate_variant_fields_kernel( if (!cudf::bit_is_set(d_null_mask, row)) { d_sizes[row] = 0; d_src_offsets[row] = 0; + if constexpr (HasStatus) { cudf::clear_bit(d_status_null_mask, row); } continue; } auto const [meta, val] = metadata_and_value_at(metadata, values, row); + auto const [field, st] = resolve_path(meta, val, path); - auto const field = resolve_path(meta, val, path); if (field.empty()) { d_sizes[row] = 0; d_src_offsets[row] = 0; cudf::clear_bit(d_null_mask, row); - continue; - } - - d_sizes[row] = static_cast(field.size()); - d_src_offsets[row] = static_cast(field.data() - val.data()); - } -} - -/** - * @brief Status-tracking version of locate_variant_fields_kernel. - * - * Fills `d_status[row]` with the per-row `variant_operation_status`. SQL-null rows get status null - * (their null bit in `d_status_null_mask` is cleared). VARIANT-null terminal values are preserved - * in the output (null bit NOT cleared) and receive `variant_null` status. - */ -CUDF_KERNEL __launch_bounds__(block_size) void locate_variant_fields_with_status_kernel( - cudf::lists_column_device_view metadata, - cudf::lists_column_device_view values, - column_device_view path, - device_span d_sizes, - device_span d_src_offsets, - bitmask_type* d_null_mask, - device_span d_status, - bitmask_type* d_status_null_mask) -{ - auto const num_rows = static_cast(d_sizes.size()); - auto const tid = cudf::detail::grid_1d::global_thread_id(); - auto const stride = cudf::detail::grid_1d::grid_stride(); - - for (auto row = tid; row < num_rows; row += stride) { - if (!cudf::bit_is_set(d_null_mask, row)) { - // SQL-null input: null output, null status - d_sizes[row] = 0; - d_src_offsets[row] = 0; - cudf::clear_bit(d_status_null_mask, row); - continue; - } - - auto const [meta, val] = metadata_and_value_at(metadata, values, row); - auto const [field, st] = resolve_path_with_status(meta, val, path); - - d_status[row] = st; - - if (st == op_status::success) { - d_sizes[row] = static_cast(field.size()); - d_src_offsets[row] = static_cast(field.data() - val.data()); - } else if (st == op_status::variant_null) { - // Keep the VARIANT null bytes in the output; do NOT clear the null bit. + if constexpr (HasStatus) { d_status[row] = st; } + } else { d_sizes[row] = static_cast(field.size()); d_src_offsets[row] = static_cast(field.data() - val.data()); - } else { - // missing_path or malformed_variant: SQL null output - d_sizes[row] = 0; - d_src_offsets[row] = 0; - cudf::clear_bit(d_null_mask, row); + if constexpr (HasStatus) { d_status[row] = st; } } } } -/** - * @brief Per-row kernel: decode each VARIANT value blob into a fixed-width primitive of type `T`. - * - * Writes the decoded value to `d_output[row]` for non-null rows whose blob is a variant primitive - * whose physical type id matches `T` exactly (e.g. an int16 value does not decode into an int32 - * output, and a float32 value does not decode into a float64 output; there is no widening). Rows - * that are null, or whose value is not an exact-width match for `T`, are marked null in - * `d_null_mask` with an output of 0. - */ -template -CUDF_KERNEL __launch_bounds__(block_size) void cast_variant_primitive_kernel( - cudf::lists_column_device_view values, device_span d_output, bitmask_type* d_null_mask) -{ - auto const num_rows = static_cast(d_output.size()); - auto const tid = cudf::detail::grid_1d::global_thread_id(); - auto const stride = cudf::detail::grid_1d::grid_stride(); - - for (auto row = tid; row < num_rows; row += stride) { - if (!cudf::bit_is_set(d_null_mask, row)) { - d_output[row] = 0; - continue; - } - - auto const val = list_row_span(values, row); - - auto const decoded = decode_primitive(val); - if (decoded.has_value()) { - d_output[row] = *decoded; - } else { - d_output[row] = 0; - cudf::clear_bit(d_null_mask, row); - } - } -} - -// Compute the cast status for a single non-null VARIANT value blob targeting a fixed-width -// primitive type T, given that the row's null bit is already set (i.e. the row is valid). +// Status helper for fixed-width primitive targets: returns the failure reason when decode fails. template requires(is_variant_numerical) __device__ op_status cast_status_for_primitive(device_span val) @@ -914,10 +664,8 @@ __device__ op_status cast_status_for_primitive(device_span val) if (val.empty()) { return op_status::malformed_variant; } if (is_variant_null(val)) { return op_status::variant_null; } if (decode_primitive(val).has_value()) { return op_status::success; } - // Has a value but not the right type if (decode_basic_type(val[0]) != basic_type::PRIMITIVE) { return op_status::type_mismatch; } auto const vhdr = variant_value_header(val[0]); - // Check if it's a valid primitive type at all (otherwise malformed) switch (static_cast(vhdr)) { case primitive_type::NULLVAL: case primitive_type::BOOLEAN_TRUE: @@ -934,6 +682,68 @@ __device__ op_status cast_status_for_primitive(device_span val) } } +// `HasStatus=false`: decode-only; SQL-null rows produce null output, other failure types produce +// null. `HasStatus=true`: also fills `d_status`/`d_status_null_mask` and honours `has_incoming`. +// `has_incoming`: when true, `incoming_status` gates decoding instead of the null mask. +template +CUDF_KERNEL __launch_bounds__(block_size) void cast_variant_primitive_kernel( + cudf::lists_column_device_view values, + device_span d_output, + bitmask_type* d_null_mask, + column_device_view incoming_status, // only used when HasStatus + bool has_incoming, // only meaningful when HasStatus + device_span d_status, // only used when HasStatus + bitmask_type* d_status_null_mask) // only used when HasStatus +{ + auto const num_rows = static_cast(d_output.size()); + auto const tid = cudf::detail::grid_1d::global_thread_id(); + auto const stride = cudf::detail::grid_1d::grid_stride(); + + for (auto row = tid; row < num_rows; row += stride) { + if constexpr (HasStatus) { + if (has_incoming) { + // Incoming status is the sole authority; the value null mask may already be pre-cleared. + if (incoming_status.is_null(row)) { + d_output[row] = T{}; + if (cudf::bit_is_set(d_null_mask, row)) { cudf::clear_bit(d_null_mask, row); } + cudf::clear_bit(d_status_null_mask, row); + continue; + } + auto const s = static_cast(incoming_status.element(row)); + if (s != op_status::success) { + d_output[row] = T{}; + if (cudf::bit_is_set(d_null_mask, row)) { cudf::clear_bit(d_null_mask, row); } + d_status[row] = s; + continue; + } + // incoming success → fall through to decode (value null bit is set) + } else { + if (!cudf::bit_is_set(d_null_mask, row)) { + d_output[row] = T{}; + cudf::clear_bit(d_status_null_mask, row); + continue; + } + } + } else { + if (!cudf::bit_is_set(d_null_mask, row)) { + d_output[row] = T{}; + continue; + } + } + + auto const val = list_row_span(values, row); + auto const decoded = decode_primitive(val); + if (decoded.has_value()) { + d_output[row] = *decoded; + if constexpr (HasStatus) { d_status[row] = op_status::success; } + } else { + d_output[row] = T{}; + cudf::clear_bit(d_null_mask, row); + if constexpr (HasStatus) { d_status[row] = cast_status_for_primitive(val); } + } + } +} + __device__ op_status cast_status_for_bool(device_span val) { if (val.empty()) { return op_status::malformed_variant; } @@ -958,7 +768,6 @@ __device__ op_status cast_status_for_string(device_span val) if (val.empty()) { return op_status::malformed_variant; } if (is_variant_null(val)) { return op_status::variant_null; } if (decode_string(val).has_value()) { return op_status::success; } - // long_string with truncated payload is malformed; other types are type_mismatch auto const btype = decode_basic_type(val[0]); if (btype == basic_type::PRIMITIVE && variant_value_header(val[0]) == static_cast(primitive_type::LONG_STRING)) { @@ -967,80 +776,6 @@ __device__ op_status cast_status_for_string(device_span val) return op_status::type_mismatch; } -// Determine the effective mask and status for a row when an incoming_status column is present. -// Returns (should_decode, row_status_or_nullopt). -// - If incoming status is null → should_decode=false, status is null (nullopt means "write null"). -// - If incoming status is non-success → should_decode=false, propagate that status. -// - If incoming status is success → should_decode=true, status determined by decode result. -__device__ cuda::std::pair> apply_incoming_status( - column_device_view incoming, size_type row) -{ - if (incoming.is_null(row)) { return {false, cuda::std::nullopt}; } - auto const s = incoming.element(row); - if (s != static_cast(op_status::success)) { return {false, static_cast(s)}; } - return {true, cuda::std::nullopt}; // status determined after decode -} - -// Status-tracking variant of cast_variant_primitive_kernel. -// When incoming_status is present, only rows with success status are decoded. -template -CUDF_KERNEL __launch_bounds__(block_size) void cast_variant_primitive_status_kernel( - cudf::lists_column_device_view values, - device_span d_output, - bitmask_type* d_null_mask, - column_device_view incoming_status, - bool has_incoming, - device_span d_status, - bitmask_type* d_status_null_mask) -{ - auto const num_rows = static_cast(d_output.size()); - auto const tid = cudf::detail::grid_1d::global_thread_id(); - auto const stride = cudf::detail::grid_1d::grid_stride(); - - for (auto row = tid; row < num_rows; row += stride) { - if (has_incoming) { - // When an upstream status is present, it is the sole authority: use it to gate decoding. - // The value-column null mask may have been pre-cleared by get_variant_field for non-success - // rows, so we must not rely on it to distinguish SQL-null from missing/malformed here. - auto const [decode, propagated] = apply_incoming_status(incoming_status, row); - if (!decode) { - d_output[row] = T{}; - if (!cudf::bit_is_set(d_null_mask, row)) { - // Already null in the value column (get_variant_field cleared it for non-success rows). - } else { - cudf::clear_bit(d_null_mask, row); - } - if (propagated.has_value()) { - d_status[row] = *propagated; - } else { - // Incoming status was null → this was a SQL-null input row - cudf::clear_bit(d_status_null_mask, row); - } - continue; - } - // Incoming status == success: fall through to decode using the value bytes. - } else { - if (!cudf::bit_is_set(d_null_mask, row)) { - // SQL-null input (no incoming status) → null output, null status - d_output[row] = T{}; - cudf::clear_bit(d_status_null_mask, row); - continue; - } - } - - auto const val = list_row_span(values, row); - auto const decoded = decode_primitive(val); - if (decoded.has_value()) { - d_output[row] = *decoded; - d_status[row] = op_status::success; - } else { - d_output[row] = T{}; - cudf::clear_bit(d_null_mask, row); - d_status[row] = cast_status_for_primitive(val); - } - } -} - /** * @brief Strings-children functor: decode each VARIANT value blob into a string. * @@ -1070,21 +805,21 @@ struct cast_variant_string_fn { if (has_incoming) { // Incoming status takes precedence over the value-column null mask. - auto const [decode, propagated] = apply_incoming_status(incoming_status, row); - if (!decode) { + if (incoming_status.is_null(row)) { + if (sizing) { d_sizes[row] = 0; } + if (cudf::bit_is_set(d_null_mask, row)) { cudf::clear_bit(d_null_mask, row); } + if (sizing && d_status_null_mask) { cudf::clear_bit(d_status_null_mask, row); } + return; + } + auto const s = static_cast(incoming_status.element(row)); + if (s != op_status::success) { if (sizing) { d_sizes[row] = 0; } if (!cudf::bit_is_set(d_null_mask, row)) { // already null from get_variant_field } else { cudf::clear_bit(d_null_mask, row); } - if (sizing && d_status) { - if (propagated.has_value()) { - d_status[row] = *propagated; - } else { - cudf::clear_bit(d_status_null_mask, row); - } - } + if (sizing && d_status) { d_status[row] = s; } return; } // incoming success: fall through to decode @@ -1155,30 +890,37 @@ struct cast_variant_fn { bool has_incoming{false}; std::unique_ptr* status_out{nullptr}; + // Helper: allocate status buffers and return the output status column. + // Avoids repeating this boilerplate in each cast operator. + auto alloc_status() -> cuda::std::pair + { + return {rmm::device_buffer{static_cast(num_rows) * sizeof(op_status), stream, mr}, + cudf::create_null_mask(num_rows, mask_state::ALL_VALID, stream, mr)}; + } + template std::unique_ptr operator()() requires(is_variant_numerical) { rmm::device_buffer data{num_rows * sizeof(T), stream, mr}; auto const grid = cudf::detail::grid_1d{num_rows, block_size}; - + auto const d_out = + device_span{static_cast(data.data()), static_cast(num_rows)}; if (status_out != nullptr) { - rmm::device_buffer status_data{num_rows * sizeof(op_status), stream, mr}; - auto status_null_mask = cudf::create_null_mask(num_rows, mask_state::ALL_VALID, stream, mr); - cast_variant_primitive_status_kernel<<>>( + auto [s_data, s_mask] = alloc_status(); + cast_variant_primitive_kernel<<>>( values, - {static_cast(data.data()), static_cast(num_rows)}, + d_out, d_null_mask, incoming_status_view, has_incoming, - {static_cast(status_data.data()), static_cast(num_rows)}, - static_cast(status_null_mask.data())); + {static_cast(s_data.data()), static_cast(num_rows)}, + static_cast(s_mask.data())); CUDF_CUDA_TRY(cudaGetLastError()); - *status_out = make_status_column( - std::move(status_data), std::move(status_null_mask), num_rows, stream, mr); + *status_out = make_status_column(std::move(s_data), std::move(s_mask), num_rows, stream, mr); } else { - cast_variant_primitive_kernel<<>>( - values, {static_cast(data.data()), static_cast(num_rows)}, d_null_mask); + cast_variant_primitive_kernel<<>>( + values, d_out, d_null_mask, incoming_status_view, false, {}, nullptr); CUDF_CUDA_TRY(cudaGetLastError()); } @@ -1197,73 +939,65 @@ struct cast_variant_fn { { rmm::device_buffer data{num_rows * sizeof(bool), stream, mr}; + rmm::device_buffer s_data, s_mask; + op_status* dp_s{nullptr}; + bitmask_type* dp_sm{nullptr}; if (status_out != nullptr) { - rmm::device_buffer status_data{num_rows * sizeof(op_status), stream, mr}; - auto status_null_mask = cudf::create_null_mask(num_rows, mask_state::ALL_VALID, stream, mr); - auto* d_status = static_cast(status_data.data()); - auto* d_status_null_mask = static_cast(status_null_mask.data()); - auto const inc_view = incoming_status_view; - auto const hi = has_incoming; - thrust::for_each(rmm::exec_policy_nosync(stream, cudf::get_current_device_resource_ref()), - cuda::counting_iterator(0), - cuda::counting_iterator(num_rows), - [vals = this->values, - d_out = static_cast(data.data()), - d_null_mask = this->d_null_mask, - d_status, - d_status_null_mask, - inc_view, - hi] __device__(size_type row) { - if (hi) { - auto const [decode, propagated] = apply_incoming_status(inc_view, row); - if (!decode) { - d_out[row] = false; - if (!cudf::bit_is_set(d_null_mask, row)) { - // already null from get_variant_field - } else { - cudf::clear_bit(d_null_mask, row); - } - if (propagated.has_value()) { - d_status[row] = *propagated; - } else { - cudf::clear_bit(d_status_null_mask, row); - } - return; - } - } else { - if (!cudf::bit_is_set(d_null_mask, row)) { - d_out[row] = false; - cudf::clear_bit(d_status_null_mask, row); - return; - } + auto [sd, sm] = alloc_status(); + s_data = std::move(sd); + s_mask = std::move(sm); + dp_s = static_cast(s_data.data()); + dp_sm = static_cast(s_mask.data()); + } + + auto const inc_view = incoming_status_view; + auto const hi = has_incoming; + thrust::for_each(rmm::exec_policy_nosync(stream, cudf::get_current_device_resource_ref()), + cuda::counting_iterator(0), + cuda::counting_iterator(num_rows), + [vals = this->values, + d_out = static_cast(data.data()), + dnm = this->d_null_mask, + dp_s, + dp_sm, + inc_view, + hi] __device__(size_type row) { + // Gate on incoming status when present, otherwise on the null mask. + if (hi) { + if (inc_view.is_null(row)) { + d_out[row] = false; + if (cudf::bit_is_set(dnm, row)) { cudf::clear_bit(dnm, row); } + if (dp_sm) { cudf::clear_bit(dp_sm, row); } + return; } - auto const val = list_row_span(vals, row); - auto const decoded = decode_bool(val); - if (decoded.has_value()) { - d_out[row] = *decoded; - d_status[row] = op_status::success; - } else { + auto const s = static_cast(inc_view.element(row)); + if (s != op_status::success) { d_out[row] = false; - cudf::clear_bit(d_null_mask, row); - d_status[row] = cast_status_for_bool(val); + if (cudf::bit_is_set(dnm, row)) { cudf::clear_bit(dnm, row); } + if (dp_s) { dp_s[row] = s; } + return; } - }); - *status_out = make_status_column( - std::move(status_data), std::move(status_null_mask), num_rows, stream, mr); - } else { - thrust::transform( - rmm::exec_policy_nosync(stream, cudf::get_current_device_resource_ref()), - cuda::counting_iterator(0), - cuda::counting_iterator(num_rows), - static_cast(data.data()), - [vals = this->values, d_null_mask = this->d_null_mask] __device__(size_type row) -> bool { - if (!cudf::bit_is_set(d_null_mask, row)) { return false; } - auto const val = list_row_span(vals, row); - auto const decoded = decode_bool(val); - if (decoded.has_value()) { return *decoded; } - cudf::clear_bit(d_null_mask, row); - return false; - }); + } else { + if (!cudf::bit_is_set(dnm, row)) { + d_out[row] = false; + if (dp_sm) { cudf::clear_bit(dp_sm, row); } + return; + } + } + auto const val = list_row_span(vals, row); + auto const decoded = decode_bool(val); + if (decoded.has_value()) { + d_out[row] = *decoded; + if (dp_s) { dp_s[row] = op_status::success; } + } else { + d_out[row] = false; + cudf::clear_bit(dnm, row); + if (dp_s) { dp_s[row] = cast_status_for_bool(val); } + } + }); + + if (status_out != nullptr) { + *status_out = make_status_column(std::move(s_data), std::move(s_mask), num_rows, stream, mr); } auto const null_count = @@ -1408,11 +1142,13 @@ std::unique_ptr get_variant_field(column_view const& variant_column, auto grid = cudf::detail::grid_1d{num_rows, block_size}; + auto const null_spans = + device_span{}; // placeholder for no-status kernel instantiation + if (status_out != nullptr) { rmm::device_buffer status_data{num_rows * sizeof(op_status), stream, mr}; auto status_null_mask = cudf::create_null_mask(num_rows, mask_state::ALL_VALID, stream, mr); - - locate_variant_fields_with_status_kernel<<>>( + locate_variant_fields_kernel<<>>( meta_lists_device_view, val_lists_device_view, *path_device_view, @@ -1422,17 +1158,18 @@ std::unique_ptr get_variant_field(column_view const& variant_column, {static_cast(status_data.data()), static_cast(num_rows)}, static_cast(status_null_mask.data())); CUDF_CUDA_TRY(cudaGetLastError()); - *status_out = make_status_column(std::move(status_data), std::move(status_null_mask), num_rows, stream, mr); } else { - locate_variant_fields_kernel<<>>( - meta_lists_device_view, - val_lists_device_view, - *path_device_view, - d_sizes, - d_src_offsets, - d_null_mask); + locate_variant_fields_kernel + <<>>(meta_lists_device_view, + val_lists_device_view, + *path_device_view, + d_sizes, + d_src_offsets, + d_null_mask, + null_spans, + nullptr); CUDF_CUDA_TRY(cudaGetLastError()); } From 11029ea645052826efea4bca52f6dce91588d0f0 Mon Sep 17 00:00:00 2001 From: Abigale Kim Date: Wed, 5 Aug 2026 17:05:32 -0500 Subject: [PATCH 03/23] Update cpp/src/io/parquet/experimental/variant_extract.cu Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- cpp/src/io/parquet/experimental/variant_extract.cu | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cpp/src/io/parquet/experimental/variant_extract.cu b/cpp/src/io/parquet/experimental/variant_extract.cu index 03b235584835..9259d862fe60 100644 --- a/cpp/src/io/parquet/experimental/variant_extract.cu +++ b/cpp/src/io/parquet/experimental/variant_extract.cu @@ -1238,6 +1238,15 @@ std::unique_ptr cast_variant(column_view const& values, // cast_variant_fn always holds a valid column_device_view (kernel ignores it when !has_incoming). auto placeholder_col = make_empty_column(data_type{type_id::UINT8}); auto placeholder_dev_view = column_device_view::create(*placeholder_col, stream); + if (incoming_status != nullptr) { + CUDF_EXPECTS(incoming_status->size() == num_rows, + "incoming status column must have the same number of rows as the values column", + std::invalid_argument); + CUDF_EXPECTS(incoming_status->type().id() == type_id::UINT8, + "incoming status column must be UINT8", + std::invalid_argument); + } + auto incoming_dev_view = (incoming_status != nullptr) ? column_device_view::create(*incoming_status, stream) : column_device_view::create(*placeholder_col, stream); From b6a421d764da5fb08ab53d1837495b1a702542cb Mon Sep 17 00:00:00 2001 From: Abigale Kim Date: Wed, 5 Aug 2026 17:05:47 -0500 Subject: [PATCH 04/23] Update cpp/src/io/parquet/experimental/variant_extract.cu Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- cpp/src/io/parquet/experimental/variant_extract.cu | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/cpp/src/io/parquet/experimental/variant_extract.cu b/cpp/src/io/parquet/experimental/variant_extract.cu index 9259d862fe60..061a23108af3 100644 --- a/cpp/src/io/parquet/experimental/variant_extract.cu +++ b/cpp/src/io/parquet/experimental/variant_extract.cu @@ -536,26 +536,28 @@ __device__ cuda::std::pair, op_status> resolve_path( device_span sub_val = val; for (size_type i = 0; i < path.size(); ++i) { auto const step = path.element(i); - if (step.size_bytes() >= 1 && step.data()[0] == '[') { auto const index = parse_index_step(step); if (!index.has_value()) { return {{}, op_status::missing_path}; } - auto const [span, st] = locate_array_element(sub_val, index.value()); + auto const [span, st] = locate_array_element_with_status(sub_val, index.value()); if (st != op_status::success) { return {{}, st}; } sub_val = span; } else { - auto const [field_id, meta_st] = find_key_in_metadata(meta, step); + auto const [field_id, meta_st] = find_key_in_metadata_with_status(meta, step); if (meta_st == op_status::malformed_variant) { return {{}, op_status::malformed_variant}; } if (!field_id.has_value()) { return {{}, op_status::missing_path}; } - auto const [span, st] = locate_object_field(sub_val, field_id.value()); + auto const [span, st] = locate_object_field_with_status(sub_val, field_id.value()); if (st != op_status::success) { return {{}, st}; } sub_val = span; } // VARIANT null before the end of the path is missing_path per spec. if (i + 1 < path.size() && is_variant_null(sub_val)) { return {{}, op_status::missing_path}; } + // A zero-length resolved value is not decodable; the value-only path drops the row. + if (sub_val.empty()) { return {{}, op_status::malformed_variant}; } } + // Terminal VARIANT null: return the bytes with variant_null status. if (is_variant_null(sub_val)) { return {sub_val, op_status::variant_null}; } return {sub_val, op_status::success}; } From 7f339cfd6ed88ec1db22aeca30c09b65a24fdc33 Mon Sep 17 00:00:00 2001 From: Abigale Kim Date: Wed, 5 Aug 2026 22:26:14 +0000 Subject: [PATCH 05/23] reviews --- cpp/include/cudf/io/experimental/variant.hpp | 14 +- .../parquet/experimental/variant_extract.cu | 52 ++-- .../io/experimental/variant_extract_test.cpp | 235 ++++++++++-------- 3 files changed, 160 insertions(+), 141 deletions(-) diff --git a/cpp/include/cudf/io/experimental/variant.hpp b/cpp/include/cudf/io/experimental/variant.hpp index f0baa16604e4..c90df32259fe 100644 --- a/cpp/include/cudf/io/experimental/variant.hpp +++ b/cpp/include/cudf/io/experimental/variant.hpp @@ -69,9 +69,9 @@ namespace io::parquet::experimental { [[nodiscard]] std::unique_ptr get_variant_field( column_view const& variant_column, std::string_view path, + std::unique_ptr* status_out = nullptr, rmm::cuda_stream_view stream = cudf::get_default_stream(), - rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref(), - std::unique_ptr* status_out = nullptr); + rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()); /** * @brief Decode a VARIANT value column's blobs into a typed cuDF column. @@ -100,10 +100,10 @@ namespace io::parquet::experimental { [[nodiscard]] std::unique_ptr cast_variant( column_view const& values, data_type desired_type, - rmm::cuda_stream_view stream = cudf::get_default_stream(), - rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref(), column_view const* incoming_status = nullptr, - std::unique_ptr* status_out = nullptr); + std::unique_ptr* status_out = nullptr, + rmm::cuda_stream_view stream = cudf::get_default_stream(), + rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()); /** * @brief Convenience wrapper: extract a nested object value by path and decode into a typed column. @@ -128,9 +128,9 @@ namespace io::parquet::experimental { column_view const& variant_column, std::string_view path, data_type desired_type, + std::unique_ptr* status_out = nullptr, rmm::cuda_stream_view stream = cudf::get_default_stream(), - rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref(), - std::unique_ptr* status_out = nullptr); + rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()); /** @} */ } // namespace io::parquet::experimental diff --git a/cpp/src/io/parquet/experimental/variant_extract.cu b/cpp/src/io/parquet/experimental/variant_extract.cu index 061a23108af3..8962b329f005 100644 --- a/cpp/src/io/parquet/experimental/variant_extract.cu +++ b/cpp/src/io/parquet/experimental/variant_extract.cu @@ -539,14 +539,14 @@ __device__ cuda::std::pair, op_status> resolve_path( if (step.size_bytes() >= 1 && step.data()[0] == '[') { auto const index = parse_index_step(step); if (!index.has_value()) { return {{}, op_status::missing_path}; } - auto const [span, st] = locate_array_element_with_status(sub_val, index.value()); + auto const [span, st] = locate_array_element(sub_val, index.value()); if (st != op_status::success) { return {{}, st}; } sub_val = span; } else { - auto const [field_id, meta_st] = find_key_in_metadata_with_status(meta, step); + auto const [field_id, meta_st] = find_key_in_metadata(meta, step); if (meta_st == op_status::malformed_variant) { return {{}, op_status::malformed_variant}; } if (!field_id.has_value()) { return {{}, op_status::missing_path}; } - auto const [span, st] = locate_object_field_with_status(sub_val, field_id.value()); + auto const [span, st] = locate_object_field(sub_val, field_id.value()); if (st != op_status::success) { return {{}, st}; } sub_val = span; } @@ -1094,9 +1094,9 @@ namespace detail { std::unique_ptr get_variant_field(column_view const& variant_column, std::string_view path, + std::unique_ptr* status_out, rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr, - std::unique_ptr* status_out) + rmm::device_async_resource_ref mr) { // Validate the variant column CUDF_EXPECTS(variant_column.type().id() == type_id::STRUCT, @@ -1215,10 +1215,10 @@ std::unique_ptr get_variant_field(column_view const& variant_column, std::unique_ptr cast_variant(column_view const& values, data_type desired_type, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr, column_view const* incoming_status, - std::unique_ptr* status_out) + std::unique_ptr* status_out, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) { validate_variant_child(values); size_type const num_rows = values.size(); @@ -1249,10 +1249,10 @@ std::unique_ptr cast_variant(column_view const& values, std::invalid_argument); } - auto incoming_dev_view = (incoming_status != nullptr) - ? column_device_view::create(*incoming_status, stream) - : column_device_view::create(*placeholder_col, stream); - bool const has_incoming = (incoming_status != nullptr); + auto incoming_dev_view = (incoming_status != nullptr) + ? column_device_view::create(*incoming_status, stream) + : column_device_view::create(*placeholder_col, stream); + bool const has_incoming = (incoming_status != nullptr); return cudf::type_dispatcher(desired_type, cast_variant_fn{val_lists_device_view, @@ -1271,47 +1271,45 @@ std::unique_ptr cast_variant(column_view const& values, std::unique_ptr get_variant_field(column_view const& variant_column, std::string_view path, + std::unique_ptr* status_out, rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr, - std::unique_ptr* status_out) + rmm::device_async_resource_ref mr) { CUDF_FUNC_RANGE(); - return detail::get_variant_field(variant_column, path, stream, mr, status_out); + return detail::get_variant_field(variant_column, path, status_out, stream, mr); } std::unique_ptr cast_variant(column_view const& values, data_type desired_type, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr, column_view const* incoming_status, - std::unique_ptr* status_out) + std::unique_ptr* status_out, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) { CUDF_FUNC_RANGE(); - return detail::cast_variant(values, desired_type, stream, mr, incoming_status, status_out); + return detail::cast_variant(values, desired_type, incoming_status, status_out, stream, mr); } std::unique_ptr extract_variant_field(column_view const& variant_column, std::string_view path, data_type desired_type, + std::unique_ptr* status_out, rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr, - std::unique_ptr* status_out) + rmm::device_async_resource_ref mr) { CUDF_FUNC_RANGE(); auto const temp_mr = cudf::get_current_device_resource_ref(); if (status_out != nullptr) { - // Get the raw field bytes, capturing intermediate extraction status std::unique_ptr extract_status; - auto value = detail::get_variant_field(variant_column, path, stream, temp_mr, &extract_status); - // Cast using the extraction status as incoming; cast populates the final status + auto value = detail::get_variant_field(variant_column, path, &extract_status, stream, temp_mr); auto const extract_status_view = extract_status->view(); return detail::cast_variant( - value->view(), desired_type, stream, mr, &extract_status_view, status_out); + value->view(), desired_type, &extract_status_view, status_out, stream, mr); } - auto value = detail::get_variant_field(variant_column, path, stream, temp_mr, nullptr); - return detail::cast_variant(value->view(), desired_type, stream, mr, nullptr, nullptr); + auto value = detail::get_variant_field(variant_column, path, nullptr, stream, temp_mr); + return detail::cast_variant(value->view(), desired_type, nullptr, nullptr, stream, mr); } } // namespace io::parquet::experimental diff --git a/cpp/tests/io/experimental/variant_extract_test.cpp b/cpp/tests/io/experimental/variant_extract_test.cpp index 84991bca374e..fbb705faaef1 100644 --- a/cpp/tests/io/experimental/variant_extract_test.cpp +++ b/cpp/tests/io/experimental/variant_extract_test.cpp @@ -152,7 +152,7 @@ TEST_F(ExtractVariantFieldTest, NullStructRow) cudf::test::structs_column_wrapper col{{meta, val}, std::vector{true, false}}; auto got = cudf::io::parquet::experimental::extract_variant_field( - col, "x", cudf::data_type{cudf::type_id::INT32}, cudf::test::get_default_stream()); + col, "x", cudf::data_type{cudf::type_id::INT32}, nullptr, cudf::test::get_default_stream()); cudf::test::fixed_width_column_wrapper expected({7, 0}, {true, false}); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); @@ -166,7 +166,7 @@ TEST_F(ExtractVariantFieldTest, NonObjectValueYieldsNull) auto col = wrap_single_variant(metab, valb); auto got = cudf::io::parquet::experimental::extract_variant_field( - col, "x", cudf::data_type{cudf::type_id::INT32}, cudf::test::get_default_stream()); + col, "x", cudf::data_type{cudf::type_id::INT32}, nullptr, cudf::test::get_default_stream()); cudf::test::fixed_width_column_wrapper expected({0}, {false}); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); @@ -180,7 +180,7 @@ TEST_F(ExtractVariantFieldTest, InvalidMetadataYieldsNull) auto col = wrap_single_variant(metab, valb); auto got = cudf::io::parquet::experimental::extract_variant_field( - col, "x", cudf::data_type{cudf::type_id::INT32}, cudf::test::get_default_stream()); + col, "x", cudf::data_type{cudf::type_id::INT32}, nullptr, cudf::test::get_default_stream()); cudf::test::fixed_width_column_wrapper expected({0}, {false}); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); @@ -194,7 +194,7 @@ TEST_F(ExtractVariantFieldTest, UnsupportedMetadataVersionYieldsNull) auto col = wrap_single_variant(metab, valb); auto got = cudf::io::parquet::experimental::extract_variant_field( - col, "x", cudf::data_type{cudf::type_id::INT32}, cudf::test::get_default_stream()); + col, "x", cudf::data_type{cudf::type_id::INT32}, nullptr, cudf::test::get_default_stream()); cudf::test::fixed_width_column_wrapper expected({0}, {false}); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); @@ -208,7 +208,7 @@ TEST_F(ExtractVariantFieldTest, TruncatedObjectValueYieldsNull) auto col = wrap_single_variant(metab, valb); auto got = cudf::io::parquet::experimental::extract_variant_field( - col, "x", cudf::data_type{cudf::type_id::INT32}, cudf::test::get_default_stream()); + col, "x", cudf::data_type{cudf::type_id::INT32}, nullptr, cudf::test::get_default_stream()); cudf::test::fixed_width_column_wrapper expected({0}, {false}); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); @@ -219,17 +219,17 @@ TEST_F(ExtractVariantFieldTest, MultiRow) auto col = make_xyz_three_row_variant(); auto stream = cudf::test::get_default_stream(); auto x = cudf::io::parquet::experimental::extract_variant_field( - col, "x", cudf::data_type{cudf::type_id::INT32}, stream); + col, "x", cudf::data_type{cudf::type_id::INT32}, nullptr, stream); cudf::test::fixed_width_column_wrapper x_exp({7, 42, 0}, {true, true, false}); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*x, x_exp); auto y = cudf::io::parquet::experimental::extract_variant_field( - col, "y", cudf::data_type{cudf::type_id::STRING}, stream); + col, "y", cudf::data_type{cudf::type_id::STRING}, nullptr, stream); cudf::test::strings_column_wrapper y_exp({"hi", "", "zzz"}, {true, false, true}); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*y, y_exp); auto z = cudf::io::parquet::experimental::extract_variant_field( - col, "z", cudf::data_type{cudf::type_id::INT32}, stream); + col, "z", cudf::data_type{cudf::type_id::INT32}, nullptr, stream); cudf::test::fixed_width_column_wrapper z_exp({0, 99, 0}, {false, true, false}); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*z, z_exp); } @@ -241,7 +241,7 @@ TEST_F(ExtractVariantFieldTest, SlicedInput) auto const sliced = cudf::slice(col, {1, 3}).front(); auto got = cudf::io::parquet::experimental::extract_variant_field( - sliced, "x", cudf::data_type{cudf::type_id::INT32}, cudf::test::get_default_stream()); + sliced, "x", cudf::data_type{cudf::type_id::INT32}, nullptr, cudf::test::get_default_stream()); cudf::test::fixed_width_column_wrapper expected({42, 0}, {true, false}); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); @@ -257,7 +257,8 @@ TEST_F(ExtractVariantFieldTest, ApacheObjectPrimitiveStringFields) {std::pair{"string_field", "Apache Parquet"}, std::pair{"timestamp_field", "2025-04-16T12:34:56.78"}}) { SCOPED_TRACE(std::string{"field: "} + field); - auto got = cudf::io::parquet::experimental::extract_variant_field(col, field, s, stream); + auto got = + cudf::io::parquet::experimental::extract_variant_field(col, field, s, nullptr, stream); cudf::test::strings_column_wrapper expected({expected_str}); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); } @@ -271,7 +272,8 @@ TEST_F(ExtractVariantFieldTest, ApacheObjectPrimitiveNullCases) for (auto const& field : {"no_such_field", "null_field"}) { SCOPED_TRACE(std::string{"field: "} + field); - auto got = cudf::io::parquet::experimental::extract_variant_field(col, field, s, stream); + auto got = + cudf::io::parquet::experimental::extract_variant_field(col, field, s, nullptr, stream); ASSERT_EQ(got->size(), 1); EXPECT_EQ(got->null_count(), 1); } @@ -280,8 +282,12 @@ TEST_F(ExtractVariantFieldTest, ApacheObjectPrimitiveNullCases) TEST_F(ExtractVariantFieldTest, ApacheObjectPrimitiveIntField) { auto col = make_apache_variant(avf::object_primitive); - auto got = cudf::io::parquet::experimental::extract_variant_field( - col, "int_field", cudf::data_type{cudf::type_id::INT8}, cudf::test::get_default_stream()); + auto got = + cudf::io::parquet::experimental::extract_variant_field(col, + "int_field", + cudf::data_type{cudf::type_id::INT8}, + nullptr, + cudf::test::get_default_stream()); cudf::test::fixed_width_column_wrapper expected{int8_t{1}}; CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); } @@ -295,12 +301,12 @@ TEST_F(ExtractVariantFieldTest, ApacheObjectNested) SCOPED_TRACE(std::string{"path: "} + path); if constexpr (std::is_same_v) { auto got = cudf::io::parquet::experimental::extract_variant_field( - col, path, cudf::data_type{cudf::type_id::STRING}, stream); + col, path, cudf::data_type{cudf::type_id::STRING}, nullptr, stream); cudf::test::strings_column_wrapper expected({expected_val}); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); } else { auto got = cudf::io::parquet::experimental::extract_variant_field( - col, path, cudf::data_type{cudf::type_to_id()}, stream); + col, path, cudf::data_type{cudf::type_to_id()}, nullptr, stream); cudf::test::fixed_width_column_wrapper expected{expected_val}; CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); } @@ -318,7 +324,7 @@ TEST_F(ExtractVariantFieldTest, ApacheObjectEmpty) { auto col = make_apache_variant(avf::object_empty); auto got = cudf::io::parquet::experimental::extract_variant_field( - col, "foo", cudf::data_type{cudf::type_id::STRING}, cudf::test::get_default_stream()); + col, "foo", cudf::data_type{cudf::type_id::STRING}, nullptr, cudf::test::get_default_stream()); ASSERT_EQ(got->size(), 1); EXPECT_EQ(got->null_count(), 1); } @@ -329,14 +335,15 @@ TEST_F(ExtractVariantFieldTest, ApacheObjectNestedChainedCalls) auto stream = cudf::test::get_default_stream(); auto single = cudf::io::parquet::experimental::get_variant_field( - col, "$.observation.value.temperature", stream); + col, "$.observation.value.temperature", nullptr, stream); auto const meta_v = cudf::structs_column_view{col}.get_sliced_child(0, stream); - auto obs = cudf::io::parquet::experimental::get_variant_field(col, "observation", stream); + auto obs = + cudf::io::parquet::experimental::get_variant_field(col, "observation", nullptr, stream); auto vobj = cudf::io::parquet::experimental::get_variant_field( - wrap_variant_view(meta_v, obs->view()), "value", stream); + wrap_variant_view(meta_v, obs->view()), "value", nullptr, stream); auto chained = cudf::io::parquet::experimental::get_variant_field( - wrap_variant_view(meta_v, vobj->view()), "temperature", stream); + wrap_variant_view(meta_v, vobj->view()), "temperature", nullptr, stream); EXPECT_EQ(single->type().id(), cudf::type_id::LIST); EXPECT_EQ(chained->type().id(), cudf::type_id::LIST); @@ -349,7 +356,7 @@ TEST_F(ExtractVariantFieldTest, ApacheObjectNestedMissingIntermediate) auto stream = cudf::test::get_default_stream(); auto got = cudf::io::parquet::experimental::extract_variant_field( - col, "$.species.nope", cudf::data_type{cudf::type_id::STRING}, stream); + col, "$.species.nope", cudf::data_type{cudf::type_id::STRING}, nullptr, stream); cudf::test::strings_column_wrapper expected({"donotread"}, {false}); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); @@ -365,7 +372,7 @@ TEST_F(ExtractVariantFieldTest, NestedPathNonObjectIntermediate) auto col = wrap_single_variant(metab, valb); // Descending into "a" fails because it is a primitive, not an object. auto got = cudf::io::parquet::experimental::extract_variant_field( - col, "$.a.b", cudf::data_type{cudf::type_id::INT32}, cudf::test::get_default_stream()); + col, "$.a.b", cudf::data_type{cudf::type_id::INT32}, nullptr, cudf::test::get_default_stream()); cudf::test::fixed_width_column_wrapper expected({0}, {false}); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); @@ -376,8 +383,8 @@ TEST_F(ExtractVariantFieldTest, BareNameEqualsDollarPath) auto col = make_xyz_three_row_variant(); auto stream = cudf::test::get_default_stream(); - auto bare = cudf::io::parquet::experimental::get_variant_field(col, "x", stream); - auto dollar = cudf::io::parquet::experimental::get_variant_field(col, "$.x", stream); + auto bare = cudf::io::parquet::experimental::get_variant_field(col, "x", nullptr, stream); + auto dollar = cudf::io::parquet::experimental::get_variant_field(col, "$.x", nullptr, stream); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*bare, *dollar); } @@ -574,8 +581,12 @@ TEST_F(ExtractVariantFieldTest, NestedPathMultiRowMixedNulls) {v0.begin(), v0.end()}, {v1.begin(), v1.end()}, {v2.begin(), v2.end()}}; cudf::test::structs_column_wrapper col{{meta, val}}; - auto got = cudf::io::parquet::experimental::extract_variant_field( - col, "$.1st.foo-bar", cudf::data_type{cudf::type_id::INT32}, cudf::test::get_default_stream()); + auto got = + cudf::io::parquet::experimental::extract_variant_field(col, + "$.1st.foo-bar", + cudf::data_type{cudf::type_id::INT32}, + nullptr, + cudf::test::get_default_stream()); cudf::test::fixed_width_column_wrapper expected({1, 0, 0}, {true, false, false}); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); @@ -586,13 +597,13 @@ TEST_F(ExtractVariantFieldTest, EmptyPathRejected) auto col = wrap_single_variant(build_metadata({}), enc_int32(1)); auto stream = cudf::test::get_default_stream(); EXPECT_THROW( - static_cast(cudf::io::parquet::experimental::get_variant_field(col, "", stream)), - std::invalid_argument); - EXPECT_THROW( - static_cast(cudf::io::parquet::experimental::get_variant_field(col, "$", stream)), + static_cast(cudf::io::parquet::experimental::get_variant_field(col, "", nullptr, stream)), std::invalid_argument); + EXPECT_THROW(static_cast( + cudf::io::parquet::experimental::get_variant_field(col, "$", nullptr, stream)), + std::invalid_argument); EXPECT_THROW(static_cast(cudf::io::parquet::experimental::extract_variant_field( - col, "", cudf::data_type{cudf::type_id::INT32}, stream)), + col, "", cudf::data_type{cudf::type_id::INT32}, nullptr, stream)), std::invalid_argument); } @@ -614,9 +625,9 @@ TEST_F(ExtractVariantFieldTest, SyntaxErrors) "$.a[01x]", "$.a[1", "$.a[99999999999999999999]"}) { - EXPECT_THROW( - static_cast(cudf::io::parquet::experimental::get_variant_field(col, bad, stream)), - std::invalid_argument) + EXPECT_THROW(static_cast( + cudf::io::parquet::experimental::get_variant_field(col, bad, nullptr, stream)), + std::invalid_argument) << "path that should have thrown: " << bad; } } @@ -628,7 +639,7 @@ TEST_F(ExtractVariantFieldTest, ApacheArrayPrimitiveIndexing) auto stream = cudf::test::get_default_stream(); auto const i8 = cudf::data_type{cudf::type_id::INT8}; auto const get = [&](char const* path) { - return cudf::io::parquet::experimental::extract_variant_field(col, path, i8, stream); + return cudf::io::parquet::experimental::extract_variant_field(col, path, i8, nullptr, stream); }; CUDF_TEST_EXPECT_COLUMNS_EQUAL(*get("$[0]"), @@ -658,7 +669,8 @@ TEST_F(ExtractVariantFieldTest, ApacheArrayPrimitiveIndexing) value.insert(value.end(), {0x0c, 42}); // INT8(42) auto wide_col = wrap_single_variant(build_metadata({}), value); - auto got = cudf::io::parquet::experimental::extract_variant_field(wide_col, "$[0]", i8, stream); + auto got = + cudf::io::parquet::experimental::extract_variant_field(wide_col, "$[0]", i8, nullptr, stream); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, cudf::test::fixed_width_column_wrapper{int8_t{42}}); } @@ -675,12 +687,12 @@ TEST_F(ExtractVariantFieldTest, ArrayIndexingTypeMismatchAndBounds) // Object-key descent into an array value: no such key -> null. auto key_on_array = - cudf::io::parquet::experimental::extract_variant_field(col, "$.foo", i8, stream); + cudf::io::parquet::experimental::extract_variant_field(col, "$.foo", i8, nullptr, stream); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*key_on_array, null_expected); // Index step against a primitive element (after first descending into it): non-array -> null. auto index_on_primitive = - cudf::io::parquet::experimental::extract_variant_field(col, "$[0][0]", i8, stream); + cudf::io::parquet::experimental::extract_variant_field(col, "$[0][0]", i8, nullptr, stream); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*index_on_primitive, null_expected); } @@ -693,7 +705,8 @@ TEST_F(ExtractVariantFieldTest, EmptyArrayIndexing) for (auto const* path : {"$[0]", "$[1]"}) { SCOPED_TRACE(std::string{"path: "} + path); - auto got = cudf::io::parquet::experimental::extract_variant_field(col, path, i8, stream); + auto got = + cudf::io::parquet::experimental::extract_variant_field(col, path, i8, nullptr, stream); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, null_expected); } @@ -703,8 +716,8 @@ TEST_F(ExtractVariantFieldTest, EmptyArrayIndexing) {0x03, 0x01, 0x02, 0x01, 0x0c, 42}, {0x03, 0x01, 0x00, 0x03, 0x0c, 42}}) { auto malformed_col = wrap_single_variant(build_metadata({}), value); - auto got = - cudf::io::parquet::experimental::extract_variant_field(malformed_col, "$[0]", i8, stream); + auto got = cudf::io::parquet::experimental::extract_variant_field( + malformed_col, "$[0]", i8, nullptr, stream); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, null_expected); } } @@ -721,14 +734,14 @@ TEST_F(ExtractVariantFieldTest, MixedObjectArrayTraversal) auto const check_str = [&](char const* path, char const* expected) { SCOPED_TRACE(std::string{"path: "} + path); auto got = cudf::io::parquet::experimental::extract_variant_field( - col, path, cudf::data_type{cudf::type_id::STRING}, stream); + col, path, cudf::data_type{cudf::type_id::STRING}, nullptr, stream); cudf::test::strings_column_wrapper const expected_col({expected}); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected_col); }; auto const check_null = [&](char const* path) { SCOPED_TRACE(std::string{"path: "} + path); auto got = cudf::io::parquet::experimental::extract_variant_field( - col, path, cudf::data_type{cudf::type_id::STRING}, stream); + col, path, cudf::data_type{cudf::type_id::STRING}, nullptr, stream); cudf::test::strings_column_wrapper const null_col({""}, {false}); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, null_col); }; @@ -755,12 +768,12 @@ TEST_F(ExtractVariantFieldTest, LargeDictionaryAndObjectScan) auto const int32_dtype = cudf::data_type{cudf::type_id::INT32}; // First, middle, and last keys each decode to their own field id. - auto first = - cudf::io::parquet::experimental::extract_variant_field(col, "k00", int32_dtype, stream); - auto mid = - cudf::io::parquet::experimental::extract_variant_field(col, "k24", int32_dtype, stream); - auto last = - cudf::io::parquet::experimental::extract_variant_field(col, "k49", int32_dtype, stream); + auto first = cudf::io::parquet::experimental::extract_variant_field( + col, "k00", int32_dtype, nullptr, stream); + auto mid = cudf::io::parquet::experimental::extract_variant_field( + col, "k24", int32_dtype, nullptr, stream); + auto last = cudf::io::parquet::experimental::extract_variant_field( + col, "k49", int32_dtype, nullptr, stream); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*first, cudf::test::fixed_width_column_wrapper{0}); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*mid, cudf::test::fixed_width_column_wrapper{24}); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*last, cudf::test::fixed_width_column_wrapper{49}); @@ -794,8 +807,8 @@ TEST_F(ExtractVariantFieldTest, MalformedVariantDataYieldsNull) for (auto const& c : cases) { SCOPED_TRACE(c.label); auto col = wrap_single_variant(c.meta, c.val); - auto got = - cudf::io::parquet::experimental::extract_variant_field(col, "x", int32_dtype, stream); + auto got = cudf::io::parquet::experimental::extract_variant_field( + col, "x", int32_dtype, nullptr, stream); ASSERT_EQ(got->size(), 1); EXPECT_EQ(got->null_count(), 1); } @@ -844,8 +857,12 @@ TEST_F(ExtractVariantFieldTest, NullsAtDifferentDepths) auto col = wrap_multi_row_variant(meta_rows, val_rows); - auto got = cudf::io::parquet::experimental::extract_variant_field( - col, "$.a.b.c.d", cudf::data_type{cudf::type_id::STRING}, cudf::test::get_default_stream()); + auto got = + cudf::io::parquet::experimental::extract_variant_field(col, + "$.a.b.c.d", + cudf::data_type{cudf::type_id::STRING}, + nullptr, + cudf::test::get_default_stream()); cudf::test::strings_column_wrapper expected(exp_strs.begin(), exp_strs.end(), exp_valid.begin()); CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(*got, expected); @@ -857,7 +874,7 @@ TEST_F(ExtractVariantFieldTest, EmptyInput) auto const variant = cudf::empty_like(make_xyz_three_row_variant()); auto got = cudf::io::parquet::experimental::extract_variant_field( - *variant, "x", cudf::data_type{cudf::type_id::INT32}, stream); + *variant, "x", cudf::data_type{cudf::type_id::INT32}, nullptr, stream); EXPECT_EQ(got->type().id(), cudf::type_id::INT32); EXPECT_EQ(got->size(), 0); EXPECT_EQ(got->null_count(), 0); @@ -870,14 +887,14 @@ TEST_F(GetVariantFieldTest, ApacheObjectPrimitive) auto col = make_apache_variant(avf::object_primitive); auto stream = cudf::test::get_default_stream(); - auto got = cudf::io::parquet::experimental::get_variant_field(col, "int_field", stream); + auto got = cudf::io::parquet::experimental::get_variant_field(col, "int_field", nullptr, stream); EXPECT_EQ(got->type().id(), cudf::type_id::LIST); EXPECT_EQ(got->size(), 1); EXPECT_EQ(cudf::lists_column_view{got->view()}.child().type().id(), cudf::type_id::UINT8); auto casted = cudf::io::parquet::experimental::cast_variant( - got->view(), cudf::data_type{cudf::type_id::INT8}, stream); + got->view(), cudf::data_type{cudf::type_id::INT8}, nullptr, nullptr, stream); cudf::test::fixed_width_column_wrapper expected{int8_t{1}}; CUDF_TEST_EXPECT_COLUMNS_EQUAL(*casted, expected); } @@ -886,7 +903,7 @@ TEST_F(GetVariantFieldTest, ApacheObjectPrimitiveMissingKeyAllNull) { auto col = make_apache_variant(avf::object_primitive); auto got = cudf::io::parquet::experimental::get_variant_field( - col, "no_such_field", cudf::test::get_default_stream()); + col, "no_such_field", nullptr, cudf::test::get_default_stream()); EXPECT_EQ(got->type().id(), cudf::type_id::LIST); EXPECT_EQ(got->size(), 1); @@ -899,11 +916,11 @@ TEST_F(GetVariantFieldTest, GetAndCastMatchesExtract) auto stream = cudf::test::get_default_stream(); auto extract_x = cudf::io::parquet::experimental::extract_variant_field( - col, "x", cudf::data_type{cudf::type_id::INT32}, stream); + col, "x", cudf::data_type{cudf::type_id::INT32}, nullptr, stream); - auto intermediate = cudf::io::parquet::experimental::get_variant_field(col, "x", stream); + auto intermediate = cudf::io::parquet::experimental::get_variant_field(col, "x", nullptr, stream); auto two_step_x = cudf::io::parquet::experimental::cast_variant( - intermediate->view(), cudf::data_type{cudf::type_id::INT32}, stream); + intermediate->view(), cudf::data_type{cudf::type_id::INT32}, nullptr, nullptr, stream); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*extract_x, *two_step_x); } @@ -913,7 +930,7 @@ TEST_F(GetVariantFieldTest, EmptyInput) auto const stream = cudf::test::get_default_stream(); auto const variant = cudf::empty_like(make_xyz_three_row_variant()); - auto got = cudf::io::parquet::experimental::get_variant_field(*variant, "x", stream); + auto got = cudf::io::parquet::experimental::get_variant_field(*variant, "x", nullptr, stream); EXPECT_EQ(got->type().id(), cudf::type_id::LIST); EXPECT_EQ(got->size(), 0); EXPECT_EQ(got->null_count(), 0); @@ -927,7 +944,7 @@ std::unique_ptr cast_apache_primitive(avf::fixture const& fi auto col = make_apache_variant(fixture); auto const value = cudf::structs_column_view{col}.get_sliced_child(1, stream); return cudf::io::parquet::experimental::cast_variant( - value, cudf::data_type{cudf::type_to_id()}, stream); + value, cudf::data_type{cudf::type_to_id()}, nullptr, nullptr, stream); } struct CastVariantTest : public cudf::test::BaseFixture {}; @@ -964,7 +981,7 @@ TEST_F(CastVariantTest, ApachePrimitiveFloats) auto col = make_apache_variant(fixture); auto const value = cudf::structs_column_view{col}.get_sliced_child(1, stream); auto got = cudf::io::parquet::experimental::cast_variant( - value, cudf::data_type{cudf::type_to_id()}, stream); + value, cudf::data_type{cudf::type_to_id()}, nullptr, nullptr, stream); cudf::test::fixed_width_column_wrapper expected{expected_val}; CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); }; @@ -980,7 +997,7 @@ TEST_F(CastVariantTest, ApachePrimitiveBooleans) auto col = make_apache_variant(fixture); auto const value = cudf::structs_column_view{col}.get_sliced_child(1, stream); auto got = cudf::io::parquet::experimental::cast_variant( - value, cudf::data_type{cudf::type_id::BOOL8}, stream); + value, cudf::data_type{cudf::type_id::BOOL8}, nullptr, nullptr, stream); cudf::test::fixed_width_column_wrapper expected{expected_val}; CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); }; @@ -993,7 +1010,7 @@ TEST_F(CastVariantTest, ApachePrimitiveBooleans) auto col = make_apache_variant(avf::primitive_null); auto const value = cudf::structs_column_view{col}.get_sliced_child(1, stream); auto got = cudf::io::parquet::experimental::cast_variant( - value, cudf::data_type{cudf::type_id::BOOL8}, stream); + value, cudf::data_type{cudf::type_id::BOOL8}, nullptr, nullptr, stream); cudf::test::fixed_width_column_wrapper expected({false}, {false}); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); } @@ -1041,7 +1058,7 @@ TEST_F(CastVariantTest, ApachePrimitiveBooleans) auto const sliced = cudf::slice(col, {slice_beg, slice_end}).front(); auto const value = cudf::structs_column_view{sliced}.get_sliced_child(1, stream); auto got = cudf::io::parquet::experimental::cast_variant( - value, cudf::data_type{cudf::type_id::BOOL8}, stream); + value, cudf::data_type{cudf::type_id::BOOL8}, nullptr, nullptr, stream); cudf::test::fixed_width_column_wrapper expected( exp_vals.begin() + slice_beg, exp_vals.begin() + slice_end, exp_valid.begin() + slice_beg); @@ -1056,7 +1073,7 @@ TEST_F(CastVariantTest, ApacheShortString) auto const value = cudf::structs_column_view{col}.get_sliced_child(1, stream); auto got = cudf::io::parquet::experimental::cast_variant( - value, cudf::data_type{cudf::type_id::STRING}, stream); + value, cudf::data_type{cudf::type_id::STRING}, nullptr, nullptr, stream); // Decoded from short_string.value: skip the 1-byte header, take the rest. std::string const expected_str(reinterpret_cast(avf::short_string.value.data() + 1), @@ -1072,7 +1089,7 @@ TEST_F(CastVariantTest, ApachePrimitiveString) auto const value = cudf::structs_column_view{col}.get_sliced_child(1, stream); auto got = cudf::io::parquet::experimental::cast_variant( - value, cudf::data_type{cudf::type_id::STRING}, stream); + value, cudf::data_type{cudf::type_id::STRING}, nullptr, nullptr, stream); // Long-string layout: 1 header byte + 4-byte LE length + payload. std::string const expected_str( @@ -1089,7 +1106,7 @@ TEST_F(CastVariantTest, MismatchedTypeYieldsNull) auto col = make_apache_variant(avf::object_primitive); auto const value = cudf::structs_column_view{col}.get_sliced_child(1, stream); auto got = cudf::io::parquet::experimental::cast_variant( - value, cudf::data_type{cudf::type_id::INT32}, stream); + value, cudf::data_type{cudf::type_id::INT32}, nullptr, nullptr, stream); ASSERT_EQ(got->size(), 1); EXPECT_EQ(got->null_count(), 1); } @@ -1105,7 +1122,8 @@ TEST_F(CastVariantTest, EmptyInput) cudf::type_id::FLOAT32, cudf::type_id::FLOAT64, cudf::type_id::BOOL8}) { - auto got = cudf::io::parquet::experimental::cast_variant(*values, cudf::data_type{id}, stream); + auto got = cudf::io::parquet::experimental::cast_variant( + *values, cudf::data_type{id}, nullptr, nullptr, stream); EXPECT_EQ(got->type().id(), id); EXPECT_EQ(got->size(), 0); EXPECT_EQ(got->null_count(), 0); @@ -1136,7 +1154,7 @@ TEST_F(CastVariantTest, CastToUnsupportedTargetThrows) for (auto const id : ids) { SCOPED_TRACE(std::string{"target type_id: "} + std::to_string(static_cast(id))); EXPECT_THROW(static_cast(cudf::io::parquet::experimental::cast_variant( - values, cudf::data_type{id}, stream)), + values, cudf::data_type{id}, nullptr, nullptr, stream)), std::invalid_argument); } } @@ -1178,7 +1196,8 @@ TEST_F(CastVariantTest, CastSourceTargetMatrix) for (auto const& src : sources) { SCOPED_TRACE(std::string{"int target "} + match_label + ", source " + src.label); auto values = values_of(src.bytes); - auto got = cudf::io::parquet::experimental::cast_variant(values, target, stream); + auto got = + cudf::io::parquet::experimental::cast_variant(values, target, nullptr, nullptr, stream); if (std::string_view{src.label} == match_label) { cudf::test::fixed_width_column_wrapper const expected{match_value}; CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); @@ -1198,7 +1217,8 @@ TEST_F(CastVariantTest, CastSourceTargetMatrix) for (auto const& src : sources) { SCOPED_TRACE(std::string{"string target, source "} + src.label); auto values = values_of(src.bytes); - auto got = cudf::io::parquet::experimental::cast_variant(values, string_type, stream); + auto got = + cudf::io::parquet::experimental::cast_variant(values, string_type, nullptr, nullptr, stream); std::string_view const label{src.label}; if (label == "short_string" || label == "long_string") { std::string const expected_str = (label == "short_string") ? "hi" : std::string(70, 'a'); @@ -1218,7 +1238,7 @@ TEST_F(CastVariantTest, ShortStringLengthZero) std::vector const val{make_variant_short_string_header(0)}; cudf::test::lists_column_wrapper values(val.begin(), val.end()); auto got = cudf::io::parquet::experimental::cast_variant( - values, cudf::data_type{cudf::type_id::STRING}, stream); + values, cudf::data_type{cudf::type_id::STRING}, nullptr, nullptr, stream); cudf::test::strings_column_wrapper expected({""}); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); } @@ -1233,7 +1253,7 @@ TEST_F(CastVariantTest, ShortStringMaxLength) val.insert(val.end(), payload.begin(), payload.end()); cudf::test::lists_column_wrapper values(val.begin(), val.end()); auto got = cudf::io::parquet::experimental::cast_variant( - values, cudf::data_type{cudf::type_id::STRING}, stream); + values, cudf::data_type{cudf::type_id::STRING}, nullptr, nullptr, stream); cudf::test::strings_column_wrapper expected({payload}); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); } @@ -1246,7 +1266,7 @@ TEST_F(CastVariantTest, LongStringLengthZero) make_variant_primitive(variant_primitive_type::LONG_STRING), 0x00, 0x00, 0x00, 0x00}; cudf::test::lists_column_wrapper values(val.begin(), val.end()); auto got = cudf::io::parquet::experimental::cast_variant( - values, cudf::data_type{cudf::type_id::STRING}, stream); + values, cudf::data_type{cudf::type_id::STRING}, nullptr, nullptr, stream); cudf::test::strings_column_wrapper expected({""}); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); } @@ -1266,7 +1286,7 @@ TEST_F(CastVariantTest, LongStringDeclaredLengthExceedsPayloadYieldsNull) SCOPED_TRACE(std::string{"payload bytes present: "} + std::to_string(val.size() - 5)); cudf::test::lists_column_wrapper values(val.begin(), val.end()); auto got = cudf::io::parquet::experimental::cast_variant( - values, cudf::data_type{cudf::type_id::STRING}, stream); + values, cudf::data_type{cudf::type_id::STRING}, nullptr, nullptr, stream); ASSERT_EQ(got->size(), 1); EXPECT_EQ(got->null_count(), 1); } @@ -1283,7 +1303,7 @@ TEST_F(CastVariantTest, LongStringPayloadExceedsDeclaredLength) hdr, 0x03, 0x00, 0x00, 0x00, 'a', 'b', 'c', 'x', 'x', 'x', 'x', 'x'}; cudf::test::lists_column_wrapper values(val.begin(), val.end()); auto got = cudf::io::parquet::experimental::cast_variant( - values, cudf::data_type{cudf::type_id::STRING}, stream); + values, cudf::data_type{cudf::type_id::STRING}, nullptr, nullptr, stream); cudf::test::strings_column_wrapper expected({"abc"}); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); } @@ -1359,7 +1379,7 @@ TEST_F(InvalidInputShapeTest, GetVariantFieldRejectsMalformedInput) for (auto const& c : cases) { SCOPED_TRACE(c.label); EXPECT_THROW(static_cast(cudf::io::parquet::experimental::get_variant_field( - c.column->view(), "x", stream)), + c.column->view(), "x", nullptr, stream)), std::invalid_argument); } } @@ -1376,9 +1396,10 @@ TEST_F(InvalidInputShapeTest, CastVariantRejectsMalformedInput) for (auto const& c : cases) { SCOPED_TRACE(c.label); - EXPECT_THROW(static_cast(cudf::io::parquet::experimental::cast_variant( - c.column->view(), cudf::data_type{cudf::type_id::INT32}, stream)), - std::invalid_argument); + EXPECT_THROW( + static_cast(cudf::io::parquet::experimental::cast_variant( + c.column->view(), cudf::data_type{cudf::type_id::INT32}, nullptr, nullptr, stream)), + std::invalid_argument); } } @@ -1419,7 +1440,7 @@ TEST_F(GetVariantFieldStatusTest, SqlNullInputProducesNullStatus) auto stream = cudf::test::get_default_stream(); std::unique_ptr status; - auto got = cudf::io::parquet::experimental::get_variant_field(col, "x", stream, cmr(), &status); + auto got = cudf::io::parquet::experimental::get_variant_field(col, "x", &status, stream, cmr()); ASSERT_NE(status, nullptr); ASSERT_EQ(status->size(), 1); @@ -1434,7 +1455,7 @@ TEST_F(GetVariantFieldStatusTest, SuccessStatus) auto stream = cudf::test::get_default_stream(); std::unique_ptr status; - auto got = cudf::io::parquet::experimental::get_variant_field(col, "x", stream, cmr(), &status); + auto got = cudf::io::parquet::experimental::get_variant_field(col, "x", &status, stream, cmr()); // Row 0: x=INT32(7) → success; Row 1: x=INT32(42) → success; Row 2: no x → missing_path ASSERT_NE(status, nullptr); @@ -1451,7 +1472,7 @@ TEST_F(GetVariantFieldStatusTest, MissingKeyProducesMissingPathStatus) std::unique_ptr status; auto got = cudf::io::parquet::experimental::get_variant_field( - col, "no_such_field", stream, cmr(), &status); + col, "no_such_field", &status, stream, cmr()); ASSERT_NE(status, nullptr); ASSERT_EQ(status->size(), 1); @@ -1471,7 +1492,7 @@ TEST_F(GetVariantFieldStatusTest, VariantNullPreservedWithStatus) std::unique_ptr status; auto got = - cudf::io::parquet::experimental::get_variant_field(col, "null_field", stream, cmr(), &status); + cudf::io::parquet::experimental::get_variant_field(col, "null_field", &status, stream, cmr()); ASSERT_NE(status, nullptr); ASSERT_EQ(status->size(), 1); @@ -1490,7 +1511,7 @@ TEST_F(GetVariantFieldStatusTest, VariantNullReturnedAsBytesWithoutStatus) auto stream = cudf::test::get_default_stream(); // No status_out: get_variant_field returns the VARIANT null bytes as a non-null list row. - auto got = cudf::io::parquet::experimental::get_variant_field(col, "null_field", stream); + auto got = cudf::io::parquet::experimental::get_variant_field(col, "null_field", nullptr, stream); EXPECT_EQ(got->null_count(), 0); EXPECT_EQ(got->size(), 1); } @@ -1504,7 +1525,7 @@ TEST_F(GetVariantFieldStatusTest, MalformedMetadataProducesMalformedStatus) auto stream = cudf::test::get_default_stream(); std::unique_ptr status; - auto got = cudf::io::parquet::experimental::get_variant_field(col, "x", stream, cmr(), &status); + auto got = cudf::io::parquet::experimental::get_variant_field(col, "x", &status, stream, cmr()); ASSERT_NE(status, nullptr); expect_status_values(*status, {ST_MALFORMED}); @@ -1522,7 +1543,7 @@ TEST_F(GetVariantFieldStatusTest, VariantNullBeforeEndIsMissingPath) std::unique_ptr status; auto got = - cudf::io::parquet::experimental::get_variant_field(col, "$.a.b", stream, cmr(), &status); + cudf::io::parquet::experimental::get_variant_field(col, "$.a.b", &status, stream, cmr()); ASSERT_NE(status, nullptr); expect_status_values(*status, {ST_MISSING}); @@ -1556,7 +1577,7 @@ TEST_F(GetVariantFieldStatusTest, MixedRows) cudf::test::structs_column_wrapper col{{meta, val}, std::vector{true, true, true, false}}; std::unique_ptr status; - auto got = cudf::io::parquet::experimental::get_variant_field(col, "x", stream, cmr(), &status); + auto got = cudf::io::parquet::experimental::get_variant_field(col, "x", &status, stream, cmr()); ASSERT_NE(status, nullptr); ASSERT_EQ(status->size(), 4); @@ -1581,7 +1602,7 @@ TEST_F(GetVariantFieldStatusTest, EmptyInput) std::unique_ptr status; auto got = - cudf::io::parquet::experimental::get_variant_field(*variant, "x", stream, cmr(), &status); + cudf::io::parquet::experimental::get_variant_field(*variant, "x", &status, stream, cmr()); ASSERT_NE(status, nullptr); EXPECT_EQ(status->size(), 0); @@ -1610,7 +1631,7 @@ TEST_F(CastVariantStatusTest, SuccessProducesSuccessStatus) auto values = make_value_col(enc_int32(42)); std::unique_ptr status; auto got = cudf::io::parquet::experimental::cast_variant( - values, cudf::data_type{cudf::type_id::INT32}, stream, cmr(), nullptr, &status); + values, cudf::data_type{cudf::type_id::INT32}, nullptr, &status, stream, cmr()); ASSERT_NE(status, nullptr); expect_status_values(*status, {ST_SUCCESS}); @@ -1625,7 +1646,7 @@ TEST_F(CastVariantStatusTest, VariantNullProducesVariantNullStatus) auto values = make_value_col(enc_null()); std::unique_ptr status; auto got = cudf::io::parquet::experimental::cast_variant( - values, cudf::data_type{cudf::type_id::INT32}, stream, cmr(), nullptr, &status); + values, cudf::data_type{cudf::type_id::INT32}, nullptr, &status, stream, cmr()); ASSERT_NE(status, nullptr); expect_status_values(*status, {ST_VNULL}); @@ -1639,7 +1660,7 @@ TEST_F(CastVariantStatusTest, TypeMismatchStatus) auto values = make_value_col(enc_int8(5)); // INT8 cast to INT32 target → mismatch std::unique_ptr status; auto got = cudf::io::parquet::experimental::cast_variant( - values, cudf::data_type{cudf::type_id::INT32}, stream, cmr(), nullptr, &status); + values, cudf::data_type{cudf::type_id::INT32}, nullptr, &status, stream, cmr()); ASSERT_NE(status, nullptr); expect_status_values(*status, {ST_MISMATCH}); @@ -1673,7 +1694,7 @@ TEST_F(CastVariantStatusTest, SqlNullInputProducesNullStatus) std::unique_ptr status; auto got = cudf::io::parquet::experimental::cast_variant( - values_col->view(), cudf::data_type{cudf::type_id::INT32}, stream, cmr(), nullptr, &status); + values_col->view(), cudf::data_type{cudf::type_id::INT32}, nullptr, &status, stream, cmr()); ASSERT_NE(status, nullptr); // Row 0: success; row 1: null status @@ -1702,7 +1723,7 @@ TEST_F(CastVariantStatusTest, IncomingStatusPropagation) std::unique_ptr status; auto got = cudf::io::parquet::experimental::cast_variant( - values, cudf::data_type{cudf::type_id::INT32}, stream, cmr(), &incoming_view1, &status); + values, cudf::data_type{cudf::type_id::INT32}, &incoming_view1, &status, stream, cmr()); ASSERT_NE(status, nullptr); // Row 0: success (decoded), Row 1: missing_path (propagated), Row 2: variant_null (propagated) @@ -1729,7 +1750,7 @@ TEST_F(CastVariantStatusTest, IncomingNullStatusProducesNullStatus) std::unique_ptr status; auto got = cudf::io::parquet::experimental::cast_variant( - values, cudf::data_type{cudf::type_id::INT32}, stream, cmr(), &incoming_view2, &status); + values, cudf::data_type{cudf::type_id::INT32}, &incoming_view2, &status, stream, cmr()); ASSERT_NE(status, nullptr); ASSERT_EQ(status->null_count(), 1); @@ -1757,7 +1778,7 @@ TEST_F(CastVariantStatusTest, BoolStatusTracking) std::unique_ptr status; auto got = cudf::io::parquet::experimental::cast_variant( - values, cudf::data_type{cudf::type_id::BOOL8}, stream, cmr(), nullptr, &status); + values, cudf::data_type{cudf::type_id::BOOL8}, nullptr, &status, stream, cmr()); ASSERT_NE(status, nullptr); expect_status_values(*status, {ST_SUCCESS, ST_VNULL, ST_MISMATCH}); @@ -1789,7 +1810,7 @@ TEST_F(CastVariantStatusTest, StringStatusTracking) std::unique_ptr status; auto got = cudf::io::parquet::experimental::cast_variant( - values, cudf::data_type{cudf::type_id::STRING}, stream, cmr(), nullptr, &status); + values, cudf::data_type{cudf::type_id::STRING}, nullptr, &status, stream, cmr()); ASSERT_NE(status, nullptr); expect_status_values(*status, {ST_SUCCESS, ST_VNULL, ST_MISMATCH, ST_MALFORMED}); @@ -1804,7 +1825,7 @@ TEST_F(CastVariantStatusTest, EmptyInput) cudf::empty_like(cudf::structs_column_view{make_xyz_three_row_variant()}.child(1)); std::unique_ptr status; auto got = cudf::io::parquet::experimental::cast_variant( - *values, cudf::data_type{cudf::type_id::INT32}, stream, cmr(), nullptr, &status); + *values, cudf::data_type{cudf::type_id::INT32}, nullptr, &status, stream, cmr()); ASSERT_NE(status, nullptr); EXPECT_EQ(status->size(), 0); @@ -1825,7 +1846,7 @@ TEST_F(ExtractVariantFieldStatusTest, SuccessStatus) std::unique_ptr status; auto got = cudf::io::parquet::experimental::extract_variant_field( - col, "x", cudf::data_type{cudf::type_id::INT32}, stream, cmr(), &status); + col, "x", cudf::data_type{cudf::type_id::INT32}, &status, stream, cmr()); ASSERT_NE(status, nullptr); // Rows 0,1 have x as INT32 → success; row 2 has no x → missing_path @@ -1844,7 +1865,7 @@ TEST_F(ExtractVariantFieldStatusTest, SqlNullInputProducesNullStatus) auto stream = cudf::test::get_default_stream(); std::unique_ptr status; auto got = cudf::io::parquet::experimental::extract_variant_field( - col, "x", cudf::data_type{cudf::type_id::INT32}, stream, cmr(), &status); + col, "x", cudf::data_type{cudf::type_id::INT32}, &status, stream, cmr()); ASSERT_NE(status, nullptr); EXPECT_EQ(status->null_count(), 1); @@ -1861,7 +1882,7 @@ TEST_F(ExtractVariantFieldStatusTest, VariantNullStatus) std::unique_ptr status; auto got = cudf::io::parquet::experimental::extract_variant_field( - col, "f", cudf::data_type{cudf::type_id::INT32}, stream, cmr(), &status); + col, "f", cudf::data_type{cudf::type_id::INT32}, &status, stream, cmr()); ASSERT_NE(status, nullptr); expect_status_values(*status, {ST_VNULL}); @@ -1878,7 +1899,7 @@ TEST_F(ExtractVariantFieldStatusTest, TypeMismatchStatus) std::unique_ptr status; auto got = cudf::io::parquet::experimental::extract_variant_field( - col, "s", cudf::data_type{cudf::type_id::INT32}, stream, cmr(), &status); + col, "s", cudf::data_type{cudf::type_id::INT32}, &status, stream, cmr()); ASSERT_NE(status, nullptr); expect_status_values(*status, {ST_MISMATCH}); @@ -1893,7 +1914,7 @@ TEST_F(ExtractVariantFieldStatusTest, MissingNestedPathStatus) std::unique_ptr status; auto got = cudf::io::parquet::experimental::extract_variant_field( - col, "$.species.nope", cudf::data_type{cudf::type_id::STRING}, stream, cmr(), &status); + col, "$.species.nope", cudf::data_type{cudf::type_id::STRING}, &status, stream, cmr()); ASSERT_NE(status, nullptr); expect_status_values(*status, {ST_MISSING}); @@ -1911,7 +1932,7 @@ TEST_F(ExtractVariantFieldStatusTest, AllNullInput) auto stream = cudf::test::get_default_stream(); std::unique_ptr status; auto got = cudf::io::parquet::experimental::extract_variant_field( - all_null, "x", cudf::data_type{cudf::type_id::INT32}, stream, cmr(), &status); + all_null, "x", cudf::data_type{cudf::type_id::INT32}, &status, stream, cmr()); ASSERT_NE(status, nullptr); EXPECT_EQ(status->null_count(), 1); @@ -1926,7 +1947,7 @@ TEST_F(ExtractVariantFieldStatusTest, ExistingCallsUnchanged) // No status_out — existing behavior auto got = cudf::io::parquet::experimental::extract_variant_field( - col, "x", cudf::data_type{cudf::type_id::INT32}, stream); + col, "x", cudf::data_type{cudf::type_id::INT32}, nullptr, stream); cudf::test::fixed_width_column_wrapper expected({7, 42, 0}, {true, true, false}); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); From a41e516f69471e13c6510d7623ce831cf9114997 Mon Sep 17 00:00:00 2001 From: Abigale Kim Date: Wed, 5 Aug 2026 22:52:54 +0000 Subject: [PATCH 06/23] reviews --- cpp/tests/io/experimental/variant_extract_test.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cpp/tests/io/experimental/variant_extract_test.cpp b/cpp/tests/io/experimental/variant_extract_test.cpp index fbb705faaef1..acc9a493cae2 100644 --- a/cpp/tests/io/experimental/variant_extract_test.cpp +++ b/cpp/tests/io/experimental/variant_extract_test.cpp @@ -1499,6 +1499,9 @@ TEST_F(GetVariantFieldStatusTest, VariantNullPreservedWithStatus) expect_status_values(*status, {ST_VNULL}); // With status requested, the VARIANT null bytes are preserved (output is NOT SQL null) EXPECT_EQ(got->null_count(), 0); + auto const null_bytes = enc_null(); + cudf::test::lists_column_wrapper expected_bytes{{null_bytes.begin(), null_bytes.end()}}; + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected_bytes); } // Without status_out, VARIANT null is returned as bytes (non-null list row), same as with status. @@ -1514,6 +1517,9 @@ TEST_F(GetVariantFieldStatusTest, VariantNullReturnedAsBytesWithoutStatus) auto got = cudf::io::parquet::experimental::get_variant_field(col, "null_field", nullptr, stream); EXPECT_EQ(got->null_count(), 0); EXPECT_EQ(got->size(), 1); + auto const null_bytes = enc_null(); + cudf::test::lists_column_wrapper expected_bytes{{null_bytes.begin(), null_bytes.end()}}; + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected_bytes); } // Malformed metadata → malformed_variant status From f018976f3572b2d1f33f52513b378359a1d24476 Mon Sep 17 00:00:00 2001 From: Abigale Kim Date: Thu, 6 Aug 2026 20:38:23 +0000 Subject: [PATCH 07/23] removing redundant things & comments --- .../parquet/experimental/variant_extract.cu | 13 ------ .../io/experimental/variant_extract_test.cpp | 41 ------------------- 2 files changed, 54 deletions(-) diff --git a/cpp/src/io/parquet/experimental/variant_extract.cu b/cpp/src/io/parquet/experimental/variant_extract.cu index 8962b329f005..26fdc5c7c878 100644 --- a/cpp/src/io/parquet/experimental/variant_extract.cu +++ b/cpp/src/io/parquet/experimental/variant_extract.cu @@ -53,13 +53,10 @@ namespace { constexpr int variant_version_v1 = 1; -// Bytes consumed by the leading metadata byte common to every Variant value. constexpr size_type variant_header_bytes = 1; -// Low 2 bits of a value's metadata byte: the basic type. using basic_type = variant_basic_type; -// For a primitive value, the value_header is the physical type id of the payload. using primitive_type = variant_primitive_type; using op_status = variant_operation_status; @@ -74,7 +71,6 @@ __device__ cuda::std::optional read_uint64(device_span return v; } -// Safely narrow a decoded value to size_type __device__ cuda::std::optional narrow_cast(cuda::std::optional value) { if (!value.has_value() || @@ -152,7 +148,6 @@ __device__ cuda::std::optional variant_value_length(device_span(value_header)) { case primitive_type::NULLVAL: @@ -418,7 +413,6 @@ __device__ cuda::std::pair, op_status> locate_array_e op_status::success}; } -// True when the value blob begins with the VARIANT null primitive header. __device__ bool is_variant_null(device_span enc) { if (enc.empty()) { return false; } @@ -598,7 +592,6 @@ __device__ device_span list_row_span(cudf::lists_column_device_vi return {col.child().data() + begin, static_cast(end - begin)}; } -// Returns the metadata and value list bytes for a given row from device views __device__ cuda::std::pair, device_span> metadata_and_value_at(cudf::lists_column_device_view const& metadata, cudf::lists_column_device_view const& values, @@ -862,7 +855,6 @@ void validate_variant_child(column_view const& child) std::invalid_argument); } -// Build the status column from d_status and d_status_null_mask buffers. std::unique_ptr make_status_column(rmm::device_buffer status_data, rmm::device_buffer status_null_mask, size_type num_rows, @@ -892,7 +884,6 @@ struct cast_variant_fn { bool has_incoming{false}; std::unique_ptr* status_out{nullptr}; - // Helper: allocate status buffers and return the output status column. // Avoids repeating this boilerplate in each cast operator. auto alloc_status() -> cuda::std::pair { @@ -1098,7 +1089,6 @@ std::unique_ptr get_variant_field(column_view const& variant_column, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) { - // Validate the variant column CUDF_EXPECTS(variant_column.type().id() == type_id::STRUCT, "VARIANT column must be struct type", std::invalid_argument); @@ -1175,7 +1165,6 @@ std::unique_ptr get_variant_field(column_view const& variant_column, CUDF_CUDA_TRY(cudaGetLastError()); } - // Convert sizes to offsets auto [offsets_column, total_bytes] = cudf::strings::detail::make_offsets_child_column(d_sizes.begin(), d_sizes.end(), stream, mr); CUDF_EXPECTS(total_bytes <= std::numeric_limits::max(), @@ -1184,7 +1173,6 @@ std::unique_ptr get_variant_field(column_view const& variant_column, device_span d_offsets{offsets_column->view().data(), static_cast(num_rows + 1)}; - // Copy values into the output buffer auto val_child = make_numeric_column( data_type{type_id::UINT8}, total_bytes, mask_state::UNALLOCATED, stream, mr); if (total_bytes > 0) { @@ -1230,7 +1218,6 @@ std::unique_ptr cast_variant(column_view const& values, auto val_device_view = column_device_view::create(values, stream); cudf::lists_column_device_view val_lists_device_view(*val_device_view); - // Initialize the null mask from the values column (or all-valid) auto null_mask = values.nullable() ? cudf::detail::copy_bitmask(values, stream, mr) : cudf::create_null_mask(num_rows, mask_state::ALL_VALID, stream, mr); diff --git a/cpp/tests/io/experimental/variant_extract_test.cpp b/cpp/tests/io/experimental/variant_extract_test.cpp index acc9a493cae2..f888d6a53ae0 100644 --- a/cpp/tests/io/experimental/variant_extract_test.cpp +++ b/cpp/tests/io/experimental/variant_extract_test.cpp @@ -51,14 +51,12 @@ namespace { using cudf::io::parquet::experimental::variant_basic_type; using cudf::io::parquet::experimental::variant_primitive_type; -// Compose a value-metadata header byte from a basic type and its 6-bit value_header. constexpr uint8_t make_variant_header(variant_basic_type basic, uint8_t value_header) { CUDF_EXPECTS(value_header <= 0x3F, "VARIANT value_header must fit in 6 bits"); return static_cast(static_cast(basic) | (value_header << 2)); } -// Header byte for a primitive value of the given physical type. constexpr uint8_t make_variant_primitive(variant_primitive_type type) { return make_variant_header(variant_basic_type::PRIMITIVE, static_cast(type)); @@ -78,7 +76,6 @@ constexpr uint8_t make_variant_object_header() return make_variant_header(variant_basic_type::OBJECT, 0); } -// Build a struct `column_view` over (metadata, value) without copying. inline cudf::column_view wrap_variant_view(cudf::column_view const& metadata, cudf::column_view const& value) { @@ -93,7 +90,6 @@ inline cudf::column_view wrap_variant_view(cudf::column_view const& metadata, {metadata, value}}; } -// Wrap a single-row (metadata, value) pair as a VARIANT struct column. inline cudf::test::structs_column_wrapper wrap_single_variant(std::vector const& meta, std::vector const& val) { @@ -102,7 +98,6 @@ inline cudf::test::structs_column_wrapper wrap_single_variant(std::vector cudf::test::structs_column_wrapper make_apache_variant(avf::fixture const& f) { @@ -402,7 +397,6 @@ inline std::vector enc_int32(int32_t v) static_cast((u >> 24) & 0xff)}; } -// Short-string primitive blob (single-byte header). inline std::vector enc_short_string(std::string_view s) { CUDF_EXPECTS(s.size() < 64, "short-string length must fit in 6 bits of the single-byte header"); @@ -1312,7 +1306,6 @@ struct InvalidInputShapeTest : public cudf::test::BaseFixture {}; namespace { -// A well-formed VARIANT child: a single-row list holding `bytes`. inline std::unique_ptr list_u8(std::vector const& bytes) { return cudf::test::lists_column_wrapper(bytes.begin(), bytes.end()).release(); @@ -1330,7 +1323,6 @@ inline std::unique_ptr scalar_i32() return cudf::test::fixed_width_column_wrapper{42}.release(); } -// A single-row STRUCT column adopting `children`. inline std::unique_ptr struct_of(std::vector> children) { return cudf::make_structs_column(1, std::move(children), 0, rmm::device_buffer{}); @@ -1345,7 +1337,6 @@ inline std::vector> two_children(std::unique_ptr column; @@ -1926,35 +1917,3 @@ TEST_F(ExtractVariantFieldStatusTest, MissingNestedPathStatus) expect_status_values(*status, {ST_MISSING}); EXPECT_EQ(got->null_count(), 1); } - -// All-null column → all-null status column -TEST_F(ExtractVariantFieldStatusTest, AllNullInput) -{ - // Single-row variant column with the row marked SQL null - cudf::test::lists_column_wrapper meta{{0x01, 0x01, 0x00, 0x01, 'x'}}; - cudf::test::lists_column_wrapper val{{0x14, 0x07, 0x00, 0x00, 0x00}}; - cudf::test::structs_column_wrapper all_null{{meta, val}, std::vector{false}}; - - auto stream = cudf::test::get_default_stream(); - std::unique_ptr status; - auto got = cudf::io::parquet::experimental::extract_variant_field( - all_null, "x", cudf::data_type{cudf::type_id::INT32}, &status, stream, cmr()); - - ASSERT_NE(status, nullptr); - EXPECT_EQ(status->null_count(), 1); - EXPECT_EQ(got->null_count(), 1); -} - -// Existing tests work without status_out (nullopt semantics) -TEST_F(ExtractVariantFieldStatusTest, ExistingCallsUnchanged) -{ - auto col = make_xyz_three_row_variant(); - auto stream = cudf::test::get_default_stream(); - - // No status_out — existing behavior - auto got = cudf::io::parquet::experimental::extract_variant_field( - col, "x", cudf::data_type{cudf::type_id::INT32}, nullptr, stream); - - cudf::test::fixed_width_column_wrapper expected({7, 42, 0}, {true, true, false}); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); -} From 5d22976785331a74406a333be8539573d916ac3f Mon Sep 17 00:00:00 2001 From: Abigale Kim Date: Thu, 6 Aug 2026 20:50:01 +0000 Subject: [PATCH 08/23] combing --- cpp/include/cudf/io/experimental/variant.hpp | 26 +++++--------------- 1 file changed, 6 insertions(+), 20 deletions(-) diff --git a/cpp/include/cudf/io/experimental/variant.hpp b/cpp/include/cudf/io/experimental/variant.hpp index c90df32259fe..482048fd9fa1 100644 --- a/cpp/include/cudf/io/experimental/variant.hpp +++ b/cpp/include/cudf/io/experimental/variant.hpp @@ -51,17 +51,10 @@ namespace io::parquet::experimental { * @param path JSONPath-like path string identifying the target field * @param stream CUDA stream * @param mr Device memory resource - * @param status_out If non-null, receives a `UINT8` column of `variant_operation_status` values - * (one per row, aligned with the returned value column). SQL-null input rows produce a null - * status entry. All other rows receive a valid status: `success`, `missing_path`, - * `variant_null`, or `malformed_variant`. Missing-path and malformed rows produce a SQL-null - * output regardless; `success` and `variant_null` rows produce non-null output with the - * resolved bytes. When `nullptr` (the default), no status column is produced. + * @param status_out Status column with variant_operation_status` values * @return `list` column with the extracted value's encoded bytes. A row is null when the - * input row is null, a name is absent, an index is out of bounds, a step does not match - * the current value, or bytes are malformed. Encoded VARIANT-null terminal values are - * always returned as the raw VARIANT-null bytes (non-null output); callers can detect them - * via the status column or by inspecting the returned bytes. + * input row is null, a name is absent, an index is out of bounds, or a step does not match + * the current value. * * @throws std::invalid_argument on empty path or malformed syntax (`[*]` wildcards, negative * indices, out-of-range indices, and quoted names inside `[...]` are not supported) @@ -84,13 +77,8 @@ namespace io::parquet::experimental { * `FLOAT32`/`FLOAT64`, or `BOOL8`) * @param stream CUDA stream * @param mr Device memory resource - * @param incoming_status Optional status column from a prior `get_variant_field` call. When - * non-null, rows whose status is null remain null in both the output and the new status - * column, and rows with a non-`success` status are propagated unchanged (output is null, - * status is preserved). Only `success` rows are decoded. - * @param status_out If non-null, receives a `UINT8` column of `variant_operation_status` values - * aligned with the output column. SQL-null input rows (or rows whose incoming status is - * null) produce null status entries. + * @param incoming_status Incoming status column from get_variant_field call + * @param status_out Status column with variant_operation_status` values * @return Typed column decoded from the VARIANT value blobs * * @throws std::invalid_argument if `values` is not a `list` column, or if `desired_type` @@ -117,9 +105,7 @@ namespace io::parquet::experimental { * `FLOAT32`/`FLOAT64`, or `BOOL8` * @param stream CUDA stream * @param mr Device memory resource - * @param status_out If non-null, receives a `UINT8` column of `variant_operation_status` values - * aligned with the output column, combining extraction and decode outcomes. SQL-null input - * rows produce null status entries. + * @param status_out Status column with variant_operation_status` values * @return Column of `desired_type` * * @throws std::invalid_argument on empty path or malformed syntax From 2345cd41c86779f1cbb329d224ae5d4a387c980c Mon Sep 17 00:00:00 2001 From: Abigale Kim Date: Thu, 6 Aug 2026 21:42:44 +0000 Subject: [PATCH 09/23] comments --- .../cudf/io/experimental/variant_spec.hpp | 20 ++++++++----------- .../parquet/experimental/variant_extract.cu | 17 +++++++++------- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/cpp/include/cudf/io/experimental/variant_spec.hpp b/cpp/include/cudf/io/experimental/variant_spec.hpp index 263e4da53d3b..a5bc4052a79a 100644 --- a/cpp/include/cudf/io/experimental/variant_spec.hpp +++ b/cpp/include/cudf/io/experimental/variant_spec.hpp @@ -47,20 +47,16 @@ enum class variant_primitive_type : uint8_t { }; /** - * @brief Per-row outcome of a VARIANT extraction or conversion operation. - * - * A SQL-null input row produces a null status (the status column entry is null). - * Every other row receives one of these valid status values. + * @brief VARIANT status types. */ enum class variant_operation_status : uint8_t { - success, ///< The requested output was produced. - missing_path, ///< Path resolution failed: key absent, index out of range, - ///< or a non-container/null value before the final step. - variant_null, ///< The resolved value is an encoded VARIANT null. - type_mismatch, ///< Source type is not accepted for the requested operation. - malformed_variant, ///< Bytes needed by the requested operation are invalid or truncated. - overflow, ///< Conversion is outside the target range or precision. - invalid_conversion, ///< Conversion failed for another value-dependent reason. + success, + missing_path, + variant_null, + type_mismatch, + malformed_variant, + overflow, + invalid_conversion, }; } // namespace cudf::io::parquet::experimental diff --git a/cpp/src/io/parquet/experimental/variant_extract.cu b/cpp/src/io/parquet/experimental/variant_extract.cu index 74b3c3e5ba0b..6bc18529e98a 100644 --- a/cpp/src/io/parquet/experimental/variant_extract.cu +++ b/cpp/src/io/parquet/experimental/variant_extract.cu @@ -53,12 +53,16 @@ namespace { constexpr int variant_version_v1 = 1; +// Bytes consumed by the leading metadata byte common to every Variant value. constexpr size_type variant_header_bytes = 1; +// Low 2 bits of a value's metadata byte: the basic type. using basic_type = variant_basic_type; +// For a primitive value, the value_header is the physical type id of the payload. using primitive_type = variant_primitive_type; +// The status of a VARIANT operation. using op_status = variant_operation_status; __device__ cuda::std::optional read_uint64(device_span data, @@ -603,15 +607,14 @@ metadata_and_value_at(cudf::lists_column_device_view const& metadata, constexpr int block_size = 256; /** - * @brief Resolves `path` in each VARIANT row and record the located field's size and source offset. + * @brief Per-row kernel: decode each VARIANT value blob into a fixed-width primitive of type `T`. * - * For each non-null row, walks `path` to the target value and writes its byte length to - * `d_sizes[row]` and its offset within the row's value blob to `d_src_offsets[row]`. Rows that are - * null, or whose path does not resolve, are marked null in `d_null_mask` with a size of 0. + * Writes the decoded value to `d_output[row]` for non-null rows whose blob is a variant primitive + * whose physical type id matches `T` exactly (e.g. an int16 value does not decode into an int32 + * output, and a float32 value does not decode into a float64 output; there is no widening). Rows + * that are null, or whose value is not an exact-width match for `T`, are marked null in + * `d_null_mask` with an output of 0. */ -// `HasStatus=false`: existing value-only behavior, no status output. -// `HasStatus=true`: also fills `d_status`/`d_status_null_mask`; SQL-null rows get null status; -// VARIANT-null terminal values are preserved in the output with `variant_null` status. template CUDF_KERNEL __launch_bounds__(block_size) void locate_variant_fields_kernel( cudf::lists_column_device_view metadata, From 2935ba6b318432d36a9f1403d6038c3771afa797 Mon Sep 17 00:00:00 2001 From: Abigale Kim Date: Thu, 6 Aug 2026 21:45:09 +0000 Subject: [PATCH 10/23] stuff --- cpp/src/io/parquet/experimental/variant_extract.cu | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/cpp/src/io/parquet/experimental/variant_extract.cu b/cpp/src/io/parquet/experimental/variant_extract.cu index 6bc18529e98a..ce29cb1128fe 100644 --- a/cpp/src/io/parquet/experimental/variant_extract.cu +++ b/cpp/src/io/parquet/experimental/variant_extract.cu @@ -607,13 +607,11 @@ metadata_and_value_at(cudf::lists_column_device_view const& metadata, constexpr int block_size = 256; /** - * @brief Per-row kernel: decode each VARIANT value blob into a fixed-width primitive of type `T`. + * @brief Resolves `path` in each VARIANT row and record the located field's size and source offset. * - * Writes the decoded value to `d_output[row]` for non-null rows whose blob is a variant primitive - * whose physical type id matches `T` exactly (e.g. an int16 value does not decode into an int32 - * output, and a float32 value does not decode into a float64 output; there is no widening). Rows - * that are null, or whose value is not an exact-width match for `T`, are marked null in - * `d_null_mask` with an output of 0. + * For each non-null row, walks `path` to the target value and writes its byte length to + * `d_sizes[row]` and its offset within the row's value blob to `d_src_offsets[row]`. Rows that are + * null, or whose path does not resolve, are marked null in `d_null_mask` with a size of 0. */ template CUDF_KERNEL __launch_bounds__(block_size) void locate_variant_fields_kernel( From f28e1267ca067df3423e976d7c416011205075b1 Mon Sep 17 00:00:00 2001 From: Abigale Kim Date: Thu, 6 Aug 2026 22:06:58 +0000 Subject: [PATCH 11/23] extract combing --- .../parquet/experimental/variant_extract.cu | 41 ++++++++++++------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/cpp/src/io/parquet/experimental/variant_extract.cu b/cpp/src/io/parquet/experimental/variant_extract.cu index ce29cb1128fe..5136acd8fdd4 100644 --- a/cpp/src/io/parquet/experimental/variant_extract.cu +++ b/cpp/src/io/parquet/experimental/variant_extract.cu @@ -228,8 +228,7 @@ __device__ cuda::std::optional variant_value_length(device_span, op_status> find_key_in_metadata( device_span meta, cudf::string_view key) @@ -295,8 +294,8 @@ __device__ cuda::std::pair, op_status> find_key_i * * @param val The object value bytes * @param id The dictionary index of the field to locate - * @return `(span, success)` when found; `(empty, missing_path)` when the value is not an object - * or the field is absent; `(empty, malformed_variant)` when the blob is malformed + * @return The encoded bytes of the field value, or an empty span if `val` is not an object, the + * field is absent, or the blob is malformed */ __device__ cuda::std::pair, op_status> locate_object_field( device_span val, int id) @@ -365,8 +364,6 @@ __device__ cuda::std::pair, op_status> locate_object_ // // Array element offsets are monotonically increasing, so the element length is taken directly from // the offset delta (o1 - o0) rather than from the element's own header. -// Returns `(span, success)` on success, `(empty, missing_path)` for out-of-bounds or non-array, -// and `(empty, malformed_variant)` for truncated data. __device__ cuda::std::pair, op_status> locate_array_element( device_span value, size_type index) { @@ -391,7 +388,10 @@ __device__ cuda::std::pair, op_status> locate_array_e position += num_elements_size; size_type const offsets_start = position; - auto const offsets_bytes = (static_cast(num_elements) + 1) * offset_size; + + // Computed in 64-bit because (num_elements + 1) * offset_size can exceed the signed `size_type` + // range (which would be UB); the check below then rejects any array that overruns the value blob. + auto const offsets_bytes = (static_cast(num_elements) + 1) * offset_size; if (cuda::std::cmp_greater(offsets_bytes, value_size - offsets_start)) { return {{}, op_status::malformed_variant}; } @@ -525,9 +525,13 @@ __device__ cuda::std::optional parse_index_step(cudf::string_view ste return index; } -// Walk a path of object-key or array-index steps. Returns `(span, status)`. -// On success the span is non-empty and status is `success` or `variant_null` (terminal null). -// On failure the span is empty and status is `missing_path` or `malformed_variant`. +// Walk a path of object-key or array-index steps level by level starting at `val` and return +// the span of the final value (subspan of `val`). Returns an empty span on failure. +// +// Each path step is encoded in the `path` strings column as either: +// - "" -> descend into an object by dictionary key, or +// - "[]" -> descend into an array by zero-based integer index. +// The step kind is inferred from the first byte (`'['` means index). __device__ cuda::std::pair, op_status> resolve_path( device_span meta, device_span val, column_device_view path) { @@ -652,7 +656,10 @@ CUDF_KERNEL __launch_bounds__(block_size) void locate_variant_fields_kernel( } } -// Status helper for fixed-width primitive targets: returns the failure reason when decode fails. +/** + * @brief Status helper for fixed-width primitive targets: returns the failure reason when decode + * fails. + */ template requires(is_variant_numerical) __device__ op_status cast_status_for_primitive(device_span val) @@ -678,9 +685,15 @@ __device__ op_status cast_status_for_primitive(device_span val) } } -// `HasStatus=false`: decode-only; SQL-null rows produce null output, other failure types produce -// null. `HasStatus=true`: also fills `d_status`/`d_status_null_mask` and honours `has_incoming`. -// `has_incoming`: when true, `incoming_status` gates decoding instead of the null mask. +/** + * @brief Per-row kernel: decode each VARIANT value blob into a fixed-width primitive of type `T`. + * + * Writes the decoded value to `d_output[row]` for non-null rows whose blob is a variant primitive + * whose physical type id matches `T` exactly (e.g. an int16 value does not decode into an int32 + * output, and a float32 value does not decode into a float64 output; there is no widening). Rows + * that are null, or whose value is not an exact-width match for `T`, are marked null in + * `d_null_mask` with an output of 0. + */ template CUDF_KERNEL __launch_bounds__(block_size) void cast_variant_primitive_kernel( cudf::lists_column_device_view values, From d901a02e38666acf61fdee7146c1c5ae400c9dbf Mon Sep 17 00:00:00 2001 From: Abigale Kim Date: Fri, 7 Aug 2026 23:41:09 +0000 Subject: [PATCH 12/23] changes --- cpp/include/cudf/io/experimental/variant.hpp | 12 +- .../cudf/io/experimental/variant_spec.hpp | 1 + .../parquet/experimental/variant_extract.cu | 260 +++++++++--------- .../io/experimental/variant_extract_test.cpp | 123 ++++----- java/src/main/native/src/VariantUtilsJni.cpp | 6 + 5 files changed, 194 insertions(+), 208 deletions(-) diff --git a/cpp/include/cudf/io/experimental/variant.hpp b/cpp/include/cudf/io/experimental/variant.hpp index 482048fd9fa1..08e75dfd1009 100644 --- a/cpp/include/cudf/io/experimental/variant.hpp +++ b/cpp/include/cudf/io/experimental/variant.hpp @@ -15,6 +15,7 @@ #include #include +#include #include /** @@ -77,7 +78,8 @@ namespace io::parquet::experimental { * `FLOAT32`/`FLOAT64`, or `BOOL8`) * @param stream CUDA stream * @param mr Device memory resource - * @param incoming_status Incoming status column from get_variant_field call + * @param incoming_status Optional status column from a prior `get_variant_field` call. When + * provided, non-success rows are propagated directly to the output without decoding. * @param status_out Status column with variant_operation_status` values * @return Typed column decoded from the VARIANT value blobs * @@ -88,10 +90,10 @@ namespace io::parquet::experimental { [[nodiscard]] std::unique_ptr cast_variant( column_view const& values, data_type desired_type, - column_view const* incoming_status = nullptr, - std::unique_ptr* status_out = nullptr, - rmm::cuda_stream_view stream = cudf::get_default_stream(), - rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()); + std::optional incoming_status = std::nullopt, + std::unique_ptr* status_out = nullptr, + rmm::cuda_stream_view stream = cudf::get_default_stream(), + rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()); /** * @brief Convenience wrapper: extract a nested object value by path and decode into a typed column. diff --git a/cpp/include/cudf/io/experimental/variant_spec.hpp b/cpp/include/cudf/io/experimental/variant_spec.hpp index a5bc4052a79a..2f37c17887a6 100644 --- a/cpp/include/cudf/io/experimental/variant_spec.hpp +++ b/cpp/include/cudf/io/experimental/variant_spec.hpp @@ -51,6 +51,7 @@ enum class variant_primitive_type : uint8_t { */ enum class variant_operation_status : uint8_t { success, + row_null, ///< the SQL row itself was null (no VARIANT data to decode) missing_path, variant_null, type_mismatch, diff --git a/cpp/src/io/parquet/experimental/variant_extract.cu b/cpp/src/io/parquet/experimental/variant_extract.cu index 5136acd8fdd4..5604c3fb9a4d 100644 --- a/cpp/src/io/parquet/experimental/variant_extract.cu +++ b/cpp/src/io/parquet/experimental/variant_extract.cu @@ -44,6 +44,7 @@ #include #include +#include #include #include @@ -326,6 +327,16 @@ __device__ cuda::std::pair, op_status> locate_object_ auto const values_base = offsets_start + static_cast(offsets_bytes); auto const values_extent = val_len - values_base; + // Read the sentinel (terminal offset at index num_fields) to get the authoritative end of the + // values region. Using the physical remainder (values_extent) would allow a malformed object + // to reference bytes beyond the sentinel, passing validation despite corrupt data. + auto const sentinel_raw = + read_uint64(val, offsets_start + num_fields.value() * offset_size, offset_size); + if (!sentinel_raw.has_value() || sentinel_raw.value() > static_cast(values_extent)) { + return {{}, op_status::malformed_variant}; + } + auto const values_region = static_cast(sentinel_raw.value()); + bool found = false; uint64_t match_start = 0; for (size_type i = 0; i < num_fields.value(); ++i) { @@ -335,7 +346,9 @@ __device__ cuda::std::pair, op_status> locate_object_ auto const match_offset = read_uint64(val, offsets_start + i * offset_size, offset_size); if (!match_offset.has_value()) { return {{}, op_status::malformed_variant}; } - if (match_offset.value() > values_extent) { return {{}, op_status::malformed_variant}; } + if (match_offset.value() > static_cast(values_region)) { + return {{}, op_status::malformed_variant}; + } match_start = match_offset.value(); found = true; break; @@ -346,7 +359,9 @@ __device__ cuda::std::pair, op_status> locate_object_ auto const value_len = variant_value_length(value); if (!value_len.has_value()) { return {{}, op_status::malformed_variant}; } auto const match_end = match_start + value_len.value(); - if (match_end > values_extent) { return {{}, op_status::malformed_variant}; } + if (match_end > static_cast(values_region)) { + return {{}, op_status::malformed_variant}; + } return {val.subspan(values_base + match_start, value_len.value()), op_status::success}; } @@ -625,8 +640,7 @@ CUDF_KERNEL __launch_bounds__(block_size) void locate_variant_fields_kernel( device_span d_sizes, device_span d_src_offsets, bitmask_type* d_null_mask, - device_span d_status, // only read/written when HasStatus - bitmask_type* d_status_null_mask) // only read/written when HasStatus + device_span d_status) // only read/written when HasStatus { auto const num_rows = static_cast(d_sizes.size()); auto const tid = cudf::detail::grid_1d::global_thread_id(); @@ -636,7 +650,7 @@ CUDF_KERNEL __launch_bounds__(block_size) void locate_variant_fields_kernel( if (!cudf::bit_is_set(d_null_mask, row)) { d_sizes[row] = 0; d_src_offsets[row] = 0; - if constexpr (HasStatus) { cudf::clear_bit(d_status_null_mask, row); } + if constexpr (HasStatus) { d_status[row] = op_status::row_null; } continue; } @@ -656,9 +670,45 @@ CUDF_KERNEL __launch_bounds__(block_size) void locate_variant_fields_kernel( } } +// Returns true for every primitive_type ID that variant_value_length recognises (i.e. every ID +// that does not fall through to its `default: return nullopt` branch). Used by both +// cast_status_for_primitive and cast_status_for_bool so the two classifiers stay in sync. +__device__ bool is_recognized_primitive_type(primitive_type ptype) +{ + switch (ptype) { + case primitive_type::NULLVAL: + case primitive_type::BOOLEAN_TRUE: + case primitive_type::BOOLEAN_FALSE: + case primitive_type::INT8: + case primitive_type::INT16: + case primitive_type::INT32: + case primitive_type::INT64: + case primitive_type::FLOAT64: + case primitive_type::DECIMAL4: + case primitive_type::DECIMAL8: + case primitive_type::DECIMAL16: + case primitive_type::DATE: + case primitive_type::TIMESTAMP_MICROS: + case primitive_type::TIMESTAMP_NTZ_MICROS: + case primitive_type::FLOAT32: + case primitive_type::BINARY: + case primitive_type::LONG_STRING: + case primitive_type::TIME_NTZ_MICROS: + case primitive_type::TIMESTAMP_NANOS: + case primitive_type::TIMESTAMP_NTZ_NANOS: + case primitive_type::UUID: return true; + default: return false; + } +} + /** * @brief Status helper for fixed-width primitive targets: returns the failure reason when decode * fails. + * + * Three cases after `decode_primitive` returns nullopt and the basic type is PRIMITIVE: + * - Header type == primitive_type_for() but payload too short → truncated → malformed_variant + * - Header type is a different recognized type → type_mismatch + * - Header type is unrecognized → malformed_variant */ template requires(is_variant_numerical) @@ -668,21 +718,10 @@ __device__ op_status cast_status_for_primitive(device_span val) if (is_variant_null(val)) { return op_status::variant_null; } if (decode_primitive(val).has_value()) { return op_status::success; } if (decode_basic_type(val[0]) != basic_type::PRIMITIVE) { return op_status::type_mismatch; } - auto const vhdr = variant_value_header(val[0]); - switch (static_cast(vhdr)) { - case primitive_type::NULLVAL: - case primitive_type::BOOLEAN_TRUE: - case primitive_type::BOOLEAN_FALSE: - case primitive_type::INT8: - case primitive_type::INT16: - case primitive_type::INT32: - case primitive_type::INT64: - case primitive_type::FLOAT32: - case primitive_type::FLOAT64: - case primitive_type::LONG_STRING: - case primitive_type::BINARY: return op_status::type_mismatch; - default: return op_status::malformed_variant; - } + auto const ptype = static_cast(variant_value_header(val[0])); + if (ptype == primitive_type_for()) { return op_status::malformed_variant; } + return is_recognized_primitive_type(ptype) ? op_status::type_mismatch + : op_status::malformed_variant; } /** @@ -694,6 +733,9 @@ __device__ op_status cast_status_for_primitive(device_span val) * that are null, or whose value is not an exact-width match for `T`, are marked null in * `d_null_mask` with an output of 0. */ +// `d_status` may be an empty span when HasStatus=true but no status_out was requested (only +// incoming_status propagation to the null mask is desired). All writes to d_status are guarded +// by d_status.data() so the empty-span case is safe. template CUDF_KERNEL __launch_bounds__(block_size) void cast_variant_primitive_kernel( cudf::lists_column_device_view values, @@ -701,8 +743,7 @@ CUDF_KERNEL __launch_bounds__(block_size) void cast_variant_primitive_kernel( bitmask_type* d_null_mask, column_device_view incoming_status, // only used when HasStatus bool has_incoming, // only meaningful when HasStatus - device_span d_status, // only used when HasStatus - bitmask_type* d_status_null_mask) // only used when HasStatus + device_span d_status) // only used when HasStatus; may be empty { auto const num_rows = static_cast(d_output.size()); auto const tid = cudf::detail::grid_1d::global_thread_id(); @@ -711,25 +752,19 @@ CUDF_KERNEL __launch_bounds__(block_size) void cast_variant_primitive_kernel( for (auto row = tid; row < num_rows; row += stride) { if constexpr (HasStatus) { if (has_incoming) { - // Incoming status is the sole authority; the value null mask may already be pre-cleared. - if (incoming_status.is_null(row)) { - d_output[row] = T{}; - if (cudf::bit_is_set(d_null_mask, row)) { cudf::clear_bit(d_null_mask, row); } - cudf::clear_bit(d_status_null_mask, row); - continue; - } + // Status column is always non-nullable; row_null replaces the null bit. auto const s = static_cast(incoming_status.element(row)); if (s != op_status::success) { d_output[row] = T{}; if (cudf::bit_is_set(d_null_mask, row)) { cudf::clear_bit(d_null_mask, row); } - d_status[row] = s; + if (d_status.data()) { d_status[row] = s; } continue; } // incoming success → fall through to decode (value null bit is set) } else { if (!cudf::bit_is_set(d_null_mask, row)) { d_output[row] = T{}; - cudf::clear_bit(d_status_null_mask, row); + if (d_status.data()) { d_status[row] = op_status::row_null; } continue; } } @@ -744,11 +779,15 @@ CUDF_KERNEL __launch_bounds__(block_size) void cast_variant_primitive_kernel( auto const decoded = decode_primitive(val); if (decoded.has_value()) { d_output[row] = *decoded; - if constexpr (HasStatus) { d_status[row] = op_status::success; } + if constexpr (HasStatus) { + if (d_status.data()) { d_status[row] = op_status::success; } + } } else { d_output[row] = T{}; cudf::clear_bit(d_null_mask, row); - if constexpr (HasStatus) { d_status[row] = cast_status_for_primitive(val); } + if constexpr (HasStatus) { + if (d_status.data()) { d_status[row] = cast_status_for_primitive(val); } + } } } } @@ -759,17 +798,12 @@ __device__ op_status cast_status_for_bool(device_span val) if (is_variant_null(val)) { return op_status::variant_null; } if (decode_bool(val).has_value()) { return op_status::success; } if (decode_basic_type(val[0]) != basic_type::PRIMITIVE) { return op_status::type_mismatch; } - auto const vhdr = variant_value_header(val[0]); - switch (static_cast(vhdr)) { - case primitive_type::INT8: - case primitive_type::INT16: - case primitive_type::INT32: - case primitive_type::INT64: - case primitive_type::FLOAT32: - case primitive_type::FLOAT64: - case primitive_type::LONG_STRING: return op_status::type_mismatch; - default: return op_status::malformed_variant; - } + // Boolean values carry no payload, so a BOOLEAN_TRUE/FALSE header can never be truncated; + // decode_bool would have succeeded above. Any remaining primitive ID is a type mismatch when + // recognised, or malformed when not. + auto const ptype = static_cast(variant_value_header(val[0])); + return is_recognized_primitive_type(ptype) ? op_status::type_mismatch + : op_status::malformed_variant; } __device__ op_status cast_status_for_string(device_span val) @@ -799,35 +833,24 @@ struct cast_variant_string_fn { size_type* d_sizes; char* d_chars; cudf::detail::input_offsetalator d_offsets; - // Status tracking (optional: both must be non-null to enable) + // Status tracking (optional: d_status non-null to enable; status is always non-nullable) op_status* d_status{nullptr}; - bitmask_type* d_status_null_mask{nullptr}; column_device_view incoming_status; bool has_incoming{false}; __device__ void operator()(size_type row) { - // Status and the status null mask are only written on the sizing pass (d_chars == nullptr). - // On the writing pass the null mask may already be cleared from the sizing pass, so we must - // not re-inspect it to write status (that would misidentify a decode-failed row as SQL null). + // Status is only written on the sizing pass (d_chars == nullptr). On the writing pass the + // null mask may already be cleared from the sizing pass, so we must not re-inspect it to + // write status (that would misidentify a decode-failed row as a SQL-null row). bool const sizing = (d_chars == nullptr); if (has_incoming) { - // Incoming status takes precedence over the value-column null mask. - if (incoming_status.is_null(row)) { - if (sizing) { d_sizes[row] = 0; } - if (cudf::bit_is_set(d_null_mask, row)) { cudf::clear_bit(d_null_mask, row); } - if (sizing && d_status_null_mask) { cudf::clear_bit(d_status_null_mask, row); } - return; - } + // Status column is always non-nullable; row_null replaces the null bit. auto const s = static_cast(incoming_status.element(row)); if (s != op_status::success) { if (sizing) { d_sizes[row] = 0; } - if (!cudf::bit_is_set(d_null_mask, row)) { - // already null from get_variant_field - } else { - cudf::clear_bit(d_null_mask, row); - } + if (cudf::bit_is_set(d_null_mask, row)) { cudf::clear_bit(d_null_mask, row); } if (sizing && d_status) { d_status[row] = s; } return; } @@ -835,7 +858,7 @@ struct cast_variant_string_fn { } else { if (!cudf::bit_is_set(d_null_mask, row)) { if (sizing) { d_sizes[row] = 0; } - if (sizing && d_status) { cudf::clear_bit(d_status_null_mask, row); } + if (sizing && d_status) { d_status[row] = op_status::row_null; } return; } } @@ -869,20 +892,12 @@ void validate_variant_child(column_view const& child) std::invalid_argument); } -std::unique_ptr make_status_column(rmm::device_buffer status_data, - rmm::device_buffer status_null_mask, - size_type num_rows, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) +// Status columns are always non-nullable: SQL-null rows are represented by the row_null enum +// value rather than by a null bit, so no null mask needs to be allocated or counted. +std::unique_ptr make_status_column(rmm::device_buffer status_data, size_type num_rows) { - auto const null_count = cudf::detail::count_unset_bits( - static_cast(status_null_mask.data()), 0, num_rows, stream); return std::make_unique( - data_type{type_id::UINT8}, - num_rows, - std::move(status_data), - null_count > 0 ? std::move(status_null_mask) : rmm::device_buffer{}, - null_count); + data_type{type_id::UINT8}, num_rows, std::move(status_data), rmm::device_buffer{}, 0); } struct cast_variant_fn { @@ -898,11 +913,10 @@ struct cast_variant_fn { bool has_incoming{false}; std::unique_ptr* status_out{nullptr}; - // Avoids repeating this boilerplate in each cast operator. - auto alloc_status() -> cuda::std::pair + // Allocates an uninitialized device buffer large enough for num_rows status bytes. + rmm::device_buffer alloc_status() { - return {rmm::device_buffer{static_cast(num_rows) * sizeof(op_status), stream, mr}, - cudf::create_null_mask(num_rows, mask_state::ALL_VALID, stream, mr)}; + return rmm::device_buffer{static_cast(num_rows) * sizeof(op_status), stream, mr}; } template @@ -914,20 +928,25 @@ struct cast_variant_fn { auto const d_out = device_span{static_cast(data.data()), static_cast(num_rows)}; if (status_out != nullptr) { - auto [s_data, s_mask] = alloc_status(); + auto s_data = alloc_status(); cast_variant_primitive_kernel<<>>( values, d_out, d_null_mask, incoming_status_view, has_incoming, - {static_cast(s_data.data()), static_cast(num_rows)}, - static_cast(s_mask.data())); + {static_cast(s_data.data()), static_cast(num_rows)}); + CUDF_CUDA_TRY(cudaGetLastError()); + *status_out = make_status_column(std::move(s_data), num_rows); + } else if (has_incoming) { + // No status_out requested, but incoming_status still needs to be applied to the null mask. + // Use HasStatus=true with an empty d_status span so writes are guarded (no allocation). + cast_variant_primitive_kernel<<>>( + values, d_out, d_null_mask, incoming_status_view, true, {}); CUDF_CUDA_TRY(cudaGetLastError()); - *status_out = make_status_column(std::move(s_data), std::move(s_mask), num_rows, stream, mr); } else { cast_variant_primitive_kernel<<>>( - values, d_out, d_null_mask, incoming_status_view, false, {}, nullptr); + values, d_out, d_null_mask, incoming_status_view, false, {}); CUDF_CUDA_TRY(cudaGetLastError()); } @@ -946,15 +965,11 @@ struct cast_variant_fn { { rmm::device_buffer data{num_rows * sizeof(bool), stream, mr}; - rmm::device_buffer s_data, s_mask; + rmm::device_buffer s_data; op_status* dp_s{nullptr}; - bitmask_type* dp_sm{nullptr}; if (status_out != nullptr) { - auto [sd, sm] = alloc_status(); - s_data = std::move(sd); - s_mask = std::move(sm); - dp_s = static_cast(s_data.data()); - dp_sm = static_cast(s_mask.data()); + s_data = alloc_status(); + dp_s = static_cast(s_data.data()); } auto const inc_view = incoming_status_view; @@ -966,17 +981,10 @@ struct cast_variant_fn { d_out = static_cast(data.data()), dnm = this->d_null_mask, dp_s, - dp_sm, inc_view, hi] __device__(size_type row) { - // Gate on incoming status when present, otherwise on the null mask. + // Status column is always non-nullable; row_null replaces the null bit. if (hi) { - if (inc_view.is_null(row)) { - d_out[row] = false; - if (cudf::bit_is_set(dnm, row)) { cudf::clear_bit(dnm, row); } - if (dp_sm) { cudf::clear_bit(dp_sm, row); } - return; - } auto const s = static_cast(inc_view.element(row)); if (s != op_status::success) { d_out[row] = false; @@ -987,7 +995,7 @@ struct cast_variant_fn { } else { if (!cudf::bit_is_set(dnm, row)) { d_out[row] = false; - if (dp_sm) { cudf::clear_bit(dp_sm, row); } + if (dp_s) { dp_s[row] = op_status::row_null; } return; } } @@ -1003,9 +1011,7 @@ struct cast_variant_fn { } }); - if (status_out != nullptr) { - *status_out = make_status_column(std::move(s_data), std::move(s_mask), num_rows, stream, mr); - } + if (status_out != nullptr) { *status_out = make_status_column(std::move(s_data), num_rows); } auto const null_count = num_rows - cudf::detail::count_set_bits(d_null_mask, 0, num_rows, stream); @@ -1021,32 +1027,20 @@ struct cast_variant_fn { requires(cuda::std::is_same_v) { rmm::device_buffer status_data; - rmm::device_buffer status_null_mask; op_status* d_status_ptr{nullptr}; - bitmask_type* d_status_null_mask_ptr{nullptr}; if (status_out != nullptr) { - status_data = rmm::device_buffer{num_rows * sizeof(op_status), stream, mr}; - status_null_mask = cudf::create_null_mask(num_rows, mask_state::ALL_VALID, stream, mr); - d_status_ptr = static_cast(status_data.data()); - d_status_null_mask_ptr = static_cast(status_null_mask.data()); + status_data = alloc_status(); + d_status_ptr = static_cast(status_data.data()); } - cast_variant_string_fn fn{values, - d_null_mask, - nullptr, - nullptr, - {}, - d_status_ptr, - d_status_null_mask_ptr, - incoming_status_view, - has_incoming}; + cast_variant_string_fn fn{ + values, d_null_mask, nullptr, nullptr, {}, d_status_ptr, incoming_status_view, has_incoming}; auto [offsets_column, chars] = cudf::strings::detail::make_strings_children(fn, num_rows, stream, mr); if (status_out != nullptr) { - *status_out = make_status_column( - std::move(status_data), std::move(status_null_mask), num_rows, stream, mr); + *status_out = make_status_column(std::move(status_data), num_rows); } auto const null_count = @@ -1148,12 +1142,8 @@ std::unique_ptr get_variant_field(column_view const& variant_column, auto grid = cudf::detail::grid_1d{num_rows, block_size}; - auto const null_spans = - device_span{}; // placeholder for no-status kernel instantiation - if (status_out != nullptr) { rmm::device_buffer status_data{num_rows * sizeof(op_status), stream, mr}; - auto status_null_mask = cudf::create_null_mask(num_rows, mask_state::ALL_VALID, stream, mr); locate_variant_fields_kernel<<>>( meta_lists_device_view, val_lists_device_view, @@ -1161,11 +1151,9 @@ std::unique_ptr get_variant_field(column_view const& variant_column, d_sizes, d_src_offsets, d_null_mask, - {static_cast(status_data.data()), static_cast(num_rows)}, - static_cast(status_null_mask.data())); + {static_cast(status_data.data()), static_cast(num_rows)}); CUDF_CUDA_TRY(cudaGetLastError()); - *status_out = - make_status_column(std::move(status_data), std::move(status_null_mask), num_rows, stream, mr); + *status_out = make_status_column(std::move(status_data), num_rows); } else { locate_variant_fields_kernel <<>>(meta_lists_device_view, @@ -1174,8 +1162,7 @@ std::unique_ptr get_variant_field(column_view const& variant_column, d_sizes, d_src_offsets, d_null_mask, - null_spans, - nullptr); + {}); CUDF_CUDA_TRY(cudaGetLastError()); } @@ -1217,7 +1204,7 @@ std::unique_ptr get_variant_field(column_view const& variant_column, std::unique_ptr cast_variant(column_view const& values, data_type desired_type, - column_view const* incoming_status, + std::optional incoming_status, std::unique_ptr* status_out, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) @@ -1254,7 +1241,7 @@ std::unique_ptr cast_variant(column_view const& values, // cast_variant_fn always holds a valid column_device_view (kernel ignores it when !has_incoming). auto placeholder_col = make_empty_column(data_type{type_id::UINT8}); auto placeholder_dev_view = column_device_view::create(*placeholder_col, stream); - if (incoming_status != nullptr) { + if (incoming_status.has_value()) { CUDF_EXPECTS(incoming_status->size() == num_rows, "incoming status column must have the same number of rows as the values column", std::invalid_argument); @@ -1263,10 +1250,10 @@ std::unique_ptr cast_variant(column_view const& values, std::invalid_argument); } - auto incoming_dev_view = (incoming_status != nullptr) + auto incoming_dev_view = incoming_status.has_value() ? column_device_view::create(*incoming_status, stream) : column_device_view::create(*placeholder_col, stream); - bool const has_incoming = (incoming_status != nullptr); + bool const has_incoming = incoming_status.has_value(); return cudf::type_dispatcher(desired_type, cast_variant_fn{val_lists_device_view, @@ -1295,7 +1282,7 @@ std::unique_ptr get_variant_field(column_view const& variant_column, std::unique_ptr cast_variant(column_view const& values, data_type desired_type, - column_view const* incoming_status, + std::optional incoming_status, std::unique_ptr* status_out, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) @@ -1317,13 +1304,12 @@ std::unique_ptr extract_variant_field(column_view const& variant_column, if (status_out != nullptr) { std::unique_ptr extract_status; auto value = detail::get_variant_field(variant_column, path, &extract_status, stream, temp_mr); - auto const extract_status_view = extract_status->view(); return detail::cast_variant( - value->view(), desired_type, &extract_status_view, status_out, stream, mr); + value->view(), desired_type, extract_status->view(), status_out, stream, mr); } auto value = detail::get_variant_field(variant_column, path, nullptr, stream, temp_mr); - return detail::cast_variant(value->view(), desired_type, nullptr, nullptr, stream, mr); + return detail::cast_variant(value->view(), desired_type, std::nullopt, nullptr, stream, mr); } } // namespace io::parquet::experimental diff --git a/cpp/tests/io/experimental/variant_extract_test.cpp b/cpp/tests/io/experimental/variant_extract_test.cpp index bc057471c20a..a1f0b1e21c5b 100644 --- a/cpp/tests/io/experimental/variant_extract_test.cpp +++ b/cpp/tests/io/experimental/variant_extract_test.cpp @@ -890,7 +890,7 @@ TEST_F(GetVariantFieldTest, ApacheObjectPrimitive) EXPECT_EQ(cudf::lists_column_view{got->view()}.child().type().id(), cudf::type_id::UINT8); auto casted = cudf::io::parquet::experimental::cast_variant( - got->view(), cudf::data_type{cudf::type_id::INT8}, nullptr, nullptr, stream); + got->view(), cudf::data_type{cudf::type_id::INT8}, {}, nullptr, stream); cudf::test::fixed_width_column_wrapper expected{int8_t{1}}; CUDF_TEST_EXPECT_COLUMNS_EQUAL(*casted, expected); } @@ -916,7 +916,7 @@ TEST_F(GetVariantFieldTest, GetAndCastMatchesExtract) auto intermediate = cudf::io::parquet::experimental::get_variant_field(col, "x", nullptr, stream); auto two_step_x = cudf::io::parquet::experimental::cast_variant( - intermediate->view(), cudf::data_type{cudf::type_id::INT32}, nullptr, nullptr, stream); + intermediate->view(), cudf::data_type{cudf::type_id::INT32}, {}, nullptr, stream); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*extract_x, *two_step_x); } @@ -940,7 +940,7 @@ std::unique_ptr cast_apache_primitive(avf::fixture const& fi auto col = make_apache_variant(fixture); auto const value = cudf::structs_column_view{col}.get_sliced_child(1, stream); return cudf::io::parquet::experimental::cast_variant( - value, cudf::data_type{cudf::type_to_id()}, nullptr, nullptr, stream); + value, cudf::data_type{cudf::type_to_id()}, {}, nullptr, stream); } struct CastVariantTest : public cudf::test::BaseFixture {}; @@ -977,7 +977,7 @@ TEST_F(CastVariantTest, ApachePrimitiveFloats) auto col = make_apache_variant(fixture); auto const value = cudf::structs_column_view{col}.get_sliced_child(1, stream); auto got = cudf::io::parquet::experimental::cast_variant( - value, cudf::data_type{cudf::type_to_id()}, nullptr, nullptr, stream); + value, cudf::data_type{cudf::type_to_id()}, {}, nullptr, stream); cudf::test::fixed_width_column_wrapper expected{expected_val}; CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); }; @@ -993,7 +993,7 @@ TEST_F(CastVariantTest, ApachePrimitiveBooleans) auto col = make_apache_variant(fixture); auto const value = cudf::structs_column_view{col}.get_sliced_child(1, stream); auto got = cudf::io::parquet::experimental::cast_variant( - value, cudf::data_type{cudf::type_id::BOOL8}, nullptr, nullptr, stream); + value, cudf::data_type{cudf::type_id::BOOL8}, {}, nullptr, stream); cudf::test::fixed_width_column_wrapper expected{expected_val}; CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); }; @@ -1006,7 +1006,7 @@ TEST_F(CastVariantTest, ApachePrimitiveBooleans) auto col = make_apache_variant(avf::primitive_null); auto const value = cudf::structs_column_view{col}.get_sliced_child(1, stream); auto got = cudf::io::parquet::experimental::cast_variant( - value, cudf::data_type{cudf::type_id::BOOL8}, nullptr, nullptr, stream); + value, cudf::data_type{cudf::type_id::BOOL8}, {}, nullptr, stream); cudf::test::fixed_width_column_wrapper expected({false}, {false}); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); } @@ -1054,7 +1054,7 @@ TEST_F(CastVariantTest, ApachePrimitiveBooleans) auto const sliced = cudf::slice(col, {slice_beg, slice_end}).front(); auto const value = cudf::structs_column_view{sliced}.get_sliced_child(1, stream); auto got = cudf::io::parquet::experimental::cast_variant( - value, cudf::data_type{cudf::type_id::BOOL8}, nullptr, nullptr, stream); + value, cudf::data_type{cudf::type_id::BOOL8}, {}, nullptr, stream); cudf::test::fixed_width_column_wrapper expected( exp_vals.begin() + slice_beg, exp_vals.begin() + slice_end, exp_valid.begin() + slice_beg); @@ -1069,7 +1069,7 @@ TEST_F(CastVariantTest, ApacheShortString) auto const value = cudf::structs_column_view{col}.get_sliced_child(1, stream); auto got = cudf::io::parquet::experimental::cast_variant( - value, cudf::data_type{cudf::type_id::STRING}, nullptr, nullptr, stream); + value, cudf::data_type{cudf::type_id::STRING}, {}, nullptr, stream); // Decoded from short_string.value: skip the 1-byte header, take the rest. std::string const expected_str(reinterpret_cast(avf::short_string.value.data() + 1), @@ -1085,7 +1085,7 @@ TEST_F(CastVariantTest, ApachePrimitiveString) auto const value = cudf::structs_column_view{col}.get_sliced_child(1, stream); auto got = cudf::io::parquet::experimental::cast_variant( - value, cudf::data_type{cudf::type_id::STRING}, nullptr, nullptr, stream); + value, cudf::data_type{cudf::type_id::STRING}, {}, nullptr, stream); // Long-string layout: 1 header byte + 4-byte LE length + payload. std::string const expected_str( @@ -1102,7 +1102,7 @@ TEST_F(CastVariantTest, MismatchedTypeYieldsNull) auto col = make_apache_variant(avf::object_primitive); auto const value = cudf::structs_column_view{col}.get_sliced_child(1, stream); auto got = cudf::io::parquet::experimental::cast_variant( - value, cudf::data_type{cudf::type_id::INT32}, nullptr, nullptr, stream); + value, cudf::data_type{cudf::type_id::INT32}, {}, nullptr, stream); ASSERT_EQ(got->size(), 1); EXPECT_EQ(got->null_count(), 1); } @@ -1119,7 +1119,7 @@ TEST_F(CastVariantTest, EmptyInput) cudf::type_id::FLOAT64, cudf::type_id::BOOL8}) { auto got = cudf::io::parquet::experimental::cast_variant( - *values, cudf::data_type{id}, nullptr, nullptr, stream); + *values, cudf::data_type{id}, {}, nullptr, stream); EXPECT_EQ(got->type().id(), id); EXPECT_EQ(got->size(), 0); EXPECT_EQ(got->null_count(), 0); @@ -1148,7 +1148,7 @@ TEST_F(CastVariantTest, UnsupportedTypeThrows) cudf::empty_like(cudf::structs_column_view{make_xyz_three_row_variant()}.child(1)); for (auto const id : ids) { EXPECT_THROW(static_cast(cudf::io::parquet::experimental::cast_variant( - *empty_values, cudf::data_type{id}, nullptr, nullptr, stream)), + *empty_values, cudf::data_type{id}, {}, nullptr, stream)), std::invalid_argument) << std::format("expected throw for type_id {} on empty input", static_cast(id)); } @@ -1158,7 +1158,7 @@ TEST_F(CastVariantTest, UnsupportedTypeThrows) auto const value = cudf::structs_column_view{col}.get_sliced_child(1, stream); for (auto const id : ids) { EXPECT_THROW(static_cast(cudf::io::parquet::experimental::cast_variant( - value, cudf::data_type{id}, nullptr, nullptr, stream)), + value, cudf::data_type{id}, {}, nullptr, stream)), std::invalid_argument) << std::format("expected throw for type_id {} on non-empty input", static_cast(id)); } @@ -1201,8 +1201,7 @@ TEST_F(CastVariantTest, CastSourceTargetMatrix) for (auto const& src : sources) { SCOPED_TRACE(std::string{"int target "} + match_label + ", source " + src.label); auto values = values_of(src.bytes); - auto got = - cudf::io::parquet::experimental::cast_variant(values, target, nullptr, nullptr, stream); + auto got = cudf::io::parquet::experimental::cast_variant(values, target, {}, nullptr, stream); if (std::string_view{src.label} == match_label) { cudf::test::fixed_width_column_wrapper const expected{match_value}; CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); @@ -1223,7 +1222,7 @@ TEST_F(CastVariantTest, CastSourceTargetMatrix) SCOPED_TRACE(std::string{"string target, source "} + src.label); auto values = values_of(src.bytes); auto got = - cudf::io::parquet::experimental::cast_variant(values, string_type, nullptr, nullptr, stream); + cudf::io::parquet::experimental::cast_variant(values, string_type, {}, nullptr, stream); std::string_view const label{src.label}; if (label == "short_string" || label == "long_string") { std::string const expected_str = (label == "short_string") ? "hi" : std::string(70, 'a'); @@ -1243,7 +1242,7 @@ TEST_F(CastVariantTest, ShortStringLengthZero) std::vector const val{make_variant_short_string_header(0)}; cudf::test::lists_column_wrapper values(val.begin(), val.end()); auto got = cudf::io::parquet::experimental::cast_variant( - values, cudf::data_type{cudf::type_id::STRING}, nullptr, nullptr, stream); + values, cudf::data_type{cudf::type_id::STRING}, {}, nullptr, stream); cudf::test::strings_column_wrapper expected({""}); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); } @@ -1258,7 +1257,7 @@ TEST_F(CastVariantTest, ShortStringMaxLength) val.insert(val.end(), payload.begin(), payload.end()); cudf::test::lists_column_wrapper values(val.begin(), val.end()); auto got = cudf::io::parquet::experimental::cast_variant( - values, cudf::data_type{cudf::type_id::STRING}, nullptr, nullptr, stream); + values, cudf::data_type{cudf::type_id::STRING}, {}, nullptr, stream); cudf::test::strings_column_wrapper expected({payload}); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); } @@ -1271,7 +1270,7 @@ TEST_F(CastVariantTest, LongStringLengthZero) make_variant_primitive(variant_primitive_type::LONG_STRING), 0x00, 0x00, 0x00, 0x00}; cudf::test::lists_column_wrapper values(val.begin(), val.end()); auto got = cudf::io::parquet::experimental::cast_variant( - values, cudf::data_type{cudf::type_id::STRING}, nullptr, nullptr, stream); + values, cudf::data_type{cudf::type_id::STRING}, {}, nullptr, stream); cudf::test::strings_column_wrapper expected({""}); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); } @@ -1291,7 +1290,7 @@ TEST_F(CastVariantTest, LongStringDeclaredLengthExceedsPayloadYieldsNull) SCOPED_TRACE(std::string{"payload bytes present: "} + std::to_string(val.size() - 5)); cudf::test::lists_column_wrapper values(val.begin(), val.end()); auto got = cudf::io::parquet::experimental::cast_variant( - values, cudf::data_type{cudf::type_id::STRING}, nullptr, nullptr, stream); + values, cudf::data_type{cudf::type_id::STRING}, {}, nullptr, stream); ASSERT_EQ(got->size(), 1); EXPECT_EQ(got->null_count(), 1); } @@ -1308,7 +1307,7 @@ TEST_F(CastVariantTest, LongStringPayloadExceedsDeclaredLength) hdr, 0x03, 0x00, 0x00, 0x00, 'a', 'b', 'c', 'x', 'x', 'x', 'x', 'x'}; cudf::test::lists_column_wrapper values(val.begin(), val.end()); auto got = cudf::io::parquet::experimental::cast_variant( - values, cudf::data_type{cudf::type_id::STRING}, nullptr, nullptr, stream); + values, cudf::data_type{cudf::type_id::STRING}, {}, nullptr, stream); cudf::test::strings_column_wrapper expected({"abc"}); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); } @@ -1398,10 +1397,9 @@ TEST_F(InvalidInputShapeTest, CastVariantRejectsMalformedInput) for (auto const& c : cases) { SCOPED_TRACE(c.label); - EXPECT_THROW( - static_cast(cudf::io::parquet::experimental::cast_variant( - c.column->view(), cudf::data_type{cudf::type_id::INT32}, nullptr, nullptr, stream)), - std::invalid_argument); + EXPECT_THROW(static_cast(cudf::io::parquet::experimental::cast_variant( + c.column->view(), cudf::data_type{cudf::type_id::INT32}, {}, nullptr, stream)), + std::invalid_argument); } } @@ -1422,6 +1420,7 @@ static void expect_status_values(cudf::column_view const& status, } constexpr uint8_t ST_SUCCESS = static_cast(op_status::success); +constexpr uint8_t ST_ROW_NULL = static_cast(op_status::row_null); constexpr uint8_t ST_MISSING = static_cast(op_status::missing_path); constexpr uint8_t ST_VNULL = static_cast(op_status::variant_null); constexpr uint8_t ST_MISMATCH = static_cast(op_status::type_mismatch); @@ -1433,8 +1432,8 @@ constexpr uint8_t ST_MALFORMED = static_cast(op_status::malformed_varia struct GetVariantFieldStatusTest : public cudf::test::BaseFixture {}; -// SQL-null input row → null output + null status -TEST_F(GetVariantFieldStatusTest, SqlNullInputProducesNullStatus) +// SQL-null input row → null output + row_null status (status column is always non-nullable) +TEST_F(GetVariantFieldStatusTest, SqlNullInputProducesRowNullStatus) { cudf::test::lists_column_wrapper meta{{0x01, 0x01, 0x00, 0x01, 'x'}}; cudf::test::lists_column_wrapper val{{0x14, 0x07, 0x00, 0x00, 0x00}}; @@ -1446,7 +1445,8 @@ TEST_F(GetVariantFieldStatusTest, SqlNullInputProducesNullStatus) ASSERT_NE(status, nullptr); ASSERT_EQ(status->size(), 1); - ASSERT_EQ(status->null_count(), 1); + ASSERT_EQ(status->null_count(), 0); + expect_status_values(*status, {ST_ROW_NULL}); ASSERT_EQ(got->null_count(), 1); } @@ -1572,7 +1572,7 @@ TEST_F(GetVariantFieldStatusTest, MixedRows) // Row 2: {} (no x key) → missing_path auto const m2 = build_metadata({}); auto const v2 = build_single_field_object(/*fid=*/0, enc_int32(0)); // fid 0 but dict empty - // Row 3: SQL null → null status + // Row 3: SQL null → row_null status (status column is always non-nullable) auto const v3 = enc_int32(0); cudf::test::lists_column_wrapper meta{{dict.begin(), dict.end()}, @@ -1589,14 +1589,8 @@ TEST_F(GetVariantFieldStatusTest, MixedRows) ASSERT_NE(status, nullptr); ASSERT_EQ(status->size(), 4); - // Row 3 (SQL null input) → null status; rows 0-2 → valid status - ASSERT_EQ(status->null_count(), 1); - // Check per-row status values via expected column (rows 0-2 valid, row 3 null) - std::vector const exp_sv{ST_SUCCESS, ST_VNULL, ST_MISSING, 0}; - std::vector const exp_sv_valid{true, true, true, false}; - cudf::test::fixed_width_column_wrapper expected_status( - exp_sv.begin(), exp_sv.end(), exp_sv_valid.begin()); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(*status, expected_status); + ASSERT_EQ(status->null_count(), 0); + expect_status_values(*status, {ST_SUCCESS, ST_VNULL, ST_MISSING, ST_ROW_NULL}); // Row 0: valid (INT32 bytes), Row 1: valid (VARIANT null bytes preserved), Row 2+3: null EXPECT_EQ(got->null_count(), 2); @@ -1639,7 +1633,7 @@ TEST_F(CastVariantStatusTest, SuccessProducesSuccessStatus) auto values = make_value_col(enc_int32(42)); std::unique_ptr status; auto got = cudf::io::parquet::experimental::cast_variant( - values, cudf::data_type{cudf::type_id::INT32}, nullptr, &status, stream, cmr()); + values, cudf::data_type{cudf::type_id::INT32}, {}, &status, stream, cmr()); ASSERT_NE(status, nullptr); expect_status_values(*status, {ST_SUCCESS}); @@ -1654,7 +1648,7 @@ TEST_F(CastVariantStatusTest, VariantNullProducesVariantNullStatus) auto values = make_value_col(enc_null()); std::unique_ptr status; auto got = cudf::io::parquet::experimental::cast_variant( - values, cudf::data_type{cudf::type_id::INT32}, nullptr, &status, stream, cmr()); + values, cudf::data_type{cudf::type_id::INT32}, {}, &status, stream, cmr()); ASSERT_NE(status, nullptr); expect_status_values(*status, {ST_VNULL}); @@ -1668,15 +1662,15 @@ TEST_F(CastVariantStatusTest, TypeMismatchStatus) auto values = make_value_col(enc_int8(5)); // INT8 cast to INT32 target → mismatch std::unique_ptr status; auto got = cudf::io::parquet::experimental::cast_variant( - values, cudf::data_type{cudf::type_id::INT32}, nullptr, &status, stream, cmr()); + values, cudf::data_type{cudf::type_id::INT32}, {}, &status, stream, cmr()); ASSERT_NE(status, nullptr); expect_status_values(*status, {ST_MISMATCH}); EXPECT_EQ(got->null_count(), 1); } -// SQL-null input (null list row) → null status -TEST_F(CastVariantStatusTest, SqlNullInputProducesNullStatus) +// SQL-null input (null list row) → row_null status (status column is always non-nullable) +TEST_F(CastVariantStatusTest, SqlNullInputProducesRowNullStatus) { auto stream = cudf::test::get_default_stream(); @@ -1702,12 +1696,12 @@ TEST_F(CastVariantStatusTest, SqlNullInputProducesNullStatus) std::unique_ptr status; auto got = cudf::io::parquet::experimental::cast_variant( - values_col->view(), cudf::data_type{cudf::type_id::INT32}, nullptr, &status, stream, cmr()); + values_col->view(), cudf::data_type{cudf::type_id::INT32}, {}, &status, stream, cmr()); ASSERT_NE(status, nullptr); - // Row 0: success; row 1: null status - ASSERT_EQ(status->null_count(), 1); - EXPECT_EQ(cudf::test::to_host(*status).first[0], ST_SUCCESS); + // Row 0: success; row 1: row_null (status column is always non-nullable) + ASSERT_EQ(status->null_count(), 0); + expect_status_values(*status, {ST_SUCCESS, ST_ROW_NULL}); EXPECT_EQ(got->null_count(), 1); } @@ -1731,7 +1725,7 @@ TEST_F(CastVariantStatusTest, IncomingStatusPropagation) std::unique_ptr status; auto got = cudf::io::parquet::experimental::cast_variant( - values, cudf::data_type{cudf::type_id::INT32}, &incoming_view1, &status, stream, cmr()); + values, cudf::data_type{cudf::type_id::INT32}, incoming_view1, &status, stream, cmr()); ASSERT_NE(status, nullptr); // Row 0: success (decoded), Row 1: missing_path (propagated), Row 2: variant_null (propagated) @@ -1740,8 +1734,9 @@ TEST_F(CastVariantStatusTest, IncomingStatusPropagation) CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); } -// Incoming status with null entry → null output and null status for that row -TEST_F(CastVariantStatusTest, IncomingNullStatusProducesNullStatus) +// Incoming row_null status → null output and row_null status for that row. +// The status column produced by get_variant_field is non-nullable; SQL-null rows carry row_null. +TEST_F(CastVariantStatusTest, IncomingRowNullStatusProducesRowNullStatus) { auto stream = cudf::test::get_default_stream(); @@ -1750,24 +1745,19 @@ TEST_F(CastVariantStatusTest, IncomingNullStatusProducesNullStatus) wrap_multi_row_variant(std::vector>(2, build_metadata({})), val_rows); auto const values = cudf::structs_column_view{col}.get_sliced_child(1, stream); - // Row 0: success, Row 1: null incoming status - cudf::test::fixed_width_column_wrapper incoming_status_w2({ST_SUCCESS, ST_SUCCESS}, - {true, false}); + // Row 0: success, Row 1: row_null (non-nullable incoming status, as produced by + // get_variant_field) + cudf::test::fixed_width_column_wrapper incoming_status_w2({ST_SUCCESS, ST_ROW_NULL}); auto incoming_status_col2 = incoming_status_w2.release(); auto const incoming_view2 = incoming_status_col2->view(); std::unique_ptr status; auto got = cudf::io::parquet::experimental::cast_variant( - values, cudf::data_type{cudf::type_id::INT32}, &incoming_view2, &status, stream, cmr()); + values, cudf::data_type{cudf::type_id::INT32}, incoming_view2, &status, stream, cmr()); ASSERT_NE(status, nullptr); - ASSERT_EQ(status->null_count(), 1); - // Row 0: success; Row 1: null status - std::vector const exp_s2{ST_SUCCESS, 0}; - std::vector const exp_s2_valid{true, false}; - cudf::test::fixed_width_column_wrapper expected_status2( - exp_s2.begin(), exp_s2.end(), exp_s2_valid.begin()); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(*status, expected_status2); + ASSERT_EQ(status->null_count(), 0); + expect_status_values(*status, {ST_SUCCESS, ST_ROW_NULL}); cudf::test::fixed_width_column_wrapper expected({7, 0}, {true, false}); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); @@ -1786,7 +1776,7 @@ TEST_F(CastVariantStatusTest, BoolStatusTracking) std::unique_ptr status; auto got = cudf::io::parquet::experimental::cast_variant( - values, cudf::data_type{cudf::type_id::BOOL8}, nullptr, &status, stream, cmr()); + values, cudf::data_type{cudf::type_id::BOOL8}, {}, &status, stream, cmr()); ASSERT_NE(status, nullptr); expect_status_values(*status, {ST_SUCCESS, ST_VNULL, ST_MISMATCH}); @@ -1818,7 +1808,7 @@ TEST_F(CastVariantStatusTest, StringStatusTracking) std::unique_ptr status; auto got = cudf::io::parquet::experimental::cast_variant( - values, cudf::data_type{cudf::type_id::STRING}, nullptr, &status, stream, cmr()); + values, cudf::data_type{cudf::type_id::STRING}, {}, &status, stream, cmr()); ASSERT_NE(status, nullptr); expect_status_values(*status, {ST_SUCCESS, ST_VNULL, ST_MISMATCH, ST_MALFORMED}); @@ -1833,7 +1823,7 @@ TEST_F(CastVariantStatusTest, EmptyInput) cudf::empty_like(cudf::structs_column_view{make_xyz_three_row_variant()}.child(1)); std::unique_ptr status; auto got = cudf::io::parquet::experimental::cast_variant( - *values, cudf::data_type{cudf::type_id::INT32}, nullptr, &status, stream, cmr()); + *values, cudf::data_type{cudf::type_id::INT32}, {}, &status, stream, cmr()); ASSERT_NE(status, nullptr); EXPECT_EQ(status->size(), 0); @@ -1863,8 +1853,8 @@ TEST_F(ExtractVariantFieldStatusTest, SuccessStatus) CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); } -// SQL null input → null status -TEST_F(ExtractVariantFieldStatusTest, SqlNullInputProducesNullStatus) +// SQL null input → row_null status (status column is always non-nullable) +TEST_F(ExtractVariantFieldStatusTest, SqlNullInputProducesRowNullStatus) { cudf::test::lists_column_wrapper meta{{0x01, 0x01, 0x00, 0x01, 'x'}}; cudf::test::lists_column_wrapper val{{0x14, 0x07, 0x00, 0x00, 0x00}}; @@ -1876,7 +1866,8 @@ TEST_F(ExtractVariantFieldStatusTest, SqlNullInputProducesNullStatus) col, "x", cudf::data_type{cudf::type_id::INT32}, &status, stream, cmr()); ASSERT_NE(status, nullptr); - EXPECT_EQ(status->null_count(), 1); + EXPECT_EQ(status->null_count(), 0); + expect_status_values(*status, {ST_ROW_NULL}); EXPECT_EQ(got->null_count(), 1); } diff --git a/java/src/main/native/src/VariantUtilsJni.cpp b/java/src/main/native/src/VariantUtilsJni.cpp index d427da7b6bd3..84c72bea1675 100644 --- a/java/src/main/native/src/VariantUtilsJni.cpp +++ b/java/src/main/native/src/VariantUtilsJni.cpp @@ -10,6 +10,8 @@ #include #include +#include + extern "C" { JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_VariantUtils_getVariantFieldValue( @@ -25,6 +27,7 @@ JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_VariantUtils_getVariantFieldValue( return cudf::jni::release_as_jlong( cudf::io::parquet::experimental::get_variant_field(variant_struct, path.get(), + nullptr, cudf::get_default_stream(), cudf::get_current_device_resource_ref())); } @@ -44,6 +47,8 @@ JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_VariantUtils_castVariantValue(JNIEnv return cudf::jni::release_as_jlong(cudf::io::parquet::experimental::cast_variant( value_bytes, cudf::data_type{static_cast(cudf_type_id)}, + std::nullopt, + nullptr, cudf::get_default_stream(), cudf::get_current_device_resource_ref())); } @@ -64,6 +69,7 @@ JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_VariantUtils_extractVariantField( variant_struct, path.get(), cudf::data_type{static_cast(cudf_type_id)}, + nullptr, cudf::get_default_stream(), cudf::get_current_device_resource_ref())); } From 54c720a58004c8dbf68a970d32e602c802d12906 Mon Sep 17 00:00:00 2001 From: Abigale Kim Date: Sat, 8 Aug 2026 01:59:23 +0000 Subject: [PATCH 13/23] changes --- cpp/include/cudf/io/experimental/variant.hpp | 19 ++++++++++------- .../parquet/experimental/variant_extract.cu | 3 +++ .../io/experimental/variant_extract_test.cpp | 21 +++++++++++++++++++ 3 files changed, 36 insertions(+), 7 deletions(-) diff --git a/cpp/include/cudf/io/experimental/variant.hpp b/cpp/include/cudf/io/experimental/variant.hpp index 08e75dfd1009..8b9e7bea2f51 100644 --- a/cpp/include/cudf/io/experimental/variant.hpp +++ b/cpp/include/cudf/io/experimental/variant.hpp @@ -50,9 +50,10 @@ namespace io::parquet::experimental { * @param variant_column Struct column (VARIANT materialization) with `list` children * (`metadata`, `value`), plus optional shredded siblings * @param path JSONPath-like path string identifying the target field + * @param status_out Optional. When non-null, receives a non-nullable `UINT8` column of + * `variant_operation_status` values, one per row * @param stream CUDA stream * @param mr Device memory resource - * @param status_out Status column with variant_operation_status` values * @return `list` column with the extracted value's encoded bytes. A row is null when the * input row is null, a name is absent, an index is out of bounds, or a step does not match * the current value. @@ -76,16 +77,19 @@ namespace io::parquet::experimental { * @param values `list` column of VARIANT-encoded value bytes * @param desired_type Target cuDF type (`STRING`, `INT8`/`INT16`/`INT32`/`INT64`, * `FLOAT32`/`FLOAT64`, or `BOOL8`) - * @param stream CUDA stream - * @param mr Device memory resource * @param incoming_status Optional status column from a prior `get_variant_field` call. When * provided, non-success rows are propagated directly to the output without decoding. - * @param status_out Status column with variant_operation_status` values + * Must be non-nullable, `UINT8`, and have the same row count as `values`. + * @param status_out Optional. When non-null, receives a non-nullable `UINT8` column of + * `variant_operation_status` values, one per row + * @param stream CUDA stream + * @param mr Device memory resource * @return Typed column decoded from the VARIANT value blobs * - * @throws std::invalid_argument if `values` is not a `list` column, or if `desired_type` + * @throws std::invalid_argument if `values` is not a `list` column; if `desired_type` * is not one of the supported types (`STRING`, `INT8`/`INT16`/`INT32`/`INT64`, - * `FLOAT32`/`FLOAT64`, or `BOOL8`) + * `FLOAT32`/`FLOAT64`, or `BOOL8`); or if `incoming_status` is provided but is nullable, + * not `UINT8`, or has a different row count than `values` */ [[nodiscard]] std::unique_ptr cast_variant( column_view const& values, @@ -105,9 +109,10 @@ namespace io::parquet::experimental { * @param path JSONPath-like path string (see `get_variant_field` for syntax) * @param desired_type Target type: `STRING`, `INT8`/`INT16`/`INT32`/`INT64`, * `FLOAT32`/`FLOAT64`, or `BOOL8` + * @param status_out Optional. When non-null, receives a non-nullable `UINT8` column of + * `variant_operation_status` values, one per row * @param stream CUDA stream * @param mr Device memory resource - * @param status_out Status column with variant_operation_status` values * @return Column of `desired_type` * * @throws std::invalid_argument on empty path or malformed syntax diff --git a/cpp/src/io/parquet/experimental/variant_extract.cu b/cpp/src/io/parquet/experimental/variant_extract.cu index 5604c3fb9a4d..3e0a870b1e1f 100644 --- a/cpp/src/io/parquet/experimental/variant_extract.cu +++ b/cpp/src/io/parquet/experimental/variant_extract.cu @@ -1248,6 +1248,9 @@ std::unique_ptr cast_variant(column_view const& values, CUDF_EXPECTS(incoming_status->type().id() == type_id::UINT8, "incoming status column must be UINT8", std::invalid_argument); + CUDF_EXPECTS(!incoming_status->nullable(), + "incoming status column must not be nullable; use row_null for SQL-null rows", + std::invalid_argument); } auto incoming_dev_view = incoming_status.has_value() diff --git a/cpp/tests/io/experimental/variant_extract_test.cpp b/cpp/tests/io/experimental/variant_extract_test.cpp index a1f0b1e21c5b..4eb47ddf7556 100644 --- a/cpp/tests/io/experimental/variant_extract_test.cpp +++ b/cpp/tests/io/experimental/variant_extract_test.cpp @@ -1403,6 +1403,27 @@ TEST_F(InvalidInputShapeTest, CastVariantRejectsMalformedInput) } } +// cast_variant must reject a nullable incoming_status column (SQL-null rows must be represented +// by the row_null enum value, not by null bits). +TEST_F(InvalidInputShapeTest, CastVariantRejectsNullableIncomingStatus) +{ + auto stream = cudf::test::get_default_stream(); + // One-row valid values column. + auto values = + list_u8({make_variant_primitive(variant_primitive_type::INT32), 0x01, 0x00, 0x00, 0x00}); + // Incoming status with a null entry (row 0 is null) — must be rejected. + // Use uint8_t{0} (== op_status::success) directly; ST_SUCCESS is not in scope here. + std::vector const sv{uint8_t{0}}; + std::vector const sv_valid{false}; + cudf::test::fixed_width_column_wrapper nullable_status( + sv.begin(), sv.end(), sv_valid.begin()); + auto const status_view = nullable_status.release()->view(); + EXPECT_THROW( + static_cast(cudf::io::parquet::experimental::cast_variant( + values->view(), cudf::data_type{cudf::type_id::INT32}, status_view, nullptr, stream)), + std::invalid_argument); +} + // --------------------------------------------------------------------------- // Status column tests // --------------------------------------------------------------------------- From 207742dd941c5d3b8406ecb3a60ece8055eb56a9 Mon Sep 17 00:00:00 2001 From: Abigale Kim Date: Sat, 8 Aug 2026 02:15:13 +0000 Subject: [PATCH 14/23] reviews --- .../parquet/experimental/variant_extract.cu | 30 ++++++------ .../io/experimental/variant_extract_test.cpp | 47 +++++++++++++++++++ 2 files changed, 63 insertions(+), 14 deletions(-) diff --git a/cpp/src/io/parquet/experimental/variant_extract.cu b/cpp/src/io/parquet/experimental/variant_extract.cu index 3e0a870b1e1f..9d2c6b0e9c4a 100644 --- a/cpp/src/io/parquet/experimental/variant_extract.cu +++ b/cpp/src/io/parquet/experimental/variant_extract.cu @@ -1224,6 +1224,21 @@ std::unique_ptr cast_variant(column_view const& values, } size_type const num_rows = values.size(); + + // Validate incoming_status before the empty-values fast path so callers always get + // std::invalid_argument for a malformed status column, even when values is empty. + if (incoming_status.has_value()) { + CUDF_EXPECTS(!incoming_status->nullable(), + "incoming status column must not be nullable; use row_null for SQL-null rows", + std::invalid_argument); + CUDF_EXPECTS(incoming_status->type().id() == type_id::UINT8, + "incoming status column must be UINT8", + std::invalid_argument); + CUDF_EXPECTS(incoming_status->size() == num_rows, + "incoming status column must have the same number of rows as the values column", + std::invalid_argument); + } + if (num_rows == 0) { if (status_out != nullptr) { *status_out = make_empty_column(data_type{type_id::UINT8}); } return make_empty_column(desired_type); @@ -1239,20 +1254,7 @@ std::unique_ptr cast_variant(column_view const& values, // Build device view for incoming status if provided; keep a placeholder when absent so that // cast_variant_fn always holds a valid column_device_view (kernel ignores it when !has_incoming). - auto placeholder_col = make_empty_column(data_type{type_id::UINT8}); - auto placeholder_dev_view = column_device_view::create(*placeholder_col, stream); - if (incoming_status.has_value()) { - CUDF_EXPECTS(incoming_status->size() == num_rows, - "incoming status column must have the same number of rows as the values column", - std::invalid_argument); - CUDF_EXPECTS(incoming_status->type().id() == type_id::UINT8, - "incoming status column must be UINT8", - std::invalid_argument); - CUDF_EXPECTS(!incoming_status->nullable(), - "incoming status column must not be nullable; use row_null for SQL-null rows", - std::invalid_argument); - } - + auto placeholder_col = make_empty_column(data_type{type_id::UINT8}); auto incoming_dev_view = incoming_status.has_value() ? column_device_view::create(*incoming_status, stream) : column_device_view::create(*placeholder_col, stream); diff --git a/cpp/tests/io/experimental/variant_extract_test.cpp b/cpp/tests/io/experimental/variant_extract_test.cpp index 4eb47ddf7556..37124855c0d6 100644 --- a/cpp/tests/io/experimental/variant_extract_test.cpp +++ b/cpp/tests/io/experimental/variant_extract_test.cpp @@ -1424,6 +1424,53 @@ TEST_F(InvalidInputShapeTest, CastVariantRejectsNullableIncomingStatus) std::invalid_argument); } +// Regression: incoming_status validation must fire even when values is empty (zero rows). +// Prior to the fix, the empty-values fast path returned before the validation block, so a +// nullable, non-UINT8, or row-count-mismatched status column was silently accepted. +TEST_F(InvalidInputShapeTest, CastVariantRejectsInvalidIncomingStatusOnEmptyValues) +{ + auto stream = cudf::test::get_default_stream(); + // Build a zero-row list values column. + auto const empty_values = + cudf::empty_like(cudf::structs_column_view{make_xyz_three_row_variant()}.child(1)); + + // Case 1: nullable incoming_status (one row, but values has zero rows — catch nullable first). + { + std::vector const sv{uint8_t{0}}; + std::vector const sv_valid{false}; + cudf::test::fixed_width_column_wrapper nullable_status( + sv.begin(), sv.end(), sv_valid.begin()); + auto const status_view = nullable_status.release()->view(); + EXPECT_THROW( + static_cast(cudf::io::parquet::experimental::cast_variant( + *empty_values, cudf::data_type{cudf::type_id::INT32}, status_view, nullptr, stream)), + std::invalid_argument) + << "nullable incoming_status must be rejected even when values is empty"; + } + + // Case 2: non-UINT8 incoming_status (zero-row INT32 column, non-nullable). + { + cudf::test::fixed_width_column_wrapper wrong_type_status{}; + auto const status_view = wrong_type_status.release()->view(); + EXPECT_THROW( + static_cast(cudf::io::parquet::experimental::cast_variant( + *empty_values, cudf::data_type{cudf::type_id::INT32}, status_view, nullptr, stream)), + std::invalid_argument) + << "non-UINT8 incoming_status must be rejected even when values is empty"; + } + + // Case 3: row-count mismatch (one-row status vs zero-row values). + { + cudf::test::fixed_width_column_wrapper mismatched_status({uint8_t{0}}); + auto const status_view = mismatched_status.release()->view(); + EXPECT_THROW( + static_cast(cudf::io::parquet::experimental::cast_variant( + *empty_values, cudf::data_type{cudf::type_id::INT32}, status_view, nullptr, stream)), + std::invalid_argument) + << "row-count-mismatched incoming_status must be rejected even when values is empty"; + } +} + // --------------------------------------------------------------------------- // Status column tests // --------------------------------------------------------------------------- From 639acc3c94d38d875fdd6f193116bbca00e7096f Mon Sep 17 00:00:00 2001 From: Abigale Kim Date: Sat, 8 Aug 2026 02:45:20 +0000 Subject: [PATCH 15/23] addressing comments --- .../io/experimental/variant_extract_test.cpp | 153 +++++++++++------- 1 file changed, 92 insertions(+), 61 deletions(-) diff --git a/cpp/tests/io/experimental/variant_extract_test.cpp b/cpp/tests/io/experimental/variant_extract_test.cpp index 37124855c0d6..814b8ab90603 100644 --- a/cpp/tests/io/experimental/variant_extract_test.cpp +++ b/cpp/tests/io/experimental/variant_extract_test.cpp @@ -64,15 +64,19 @@ constexpr uint8_t make_variant_primitive(variant_primitive_type type) return make_variant_header(variant_basic_type::PRIMITIVE, static_cast(type)); } -// Header byte for a short string of the given length (must fit in 6 bits: 0..63). +/** + * @brief Header byte for a short string of the given length (must fit in 6 bits: 0..63). + */ constexpr uint8_t make_variant_short_string_header(std::size_t length) { CUDF_EXPECTS(length <= 0x3F, "VARIANT short string length must fit in 6 bits"); return make_variant_header(variant_basic_type::SHORT_STRING, static_cast(length)); } -// Header byte for an object value with 1-byte field ids and 1-byte offsets -// (is_large=false), i.e. value_header == 0. +/** + * @brief Header byte for an object value with 1-byte field ids and 1-byte offsets + * (is_large=false), i.e. value_header == 0. + */ constexpr uint8_t make_variant_object_header() { return make_variant_header(variant_basic_type::OBJECT, 0); @@ -108,10 +112,12 @@ cudf::test::structs_column_wrapper make_apache_variant(avf::fixture const& return cudf::test::structs_column_wrapper{{m, v}}; } -// Three-row VARIANT fixture reused by multiple multi-row tests below. -// Row 0: dict {x,y}, value { x: INT32(7), y: "hi" } -// Row 1: dict {x,z}, value { x: INT32(42), z: INT32(99) } -// Row 2: dict {y}, value { y: "zzz" } +/** + * @brief Three-row VARIANT fixture reused by multiple multi-row tests below. + * Row 0: dict {x,y}, value { x: INT32(7), y: "hi" } + * Row 1: dict {x,z}, value { x: INT32(42), z: INT32(99) } + * Row 2: dict {y}, value { y: "zzz" } + */ inline cudf::test::structs_column_wrapper make_xyz_three_row_variant() { std::vector const m1 = {0x01, 0x02, 0x00, 0x01, 0x02, 'x', 'y'}; @@ -388,7 +394,9 @@ TEST_F(ExtractVariantFieldTest, BareNameEqualsDollarPath) namespace { -// INT32 primitive blob: primitive int32 header + little-endian 4-byte payload. +/** + * @brief INT32 primitive blob: primitive int32 header + little-endian 4-byte payload. + */ inline std::vector enc_int32(int32_t v) { auto const u = static_cast(v); @@ -407,7 +415,9 @@ inline std::vector enc_short_string(std::string_view s) return out; } -// Append `width` little-endian bytes of `bits` to `out`. +/** + * @brief Append `width` little-endian bytes of `bits` to `out`. + */ inline void append_le(std::vector& out, uint64_t bits, int width) { for (int i = 0; i < width; ++i) { @@ -415,7 +425,10 @@ inline void append_le(std::vector& out, uint64_t bits, int width) } } -// Primitive value blobs (header + fixed payload) for every physical type the cast matrix exercises. +/** + * @brief Primitive value blobs (header + fixed payload) for every physical type the cast matrix + * exercises. + */ inline std::vector enc_null() { return {make_variant_primitive(variant_primitive_type::NULLVAL)}; @@ -453,7 +466,9 @@ inline std::vector enc_float64(double v) return out; } -// Long-string primitive blob: header + 4-byte LE length + payload. +/** + * @brief Long-string primitive blob: header + 4-byte LE length + payload. + */ inline std::vector enc_long_string(std::string_view s) { std::vector out{make_variant_primitive(variant_primitive_type::LONG_STRING)}; @@ -464,8 +479,10 @@ inline std::vector enc_long_string(std::string_view s) return out; } -// Build a single-field object value wrapping `inner` under field id `fid`. -// field_off_size=1, field_id_size=1, is_large=false. +/** + * @brief Build a single-field object value wrapping `inner` under field id `fid`. + * field_off_size=1, field_id_size=1, is_large=false. + */ inline std::vector build_single_field_object(uint8_t fid, std::vector const& inner) { @@ -477,10 +494,12 @@ inline std::vector build_single_field_object(uint8_t fid, return out; } -// Build a VARIANT object blob with `n_fields` fields. Field ids are 0..n_fields-1 -// (in ascending order, matching the dictionary positions) and each field holds a bare INT32 equal -// to its field id. Uses 1-byte field_id_size and 1-byte field_off_size; n_fields must be -// <= 51 so the total value bytes (5 * n_fields) still fit in 1-byte offsets. +/** + * @brief Build a VARIANT object blob with `n_fields` fields. Field ids are 0..n_fields-1 + * (in ascending order, matching the dictionary positions) and each field holds a bare INT32 equal + * to its field id. Uses 1-byte field_id_size and 1-byte field_off_size; n_fields must be + * <= 51 so the total value bytes (5 * n_fields) still fit in 1-byte offsets. + */ inline std::vector build_sequential_int32_object(int n_fields) { std::vector out{make_variant_object_header(), static_cast(n_fields)}; @@ -497,7 +516,9 @@ inline std::vector build_sequential_int32_object(int n_fields) return out; } -// Lexicographically ordered dictionary of N zero-padded two-digit keys "k". +/** + * @brief Lexicographically ordered dictionary of N zero-padded two-digit keys "k". + */ inline std::vector make_numeric_keys(int n) { std::vector out; @@ -510,9 +531,11 @@ inline std::vector make_numeric_keys(int n) return out; } -// Wrap per-row (metadata, value) byte vectors into a VARIANT struct column. Built -// with make_lists_column + structs_column_wrapper directly so the helper stays -// self-contained within this test file for dynamic row counts. +/** + * @brief Wrap per-row (metadata, value) byte vectors into a VARIANT struct column. Built + * with make_lists_column + structs_column_wrapper directly so the helper stays + * self-contained within this test file for dynamic row counts. + */ inline cudf::test::structs_column_wrapper wrap_multi_row_variant( std::vector> const& meta_rows, std::vector> const& val_rows) @@ -536,7 +559,9 @@ inline cudf::test::structs_column_wrapper wrap_multi_row_variant( return cudf::test::structs_column_wrapper{std::move(children)}; } -// Build a metadata blob (version 1, offset_size=1) for the given ordered string dictionary. +/** + * @brief Build a metadata blob (version 1, offset_size=1) for the given ordered string dictionary. + */ inline std::vector build_metadata(std::vector const& keys) { std::vector out{0x01, static_cast(keys.size())}; @@ -1321,13 +1346,17 @@ inline std::unique_ptr list_u8(std::vector const& bytes) return cudf::test::lists_column_wrapper(bytes.begin(), bytes.end()).release(); } -// A single-row list (wrong element type for a VARIANT child). +/** + * @brief A single-row list (wrong element type for a VARIANT child). + */ inline std::unique_ptr list_i32(std::vector const& values) { return cudf::test::lists_column_wrapper(values.begin(), values.end()).release(); } -// A single-row fixed-width int32 column (a non-list child). +/** + * @brief A single-row fixed-width int32 column (a non-list child). + */ inline std::unique_ptr scalar_i32() { return cudf::test::fixed_width_column_wrapper{42}.release(); @@ -1354,11 +1383,11 @@ struct broken_shape { } // namespace -// A VARIANT column must be a STRUCT whose first two children are each a list. Enumerate the -// distinct ways that column-shape contract can be broken; get_variant_field must reject every one -// with std::invalid_argument. TEST_F(InvalidInputShapeTest, GetVariantFieldRejectsMalformedInput) { + // A VARIANT column must be a STRUCT whose first two children are each a list. Enumerate + // the distinct ways that column-shape contract can be broken; get_variant_field must reject every + // one with std::invalid_argument. auto stream = cudf::test::get_default_stream(); std::vector cases; @@ -1385,10 +1414,10 @@ TEST_F(InvalidInputShapeTest, GetVariantFieldRejectsMalformedInput) } } -// cast_variant requires a list input; every other shape must be rejected with -// std::invalid_argument. TEST_F(InvalidInputShapeTest, CastVariantRejectsMalformedInput) { + // cast_variant requires a list input; every other shape must be rejected with + // std::invalid_argument. auto stream = cudf::test::get_default_stream(); std::vector cases; @@ -1403,10 +1432,10 @@ TEST_F(InvalidInputShapeTest, CastVariantRejectsMalformedInput) } } -// cast_variant must reject a nullable incoming_status column (SQL-null rows must be represented -// by the row_null enum value, not by null bits). TEST_F(InvalidInputShapeTest, CastVariantRejectsNullableIncomingStatus) { + // cast_variant must reject a nullable incoming_status column (SQL-null rows must be represented + // by the row_null enum value, not by null bits). auto stream = cudf::test::get_default_stream(); // One-row valid values column. auto values = @@ -1424,11 +1453,11 @@ TEST_F(InvalidInputShapeTest, CastVariantRejectsNullableIncomingStatus) std::invalid_argument); } -// Regression: incoming_status validation must fire even when values is empty (zero rows). -// Prior to the fix, the empty-values fast path returned before the validation block, so a -// nullable, non-UINT8, or row-count-mismatched status column was silently accepted. TEST_F(InvalidInputShapeTest, CastVariantRejectsInvalidIncomingStatusOnEmptyValues) { + // Regression: incoming_status validation must fire even when values is empty (zero rows). + // Prior to the fix, the empty-values fast path returned before the validation block, so a + // nullable, non-UINT8, or row-count-mismatched status column was silently accepted. auto stream = cudf::test::get_default_stream(); // Build a zero-row list values column. auto const empty_values = @@ -1478,8 +1507,10 @@ using op_status = cudf::io::parquet::experimental::variant_operation_status; namespace expns = cudf::io::parquet::experimental; auto const& cmr = cudf::get_current_device_resource_ref; -// Helper using fixed_width_column_wrapper comparison for the common case where the status -// column has no nulls. +/** + * @brief Helper using fixed_width_column_wrapper comparison for the common case where the status + * column has no nulls. + */ static void expect_status_values(cudf::column_view const& status, std::vector const& expected) { @@ -1500,9 +1531,9 @@ constexpr uint8_t ST_MALFORMED = static_cast(op_status::malformed_varia struct GetVariantFieldStatusTest : public cudf::test::BaseFixture {}; -// SQL-null input row → null output + row_null status (status column is always non-nullable) TEST_F(GetVariantFieldStatusTest, SqlNullInputProducesRowNullStatus) { + // SQL-null input row → null output + row_null status (status column is always non-nullable) cudf::test::lists_column_wrapper meta{{0x01, 0x01, 0x00, 0x01, 'x'}}; cudf::test::lists_column_wrapper val{{0x14, 0x07, 0x00, 0x00, 0x00}}; cudf::test::structs_column_wrapper col{{meta, val}, std::vector{false}}; @@ -1518,9 +1549,9 @@ TEST_F(GetVariantFieldStatusTest, SqlNullInputProducesRowNullStatus) ASSERT_EQ(got->null_count(), 1); } -// Successful extraction → success status TEST_F(GetVariantFieldStatusTest, SuccessStatus) { + // Successful extraction → success status auto col = make_xyz_three_row_variant(); auto stream = cudf::test::get_default_stream(); @@ -1534,9 +1565,9 @@ TEST_F(GetVariantFieldStatusTest, SuccessStatus) EXPECT_EQ(got->null_count(), 1); } -// Missing key → missing_path status TEST_F(GetVariantFieldStatusTest, MissingKeyProducesMissingPathStatus) { + // Missing key → missing_path status auto col = make_apache_variant(avf::object_primitive); auto stream = cudf::test::get_default_stream(); @@ -1550,9 +1581,9 @@ TEST_F(GetVariantFieldStatusTest, MissingKeyProducesMissingPathStatus) EXPECT_EQ(got->null_count(), 1); } -// VARIANT null terminal value → variant_null status, preserved bytes (non-null output) TEST_F(GetVariantFieldStatusTest, VariantNullPreservedWithStatus) { + // VARIANT null terminal value → variant_null status, preserved bytes (non-null output) // Build a single-row VARIANT: object {null_field: VARIANT_NULL} // metadata: {null_field}, value: object wrapping NULLVAL primitive auto const m = build_metadata({"null_field"}); @@ -1574,10 +1605,10 @@ TEST_F(GetVariantFieldStatusTest, VariantNullPreservedWithStatus) CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected_bytes); } -// Without status_out, VARIANT null is returned as bytes (non-null list row), same as with status. -// Only cast_variant turns a VARIANT null blob into a SQL null. TEST_F(GetVariantFieldStatusTest, VariantNullReturnedAsBytesWithoutStatus) { + // Without status_out, VARIANT null is returned as bytes (non-null list row), same as with status. + // Only cast_variant turns a VARIANT null blob into a SQL null. auto const m = build_metadata({"null_field"}); auto const v = build_single_field_object(/*fid=*/0, enc_null()); auto col = wrap_single_variant(m, v); @@ -1592,9 +1623,9 @@ TEST_F(GetVariantFieldStatusTest, VariantNullReturnedAsBytesWithoutStatus) CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected_bytes); } -// Malformed metadata → malformed_variant status TEST_F(GetVariantFieldStatusTest, MalformedMetadataProducesMalformedStatus) { + // Malformed metadata → malformed_variant status std::vector const bad_meta = {0x02}; // too short / version ≠ 1 std::vector const val = {0x14, 0x07, 0x00, 0x00, 0x00}; auto col = wrap_single_variant(bad_meta, val); @@ -1608,9 +1639,9 @@ TEST_F(GetVariantFieldStatusTest, MalformedMetadataProducesMalformedStatus) EXPECT_EQ(got->null_count(), 1); } -// VARIANT null before end of a nested path → missing_path TEST_F(GetVariantFieldStatusTest, VariantNullBeforeEndIsMissingPath) { + // VARIANT null before end of a nested path → missing_path // Object {a: VARIANT_NULL}; path "$.a.b" should be missing_path (null intermediate) auto const m = build_metadata({"a"}); auto const v = build_single_field_object(/*fid=*/0, enc_null()); @@ -1626,9 +1657,9 @@ TEST_F(GetVariantFieldStatusTest, VariantNullBeforeEndIsMissingPath) EXPECT_EQ(got->null_count(), 1); } -// Mixed rows: success / missing / variant_null / malformed / SQL null TEST_F(GetVariantFieldStatusTest, MixedRows) { + // Mixed rows: success / missing / variant_null / malformed / SQL null auto stream = cudf::test::get_default_stream(); auto const dict = build_metadata({"x"}); @@ -1664,9 +1695,9 @@ TEST_F(GetVariantFieldStatusTest, MixedRows) EXPECT_EQ(got->null_count(), 2); } -// Empty input → empty status column TEST_F(GetVariantFieldStatusTest, EmptyInput) { + // Empty input → empty status column auto const stream = cudf::test::get_default_stream(); auto const variant = cudf::empty_like(make_xyz_three_row_variant()); @@ -1694,9 +1725,9 @@ inline cudf::test::lists_column_wrapper make_value_col(std::vector status; @@ -1709,9 +1740,9 @@ TEST_F(CastVariantStatusTest, SuccessProducesSuccessStatus) CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); } -// VARIANT null → variant_null status TEST_F(CastVariantStatusTest, VariantNullProducesVariantNullStatus) { + // VARIANT null → variant_null status auto stream = cudf::test::get_default_stream(); auto values = make_value_col(enc_null()); std::unique_ptr status; @@ -1723,9 +1754,9 @@ TEST_F(CastVariantStatusTest, VariantNullProducesVariantNullStatus) EXPECT_EQ(got->null_count(), 1); } -// Type mismatch → type_mismatch status TEST_F(CastVariantStatusTest, TypeMismatchStatus) { + // Type mismatch → type_mismatch status auto stream = cudf::test::get_default_stream(); auto values = make_value_col(enc_int8(5)); // INT8 cast to INT32 target → mismatch std::unique_ptr status; @@ -1737,9 +1768,9 @@ TEST_F(CastVariantStatusTest, TypeMismatchStatus) EXPECT_EQ(got->null_count(), 1); } -// SQL-null input (null list row) → row_null status (status column is always non-nullable) TEST_F(CastVariantStatusTest, SqlNullInputProducesRowNullStatus) { + // SQL-null input (null list row) → row_null status (status column is always non-nullable) auto stream = cudf::test::get_default_stream(); // Build the values list column directly (two rows), then mask row 1 null. @@ -1773,9 +1804,9 @@ TEST_F(CastVariantStatusTest, SqlNullInputProducesRowNullStatus) EXPECT_EQ(got->null_count(), 1); } -// Incoming status propagation: non-success upstream → propagated status TEST_F(CastVariantStatusTest, IncomingStatusPropagation) { + // Incoming status propagation: non-success upstream → propagated status auto stream = cudf::test::get_default_stream(); // 3 rows: success, missing_path, variant_null (from a prior get_variant_field) @@ -1802,10 +1833,10 @@ TEST_F(CastVariantStatusTest, IncomingStatusPropagation) CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); } -// Incoming row_null status → null output and row_null status for that row. -// The status column produced by get_variant_field is non-nullable; SQL-null rows carry row_null. TEST_F(CastVariantStatusTest, IncomingRowNullStatusProducesRowNullStatus) { + // Incoming row_null status → null output and row_null status for that row. + // The status column produced by get_variant_field is non-nullable; SQL-null rows carry row_null. auto stream = cudf::test::get_default_stream(); std::vector> const val_rows{enc_int32(7), enc_int32(1)}; @@ -1831,9 +1862,9 @@ TEST_F(CastVariantStatusTest, IncomingRowNullStatusProducesRowNullStatus) CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); } -// Status for bool target TEST_F(CastVariantStatusTest, BoolStatusTracking) { + // Status for bool target auto stream = cudf::test::get_default_stream(); // 3 rows: bool_true (success), null (variant_null), int32 (type_mismatch) @@ -1852,9 +1883,9 @@ TEST_F(CastVariantStatusTest, BoolStatusTracking) CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); } -// Status for string target: short_string, variant_null, type_mismatch, malformed long_string TEST_F(CastVariantStatusTest, StringStatusTracking) { + // Status for string target: short_string, variant_null, type_mismatch, malformed long_string auto stream = cudf::test::get_default_stream(); std::vector> const val_rows{ @@ -1883,9 +1914,9 @@ TEST_F(CastVariantStatusTest, StringStatusTracking) EXPECT_EQ(got->null_count(), 3); // all but row 0 are null } -// Empty input → empty status column TEST_F(CastVariantStatusTest, EmptyInput) { + // Empty input → empty status column auto const stream = cudf::test::get_default_stream(); auto const values = cudf::empty_like(cudf::structs_column_view{make_xyz_three_row_variant()}.child(1)); @@ -1904,9 +1935,9 @@ TEST_F(CastVariantStatusTest, EmptyInput) struct ExtractVariantFieldStatusTest : public cudf::test::BaseFixture {}; -// Success path: object {x: INT32(7)} extracted as INT32 TEST_F(ExtractVariantFieldStatusTest, SuccessStatus) { + // Success path: object {x: INT32(7)} extracted as INT32 auto col = make_xyz_three_row_variant(); auto stream = cudf::test::get_default_stream(); @@ -1921,9 +1952,9 @@ TEST_F(ExtractVariantFieldStatusTest, SuccessStatus) CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); } -// SQL null input → row_null status (status column is always non-nullable) TEST_F(ExtractVariantFieldStatusTest, SqlNullInputProducesRowNullStatus) { + // SQL null input → row_null status (status column is always non-nullable) cudf::test::lists_column_wrapper meta{{0x01, 0x01, 0x00, 0x01, 'x'}}; cudf::test::lists_column_wrapper val{{0x14, 0x07, 0x00, 0x00, 0x00}}; cudf::test::structs_column_wrapper col{{meta, val}, std::vector{false}}; @@ -1939,9 +1970,9 @@ TEST_F(ExtractVariantFieldStatusTest, SqlNullInputProducesRowNullStatus) EXPECT_EQ(got->null_count(), 1); } -// VARIANT null → variant_null status (from extraction phase) TEST_F(ExtractVariantFieldStatusTest, VariantNullStatus) { + // VARIANT null → variant_null status (from extraction phase) auto const m = build_metadata({"f"}); auto const v = build_single_field_object(/*fid=*/0, enc_null()); auto col = wrap_single_variant(m, v); @@ -1956,9 +1987,9 @@ TEST_F(ExtractVariantFieldStatusTest, VariantNullStatus) EXPECT_EQ(got->null_count(), 1); } -// Type mismatch: field exists but is a string, requested as INT32 TEST_F(ExtractVariantFieldStatusTest, TypeMismatchStatus) { + // Type mismatch: field exists but is a string, requested as INT32 auto const m = build_metadata({"s"}); auto const v = build_single_field_object(/*fid=*/0, enc_short_string("hello")); auto col = wrap_single_variant(m, v); @@ -1973,9 +2004,9 @@ TEST_F(ExtractVariantFieldStatusTest, TypeMismatchStatus) EXPECT_EQ(got->null_count(), 1); } -// Missing path for a multi-step path TEST_F(ExtractVariantFieldStatusTest, MissingNestedPathStatus) { + // Missing path for a multi-step path auto col = make_apache_variant(avf::object_nested); auto stream = cudf::test::get_default_stream(); From 3cb0e7f18892d5f5aca61c459404f6fe0d8bfea4 Mon Sep 17 00:00:00 2001 From: Abigale Kim Date: Sat, 8 Aug 2026 02:54:03 +0000 Subject: [PATCH 16/23] reviews --- cpp/tests/io/experimental/variant_extract_test.cpp | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/cpp/tests/io/experimental/variant_extract_test.cpp b/cpp/tests/io/experimental/variant_extract_test.cpp index 814b8ab90603..dfbd900350ae 100644 --- a/cpp/tests/io/experimental/variant_extract_test.cpp +++ b/cpp/tests/io/experimental/variant_extract_test.cpp @@ -1446,7 +1446,8 @@ TEST_F(InvalidInputShapeTest, CastVariantRejectsNullableIncomingStatus) std::vector const sv_valid{false}; cudf::test::fixed_width_column_wrapper nullable_status( sv.begin(), sv.end(), sv_valid.begin()); - auto const status_view = nullable_status.release()->view(); + auto const status_col = nullable_status.release(); + auto const status_view = status_col->view(); EXPECT_THROW( static_cast(cudf::io::parquet::experimental::cast_variant( values->view(), cudf::data_type{cudf::type_id::INT32}, status_view, nullptr, stream)), @@ -1469,7 +1470,8 @@ TEST_F(InvalidInputShapeTest, CastVariantRejectsInvalidIncomingStatusOnEmptyValu std::vector const sv_valid{false}; cudf::test::fixed_width_column_wrapper nullable_status( sv.begin(), sv.end(), sv_valid.begin()); - auto const status_view = nullable_status.release()->view(); + auto const status_col = nullable_status.release(); + auto const status_view = status_col->view(); EXPECT_THROW( static_cast(cudf::io::parquet::experimental::cast_variant( *empty_values, cudf::data_type{cudf::type_id::INT32}, status_view, nullptr, stream)), @@ -1480,7 +1482,8 @@ TEST_F(InvalidInputShapeTest, CastVariantRejectsInvalidIncomingStatusOnEmptyValu // Case 2: non-UINT8 incoming_status (zero-row INT32 column, non-nullable). { cudf::test::fixed_width_column_wrapper wrong_type_status{}; - auto const status_view = wrong_type_status.release()->view(); + auto const status_col = wrong_type_status.release(); + auto const status_view = status_col->view(); EXPECT_THROW( static_cast(cudf::io::parquet::experimental::cast_variant( *empty_values, cudf::data_type{cudf::type_id::INT32}, status_view, nullptr, stream)), @@ -1491,7 +1494,8 @@ TEST_F(InvalidInputShapeTest, CastVariantRejectsInvalidIncomingStatusOnEmptyValu // Case 3: row-count mismatch (one-row status vs zero-row values). { cudf::test::fixed_width_column_wrapper mismatched_status({uint8_t{0}}); - auto const status_view = mismatched_status.release()->view(); + auto const status_col = mismatched_status.release(); + auto const status_view = status_col->view(); EXPECT_THROW( static_cast(cudf::io::parquet::experimental::cast_variant( *empty_values, cudf::data_type{cudf::type_id::INT32}, status_view, nullptr, stream)), From d07f24f865507a1833d7bf8bf62c920240fce92a Mon Sep 17 00:00:00 2001 From: Abigale Kim Date: Tue, 11 Aug 2026 01:35:30 +0000 Subject: [PATCH 17/23] changes --- .../parquet/experimental/variant_extract.cu | 22 +++++++++++++++-- .../io/experimental/variant_extract_test.cpp | 24 +++++++++++++++---- 2 files changed, 39 insertions(+), 7 deletions(-) diff --git a/cpp/src/io/parquet/experimental/variant_extract.cu b/cpp/src/io/parquet/experimental/variant_extract.cu index 9d2c6b0e9c4a..4dc20e62659f 100644 --- a/cpp/src/io/parquet/experimental/variant_extract.cu +++ b/cpp/src/io/parquet/experimental/variant_extract.cu @@ -257,10 +257,19 @@ __device__ cuda::std::pair, op_status> find_key_i if (!start_off.has_value()) { return {cuda::std::nullopt, op_status::malformed_variant}; } auto const strings_base = offsets_start + static_cast(offsets_bytes); auto const strings_extent = meta_len - strings_base; + // Read the terminal offset offsets[num_entries] before scanning entries. An early key match + // must be bounded by this declared extent, not the physical buffer, so validate it upfront. + auto const terminal_off_pos = + offsets_start + static_cast(num_entries.value()) * offset_size; + auto const terminal_off = read_uint64(meta, terminal_off_pos, offset_size); + if (!terminal_off.has_value() || cuda::std::cmp_greater(terminal_off.value(), strings_extent)) { + return {cuda::std::nullopt, op_status::malformed_variant}; + } + auto const strings_declared = static_cast(terminal_off.value()); for (size_type i = 0; i < num_entries.value(); ++i) { auto const end_off = read_uint64(meta, offsets_start + (i + 1) * offset_size, offset_size); if (!end_off.has_value()) { return {cuda::std::nullopt, op_status::malformed_variant}; } - if (end_off.value() < start_off.value() || end_off.value() > strings_extent) { + if (end_off.value() < start_off.value() || end_off.value() > strings_declared) { return {cuda::std::nullopt, op_status::malformed_variant}; } cudf::string_view const entry{ @@ -412,6 +421,15 @@ __device__ cuda::std::pair, op_status> locate_array_e } size_type const values_base = offsets_start + static_cast(offsets_bytes); auto const values_extent = value_size - values_base; + // Read the terminal offset offsets[num_elements]; it is the spec-declared bound on the + // values region and must be used instead of the physical extent so that an element whose + // offset escapes the declared boundary is caught as malformed even when physical bytes + // are present beyond it. + auto const terminal_off_pos = offsets_start + static_cast(num_elements) * offset_size; + auto const terminal_off = read_uint64(value, terminal_off_pos, offset_size); + if (!terminal_off.has_value() || cuda::std::cmp_greater(*terminal_off, values_extent)) { + return {{}, op_status::malformed_variant}; + } auto const start_offset_pos = offsets_start + static_cast(index) * offset_size; auto const end_offset_pos = offsets_start + (static_cast(index) + 1) * offset_size; @@ -425,7 +443,7 @@ __device__ cuda::std::pair, op_status> locate_array_e } auto const element_start = *start_offset; auto const element_end = *end_offset; - if (element_end < element_start || cuda::std::cmp_greater(element_end, values_extent)) { + if (element_end < element_start || cuda::std::cmp_greater(element_end, *terminal_off)) { return {{}, op_status::malformed_variant}; } return {value.subspan(values_base + element_start, element_end - element_start), diff --git a/cpp/tests/io/experimental/variant_extract_test.cpp b/cpp/tests/io/experimental/variant_extract_test.cpp index dfbd900350ae..6a2d4c90ce60 100644 --- a/cpp/tests/io/experimental/variant_extract_test.cpp +++ b/cpp/tests/io/experimental/variant_extract_test.cpp @@ -731,11 +731,18 @@ TEST_F(ExtractVariantFieldTest, EmptyArrayIndexing) CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, null_expected); } - // Truncated counts/tables, decreasing offsets, and offsets beyond the values region yield null. - for (auto const& value : std::vector>{{0x13}, - {0x03, 0x01, 0x00}, - {0x03, 0x01, 0x02, 0x01, 0x0c, 42}, - {0x03, 0x01, 0x00, 0x03, 0x0c, 42}}) { + // Truncated counts/tables, decreasing offsets, offsets beyond the values region, and an + // element end that escapes the terminal offset all yield null. + for (auto const& value : std::vector>{ + {0x13}, + {0x03, 0x01, 0x00}, + {0x03, 0x01, 0x02, 0x01, 0x0c, 42}, + {0x03, 0x01, 0x00, 0x03, 0x0c, 42}, + // 2-element array: offsets[0]=0, offsets[1]=5, terminal offsets[2]=1. + // 5 physical value bytes are present (passes the old physical-extent check), + // but the terminal offset declares only 1 value byte, so element 0's end (5) + // escapes the declared boundary → malformed. + {0x03, 0x02, 0x00, 0x05, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00}}) { auto malformed_col = wrap_single_variant(build_metadata({}), value); auto got = cudf::io::parquet::experimental::extract_variant_field( malformed_col, "$[0]", i8, nullptr, stream); @@ -823,6 +830,13 @@ TEST_F(ExtractVariantFieldTest, MalformedVariantDataYieldsNull) {"object declares more fields than the value buffer holds", build_metadata({"x"}), {make_variant_object_header(), 0xFF}}, + // Two-key dict: offsets[0]=0, offsets[1]=1 ("x" is key 0), terminal offsets[2]=0. + // 2 physical string bytes are present so the old per-entry check passes (1 <= 2), + // but the terminal offset declares the string region as 0 bytes. The key "x" + // matches at i=0 before the terminal is consulted → must be malformed. + {"metadata terminal offset below first key's declared end", + {0x01, 0x02, 0x00, 0x01, 0x00, 'x', 'y'}, + build_single_field_object(0, enc_int32(42))}, }; for (auto const& c : cases) { From bcd3083cb5fd8d572092aaf146b0b5189d9b04a9 Mon Sep 17 00:00:00 2001 From: Abigale Kim Date: Tue, 11 Aug 2026 20:39:14 +0000 Subject: [PATCH 18/23] changes --- .../parquet/experimental/variant_extract.cu | 21 ++++++++++++--- .../io/experimental/variant_extract_test.cpp | 26 +++++++++++++++---- 2 files changed, 38 insertions(+), 9 deletions(-) diff --git a/cpp/src/io/parquet/experimental/variant_extract.cu b/cpp/src/io/parquet/experimental/variant_extract.cu index 4dc20e62659f..428936c72367 100644 --- a/cpp/src/io/parquet/experimental/variant_extract.cu +++ b/cpp/src/io/parquet/experimental/variant_extract.cu @@ -254,7 +254,10 @@ __device__ cuda::std::pair, op_status> find_key_i } auto start_off = read_uint64(meta, offsets_start, offset_size); - if (!start_off.has_value()) { return {cuda::std::nullopt, op_status::malformed_variant}; } + // Parquet VARIANT spec requires offsets[0] == 0; any other value is malformed. + if (!start_off.has_value() || start_off.value() != 0) { + return {cuda::std::nullopt, op_status::malformed_variant}; + } auto const strings_base = offsets_start + static_cast(offsets_bytes); auto const strings_extent = meta_len - strings_base; // Read the terminal offset offsets[num_entries] before scanning entries. An early key match @@ -829,11 +832,21 @@ __device__ op_status cast_status_for_string(device_span val) if (val.empty()) { return op_status::malformed_variant; } if (is_variant_null(val)) { return op_status::variant_null; } if (decode_string(val).has_value()) { return op_status::success; } + // decode_string failed. Classify the failure: + // SHORT_STRING with truncated payload → malformed_variant (valid encoding type, bad payload) + // LONG_STRING (PRIMITIVE header) → malformed_variant (already confirmed truncated above) + // Other recognized primitive type → type_mismatch (well-formed but wrong type) + // Unrecognized primitive/basic type → malformed_variant auto const btype = decode_basic_type(val[0]); - if (btype == basic_type::PRIMITIVE && - variant_value_header(val[0]) == static_cast(primitive_type::LONG_STRING)) { - return op_status::malformed_variant; + if (btype == basic_type::SHORT_STRING) { return op_status::malformed_variant; } + if (btype == basic_type::PRIMITIVE) { + auto const ptype = static_cast(variant_value_header(val[0])); + // LONG_STRING is a recognized string type whose payload was truncated. + if (ptype == primitive_type::LONG_STRING) { return op_status::malformed_variant; } + return is_recognized_primitive_type(ptype) ? op_status::type_mismatch + : op_status::malformed_variant; } + // OBJECT, ARRAY, or other non-primitive basic types: well-formed, just not a string. return op_status::type_mismatch; } diff --git a/cpp/tests/io/experimental/variant_extract_test.cpp b/cpp/tests/io/experimental/variant_extract_test.cpp index 6a2d4c90ce60..9124d586c3e0 100644 --- a/cpp/tests/io/experimental/variant_extract_test.cpp +++ b/cpp/tests/io/experimental/variant_extract_test.cpp @@ -837,6 +837,10 @@ TEST_F(ExtractVariantFieldTest, MalformedVariantDataYieldsNull) {"metadata terminal offset below first key's declared end", {0x01, 0x02, 0x00, 0x01, 0x00, 'x', 'y'}, build_single_field_object(0, enc_int32(42))}, + // Single-key dict where offsets[0] != 0. The Parquet VARIANT spec requires offsets[0] == 0; + // a non-zero first offset makes the string region ill-defined. + // Layout: num_entries=1, offsets[0]=1 (invalid), offsets[1]=2, string bytes "x". + {"metadata first offset non-zero", {0x01, 0x01, 0x01, 0x02, 'x'}, valid_object}, }; for (auto const& c : cases) { @@ -1903,13 +1907,22 @@ TEST_F(CastVariantStatusTest, BoolStatusTracking) TEST_F(CastVariantStatusTest, StringStatusTracking) { - // Status for string target: short_string, variant_null, type_mismatch, malformed long_string + // Status for string target: short_string, variant_null, type_mismatch, malformed long_string, + // truncated short_string, and unrecognized primitive id. auto stream = cudf::test::get_default_stream(); + // A SHORT_STRING header that claims 5 bytes of content but provides none. + std::vector const truncated_short_string{make_variant_short_string_header(5)}; + + // An unrecognized primitive type id (0x3F maps to the value_header field of a PRIMITIVE byte + // and does not correspond to any defined primitive_type enum value). + std::vector const unknown_primitive_id{ + make_variant_header(variant_basic_type::PRIMITIVE, 0x3F)}; + std::vector> const val_rows{ enc_short_string("hi"), // success enc_null(), // variant_null - enc_int32(5), // type_mismatch + enc_int32(5), // type_mismatch (recognized non-string primitive) // malformed long_string: header + declares 10 bytes but only 2 present {make_variant_primitive(cudf::io::parquet::experimental::variant_primitive_type::LONG_STRING), 0x0A, @@ -1918,9 +1931,11 @@ TEST_F(CastVariantStatusTest, StringStatusTracking) 0x00, 'a', 'b'}, + truncated_short_string, // malformed: SHORT_STRING with truncated payload + unknown_primitive_id, // malformed: unrecognized primitive id }; auto col = - wrap_multi_row_variant(std::vector>(4, build_metadata({})), val_rows); + wrap_multi_row_variant(std::vector>(6, build_metadata({})), val_rows); auto values = cudf::structs_column_view{col}.get_sliced_child(1, stream); std::unique_ptr status; @@ -1928,8 +1943,9 @@ TEST_F(CastVariantStatusTest, StringStatusTracking) values, cudf::data_type{cudf::type_id::STRING}, {}, &status, stream, cmr()); ASSERT_NE(status, nullptr); - expect_status_values(*status, {ST_SUCCESS, ST_VNULL, ST_MISMATCH, ST_MALFORMED}); - EXPECT_EQ(got->null_count(), 3); // all but row 0 are null + expect_status_values( + *status, {ST_SUCCESS, ST_VNULL, ST_MISMATCH, ST_MALFORMED, ST_MALFORMED, ST_MALFORMED}); + EXPECT_EQ(got->null_count(), 5); // all but row 0 are null } TEST_F(CastVariantStatusTest, EmptyInput) From 3f316bfcb5ace4e0ccc05691e8d7e7dc6339e8a5 Mon Sep 17 00:00:00 2001 From: Abigale Kim Date: Wed, 12 Aug 2026 20:07:50 +0000 Subject: [PATCH 19/23] reviews --- .../cudf/io/experimental/variant_spec.hpp | 16 +- .../parquet/experimental/variant_extract.cu | 148 +++++++++--------- .../io/experimental/variant_extract_test.cpp | 14 +- 3 files changed, 89 insertions(+), 89 deletions(-) diff --git a/cpp/include/cudf/io/experimental/variant_spec.hpp b/cpp/include/cudf/io/experimental/variant_spec.hpp index 2f37c17887a6..aad48f7f1a16 100644 --- a/cpp/include/cudf/io/experimental/variant_spec.hpp +++ b/cpp/include/cudf/io/experimental/variant_spec.hpp @@ -50,14 +50,14 @@ enum class variant_primitive_type : uint8_t { * @brief VARIANT status types. */ enum class variant_operation_status : uint8_t { - success, - row_null, ///< the SQL row itself was null (no VARIANT data to decode) - missing_path, - variant_null, - type_mismatch, - malformed_variant, - overflow, - invalid_conversion, + SUCCESS = 0, ///< operation completed successfully + ROW_NULL = 1, ///< the SQL row itself was null (no VARIANT data to decode) + MISSING_PATH = 2, ///< the requested path does not exist in the VARIANT + VARIANT_NULL = 3, ///< the value at the path is a VARIANT null + TYPE_MISMATCH = 4, ///< the value's type does not match the requested type + MALFORMED_VARIANT = 5, ///< the VARIANT binary encoding is invalid + OVERFLOW = 6, ///< the value overflows the target numeric type + INVALID_CONVERSION = 7, ///< the value cannot be converted to the requested type }; } // namespace cudf::io::parquet::experimental diff --git a/cpp/src/io/parquet/experimental/variant_extract.cu b/cpp/src/io/parquet/experimental/variant_extract.cu index 428936c72367..70c170f51a16 100644 --- a/cpp/src/io/parquet/experimental/variant_extract.cu +++ b/cpp/src/io/parquet/experimental/variant_extract.cu @@ -235,28 +235,28 @@ __device__ cuda::std::pair, op_status> find_key_i device_span meta, cudf::string_view key) { auto const meta_len = static_cast(meta.size()); - if (meta_len < 1) { return {cuda::std::nullopt, op_status::malformed_variant}; } + if (meta_len < 1) { return {cuda::std::nullopt, op_status::MALFORMED_VARIANT}; } auto const header = meta[0]; int const version = header & 0x0F; - if (version != variant_version_v1) { return {cuda::std::nullopt, op_status::malformed_variant}; } + if (version != variant_version_v1) { return {cuda::std::nullopt, op_status::MALFORMED_VARIANT}; } int const offset_size = ((header >> 6) & 0x03) + 1; size_type pos = 1; auto const num_entries = narrow_cast(read_uint64(meta, pos, offset_size)); - if (!num_entries.has_value()) { return {cuda::std::nullopt, op_status::malformed_variant}; } + if (!num_entries.has_value()) { return {cuda::std::nullopt, op_status::MALFORMED_VARIANT}; } pos += offset_size; auto const offsets_start = pos; auto const offsets_bytes = (static_cast(num_entries.value()) + 1) * offset_size; if (cuda::std::cmp_greater(offsets_bytes, meta_len - offsets_start)) { - return {cuda::std::nullopt, op_status::malformed_variant}; + return {cuda::std::nullopt, op_status::MALFORMED_VARIANT}; } auto start_off = read_uint64(meta, offsets_start, offset_size); // Parquet VARIANT spec requires offsets[0] == 0; any other value is malformed. if (!start_off.has_value() || start_off.value() != 0) { - return {cuda::std::nullopt, op_status::malformed_variant}; + return {cuda::std::nullopt, op_status::MALFORMED_VARIANT}; } auto const strings_base = offsets_start + static_cast(offsets_bytes); auto const strings_extent = meta_len - strings_base; @@ -266,22 +266,22 @@ __device__ cuda::std::pair, op_status> find_key_i offsets_start + static_cast(num_entries.value()) * offset_size; auto const terminal_off = read_uint64(meta, terminal_off_pos, offset_size); if (!terminal_off.has_value() || cuda::std::cmp_greater(terminal_off.value(), strings_extent)) { - return {cuda::std::nullopt, op_status::malformed_variant}; + return {cuda::std::nullopt, op_status::MALFORMED_VARIANT}; } auto const strings_declared = static_cast(terminal_off.value()); for (size_type i = 0; i < num_entries.value(); ++i) { auto const end_off = read_uint64(meta, offsets_start + (i + 1) * offset_size, offset_size); - if (!end_off.has_value()) { return {cuda::std::nullopt, op_status::malformed_variant}; } + if (!end_off.has_value()) { return {cuda::std::nullopt, op_status::MALFORMED_VARIANT}; } if (end_off.value() < start_off.value() || end_off.value() > strings_declared) { - return {cuda::std::nullopt, op_status::malformed_variant}; + return {cuda::std::nullopt, op_status::MALFORMED_VARIANT}; } cudf::string_view const entry{ reinterpret_cast(meta.data() + strings_base + start_off.value()), static_cast(end_off.value() - start_off.value())}; - if (entry == key) { return {i, op_status::success}; } + if (entry == key) { return {i, op_status::SUCCESS}; } start_off = end_off; } - return {cuda::std::nullopt, op_status::missing_path}; + return {cuda::std::nullopt, op_status::MISSING_PATH}; } /** @@ -314,10 +314,10 @@ __device__ cuda::std::pair, op_status> locate_object_ device_span val, int id) { auto const val_len = static_cast(val.size()); - if (val_len < 1) { return {{}, op_status::malformed_variant}; } + if (val_len < 1) { return {{}, op_status::MALFORMED_VARIANT}; } auto const value_metadata = val[0]; if (decode_basic_type(value_metadata) != basic_type::OBJECT) { - return {{}, op_status::missing_path}; + return {{}, op_status::MISSING_PATH}; } auto const [offset_size, id_size, num_elements_size] = @@ -325,16 +325,16 @@ __device__ cuda::std::pair, op_status> locate_object_ size_type pos = 1; auto const num_fields = narrow_cast(read_uint64(val, pos, num_elements_size)); - if (!num_fields.has_value()) { return {{}, op_status::malformed_variant}; } + if (!num_fields.has_value()) { return {{}, op_status::MALFORMED_VARIANT}; } pos += num_elements_size; auto const ids_start = pos; auto const ids_bytes = static_cast(num_fields.value()) * id_size; - if (ids_bytes > val_len - ids_start) { return {{}, op_status::malformed_variant}; } + if (ids_bytes > val_len - ids_start) { return {{}, op_status::MALFORMED_VARIANT}; } auto const offsets_start = ids_start + static_cast(ids_bytes); auto const offsets_bytes = (static_cast(num_fields.value()) + 1) * offset_size; - if (offsets_bytes > val_len - offsets_start) { return {{}, op_status::malformed_variant}; } + if (offsets_bytes > val_len - offsets_start) { return {{}, op_status::MALFORMED_VARIANT}; } auto const values_base = offsets_start + static_cast(offsets_bytes); auto const values_extent = val_len - values_base; @@ -345,7 +345,7 @@ __device__ cuda::std::pair, op_status> locate_object_ auto const sentinel_raw = read_uint64(val, offsets_start + num_fields.value() * offset_size, offset_size); if (!sentinel_raw.has_value() || sentinel_raw.value() > static_cast(values_extent)) { - return {{}, op_status::malformed_variant}; + return {{}, op_status::MALFORMED_VARIANT}; } auto const values_region = static_cast(sentinel_raw.value()); @@ -353,28 +353,28 @@ __device__ cuda::std::pair, op_status> locate_object_ uint64_t match_start = 0; for (size_type i = 0; i < num_fields.value(); ++i) { auto const current_id = read_uint64(val, ids_start + i * id_size, id_size); - if (!current_id.has_value()) { return {{}, op_status::malformed_variant}; } + if (!current_id.has_value()) { return {{}, op_status::MALFORMED_VARIANT}; } if (cuda::std::cmp_not_equal(current_id.value(), id)) { continue; } auto const match_offset = read_uint64(val, offsets_start + i * offset_size, offset_size); - if (!match_offset.has_value()) { return {{}, op_status::malformed_variant}; } + if (!match_offset.has_value()) { return {{}, op_status::MALFORMED_VARIANT}; } if (match_offset.value() > static_cast(values_region)) { - return {{}, op_status::malformed_variant}; + return {{}, op_status::MALFORMED_VARIANT}; } match_start = match_offset.value(); found = true; break; } - if (!found) { return {{}, op_status::missing_path}; } + if (!found) { return {{}, op_status::MISSING_PATH}; } auto const value = val.subspan(values_base + match_start); auto const value_len = variant_value_length(value); - if (!value_len.has_value()) { return {{}, op_status::malformed_variant}; } + if (!value_len.has_value()) { return {{}, op_status::MALFORMED_VARIANT}; } auto const match_end = match_start + value_len.value(); if (match_end > static_cast(values_region)) { - return {{}, op_status::malformed_variant}; + return {{}, op_status::MALFORMED_VARIANT}; } - return {val.subspan(values_base + match_start, value_len.value()), op_status::success}; + return {val.subspan(values_base + match_start, value_len.value()), op_status::SUCCESS}; } // Parse an array value header and return the sub-span of the element at `index` (0-based) within @@ -394,13 +394,13 @@ __device__ cuda::std::pair, op_status> locate_object_ __device__ cuda::std::pair, op_status> locate_array_element( device_span value, size_type index) { - if (index < 0) { return {{}, op_status::missing_path}; } + if (index < 0) { return {{}, op_status::MISSING_PATH}; } auto const value_size = static_cast(value.size()); - if (value_size < 1) { return {{}, op_status::malformed_variant}; } + if (value_size < 1) { return {{}, op_status::MALFORMED_VARIANT}; } uint8_t const value_metadata = value[0]; if (decode_basic_type(value_metadata) != basic_type::ARRAY) { - return {{}, op_status::missing_path}; + return {{}, op_status::MISSING_PATH}; } int const value_header = variant_value_header(value_metadata); @@ -409,9 +409,9 @@ __device__ cuda::std::pair, op_status> locate_array_e size_type position = 1; auto const num_elements_value = narrow_cast(read_uint64(value, position, num_elements_size)); - if (!num_elements_value.has_value()) { return {{}, op_status::malformed_variant}; } + if (!num_elements_value.has_value()) { return {{}, op_status::MALFORMED_VARIANT}; } auto const num_elements = num_elements_value.value(); - if (index >= num_elements) { return {{}, op_status::missing_path}; } + if (index >= num_elements) { return {{}, op_status::MISSING_PATH}; } position += num_elements_size; size_type const offsets_start = position; @@ -420,7 +420,7 @@ __device__ cuda::std::pair, op_status> locate_array_e // range (which would be UB); the check below then rejects any array that overruns the value blob. auto const offsets_bytes = (static_cast(num_elements) + 1) * offset_size; if (cuda::std::cmp_greater(offsets_bytes, value_size - offsets_start)) { - return {{}, op_status::malformed_variant}; + return {{}, op_status::MALFORMED_VARIANT}; } size_type const values_base = offsets_start + static_cast(offsets_bytes); auto const values_extent = value_size - values_base; @@ -431,26 +431,26 @@ __device__ cuda::std::pair, op_status> locate_array_e auto const terminal_off_pos = offsets_start + static_cast(num_elements) * offset_size; auto const terminal_off = read_uint64(value, terminal_off_pos, offset_size); if (!terminal_off.has_value() || cuda::std::cmp_greater(*terminal_off, values_extent)) { - return {{}, op_status::malformed_variant}; + return {{}, op_status::MALFORMED_VARIANT}; } auto const start_offset_pos = offsets_start + static_cast(index) * offset_size; auto const end_offset_pos = offsets_start + (static_cast(index) + 1) * offset_size; if (cuda::std::cmp_greater(end_offset_pos + offset_size, value_size)) { - return {{}, op_status::malformed_variant}; + return {{}, op_status::MALFORMED_VARIANT}; } auto const start_offset = read_uint64(value, start_offset_pos, offset_size); auto const end_offset = read_uint64(value, end_offset_pos, offset_size); if (!start_offset.has_value() || !end_offset.has_value()) { - return {{}, op_status::malformed_variant}; + return {{}, op_status::MALFORMED_VARIANT}; } auto const element_start = *start_offset; auto const element_end = *end_offset; if (element_end < element_start || cuda::std::cmp_greater(element_end, *terminal_off)) { - return {{}, op_status::malformed_variant}; + return {{}, op_status::MALFORMED_VARIANT}; } return {value.subspan(values_base + element_start, element_end - element_start), - op_status::success}; + op_status::SUCCESS}; } __device__ bool is_variant_null(device_span enc) @@ -576,28 +576,28 @@ __device__ cuda::std::pair, op_status> resolve_path( auto const step = path.element(i); if (step.size_bytes() >= 1 && step.data()[0] == '[') { auto const index = parse_index_step(step); - if (!index.has_value()) { return {{}, op_status::missing_path}; } + if (!index.has_value()) { return {{}, op_status::MISSING_PATH}; } auto const [span, st] = locate_array_element(sub_val, index.value()); - if (st != op_status::success) { return {{}, st}; } + if (st != op_status::SUCCESS) { return {{}, st}; } sub_val = span; } else { auto const [field_id, meta_st] = find_key_in_metadata(meta, step); - if (meta_st == op_status::malformed_variant) { return {{}, op_status::malformed_variant}; } - if (!field_id.has_value()) { return {{}, op_status::missing_path}; } + if (meta_st == op_status::MALFORMED_VARIANT) { return {{}, op_status::MALFORMED_VARIANT}; } + if (!field_id.has_value()) { return {{}, op_status::MISSING_PATH}; } auto const [span, st] = locate_object_field(sub_val, field_id.value()); - if (st != op_status::success) { return {{}, st}; } + if (st != op_status::SUCCESS) { return {{}, st}; } sub_val = span; } // VARIANT null before the end of the path is missing_path per spec. - if (i + 1 < path.size() && is_variant_null(sub_val)) { return {{}, op_status::missing_path}; } + if (i + 1 < path.size() && is_variant_null(sub_val)) { return {{}, op_status::MISSING_PATH}; } // A zero-length resolved value is not decodable; the value-only path drops the row. - if (sub_val.empty()) { return {{}, op_status::malformed_variant}; } + if (sub_val.empty()) { return {{}, op_status::MALFORMED_VARIANT}; } } // Terminal VARIANT null: return the bytes with variant_null status. - if (is_variant_null(sub_val)) { return {sub_val, op_status::variant_null}; } - return {sub_val, op_status::success}; + if (is_variant_null(sub_val)) { return {sub_val, op_status::VARIANT_NULL}; } + return {sub_val, op_status::SUCCESS}; } __device__ cuda::std::optional> decode_string( @@ -671,7 +671,7 @@ CUDF_KERNEL __launch_bounds__(block_size) void locate_variant_fields_kernel( if (!cudf::bit_is_set(d_null_mask, row)) { d_sizes[row] = 0; d_src_offsets[row] = 0; - if constexpr (HasStatus) { d_status[row] = op_status::row_null; } + if constexpr (HasStatus) { d_status[row] = op_status::ROW_NULL; } continue; } @@ -735,14 +735,14 @@ template requires(is_variant_numerical) __device__ op_status cast_status_for_primitive(device_span val) { - if (val.empty()) { return op_status::malformed_variant; } - if (is_variant_null(val)) { return op_status::variant_null; } - if (decode_primitive(val).has_value()) { return op_status::success; } - if (decode_basic_type(val[0]) != basic_type::PRIMITIVE) { return op_status::type_mismatch; } + if (val.empty()) { return op_status::MALFORMED_VARIANT; } + if (is_variant_null(val)) { return op_status::VARIANT_NULL; } + if (decode_primitive(val).has_value()) { return op_status::SUCCESS; } + if (decode_basic_type(val[0]) != basic_type::PRIMITIVE) { return op_status::TYPE_MISMATCH; } auto const ptype = static_cast(variant_value_header(val[0])); - if (ptype == primitive_type_for()) { return op_status::malformed_variant; } - return is_recognized_primitive_type(ptype) ? op_status::type_mismatch - : op_status::malformed_variant; + if (ptype == primitive_type_for()) { return op_status::MALFORMED_VARIANT; } + return is_recognized_primitive_type(ptype) ? op_status::TYPE_MISMATCH + : op_status::MALFORMED_VARIANT; } /** @@ -775,7 +775,7 @@ CUDF_KERNEL __launch_bounds__(block_size) void cast_variant_primitive_kernel( if (has_incoming) { // Status column is always non-nullable; row_null replaces the null bit. auto const s = static_cast(incoming_status.element(row)); - if (s != op_status::success) { + if (s != op_status::SUCCESS) { d_output[row] = T{}; if (cudf::bit_is_set(d_null_mask, row)) { cudf::clear_bit(d_null_mask, row); } if (d_status.data()) { d_status[row] = s; } @@ -785,7 +785,7 @@ CUDF_KERNEL __launch_bounds__(block_size) void cast_variant_primitive_kernel( } else { if (!cudf::bit_is_set(d_null_mask, row)) { d_output[row] = T{}; - if (d_status.data()) { d_status[row] = op_status::row_null; } + if (d_status.data()) { d_status[row] = op_status::ROW_NULL; } continue; } } @@ -801,7 +801,7 @@ CUDF_KERNEL __launch_bounds__(block_size) void cast_variant_primitive_kernel( if (decoded.has_value()) { d_output[row] = *decoded; if constexpr (HasStatus) { - if (d_status.data()) { d_status[row] = op_status::success; } + if (d_status.data()) { d_status[row] = op_status::SUCCESS; } } } else { d_output[row] = T{}; @@ -815,39 +815,39 @@ CUDF_KERNEL __launch_bounds__(block_size) void cast_variant_primitive_kernel( __device__ op_status cast_status_for_bool(device_span val) { - if (val.empty()) { return op_status::malformed_variant; } - if (is_variant_null(val)) { return op_status::variant_null; } - if (decode_bool(val).has_value()) { return op_status::success; } - if (decode_basic_type(val[0]) != basic_type::PRIMITIVE) { return op_status::type_mismatch; } + if (val.empty()) { return op_status::MALFORMED_VARIANT; } + if (is_variant_null(val)) { return op_status::VARIANT_NULL; } + if (decode_bool(val).has_value()) { return op_status::SUCCESS; } + if (decode_basic_type(val[0]) != basic_type::PRIMITIVE) { return op_status::TYPE_MISMATCH; } // Boolean values carry no payload, so a BOOLEAN_TRUE/FALSE header can never be truncated; // decode_bool would have succeeded above. Any remaining primitive ID is a type mismatch when // recognised, or malformed when not. auto const ptype = static_cast(variant_value_header(val[0])); - return is_recognized_primitive_type(ptype) ? op_status::type_mismatch - : op_status::malformed_variant; + return is_recognized_primitive_type(ptype) ? op_status::TYPE_MISMATCH + : op_status::MALFORMED_VARIANT; } __device__ op_status cast_status_for_string(device_span val) { - if (val.empty()) { return op_status::malformed_variant; } - if (is_variant_null(val)) { return op_status::variant_null; } - if (decode_string(val).has_value()) { return op_status::success; } + if (val.empty()) { return op_status::MALFORMED_VARIANT; } + if (is_variant_null(val)) { return op_status::VARIANT_NULL; } + if (decode_string(val).has_value()) { return op_status::SUCCESS; } // decode_string failed. Classify the failure: // SHORT_STRING with truncated payload → malformed_variant (valid encoding type, bad payload) // LONG_STRING (PRIMITIVE header) → malformed_variant (already confirmed truncated above) // Other recognized primitive type → type_mismatch (well-formed but wrong type) // Unrecognized primitive/basic type → malformed_variant auto const btype = decode_basic_type(val[0]); - if (btype == basic_type::SHORT_STRING) { return op_status::malformed_variant; } + if (btype == basic_type::SHORT_STRING) { return op_status::MALFORMED_VARIANT; } if (btype == basic_type::PRIMITIVE) { auto const ptype = static_cast(variant_value_header(val[0])); // LONG_STRING is a recognized string type whose payload was truncated. - if (ptype == primitive_type::LONG_STRING) { return op_status::malformed_variant; } - return is_recognized_primitive_type(ptype) ? op_status::type_mismatch - : op_status::malformed_variant; + if (ptype == primitive_type::LONG_STRING) { return op_status::MALFORMED_VARIANT; } + return is_recognized_primitive_type(ptype) ? op_status::TYPE_MISMATCH + : op_status::MALFORMED_VARIANT; } // OBJECT, ARRAY, or other non-primitive basic types: well-formed, just not a string. - return op_status::type_mismatch; + return op_status::TYPE_MISMATCH; } /** @@ -879,7 +879,7 @@ struct cast_variant_string_fn { if (has_incoming) { // Status column is always non-nullable; row_null replaces the null bit. auto const s = static_cast(incoming_status.element(row)); - if (s != op_status::success) { + if (s != op_status::SUCCESS) { if (sizing) { d_sizes[row] = 0; } if (cudf::bit_is_set(d_null_mask, row)) { cudf::clear_bit(d_null_mask, row); } if (sizing && d_status) { d_status[row] = s; } @@ -889,7 +889,7 @@ struct cast_variant_string_fn { } else { if (!cudf::bit_is_set(d_null_mask, row)) { if (sizing) { d_sizes[row] = 0; } - if (sizing && d_status) { d_status[row] = op_status::row_null; } + if (sizing && d_status) { d_status[row] = op_status::ROW_NULL; } return; } } @@ -909,7 +909,7 @@ struct cast_variant_string_fn { } else { cuda::std::memcpy(d_chars + d_offsets[row], str->data(), str->size()); } - if (sizing && d_status) { d_status[row] = op_status::success; } + if (sizing && d_status) { d_status[row] = op_status::SUCCESS; } } }; @@ -1017,7 +1017,7 @@ struct cast_variant_fn { // Status column is always non-nullable; row_null replaces the null bit. if (hi) { auto const s = static_cast(inc_view.element(row)); - if (s != op_status::success) { + if (s != op_status::SUCCESS) { d_out[row] = false; if (cudf::bit_is_set(dnm, row)) { cudf::clear_bit(dnm, row); } if (dp_s) { dp_s[row] = s; } @@ -1026,7 +1026,7 @@ struct cast_variant_fn { } else { if (!cudf::bit_is_set(dnm, row)) { d_out[row] = false; - if (dp_s) { dp_s[row] = op_status::row_null; } + if (dp_s) { dp_s[row] = op_status::ROW_NULL; } return; } } @@ -1034,7 +1034,7 @@ struct cast_variant_fn { auto const decoded = decode_bool(val); if (decoded.has_value()) { d_out[row] = *decoded; - if (dp_s) { dp_s[row] = op_status::success; } + if (dp_s) { dp_s[row] = op_status::SUCCESS; } } else { d_out[row] = false; cudf::clear_bit(dnm, row); diff --git a/cpp/tests/io/experimental/variant_extract_test.cpp b/cpp/tests/io/experimental/variant_extract_test.cpp index 9124d586c3e0..55187c9eeb3a 100644 --- a/cpp/tests/io/experimental/variant_extract_test.cpp +++ b/cpp/tests/io/experimental/variant_extract_test.cpp @@ -1459,7 +1459,7 @@ TEST_F(InvalidInputShapeTest, CastVariantRejectsNullableIncomingStatus) auto values = list_u8({make_variant_primitive(variant_primitive_type::INT32), 0x01, 0x00, 0x00, 0x00}); // Incoming status with a null entry (row 0 is null) — must be rejected. - // Use uint8_t{0} (== op_status::success) directly; ST_SUCCESS is not in scope here. + // Use uint8_t{0} (== op_status::SUCCESS) directly; ST_SUCCESS is not in scope here. std::vector const sv{uint8_t{0}}; std::vector const sv_valid{false}; cudf::test::fixed_width_column_wrapper nullable_status( @@ -1540,12 +1540,12 @@ static void expect_status_values(cudf::column_view const& status, CUDF_TEST_EXPECT_COLUMNS_EQUAL(status, exp); } -constexpr uint8_t ST_SUCCESS = static_cast(op_status::success); -constexpr uint8_t ST_ROW_NULL = static_cast(op_status::row_null); -constexpr uint8_t ST_MISSING = static_cast(op_status::missing_path); -constexpr uint8_t ST_VNULL = static_cast(op_status::variant_null); -constexpr uint8_t ST_MISMATCH = static_cast(op_status::type_mismatch); -constexpr uint8_t ST_MALFORMED = static_cast(op_status::malformed_variant); +constexpr uint8_t ST_SUCCESS = static_cast(op_status::SUCCESS); +constexpr uint8_t ST_ROW_NULL = static_cast(op_status::ROW_NULL); +constexpr uint8_t ST_MISSING = static_cast(op_status::MISSING_PATH); +constexpr uint8_t ST_VNULL = static_cast(op_status::VARIANT_NULL); +constexpr uint8_t ST_MISMATCH = static_cast(op_status::TYPE_MISMATCH); +constexpr uint8_t ST_MALFORMED = static_cast(op_status::MALFORMED_VARIANT); // --------------------------------------------------------------------------- // GetVariantField status tests From f2b8a522b5f1779b16fb1446358b8bae67888aab Mon Sep 17 00:00:00 2001 From: Abigale Kim Date: Wed, 12 Aug 2026 22:36:46 +0000 Subject: [PATCH 20/23] reviews --- cpp/src/io/parquet/experimental/variant_extract.cu | 4 ++++ cpp/tests/io/experimental/variant_extract_test.cpp | 7 ++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/cpp/src/io/parquet/experimental/variant_extract.cu b/cpp/src/io/parquet/experimental/variant_extract.cu index 70c170f51a16..77be833a8db4 100644 --- a/cpp/src/io/parquet/experimental/variant_extract.cu +++ b/cpp/src/io/parquet/experimental/variant_extract.cu @@ -433,6 +433,10 @@ __device__ cuda::std::pair, op_status> locate_array_e if (!terminal_off.has_value() || cuda::std::cmp_greater(*terminal_off, values_extent)) { return {{}, op_status::MALFORMED_VARIANT}; } + // The spec requires offsets[0] == 0; a nonzero first offset silently skips leading + // value bytes and can return a plausible result from a malformed array. + auto const first_off = read_uint64(value, offsets_start, offset_size); + if (!first_off.has_value() || *first_off != 0) { return {{}, op_status::MALFORMED_VARIANT}; } auto const start_offset_pos = offsets_start + static_cast(index) * offset_size; auto const end_offset_pos = offsets_start + (static_cast(index) + 1) * offset_size; diff --git a/cpp/tests/io/experimental/variant_extract_test.cpp b/cpp/tests/io/experimental/variant_extract_test.cpp index 55187c9eeb3a..cdf6d9cdcc9b 100644 --- a/cpp/tests/io/experimental/variant_extract_test.cpp +++ b/cpp/tests/io/experimental/variant_extract_test.cpp @@ -742,7 +742,12 @@ TEST_F(ExtractVariantFieldTest, EmptyArrayIndexing) // 5 physical value bytes are present (passes the old physical-extent check), // but the terminal offset declares only 1 value byte, so element 0's end (5) // escapes the declared boundary → malformed. - {0x03, 0x02, 0x00, 0x05, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00}}) { + {0x03, 0x02, 0x00, 0x05, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00}, + // 1-element array where offsets[0]=1 (must be 0 per spec). + // offsets: [1, 2], values: [0x0c, 0x2a] (an int8 variant for 42). + // terminal_off=2 <= values_extent=2 so the terminal check passes; the + // nonzero first offset is caught by the new offsets[0]==0 guard. + {0x03, 0x01, 0x01, 0x02, 0x0c, 0x2a}}) { auto malformed_col = wrap_single_variant(build_metadata({}), value); auto got = cudf::io::parquet::experimental::extract_variant_field( malformed_col, "$[0]", i8, nullptr, stream); From 62e331926f46d6a04af809148c3b4bf52872d24b Mon Sep 17 00:00:00 2001 From: Abigale Kim Date: Thu, 13 Aug 2026 19:49:27 -0500 Subject: [PATCH 21/23] Update cpp/src/io/parquet/experimental/variant_extract.cu Co-authored-by: Lawrence Mitchell --- cpp/src/io/parquet/experimental/variant_extract.cu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/io/parquet/experimental/variant_extract.cu b/cpp/src/io/parquet/experimental/variant_extract.cu index 77be833a8db4..0dac34de8693 100644 --- a/cpp/src/io/parquet/experimental/variant_extract.cu +++ b/cpp/src/io/parquet/experimental/variant_extract.cu @@ -929,7 +929,7 @@ void validate_variant_child(column_view const& child) // Status columns are always non-nullable: SQL-null rows are represented by the row_null enum // value rather than by a null bit, so no null mask needs to be allocated or counted. -std::unique_ptr make_status_column(rmm::device_buffer status_data, size_type num_rows) +std::unique_ptr make_status_column(rmm::device_buffer&& status_data, size_type num_rows) { return std::make_unique( data_type{type_id::UINT8}, num_rows, std::move(status_data), rmm::device_buffer{}, 0); From c239a0a66f2b2e67e09e2616df642657901d3300 Mon Sep 17 00:00:00 2001 From: Abigale Kim Date: Fri, 14 Aug 2026 00:50:43 +0000 Subject: [PATCH 22/23] fixing comments --- .../parquet/experimental/variant_extract.cu | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/cpp/src/io/parquet/experimental/variant_extract.cu b/cpp/src/io/parquet/experimental/variant_extract.cu index 77be833a8db4..7c9a64c4b440 100644 --- a/cpp/src/io/parquet/experimental/variant_extract.cu +++ b/cpp/src/io/parquet/experimental/variant_extract.cu @@ -695,9 +695,8 @@ CUDF_KERNEL __launch_bounds__(block_size) void locate_variant_fields_kernel( } } -// Returns true for every primitive_type ID that variant_value_length recognises (i.e. every ID -// that does not fall through to its `default: return nullopt` branch). Used by both -// cast_status_for_primitive and cast_status_for_bool so the two classifiers stay in sync. +// Returns true for every primitive_type ID that variant_value_length maps to a known payload +// size in its `basic_type::PRIMITIVE` switch, i.e. every ID other than its `default` case. __device__ bool is_recognized_primitive_type(primitive_type ptype) { switch (ptype) { @@ -727,13 +726,8 @@ __device__ bool is_recognized_primitive_type(primitive_type ptype) } /** - * @brief Status helper for fixed-width primitive targets: returns the failure reason when decode - * fails. - * - * Three cases after `decode_primitive` returns nullopt and the basic type is PRIMITIVE: - * - Header type == primitive_type_for() but payload too short → truncated → malformed_variant - * - Header type is a different recognized type → type_mismatch - * - Header type is unrecognized → malformed_variant + * @brief Status helper for fixed-width primitive targets: classifies why `decode_primitive` + * failed to decode `val`, per `variant_operation_status` semantics. */ template requires(is_variant_numerical) @@ -836,11 +830,6 @@ __device__ op_status cast_status_for_string(device_span val) if (val.empty()) { return op_status::MALFORMED_VARIANT; } if (is_variant_null(val)) { return op_status::VARIANT_NULL; } if (decode_string(val).has_value()) { return op_status::SUCCESS; } - // decode_string failed. Classify the failure: - // SHORT_STRING with truncated payload → malformed_variant (valid encoding type, bad payload) - // LONG_STRING (PRIMITIVE header) → malformed_variant (already confirmed truncated above) - // Other recognized primitive type → type_mismatch (well-formed but wrong type) - // Unrecognized primitive/basic type → malformed_variant auto const btype = decode_basic_type(val[0]); if (btype == basic_type::SHORT_STRING) { return op_status::MALFORMED_VARIANT; } if (btype == basic_type::PRIMITIVE) { From 4d3b6ffe8da9de5a139e26f7d2941c561e7c50be Mon Sep 17 00:00:00 2001 From: Abigale Kim Date: Fri, 14 Aug 2026 21:18:42 +0000 Subject: [PATCH 23/23] stuff --- cpp/include/cudf/io/experimental/variant.hpp | 39 +- .../parquet/experimental/variant_extract.cu | 136 +++--- .../io/experimental/variant_extract_test.cpp | 402 ++++++++++-------- java/src/main/native/src/VariantUtilsJni.cpp | 6 +- 4 files changed, 309 insertions(+), 274 deletions(-) diff --git a/cpp/include/cudf/io/experimental/variant.hpp b/cpp/include/cudf/io/experimental/variant.hpp index 8b9e7bea2f51..43da7a276381 100644 --- a/cpp/include/cudf/io/experimental/variant.hpp +++ b/cpp/include/cudf/io/experimental/variant.hpp @@ -50,8 +50,9 @@ namespace io::parquet::experimental { * @param variant_column Struct column (VARIANT materialization) with `list` children * (`metadata`, `value`), plus optional shredded siblings * @param path JSONPath-like path string identifying the target field - * @param status_out Optional. When non-null, receives a non-nullable `UINT8` column of - * `variant_operation_status` values, one per row + * @param status Optional. When provided, filled with `variant_operation_status` values, one per + * row. Must be non-nullable, `UINT8`, and have the same row count as + * `variant_column` * @param stream CUDA stream * @param mr Device memory resource * @return `list` column with the extracted value's encoded bytes. A row is null when the @@ -59,14 +60,16 @@ namespace io::parquet::experimental { * the current value. * * @throws std::invalid_argument on empty path or malformed syntax (`[*]` wildcards, negative - * indices, out-of-range indices, and quoted names inside `[...]` are not supported) + * indices, out-of-range indices, and quoted names inside `[...]` are not supported); or if + * `status` is provided but is nullable, not `UINT8`, or has a different row count than + * `variant_column` */ [[nodiscard]] std::unique_ptr get_variant_field( column_view const& variant_column, std::string_view path, - std::unique_ptr* status_out = nullptr, - rmm::cuda_stream_view stream = cudf::get_default_stream(), - rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()); + std::optional status = std::nullopt, + rmm::cuda_stream_view stream = cudf::get_default_stream(), + rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()); /** * @brief Decode a VARIANT value column's blobs into a typed cuDF column. @@ -80,22 +83,22 @@ namespace io::parquet::experimental { * @param incoming_status Optional status column from a prior `get_variant_field` call. When * provided, non-success rows are propagated directly to the output without decoding. * Must be non-nullable, `UINT8`, and have the same row count as `values`. - * @param status_out Optional. When non-null, receives a non-nullable `UINT8` column of - * `variant_operation_status` values, one per row + * @param status Optional. When provided, filled with `variant_operation_status` values, one per + * row. Must be non-nullable, `UINT8`, and have the same row count as `values` * @param stream CUDA stream * @param mr Device memory resource * @return Typed column decoded from the VARIANT value blobs * * @throws std::invalid_argument if `values` is not a `list` column; if `desired_type` * is not one of the supported types (`STRING`, `INT8`/`INT16`/`INT32`/`INT64`, - * `FLOAT32`/`FLOAT64`, or `BOOL8`); or if `incoming_status` is provided but is nullable, - * not `UINT8`, or has a different row count than `values` + * `FLOAT32`/`FLOAT64`, or `BOOL8`); or if `incoming_status` or `status` is provided but is + * nullable, not `UINT8`, or has a different row count than `values` */ [[nodiscard]] std::unique_ptr cast_variant( column_view const& values, data_type desired_type, std::optional incoming_status = std::nullopt, - std::unique_ptr* status_out = nullptr, + std::optional status = std::nullopt, rmm::cuda_stream_view stream = cudf::get_default_stream(), rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()); @@ -109,21 +112,23 @@ namespace io::parquet::experimental { * @param path JSONPath-like path string (see `get_variant_field` for syntax) * @param desired_type Target type: `STRING`, `INT8`/`INT16`/`INT32`/`INT64`, * `FLOAT32`/`FLOAT64`, or `BOOL8` - * @param status_out Optional. When non-null, receives a non-nullable `UINT8` column of - * `variant_operation_status` values, one per row + * @param status Optional. When provided, filled with `variant_operation_status` values, one per + * row. Must be non-nullable, `UINT8`, and have the same row count as + * `variant_column` * @param stream CUDA stream * @param mr Device memory resource * @return Column of `desired_type` * - * @throws std::invalid_argument on empty path or malformed syntax + * @throws std::invalid_argument on empty path or malformed syntax; or if `status` is provided but + * is nullable, not `UINT8`, or has a different row count than `variant_column` */ [[nodiscard]] std::unique_ptr extract_variant_field( column_view const& variant_column, std::string_view path, data_type desired_type, - std::unique_ptr* status_out = nullptr, - rmm::cuda_stream_view stream = cudf::get_default_stream(), - rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()); + std::optional status = std::nullopt, + rmm::cuda_stream_view stream = cudf::get_default_stream(), + rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()); /** @} */ } // namespace io::parquet::experimental diff --git a/cpp/src/io/parquet/experimental/variant_extract.cu b/cpp/src/io/parquet/experimental/variant_extract.cu index 47c91d647f85..b4d227abefdf 100644 --- a/cpp/src/io/parquet/experimental/variant_extract.cu +++ b/cpp/src/io/parquet/experimental/variant_extract.cu @@ -752,7 +752,7 @@ __device__ op_status cast_status_for_primitive(device_span val) * that are null, or whose value is not an exact-width match for `T`, are marked null in * `d_null_mask` with an output of 0. */ -// `d_status` may be an empty span when HasStatus=true but no status_out was requested (only +// `d_status` may be an empty span when HasStatus=true but no `status` output was requested (only // incoming_status propagation to the null mask is desired). All writes to d_status are guarded // by d_status.data() so the empty-span case is safe. template @@ -916,14 +916,6 @@ void validate_variant_child(column_view const& child) std::invalid_argument); } -// Status columns are always non-nullable: SQL-null rows are represented by the row_null enum -// value rather than by a null bit, so no null mask needs to be allocated or counted. -std::unique_ptr make_status_column(rmm::device_buffer&& status_data, size_type num_rows) -{ - return std::make_unique( - data_type{type_id::UINT8}, num_rows, std::move(status_data), rmm::device_buffer{}, 0); -} - struct cast_variant_fn { cudf::lists_column_device_view values; size_type num_rows; @@ -935,13 +927,7 @@ struct cast_variant_fn { // Optional status tracking column_device_view incoming_status_view; bool has_incoming{false}; - std::unique_ptr* status_out{nullptr}; - - // Allocates an uninitialized device buffer large enough for num_rows status bytes. - rmm::device_buffer alloc_status() - { - return rmm::device_buffer{static_cast(num_rows) * sizeof(op_status), stream, mr}; - } + op_status* d_status{nullptr}; template std::unique_ptr operator()() @@ -951,20 +937,18 @@ struct cast_variant_fn { auto const grid = cudf::detail::grid_1d{num_rows, block_size}; auto const d_out = device_span{static_cast(data.data()), static_cast(num_rows)}; - if (status_out != nullptr) { - auto s_data = alloc_status(); + if (d_status != nullptr) { cast_variant_primitive_kernel<<>>( values, d_out, d_null_mask, incoming_status_view, has_incoming, - {static_cast(s_data.data()), static_cast(num_rows)}); + {d_status, static_cast(num_rows)}); CUDF_CUDA_TRY(cudaGetLastError()); - *status_out = make_status_column(std::move(s_data), num_rows); } else if (has_incoming) { - // No status_out requested, but incoming_status still needs to be applied to the null mask. - // Use HasStatus=true with an empty d_status span so writes are guarded (no allocation). + // No status output requested, but incoming_status still needs to be applied to the null + // mask. Use HasStatus=true with an empty d_status span so writes are guarded (no-op). cast_variant_primitive_kernel<<>>( values, d_out, d_null_mask, incoming_status_view, true, {}); CUDF_CUDA_TRY(cudaGetLastError()); @@ -989,12 +973,7 @@ struct cast_variant_fn { { rmm::device_buffer data{num_rows * sizeof(bool), stream, mr}; - rmm::device_buffer s_data; - op_status* dp_s{nullptr}; - if (status_out != nullptr) { - s_data = alloc_status(); - dp_s = static_cast(s_data.data()); - } + auto* dp_s = d_status; auto const inc_view = incoming_status_view; auto const hi = has_incoming; @@ -1035,8 +1014,6 @@ struct cast_variant_fn { } }); - if (status_out != nullptr) { *status_out = make_status_column(std::move(s_data), num_rows); } - auto const null_count = num_rows - cudf::detail::count_set_bits(d_null_mask, 0, num_rows, stream); return std::make_unique(desired_type, @@ -1050,23 +1027,11 @@ struct cast_variant_fn { std::unique_ptr operator()() requires(cuda::std::is_same_v) { - rmm::device_buffer status_data; - op_status* d_status_ptr{nullptr}; - - if (status_out != nullptr) { - status_data = alloc_status(); - d_status_ptr = static_cast(status_data.data()); - } - cast_variant_string_fn fn{ - values, d_null_mask, nullptr, nullptr, {}, d_status_ptr, incoming_status_view, has_incoming}; + values, d_null_mask, nullptr, nullptr, {}, d_status, incoming_status_view, has_incoming}; auto [offsets_column, chars] = cudf::strings::detail::make_strings_children(fn, num_rows, stream, mr); - if (status_out != nullptr) { - *status_out = make_status_column(std::move(status_data), num_rows); - } - auto const null_count = num_rows - cudf::detail::count_set_bits(d_null_mask, 0, num_rows, stream); return make_strings_column(num_rows, @@ -1117,7 +1082,7 @@ namespace detail { std::unique_ptr get_variant_field(column_view const& variant_column, std::string_view path, - std::unique_ptr* status_out, + std::optional status, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) { @@ -1134,8 +1099,19 @@ std::unique_ptr get_variant_field(column_view const& variant_column, auto const steps = parse_variant_path(path); auto const num_rows = variant_column.size(); + + if (status.has_value()) { + CUDF_EXPECTS(!status->nullable(), + "status column must not be nullable; use row_null for SQL-null rows", + std::invalid_argument); + CUDF_EXPECTS( + status->type().id() == type_id::UINT8, "status column must be UINT8", std::invalid_argument); + CUDF_EXPECTS(status->size() == num_rows, + "status column must have the same number of rows as variant_column", + std::invalid_argument); + } + if (num_rows == 0) { - if (status_out != nullptr) { *status_out = make_empty_column(data_type{type_id::UINT8}); } return cudf::make_lists_column( 0, make_empty_column(type_id::INT32), make_empty_column(type_id::UINT8), 0, {}); } @@ -1166,8 +1142,7 @@ std::unique_ptr get_variant_field(column_view const& variant_column, auto grid = cudf::detail::grid_1d{num_rows, block_size}; - if (status_out != nullptr) { - rmm::device_buffer status_data{num_rows * sizeof(op_status), stream, mr}; + if (status.has_value()) { locate_variant_fields_kernel<<>>( meta_lists_device_view, val_lists_device_view, @@ -1175,9 +1150,8 @@ std::unique_ptr get_variant_field(column_view const& variant_column, d_sizes, d_src_offsets, d_null_mask, - {static_cast(status_data.data()), static_cast(num_rows)}); + {reinterpret_cast(status->data()), static_cast(num_rows)}); CUDF_CUDA_TRY(cudaGetLastError()); - *status_out = make_status_column(std::move(status_data), num_rows); } else { locate_variant_fields_kernel <<>>(meta_lists_device_view, @@ -1229,7 +1203,7 @@ std::unique_ptr get_variant_field(column_view const& variant_column, std::unique_ptr cast_variant(column_view const& values, data_type desired_type, std::optional incoming_status, - std::unique_ptr* status_out, + std::optional status, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) { @@ -1263,11 +1237,19 @@ std::unique_ptr cast_variant(column_view const& values, std::invalid_argument); } - if (num_rows == 0) { - if (status_out != nullptr) { *status_out = make_empty_column(data_type{type_id::UINT8}); } - return make_empty_column(desired_type); + if (status.has_value()) { + CUDF_EXPECTS(!status->nullable(), + "status column must not be nullable; use row_null for SQL-null rows", + std::invalid_argument); + CUDF_EXPECTS( + status->type().id() == type_id::UINT8, "status column must be UINT8", std::invalid_argument); + CUDF_EXPECTS(status->size() == num_rows, + "status column must have the same number of rows as the values column", + std::invalid_argument); } + if (num_rows == 0) { return make_empty_column(desired_type); } + auto val_device_view = column_device_view::create(values, stream); cudf::lists_column_device_view val_lists_device_view(*val_device_view); @@ -1284,61 +1266,65 @@ std::unique_ptr cast_variant(column_view const& values, : column_device_view::create(*placeholder_col, stream); bool const has_incoming = incoming_status.has_value(); - return cudf::type_dispatcher(desired_type, - cast_variant_fn{val_lists_device_view, - num_rows, - desired_type, - d_null_mask, - std::move(null_mask), - stream, - mr, - *incoming_dev_view, - has_incoming, - status_out}); + return cudf::type_dispatcher( + desired_type, + cast_variant_fn{ + val_lists_device_view, + num_rows, + desired_type, + d_null_mask, + std::move(null_mask), + stream, + mr, + *incoming_dev_view, + has_incoming, + status.has_value() ? reinterpret_cast(status->data()) : nullptr}); } } // namespace detail std::unique_ptr get_variant_field(column_view const& variant_column, std::string_view path, - std::unique_ptr* status_out, + std::optional status, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) { CUDF_FUNC_RANGE(); - return detail::get_variant_field(variant_column, path, status_out, stream, mr); + return detail::get_variant_field(variant_column, path, status, stream, mr); } std::unique_ptr cast_variant(column_view const& values, data_type desired_type, std::optional incoming_status, - std::unique_ptr* status_out, + std::optional status, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) { CUDF_FUNC_RANGE(); - return detail::cast_variant(values, desired_type, incoming_status, status_out, stream, mr); + return detail::cast_variant(values, desired_type, incoming_status, status, stream, mr); } std::unique_ptr extract_variant_field(column_view const& variant_column, std::string_view path, data_type desired_type, - std::unique_ptr* status_out, + std::optional status, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) { CUDF_FUNC_RANGE(); auto const temp_mr = cudf::get_current_device_resource_ref(); - if (status_out != nullptr) { - std::unique_ptr extract_status; - auto value = detail::get_variant_field(variant_column, path, &extract_status, stream, temp_mr); + if (status.has_value()) { + auto extract_status = make_numeric_column( + data_type{type_id::UINT8}, variant_column.size(), mask_state::UNALLOCATED, stream, temp_mr); + auto value = detail::get_variant_field( + variant_column, path, extract_status->mutable_view(), stream, temp_mr); return detail::cast_variant( - value->view(), desired_type, extract_status->view(), status_out, stream, mr); + value->view(), desired_type, extract_status->view(), status, stream, mr); } - auto value = detail::get_variant_field(variant_column, path, nullptr, stream, temp_mr); - return detail::cast_variant(value->view(), desired_type, std::nullopt, nullptr, stream, mr); + auto value = detail::get_variant_field(variant_column, path, std::nullopt, stream, temp_mr); + return detail::cast_variant(value->view(), desired_type, std::nullopt, std::nullopt, stream, mr); } } // namespace io::parquet::experimental diff --git a/cpp/tests/io/experimental/variant_extract_test.cpp b/cpp/tests/io/experimental/variant_extract_test.cpp index cdf6d9cdcc9b..c9739f322f11 100644 --- a/cpp/tests/io/experimental/variant_extract_test.cpp +++ b/cpp/tests/io/experimental/variant_extract_test.cpp @@ -154,8 +154,12 @@ TEST_F(ExtractVariantFieldTest, NullStructRow) // Use the validity vector to mask the second row null. cudf::test::structs_column_wrapper col{{meta, val}, std::vector{true, false}}; - auto got = cudf::io::parquet::experimental::extract_variant_field( - col, "x", cudf::data_type{cudf::type_id::INT32}, nullptr, cudf::test::get_default_stream()); + auto got = + cudf::io::parquet::experimental::extract_variant_field(col, + "x", + cudf::data_type{cudf::type_id::INT32}, + std::nullopt, + cudf::test::get_default_stream()); cudf::test::fixed_width_column_wrapper expected({7, 0}, {true, false}); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); @@ -168,8 +172,12 @@ TEST_F(ExtractVariantFieldTest, NonObjectValueYieldsNull) std::vector const valb = {0x14, 0x07, 0x00, 0x00, 0x00}; auto col = wrap_single_variant(metab, valb); - auto got = cudf::io::parquet::experimental::extract_variant_field( - col, "x", cudf::data_type{cudf::type_id::INT32}, nullptr, cudf::test::get_default_stream()); + auto got = + cudf::io::parquet::experimental::extract_variant_field(col, + "x", + cudf::data_type{cudf::type_id::INT32}, + std::nullopt, + cudf::test::get_default_stream()); cudf::test::fixed_width_column_wrapper expected({0}, {false}); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); @@ -182,8 +190,12 @@ TEST_F(ExtractVariantFieldTest, InvalidMetadataYieldsNull) std::vector const valb = {0x02, 0x01, 0x00, 0x00, 0x05, 0x14, 0x07, 0x00, 0x00, 0x00}; auto col = wrap_single_variant(metab, valb); - auto got = cudf::io::parquet::experimental::extract_variant_field( - col, "x", cudf::data_type{cudf::type_id::INT32}, nullptr, cudf::test::get_default_stream()); + auto got = + cudf::io::parquet::experimental::extract_variant_field(col, + "x", + cudf::data_type{cudf::type_id::INT32}, + std::nullopt, + cudf::test::get_default_stream()); cudf::test::fixed_width_column_wrapper expected({0}, {false}); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); @@ -196,8 +208,12 @@ TEST_F(ExtractVariantFieldTest, UnsupportedMetadataVersionYieldsNull) std::vector const valb = {0x02, 0x01, 0x00, 0x00, 0x05, 0x14, 0x07, 0x00, 0x00, 0x00}; auto col = wrap_single_variant(metab, valb); - auto got = cudf::io::parquet::experimental::extract_variant_field( - col, "x", cudf::data_type{cudf::type_id::INT32}, nullptr, cudf::test::get_default_stream()); + auto got = + cudf::io::parquet::experimental::extract_variant_field(col, + "x", + cudf::data_type{cudf::type_id::INT32}, + std::nullopt, + cudf::test::get_default_stream()); cudf::test::fixed_width_column_wrapper expected({0}, {false}); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); @@ -210,8 +226,12 @@ TEST_F(ExtractVariantFieldTest, TruncatedObjectValueYieldsNull) std::vector const valb = {0x02}; auto col = wrap_single_variant(metab, valb); - auto got = cudf::io::parquet::experimental::extract_variant_field( - col, "x", cudf::data_type{cudf::type_id::INT32}, nullptr, cudf::test::get_default_stream()); + auto got = + cudf::io::parquet::experimental::extract_variant_field(col, + "x", + cudf::data_type{cudf::type_id::INT32}, + std::nullopt, + cudf::test::get_default_stream()); cudf::test::fixed_width_column_wrapper expected({0}, {false}); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); @@ -222,17 +242,17 @@ TEST_F(ExtractVariantFieldTest, MultiRow) auto col = make_xyz_three_row_variant(); auto stream = cudf::test::get_default_stream(); auto x = cudf::io::parquet::experimental::extract_variant_field( - col, "x", cudf::data_type{cudf::type_id::INT32}, nullptr, stream); + col, "x", cudf::data_type{cudf::type_id::INT32}, std::nullopt, stream); cudf::test::fixed_width_column_wrapper x_exp({7, 42, 0}, {true, true, false}); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*x, x_exp); auto y = cudf::io::parquet::experimental::extract_variant_field( - col, "y", cudf::data_type{cudf::type_id::STRING}, nullptr, stream); + col, "y", cudf::data_type{cudf::type_id::STRING}, std::nullopt, stream); cudf::test::strings_column_wrapper y_exp({"hi", "", "zzz"}, {true, false, true}); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*y, y_exp); auto z = cudf::io::parquet::experimental::extract_variant_field( - col, "z", cudf::data_type{cudf::type_id::INT32}, nullptr, stream); + col, "z", cudf::data_type{cudf::type_id::INT32}, std::nullopt, stream); cudf::test::fixed_width_column_wrapper z_exp({0, 99, 0}, {false, true, false}); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*z, z_exp); } @@ -243,8 +263,12 @@ TEST_F(ExtractVariantFieldTest, SlicedInput) auto const col = make_xyz_three_row_variant(); auto const sliced = cudf::slice(col, {1, 3}).front(); - auto got = cudf::io::parquet::experimental::extract_variant_field( - sliced, "x", cudf::data_type{cudf::type_id::INT32}, nullptr, cudf::test::get_default_stream()); + auto got = + cudf::io::parquet::experimental::extract_variant_field(sliced, + "x", + cudf::data_type{cudf::type_id::INT32}, + std::nullopt, + cudf::test::get_default_stream()); cudf::test::fixed_width_column_wrapper expected({42, 0}, {true, false}); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); @@ -261,7 +285,7 @@ TEST_F(ExtractVariantFieldTest, ApacheObjectPrimitiveStringFields) std::pair{"timestamp_field", "2025-04-16T12:34:56.78"}}) { SCOPED_TRACE(std::string{"field: "} + field); auto got = - cudf::io::parquet::experimental::extract_variant_field(col, field, s, nullptr, stream); + cudf::io::parquet::experimental::extract_variant_field(col, field, s, std::nullopt, stream); cudf::test::strings_column_wrapper expected({expected_str}); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); } @@ -276,7 +300,7 @@ TEST_F(ExtractVariantFieldTest, ApacheObjectPrimitiveNullCases) for (auto const& field : {"no_such_field", "null_field"}) { SCOPED_TRACE(std::string{"field: "} + field); auto got = - cudf::io::parquet::experimental::extract_variant_field(col, field, s, nullptr, stream); + cudf::io::parquet::experimental::extract_variant_field(col, field, s, std::nullopt, stream); ASSERT_EQ(got->size(), 1); EXPECT_EQ(got->null_count(), 1); } @@ -289,7 +313,7 @@ TEST_F(ExtractVariantFieldTest, ApacheObjectPrimitiveIntField) cudf::io::parquet::experimental::extract_variant_field(col, "int_field", cudf::data_type{cudf::type_id::INT8}, - nullptr, + std::nullopt, cudf::test::get_default_stream()); cudf::test::fixed_width_column_wrapper expected{int8_t{1}}; CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); @@ -304,12 +328,12 @@ TEST_F(ExtractVariantFieldTest, ApacheObjectNested) SCOPED_TRACE(std::string{"path: "} + path); if constexpr (std::is_same_v) { auto got = cudf::io::parquet::experimental::extract_variant_field( - col, path, cudf::data_type{cudf::type_id::STRING}, nullptr, stream); + col, path, cudf::data_type{cudf::type_id::STRING}, std::nullopt, stream); cudf::test::strings_column_wrapper expected({expected_val}); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); } else { auto got = cudf::io::parquet::experimental::extract_variant_field( - col, path, cudf::data_type{cudf::type_to_id()}, nullptr, stream); + col, path, cudf::data_type{cudf::type_to_id()}, std::nullopt, stream); cudf::test::fixed_width_column_wrapper expected{expected_val}; CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); } @@ -326,8 +350,12 @@ TEST_F(ExtractVariantFieldTest, ApacheObjectNested) TEST_F(ExtractVariantFieldTest, ApacheObjectEmpty) { auto col = make_apache_variant(avf::object_empty); - auto got = cudf::io::parquet::experimental::extract_variant_field( - col, "foo", cudf::data_type{cudf::type_id::STRING}, nullptr, cudf::test::get_default_stream()); + auto got = + cudf::io::parquet::experimental::extract_variant_field(col, + "foo", + cudf::data_type{cudf::type_id::STRING}, + std::nullopt, + cudf::test::get_default_stream()); ASSERT_EQ(got->size(), 1); EXPECT_EQ(got->null_count(), 1); } @@ -338,15 +366,15 @@ TEST_F(ExtractVariantFieldTest, ApacheObjectNestedChainedCalls) auto stream = cudf::test::get_default_stream(); auto single = cudf::io::parquet::experimental::get_variant_field( - col, "$.observation.value.temperature", nullptr, stream); + col, "$.observation.value.temperature", std::nullopt, stream); auto const meta_v = cudf::structs_column_view{col}.get_sliced_child(0, stream); auto obs = - cudf::io::parquet::experimental::get_variant_field(col, "observation", nullptr, stream); + cudf::io::parquet::experimental::get_variant_field(col, "observation", std::nullopt, stream); auto vobj = cudf::io::parquet::experimental::get_variant_field( - wrap_variant_view(meta_v, obs->view()), "value", nullptr, stream); + wrap_variant_view(meta_v, obs->view()), "value", std::nullopt, stream); auto chained = cudf::io::parquet::experimental::get_variant_field( - wrap_variant_view(meta_v, vobj->view()), "temperature", nullptr, stream); + wrap_variant_view(meta_v, vobj->view()), "temperature", std::nullopt, stream); EXPECT_EQ(single->type().id(), cudf::type_id::LIST); EXPECT_EQ(chained->type().id(), cudf::type_id::LIST); @@ -359,7 +387,7 @@ TEST_F(ExtractVariantFieldTest, ApacheObjectNestedMissingIntermediate) auto stream = cudf::test::get_default_stream(); auto got = cudf::io::parquet::experimental::extract_variant_field( - col, "$.species.nope", cudf::data_type{cudf::type_id::STRING}, nullptr, stream); + col, "$.species.nope", cudf::data_type{cudf::type_id::STRING}, std::nullopt, stream); cudf::test::strings_column_wrapper expected({"donotread"}, {false}); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); @@ -374,8 +402,12 @@ TEST_F(ExtractVariantFieldTest, NestedPathNonObjectIntermediate) auto col = wrap_single_variant(metab, valb); // Descending into "a" fails because it is a primitive, not an object. - auto got = cudf::io::parquet::experimental::extract_variant_field( - col, "$.a.b", cudf::data_type{cudf::type_id::INT32}, nullptr, cudf::test::get_default_stream()); + auto got = + cudf::io::parquet::experimental::extract_variant_field(col, + "$.a.b", + cudf::data_type{cudf::type_id::INT32}, + std::nullopt, + cudf::test::get_default_stream()); cudf::test::fixed_width_column_wrapper expected({0}, {false}); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); @@ -386,8 +418,9 @@ TEST_F(ExtractVariantFieldTest, BareNameEqualsDollarPath) auto col = make_xyz_three_row_variant(); auto stream = cudf::test::get_default_stream(); - auto bare = cudf::io::parquet::experimental::get_variant_field(col, "x", nullptr, stream); - auto dollar = cudf::io::parquet::experimental::get_variant_field(col, "$.x", nullptr, stream); + auto bare = cudf::io::parquet::experimental::get_variant_field(col, "x", std::nullopt, stream); + auto dollar = + cudf::io::parquet::experimental::get_variant_field(col, "$.x", std::nullopt, stream); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*bare, *dollar); } @@ -606,7 +639,7 @@ TEST_F(ExtractVariantFieldTest, NestedPathMultiRowMixedNulls) cudf::io::parquet::experimental::extract_variant_field(col, "$.1st.foo-bar", cudf::data_type{cudf::type_id::INT32}, - nullptr, + std::nullopt, cudf::test::get_default_stream()); cudf::test::fixed_width_column_wrapper expected({1, 0, 0}, {true, false, false}); @@ -617,14 +650,14 @@ TEST_F(ExtractVariantFieldTest, EmptyPathRejected) { auto col = wrap_single_variant(build_metadata({}), enc_int32(1)); auto stream = cudf::test::get_default_stream(); - EXPECT_THROW( - static_cast(cudf::io::parquet::experimental::get_variant_field(col, "", nullptr, stream)), - std::invalid_argument); EXPECT_THROW(static_cast( - cudf::io::parquet::experimental::get_variant_field(col, "$", nullptr, stream)), + cudf::io::parquet::experimental::get_variant_field(col, "", std::nullopt, stream)), + std::invalid_argument); + EXPECT_THROW(static_cast(cudf::io::parquet::experimental::get_variant_field( + col, "$", std::nullopt, stream)), std::invalid_argument); EXPECT_THROW(static_cast(cudf::io::parquet::experimental::extract_variant_field( - col, "", cudf::data_type{cudf::type_id::INT32}, nullptr, stream)), + col, "", cudf::data_type{cudf::type_id::INT32}, std::nullopt, stream)), std::invalid_argument); } @@ -646,8 +679,8 @@ TEST_F(ExtractVariantFieldTest, SyntaxErrors) "$.a[01x]", "$.a[1", "$.a[99999999999999999999]"}) { - EXPECT_THROW(static_cast( - cudf::io::parquet::experimental::get_variant_field(col, bad, nullptr, stream)), + EXPECT_THROW(static_cast(cudf::io::parquet::experimental::get_variant_field( + col, bad, std::nullopt, stream)), std::invalid_argument) << "path that should have thrown: " << bad; } @@ -660,7 +693,8 @@ TEST_F(ExtractVariantFieldTest, ApacheArrayPrimitiveIndexing) auto stream = cudf::test::get_default_stream(); auto const i8 = cudf::data_type{cudf::type_id::INT8}; auto const get = [&](char const* path) { - return cudf::io::parquet::experimental::extract_variant_field(col, path, i8, nullptr, stream); + return cudf::io::parquet::experimental::extract_variant_field( + col, path, i8, std::nullopt, stream); }; CUDF_TEST_EXPECT_COLUMNS_EQUAL(*get("$[0]"), @@ -690,8 +724,8 @@ TEST_F(ExtractVariantFieldTest, ApacheArrayPrimitiveIndexing) value.insert(value.end(), {0x0c, 42}); // INT8(42) auto wide_col = wrap_single_variant(build_metadata({}), value); - auto got = - cudf::io::parquet::experimental::extract_variant_field(wide_col, "$[0]", i8, nullptr, stream); + auto got = cudf::io::parquet::experimental::extract_variant_field( + wide_col, "$[0]", i8, std::nullopt, stream); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, cudf::test::fixed_width_column_wrapper{int8_t{42}}); } @@ -708,12 +742,12 @@ TEST_F(ExtractVariantFieldTest, ArrayIndexingTypeMismatchAndBounds) // Object-key descent into an array value: no such key -> null. auto key_on_array = - cudf::io::parquet::experimental::extract_variant_field(col, "$.foo", i8, nullptr, stream); + cudf::io::parquet::experimental::extract_variant_field(col, "$.foo", i8, std::nullopt, stream); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*key_on_array, null_expected); // Index step against a primitive element (after first descending into it): non-array -> null. - auto index_on_primitive = - cudf::io::parquet::experimental::extract_variant_field(col, "$[0][0]", i8, nullptr, stream); + auto index_on_primitive = cudf::io::parquet::experimental::extract_variant_field( + col, "$[0][0]", i8, std::nullopt, stream); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*index_on_primitive, null_expected); } @@ -727,7 +761,7 @@ TEST_F(ExtractVariantFieldTest, EmptyArrayIndexing) for (auto const* path : {"$[0]", "$[1]"}) { SCOPED_TRACE(std::string{"path: "} + path); auto got = - cudf::io::parquet::experimental::extract_variant_field(col, path, i8, nullptr, stream); + cudf::io::parquet::experimental::extract_variant_field(col, path, i8, std::nullopt, stream); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, null_expected); } @@ -750,7 +784,7 @@ TEST_F(ExtractVariantFieldTest, EmptyArrayIndexing) {0x03, 0x01, 0x01, 0x02, 0x0c, 0x2a}}) { auto malformed_col = wrap_single_variant(build_metadata({}), value); auto got = cudf::io::parquet::experimental::extract_variant_field( - malformed_col, "$[0]", i8, nullptr, stream); + malformed_col, "$[0]", i8, std::nullopt, stream); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, null_expected); } } @@ -767,14 +801,14 @@ TEST_F(ExtractVariantFieldTest, MixedObjectArrayTraversal) auto const check_str = [&](char const* path, char const* expected) { SCOPED_TRACE(std::string{"path: "} + path); auto got = cudf::io::parquet::experimental::extract_variant_field( - col, path, cudf::data_type{cudf::type_id::STRING}, nullptr, stream); + col, path, cudf::data_type{cudf::type_id::STRING}, std::nullopt, stream); cudf::test::strings_column_wrapper const expected_col({expected}); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected_col); }; auto const check_null = [&](char const* path) { SCOPED_TRACE(std::string{"path: "} + path); auto got = cudf::io::parquet::experimental::extract_variant_field( - col, path, cudf::data_type{cudf::type_id::STRING}, nullptr, stream); + col, path, cudf::data_type{cudf::type_id::STRING}, std::nullopt, stream); cudf::test::strings_column_wrapper const null_col({""}, {false}); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, null_col); }; @@ -802,11 +836,11 @@ TEST_F(ExtractVariantFieldTest, LargeDictionaryAndObjectScan) // First, middle, and last keys each decode to their own field id. auto first = cudf::io::parquet::experimental::extract_variant_field( - col, "k00", int32_dtype, nullptr, stream); + col, "k00", int32_dtype, std::nullopt, stream); auto mid = cudf::io::parquet::experimental::extract_variant_field( - col, "k24", int32_dtype, nullptr, stream); + col, "k24", int32_dtype, std::nullopt, stream); auto last = cudf::io::parquet::experimental::extract_variant_field( - col, "k49", int32_dtype, nullptr, stream); + col, "k49", int32_dtype, std::nullopt, stream); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*first, cudf::test::fixed_width_column_wrapper{0}); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*mid, cudf::test::fixed_width_column_wrapper{24}); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*last, cudf::test::fixed_width_column_wrapper{49}); @@ -852,7 +886,7 @@ TEST_F(ExtractVariantFieldTest, MalformedVariantDataYieldsNull) SCOPED_TRACE(c.label); auto col = wrap_single_variant(c.meta, c.val); auto got = cudf::io::parquet::experimental::extract_variant_field( - col, "x", int32_dtype, nullptr, stream); + col, "x", int32_dtype, std::nullopt, stream); ASSERT_EQ(got->size(), 1); EXPECT_EQ(got->null_count(), 1); } @@ -905,7 +939,7 @@ TEST_F(ExtractVariantFieldTest, NullsAtDifferentDepths) cudf::io::parquet::experimental::extract_variant_field(col, "$.a.b.c.d", cudf::data_type{cudf::type_id::STRING}, - nullptr, + std::nullopt, cudf::test::get_default_stream()); cudf::test::strings_column_wrapper expected(exp_strs.begin(), exp_strs.end(), exp_valid.begin()); @@ -918,7 +952,7 @@ TEST_F(ExtractVariantFieldTest, EmptyInput) auto const variant = cudf::empty_like(make_xyz_three_row_variant()); auto got = cudf::io::parquet::experimental::extract_variant_field( - *variant, "x", cudf::data_type{cudf::type_id::INT32}, nullptr, stream); + *variant, "x", cudf::data_type{cudf::type_id::INT32}, std::nullopt, stream); EXPECT_EQ(got->type().id(), cudf::type_id::INT32); EXPECT_EQ(got->size(), 0); EXPECT_EQ(got->null_count(), 0); @@ -931,14 +965,15 @@ TEST_F(GetVariantFieldTest, ApacheObjectPrimitive) auto col = make_apache_variant(avf::object_primitive); auto stream = cudf::test::get_default_stream(); - auto got = cudf::io::parquet::experimental::get_variant_field(col, "int_field", nullptr, stream); + auto got = + cudf::io::parquet::experimental::get_variant_field(col, "int_field", std::nullopt, stream); EXPECT_EQ(got->type().id(), cudf::type_id::LIST); EXPECT_EQ(got->size(), 1); EXPECT_EQ(cudf::lists_column_view{got->view()}.child().type().id(), cudf::type_id::UINT8); auto casted = cudf::io::parquet::experimental::cast_variant( - got->view(), cudf::data_type{cudf::type_id::INT8}, {}, nullptr, stream); + got->view(), cudf::data_type{cudf::type_id::INT8}, {}, std::nullopt, stream); cudf::test::fixed_width_column_wrapper expected{int8_t{1}}; CUDF_TEST_EXPECT_COLUMNS_EQUAL(*casted, expected); } @@ -947,7 +982,7 @@ TEST_F(GetVariantFieldTest, ApacheObjectPrimitiveMissingKeyAllNull) { auto col = make_apache_variant(avf::object_primitive); auto got = cudf::io::parquet::experimental::get_variant_field( - col, "no_such_field", nullptr, cudf::test::get_default_stream()); + col, "no_such_field", std::nullopt, cudf::test::get_default_stream()); EXPECT_EQ(got->type().id(), cudf::type_id::LIST); EXPECT_EQ(got->size(), 1); @@ -960,11 +995,12 @@ TEST_F(GetVariantFieldTest, GetAndCastMatchesExtract) auto stream = cudf::test::get_default_stream(); auto extract_x = cudf::io::parquet::experimental::extract_variant_field( - col, "x", cudf::data_type{cudf::type_id::INT32}, nullptr, stream); + col, "x", cudf::data_type{cudf::type_id::INT32}, std::nullopt, stream); - auto intermediate = cudf::io::parquet::experimental::get_variant_field(col, "x", nullptr, stream); - auto two_step_x = cudf::io::parquet::experimental::cast_variant( - intermediate->view(), cudf::data_type{cudf::type_id::INT32}, {}, nullptr, stream); + auto intermediate = + cudf::io::parquet::experimental::get_variant_field(col, "x", std::nullopt, stream); + auto two_step_x = cudf::io::parquet::experimental::cast_variant( + intermediate->view(), cudf::data_type{cudf::type_id::INT32}, {}, std::nullopt, stream); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*extract_x, *two_step_x); } @@ -974,7 +1010,8 @@ TEST_F(GetVariantFieldTest, EmptyInput) auto const stream = cudf::test::get_default_stream(); auto const variant = cudf::empty_like(make_xyz_three_row_variant()); - auto got = cudf::io::parquet::experimental::get_variant_field(*variant, "x", nullptr, stream); + auto got = + cudf::io::parquet::experimental::get_variant_field(*variant, "x", std::nullopt, stream); EXPECT_EQ(got->type().id(), cudf::type_id::LIST); EXPECT_EQ(got->size(), 0); EXPECT_EQ(got->null_count(), 0); @@ -988,7 +1025,7 @@ std::unique_ptr cast_apache_primitive(avf::fixture const& fi auto col = make_apache_variant(fixture); auto const value = cudf::structs_column_view{col}.get_sliced_child(1, stream); return cudf::io::parquet::experimental::cast_variant( - value, cudf::data_type{cudf::type_to_id()}, {}, nullptr, stream); + value, cudf::data_type{cudf::type_to_id()}, {}, std::nullopt, stream); } struct CastVariantTest : public cudf::test::BaseFixture {}; @@ -1025,7 +1062,7 @@ TEST_F(CastVariantTest, ApachePrimitiveFloats) auto col = make_apache_variant(fixture); auto const value = cudf::structs_column_view{col}.get_sliced_child(1, stream); auto got = cudf::io::parquet::experimental::cast_variant( - value, cudf::data_type{cudf::type_to_id()}, {}, nullptr, stream); + value, cudf::data_type{cudf::type_to_id()}, {}, std::nullopt, stream); cudf::test::fixed_width_column_wrapper expected{expected_val}; CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); }; @@ -1041,7 +1078,7 @@ TEST_F(CastVariantTest, ApachePrimitiveBooleans) auto col = make_apache_variant(fixture); auto const value = cudf::structs_column_view{col}.get_sliced_child(1, stream); auto got = cudf::io::parquet::experimental::cast_variant( - value, cudf::data_type{cudf::type_id::BOOL8}, {}, nullptr, stream); + value, cudf::data_type{cudf::type_id::BOOL8}, {}, std::nullopt, stream); cudf::test::fixed_width_column_wrapper expected{expected_val}; CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); }; @@ -1054,7 +1091,7 @@ TEST_F(CastVariantTest, ApachePrimitiveBooleans) auto col = make_apache_variant(avf::primitive_null); auto const value = cudf::structs_column_view{col}.get_sliced_child(1, stream); auto got = cudf::io::parquet::experimental::cast_variant( - value, cudf::data_type{cudf::type_id::BOOL8}, {}, nullptr, stream); + value, cudf::data_type{cudf::type_id::BOOL8}, {}, std::nullopt, stream); cudf::test::fixed_width_column_wrapper expected({false}, {false}); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); } @@ -1102,7 +1139,7 @@ TEST_F(CastVariantTest, ApachePrimitiveBooleans) auto const sliced = cudf::slice(col, {slice_beg, slice_end}).front(); auto const value = cudf::structs_column_view{sliced}.get_sliced_child(1, stream); auto got = cudf::io::parquet::experimental::cast_variant( - value, cudf::data_type{cudf::type_id::BOOL8}, {}, nullptr, stream); + value, cudf::data_type{cudf::type_id::BOOL8}, {}, std::nullopt, stream); cudf::test::fixed_width_column_wrapper expected( exp_vals.begin() + slice_beg, exp_vals.begin() + slice_end, exp_valid.begin() + slice_beg); @@ -1117,7 +1154,7 @@ TEST_F(CastVariantTest, ApacheShortString) auto const value = cudf::structs_column_view{col}.get_sliced_child(1, stream); auto got = cudf::io::parquet::experimental::cast_variant( - value, cudf::data_type{cudf::type_id::STRING}, {}, nullptr, stream); + value, cudf::data_type{cudf::type_id::STRING}, {}, std::nullopt, stream); // Decoded from short_string.value: skip the 1-byte header, take the rest. std::string const expected_str(reinterpret_cast(avf::short_string.value.data() + 1), @@ -1133,7 +1170,7 @@ TEST_F(CastVariantTest, ApachePrimitiveString) auto const value = cudf::structs_column_view{col}.get_sliced_child(1, stream); auto got = cudf::io::parquet::experimental::cast_variant( - value, cudf::data_type{cudf::type_id::STRING}, {}, nullptr, stream); + value, cudf::data_type{cudf::type_id::STRING}, {}, std::nullopt, stream); // Long-string layout: 1 header byte + 4-byte LE length + payload. std::string const expected_str( @@ -1150,7 +1187,7 @@ TEST_F(CastVariantTest, MismatchedTypeYieldsNull) auto col = make_apache_variant(avf::object_primitive); auto const value = cudf::structs_column_view{col}.get_sliced_child(1, stream); auto got = cudf::io::parquet::experimental::cast_variant( - value, cudf::data_type{cudf::type_id::INT32}, {}, nullptr, stream); + value, cudf::data_type{cudf::type_id::INT32}, {}, std::nullopt, stream); ASSERT_EQ(got->size(), 1); EXPECT_EQ(got->null_count(), 1); } @@ -1167,7 +1204,7 @@ TEST_F(CastVariantTest, EmptyInput) cudf::type_id::FLOAT64, cudf::type_id::BOOL8}) { auto got = cudf::io::parquet::experimental::cast_variant( - *values, cudf::data_type{id}, {}, nullptr, stream); + *values, cudf::data_type{id}, {}, std::nullopt, stream); EXPECT_EQ(got->type().id(), id); EXPECT_EQ(got->size(), 0); EXPECT_EQ(got->null_count(), 0); @@ -1196,7 +1233,7 @@ TEST_F(CastVariantTest, UnsupportedTypeThrows) cudf::empty_like(cudf::structs_column_view{make_xyz_three_row_variant()}.child(1)); for (auto const id : ids) { EXPECT_THROW(static_cast(cudf::io::parquet::experimental::cast_variant( - *empty_values, cudf::data_type{id}, {}, nullptr, stream)), + *empty_values, cudf::data_type{id}, {}, std::nullopt, stream)), std::invalid_argument) << std::format("expected throw for type_id {} on empty input", static_cast(id)); } @@ -1206,7 +1243,7 @@ TEST_F(CastVariantTest, UnsupportedTypeThrows) auto const value = cudf::structs_column_view{col}.get_sliced_child(1, stream); for (auto const id : ids) { EXPECT_THROW(static_cast(cudf::io::parquet::experimental::cast_variant( - value, cudf::data_type{id}, {}, nullptr, stream)), + value, cudf::data_type{id}, {}, std::nullopt, stream)), std::invalid_argument) << std::format("expected throw for type_id {} on non-empty input", static_cast(id)); } @@ -1249,7 +1286,8 @@ TEST_F(CastVariantTest, CastSourceTargetMatrix) for (auto const& src : sources) { SCOPED_TRACE(std::string{"int target "} + match_label + ", source " + src.label); auto values = values_of(src.bytes); - auto got = cudf::io::parquet::experimental::cast_variant(values, target, {}, nullptr, stream); + auto got = + cudf::io::parquet::experimental::cast_variant(values, target, {}, std::nullopt, stream); if (std::string_view{src.label} == match_label) { cudf::test::fixed_width_column_wrapper const expected{match_value}; CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); @@ -1270,7 +1308,7 @@ TEST_F(CastVariantTest, CastSourceTargetMatrix) SCOPED_TRACE(std::string{"string target, source "} + src.label); auto values = values_of(src.bytes); auto got = - cudf::io::parquet::experimental::cast_variant(values, string_type, {}, nullptr, stream); + cudf::io::parquet::experimental::cast_variant(values, string_type, {}, std::nullopt, stream); std::string_view const label{src.label}; if (label == "short_string" || label == "long_string") { std::string const expected_str = (label == "short_string") ? "hi" : std::string(70, 'a'); @@ -1290,7 +1328,7 @@ TEST_F(CastVariantTest, ShortStringLengthZero) std::vector const val{make_variant_short_string_header(0)}; cudf::test::lists_column_wrapper values(val.begin(), val.end()); auto got = cudf::io::parquet::experimental::cast_variant( - values, cudf::data_type{cudf::type_id::STRING}, {}, nullptr, stream); + values, cudf::data_type{cudf::type_id::STRING}, {}, std::nullopt, stream); cudf::test::strings_column_wrapper expected({""}); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); } @@ -1305,7 +1343,7 @@ TEST_F(CastVariantTest, ShortStringMaxLength) val.insert(val.end(), payload.begin(), payload.end()); cudf::test::lists_column_wrapper values(val.begin(), val.end()); auto got = cudf::io::parquet::experimental::cast_variant( - values, cudf::data_type{cudf::type_id::STRING}, {}, nullptr, stream); + values, cudf::data_type{cudf::type_id::STRING}, {}, std::nullopt, stream); cudf::test::strings_column_wrapper expected({payload}); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); } @@ -1318,7 +1356,7 @@ TEST_F(CastVariantTest, LongStringLengthZero) make_variant_primitive(variant_primitive_type::LONG_STRING), 0x00, 0x00, 0x00, 0x00}; cudf::test::lists_column_wrapper values(val.begin(), val.end()); auto got = cudf::io::parquet::experimental::cast_variant( - values, cudf::data_type{cudf::type_id::STRING}, {}, nullptr, stream); + values, cudf::data_type{cudf::type_id::STRING}, {}, std::nullopt, stream); cudf::test::strings_column_wrapper expected({""}); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); } @@ -1338,7 +1376,7 @@ TEST_F(CastVariantTest, LongStringDeclaredLengthExceedsPayloadYieldsNull) SCOPED_TRACE(std::string{"payload bytes present: "} + std::to_string(val.size() - 5)); cudf::test::lists_column_wrapper values(val.begin(), val.end()); auto got = cudf::io::parquet::experimental::cast_variant( - values, cudf::data_type{cudf::type_id::STRING}, {}, nullptr, stream); + values, cudf::data_type{cudf::type_id::STRING}, {}, std::nullopt, stream); ASSERT_EQ(got->size(), 1); EXPECT_EQ(got->null_count(), 1); } @@ -1355,7 +1393,7 @@ TEST_F(CastVariantTest, LongStringPayloadExceedsDeclaredLength) hdr, 0x03, 0x00, 0x00, 0x00, 'a', 'b', 'c', 'x', 'x', 'x', 'x', 'x'}; cudf::test::lists_column_wrapper values(val.begin(), val.end()); auto got = cudf::io::parquet::experimental::cast_variant( - values, cudf::data_type{cudf::type_id::STRING}, {}, nullptr, stream); + values, cudf::data_type{cudf::type_id::STRING}, {}, std::nullopt, stream); cudf::test::strings_column_wrapper expected({"abc"}); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); } @@ -1432,7 +1470,7 @@ TEST_F(InvalidInputShapeTest, GetVariantFieldRejectsMalformedInput) for (auto const& c : cases) { SCOPED_TRACE(c.label); EXPECT_THROW(static_cast(cudf::io::parquet::experimental::get_variant_field( - c.column->view(), "x", nullptr, stream)), + c.column->view(), "x", std::nullopt, stream)), std::invalid_argument); } } @@ -1449,9 +1487,10 @@ TEST_F(InvalidInputShapeTest, CastVariantRejectsMalformedInput) for (auto const& c : cases) { SCOPED_TRACE(c.label); - EXPECT_THROW(static_cast(cudf::io::parquet::experimental::cast_variant( - c.column->view(), cudf::data_type{cudf::type_id::INT32}, {}, nullptr, stream)), - std::invalid_argument); + EXPECT_THROW( + static_cast(cudf::io::parquet::experimental::cast_variant( + c.column->view(), cudf::data_type{cudf::type_id::INT32}, {}, std::nullopt, stream)), + std::invalid_argument); } } @@ -1473,7 +1512,7 @@ TEST_F(InvalidInputShapeTest, CastVariantRejectsNullableIncomingStatus) auto const status_view = status_col->view(); EXPECT_THROW( static_cast(cudf::io::parquet::experimental::cast_variant( - values->view(), cudf::data_type{cudf::type_id::INT32}, status_view, nullptr, stream)), + values->view(), cudf::data_type{cudf::type_id::INT32}, status_view, std::nullopt, stream)), std::invalid_argument); } @@ -1497,7 +1536,7 @@ TEST_F(InvalidInputShapeTest, CastVariantRejectsInvalidIncomingStatusOnEmptyValu auto const status_view = status_col->view(); EXPECT_THROW( static_cast(cudf::io::parquet::experimental::cast_variant( - *empty_values, cudf::data_type{cudf::type_id::INT32}, status_view, nullptr, stream)), + *empty_values, cudf::data_type{cudf::type_id::INT32}, status_view, std::nullopt, stream)), std::invalid_argument) << "nullable incoming_status must be rejected even when values is empty"; } @@ -1509,7 +1548,7 @@ TEST_F(InvalidInputShapeTest, CastVariantRejectsInvalidIncomingStatusOnEmptyValu auto const status_view = status_col->view(); EXPECT_THROW( static_cast(cudf::io::parquet::experimental::cast_variant( - *empty_values, cudf::data_type{cudf::type_id::INT32}, status_view, nullptr, stream)), + *empty_values, cudf::data_type{cudf::type_id::INT32}, status_view, std::nullopt, stream)), std::invalid_argument) << "non-UINT8 incoming_status must be rejected even when values is empty"; } @@ -1521,7 +1560,7 @@ TEST_F(InvalidInputShapeTest, CastVariantRejectsInvalidIncomingStatusOnEmptyValu auto const status_view = status_col->view(); EXPECT_THROW( static_cast(cudf::io::parquet::experimental::cast_variant( - *empty_values, cudf::data_type{cudf::type_id::INT32}, status_view, nullptr, stream)), + *empty_values, cudf::data_type{cudf::type_id::INT32}, status_view, std::nullopt, stream)), std::invalid_argument) << "row-count-mismatched incoming_status must be rejected even when values is empty"; } @@ -1545,6 +1584,16 @@ static void expect_status_values(cudf::column_view const& status, CUDF_TEST_EXPECT_COLUMNS_EQUAL(status, exp); } +/** + * @brief Allocates a non-nullable UINT8 column of `num_rows` rows for callers to pass as the + * `status` output parameter of `get_variant_field`/`cast_variant`/`extract_variant_field`. + */ +static std::unique_ptr make_status_buffer(cudf::size_type num_rows) +{ + return cudf::make_numeric_column( + cudf::data_type{cudf::type_id::UINT8}, num_rows, cudf::mask_state::UNALLOCATED); +} + constexpr uint8_t ST_SUCCESS = static_cast(op_status::SUCCESS); constexpr uint8_t ST_ROW_NULL = static_cast(op_status::ROW_NULL); constexpr uint8_t ST_MISSING = static_cast(op_status::MISSING_PATH); @@ -1566,11 +1615,10 @@ TEST_F(GetVariantFieldStatusTest, SqlNullInputProducesRowNullStatus) cudf::test::structs_column_wrapper col{{meta, val}, std::vector{false}}; auto stream = cudf::test::get_default_stream(); - std::unique_ptr status; - auto got = cudf::io::parquet::experimental::get_variant_field(col, "x", &status, stream, cmr()); + auto status = make_status_buffer(cudf::column_view{col}.size()); + auto got = cudf::io::parquet::experimental::get_variant_field( + col, "x", status->mutable_view(), stream, cmr()); - ASSERT_NE(status, nullptr); - ASSERT_EQ(status->size(), 1); ASSERT_EQ(status->null_count(), 0); expect_status_values(*status, {ST_ROW_NULL}); ASSERT_EQ(got->null_count(), 1); @@ -1582,11 +1630,11 @@ TEST_F(GetVariantFieldStatusTest, SuccessStatus) auto col = make_xyz_three_row_variant(); auto stream = cudf::test::get_default_stream(); - std::unique_ptr status; - auto got = cudf::io::parquet::experimental::get_variant_field(col, "x", &status, stream, cmr()); + auto status = make_status_buffer(cudf::column_view{col}.size()); + auto got = cudf::io::parquet::experimental::get_variant_field( + col, "x", status->mutable_view(), stream, cmr()); // Row 0: x=INT32(7) → success; Row 1: x=INT32(42) → success; Row 2: no x → missing_path - ASSERT_NE(status, nullptr); expect_status_values(*status, {ST_SUCCESS, ST_SUCCESS, ST_MISSING}); // Output rows 0,1 valid; row 2 null EXPECT_EQ(got->null_count(), 1); @@ -1598,12 +1646,10 @@ TEST_F(GetVariantFieldStatusTest, MissingKeyProducesMissingPathStatus) auto col = make_apache_variant(avf::object_primitive); auto stream = cudf::test::get_default_stream(); - std::unique_ptr status; - auto got = cudf::io::parquet::experimental::get_variant_field( - col, "no_such_field", &status, stream, cmr()); + auto status = make_status_buffer(cudf::column_view{col}.size()); + auto got = cudf::io::parquet::experimental::get_variant_field( + col, "no_such_field", status->mutable_view(), stream, cmr()); - ASSERT_NE(status, nullptr); - ASSERT_EQ(status->size(), 1); expect_status_values(*status, {ST_MISSING}); EXPECT_EQ(got->null_count(), 1); } @@ -1618,12 +1664,10 @@ TEST_F(GetVariantFieldStatusTest, VariantNullPreservedWithStatus) auto col = wrap_single_variant(m, v); auto stream = cudf::test::get_default_stream(); - std::unique_ptr status; - auto got = - cudf::io::parquet::experimental::get_variant_field(col, "null_field", &status, stream, cmr()); + auto status = make_status_buffer(cudf::column_view{col}.size()); + auto got = cudf::io::parquet::experimental::get_variant_field( + col, "null_field", status->mutable_view(), stream, cmr()); - ASSERT_NE(status, nullptr); - ASSERT_EQ(status->size(), 1); expect_status_values(*status, {ST_VNULL}); // With status requested, the VARIANT null bytes are preserved (output is NOT SQL null) EXPECT_EQ(got->null_count(), 0); @@ -1642,7 +1686,8 @@ TEST_F(GetVariantFieldStatusTest, VariantNullReturnedAsBytesWithoutStatus) auto stream = cudf::test::get_default_stream(); // No status_out: get_variant_field returns the VARIANT null bytes as a non-null list row. - auto got = cudf::io::parquet::experimental::get_variant_field(col, "null_field", nullptr, stream); + auto got = + cudf::io::parquet::experimental::get_variant_field(col, "null_field", std::nullopt, stream); EXPECT_EQ(got->null_count(), 0); EXPECT_EQ(got->size(), 1); auto const null_bytes = enc_null(); @@ -1658,10 +1703,10 @@ TEST_F(GetVariantFieldStatusTest, MalformedMetadataProducesMalformedStatus) auto col = wrap_single_variant(bad_meta, val); auto stream = cudf::test::get_default_stream(); - std::unique_ptr status; - auto got = cudf::io::parquet::experimental::get_variant_field(col, "x", &status, stream, cmr()); + auto status = make_status_buffer(cudf::column_view{col}.size()); + auto got = cudf::io::parquet::experimental::get_variant_field( + col, "x", status->mutable_view(), stream, cmr()); - ASSERT_NE(status, nullptr); expect_status_values(*status, {ST_MALFORMED}); EXPECT_EQ(got->null_count(), 1); } @@ -1675,11 +1720,10 @@ TEST_F(GetVariantFieldStatusTest, VariantNullBeforeEndIsMissingPath) auto col = wrap_single_variant(m, v); auto stream = cudf::test::get_default_stream(); - std::unique_ptr status; - auto got = - cudf::io::parquet::experimental::get_variant_field(col, "$.a.b", &status, stream, cmr()); + auto status = make_status_buffer(cudf::column_view{col}.size()); + auto got = cudf::io::parquet::experimental::get_variant_field( + col, "$.a.b", status->mutable_view(), stream, cmr()); - ASSERT_NE(status, nullptr); expect_status_values(*status, {ST_MISSING}); EXPECT_EQ(got->null_count(), 1); } @@ -1710,11 +1754,10 @@ TEST_F(GetVariantFieldStatusTest, MixedRows) // Row 3 is SQL null cudf::test::structs_column_wrapper col{{meta, val}, std::vector{true, true, true, false}}; - std::unique_ptr status; - auto got = cudf::io::parquet::experimental::get_variant_field(col, "x", &status, stream, cmr()); + auto status = make_status_buffer(cudf::column_view{col}.size()); + auto got = cudf::io::parquet::experimental::get_variant_field( + col, "x", status->mutable_view(), stream, cmr()); - ASSERT_NE(status, nullptr); - ASSERT_EQ(status->size(), 4); ASSERT_EQ(status->null_count(), 0); expect_status_values(*status, {ST_SUCCESS, ST_VNULL, ST_MISSING, ST_ROW_NULL}); @@ -1728,11 +1771,10 @@ TEST_F(GetVariantFieldStatusTest, EmptyInput) auto const stream = cudf::test::get_default_stream(); auto const variant = cudf::empty_like(make_xyz_three_row_variant()); - std::unique_ptr status; - auto got = - cudf::io::parquet::experimental::get_variant_field(*variant, "x", &status, stream, cmr()); + auto status = make_status_buffer(variant->size()); + auto got = cudf::io::parquet::experimental::get_variant_field( + *variant, "x", status->mutable_view(), stream, cmr()); - ASSERT_NE(status, nullptr); EXPECT_EQ(status->size(), 0); EXPECT_EQ(got->size(), 0); } @@ -1757,11 +1799,10 @@ TEST_F(CastVariantStatusTest, SuccessProducesSuccessStatus) // Success → success status auto stream = cudf::test::get_default_stream(); auto values = make_value_col(enc_int32(42)); - std::unique_ptr status; - auto got = cudf::io::parquet::experimental::cast_variant( - values, cudf::data_type{cudf::type_id::INT32}, {}, &status, stream, cmr()); + auto status = make_status_buffer(cudf::column_view{values}.size()); + auto got = cudf::io::parquet::experimental::cast_variant( + values, cudf::data_type{cudf::type_id::INT32}, {}, status->mutable_view(), stream, cmr()); - ASSERT_NE(status, nullptr); expect_status_values(*status, {ST_SUCCESS}); cudf::test::fixed_width_column_wrapper expected{42}; CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); @@ -1772,11 +1813,10 @@ TEST_F(CastVariantStatusTest, VariantNullProducesVariantNullStatus) // VARIANT null → variant_null status auto stream = cudf::test::get_default_stream(); auto values = make_value_col(enc_null()); - std::unique_ptr status; - auto got = cudf::io::parquet::experimental::cast_variant( - values, cudf::data_type{cudf::type_id::INT32}, {}, &status, stream, cmr()); + auto status = make_status_buffer(cudf::column_view{values}.size()); + auto got = cudf::io::parquet::experimental::cast_variant( + values, cudf::data_type{cudf::type_id::INT32}, {}, status->mutable_view(), stream, cmr()); - ASSERT_NE(status, nullptr); expect_status_values(*status, {ST_VNULL}); EXPECT_EQ(got->null_count(), 1); } @@ -1786,11 +1826,10 @@ TEST_F(CastVariantStatusTest, TypeMismatchStatus) // Type mismatch → type_mismatch status auto stream = cudf::test::get_default_stream(); auto values = make_value_col(enc_int8(5)); // INT8 cast to INT32 target → mismatch - std::unique_ptr status; - auto got = cudf::io::parquet::experimental::cast_variant( - values, cudf::data_type{cudf::type_id::INT32}, {}, &status, stream, cmr()); + auto status = make_status_buffer(cudf::column_view{values}.size()); + auto got = cudf::io::parquet::experimental::cast_variant( + values, cudf::data_type{cudf::type_id::INT32}, {}, status->mutable_view(), stream, cmr()); - ASSERT_NE(status, nullptr); expect_status_values(*status, {ST_MISMATCH}); EXPECT_EQ(got->null_count(), 1); } @@ -1820,11 +1859,14 @@ TEST_F(CastVariantStatusTest, SqlNullInputProducesRowNullStatus) stream.synchronize(); values_col->set_null_mask(std::move(null_mask), 1); - std::unique_ptr status; - auto got = cudf::io::parquet::experimental::cast_variant( - values_col->view(), cudf::data_type{cudf::type_id::INT32}, {}, &status, stream, cmr()); + auto status = make_status_buffer(values_col->size()); + auto got = cudf::io::parquet::experimental::cast_variant(values_col->view(), + cudf::data_type{cudf::type_id::INT32}, + {}, + status->mutable_view(), + stream, + cmr()); - ASSERT_NE(status, nullptr); // Row 0: success; row 1: row_null (status column is always non-nullable) ASSERT_EQ(status->null_count(), 0); expect_status_values(*status, {ST_SUCCESS, ST_ROW_NULL}); @@ -1849,11 +1891,14 @@ TEST_F(CastVariantStatusTest, IncomingStatusPropagation) auto incoming_status_col = incoming_status_w.release(); auto const incoming_view1 = incoming_status_col->view(); - std::unique_ptr status; - auto got = cudf::io::parquet::experimental::cast_variant( - values, cudf::data_type{cudf::type_id::INT32}, incoming_view1, &status, stream, cmr()); + auto status = make_status_buffer(values.size()); + auto got = cudf::io::parquet::experimental::cast_variant(values, + cudf::data_type{cudf::type_id::INT32}, + incoming_view1, + status->mutable_view(), + stream, + cmr()); - ASSERT_NE(status, nullptr); // Row 0: success (decoded), Row 1: missing_path (propagated), Row 2: variant_null (propagated) expect_status_values(*status, {ST_SUCCESS, ST_MISSING, ST_VNULL}); cudf::test::fixed_width_column_wrapper expected({7, 0, 0}, {true, false, false}); @@ -1877,11 +1922,14 @@ TEST_F(CastVariantStatusTest, IncomingRowNullStatusProducesRowNullStatus) auto incoming_status_col2 = incoming_status_w2.release(); auto const incoming_view2 = incoming_status_col2->view(); - std::unique_ptr status; - auto got = cudf::io::parquet::experimental::cast_variant( - values, cudf::data_type{cudf::type_id::INT32}, incoming_view2, &status, stream, cmr()); + auto status = make_status_buffer(values.size()); + auto got = cudf::io::parquet::experimental::cast_variant(values, + cudf::data_type{cudf::type_id::INT32}, + incoming_view2, + status->mutable_view(), + stream, + cmr()); - ASSERT_NE(status, nullptr); ASSERT_EQ(status->null_count(), 0); expect_status_values(*status, {ST_SUCCESS, ST_ROW_NULL}); @@ -1900,11 +1948,10 @@ TEST_F(CastVariantStatusTest, BoolStatusTracking) wrap_multi_row_variant(std::vector>(3, build_metadata({})), val_rows); auto values = cudf::structs_column_view{col}.get_sliced_child(1, stream); - std::unique_ptr status; - auto got = cudf::io::parquet::experimental::cast_variant( - values, cudf::data_type{cudf::type_id::BOOL8}, {}, &status, stream, cmr()); + auto status = make_status_buffer(values.size()); + auto got = cudf::io::parquet::experimental::cast_variant( + values, cudf::data_type{cudf::type_id::BOOL8}, {}, status->mutable_view(), stream, cmr()); - ASSERT_NE(status, nullptr); expect_status_values(*status, {ST_SUCCESS, ST_VNULL, ST_MISMATCH}); cudf::test::fixed_width_column_wrapper expected({true, false, false}, {true, false, false}); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); @@ -1943,11 +1990,10 @@ TEST_F(CastVariantStatusTest, StringStatusTracking) wrap_multi_row_variant(std::vector>(6, build_metadata({})), val_rows); auto values = cudf::structs_column_view{col}.get_sliced_child(1, stream); - std::unique_ptr status; - auto got = cudf::io::parquet::experimental::cast_variant( - values, cudf::data_type{cudf::type_id::STRING}, {}, &status, stream, cmr()); + auto status = make_status_buffer(values.size()); + auto got = cudf::io::parquet::experimental::cast_variant( + values, cudf::data_type{cudf::type_id::STRING}, {}, status->mutable_view(), stream, cmr()); - ASSERT_NE(status, nullptr); expect_status_values( *status, {ST_SUCCESS, ST_VNULL, ST_MISMATCH, ST_MALFORMED, ST_MALFORMED, ST_MALFORMED}); EXPECT_EQ(got->null_count(), 5); // all but row 0 are null @@ -1959,11 +2005,10 @@ TEST_F(CastVariantStatusTest, EmptyInput) auto const stream = cudf::test::get_default_stream(); auto const values = cudf::empty_like(cudf::structs_column_view{make_xyz_three_row_variant()}.child(1)); - std::unique_ptr status; - auto got = cudf::io::parquet::experimental::cast_variant( - *values, cudf::data_type{cudf::type_id::INT32}, {}, &status, stream, cmr()); + auto status = make_status_buffer(values->size()); + auto got = cudf::io::parquet::experimental::cast_variant( + *values, cudf::data_type{cudf::type_id::INT32}, {}, status->mutable_view(), stream, cmr()); - ASSERT_NE(status, nullptr); EXPECT_EQ(status->size(), 0); EXPECT_EQ(got->size(), 0); } @@ -1980,11 +2025,10 @@ TEST_F(ExtractVariantFieldStatusTest, SuccessStatus) auto col = make_xyz_three_row_variant(); auto stream = cudf::test::get_default_stream(); - std::unique_ptr status; - auto got = cudf::io::parquet::experimental::extract_variant_field( - col, "x", cudf::data_type{cudf::type_id::INT32}, &status, stream, cmr()); + auto status = make_status_buffer(cudf::column_view{col}.size()); + auto got = cudf::io::parquet::experimental::extract_variant_field( + col, "x", cudf::data_type{cudf::type_id::INT32}, status->mutable_view(), stream, cmr()); - ASSERT_NE(status, nullptr); // Rows 0,1 have x as INT32 → success; row 2 has no x → missing_path expect_status_values(*status, {ST_SUCCESS, ST_SUCCESS, ST_MISSING}); cudf::test::fixed_width_column_wrapper expected({7, 42, 0}, {true, true, false}); @@ -1999,12 +2043,10 @@ TEST_F(ExtractVariantFieldStatusTest, SqlNullInputProducesRowNullStatus) cudf::test::structs_column_wrapper col{{meta, val}, std::vector{false}}; auto stream = cudf::test::get_default_stream(); - std::unique_ptr status; - auto got = cudf::io::parquet::experimental::extract_variant_field( - col, "x", cudf::data_type{cudf::type_id::INT32}, &status, stream, cmr()); + auto status = make_status_buffer(cudf::column_view{col}.size()); + auto got = cudf::io::parquet::experimental::extract_variant_field( + col, "x", cudf::data_type{cudf::type_id::INT32}, status->mutable_view(), stream, cmr()); - ASSERT_NE(status, nullptr); - EXPECT_EQ(status->null_count(), 0); expect_status_values(*status, {ST_ROW_NULL}); EXPECT_EQ(got->null_count(), 1); } @@ -2017,11 +2059,10 @@ TEST_F(ExtractVariantFieldStatusTest, VariantNullStatus) auto col = wrap_single_variant(m, v); auto stream = cudf::test::get_default_stream(); - std::unique_ptr status; - auto got = cudf::io::parquet::experimental::extract_variant_field( - col, "f", cudf::data_type{cudf::type_id::INT32}, &status, stream, cmr()); + auto status = make_status_buffer(cudf::column_view{col}.size()); + auto got = cudf::io::parquet::experimental::extract_variant_field( + col, "f", cudf::data_type{cudf::type_id::INT32}, status->mutable_view(), stream, cmr()); - ASSERT_NE(status, nullptr); expect_status_values(*status, {ST_VNULL}); EXPECT_EQ(got->null_count(), 1); } @@ -2034,11 +2075,10 @@ TEST_F(ExtractVariantFieldStatusTest, TypeMismatchStatus) auto col = wrap_single_variant(m, v); auto stream = cudf::test::get_default_stream(); - std::unique_ptr status; - auto got = cudf::io::parquet::experimental::extract_variant_field( - col, "s", cudf::data_type{cudf::type_id::INT32}, &status, stream, cmr()); + auto status = make_status_buffer(cudf::column_view{col}.size()); + auto got = cudf::io::parquet::experimental::extract_variant_field( + col, "s", cudf::data_type{cudf::type_id::INT32}, status->mutable_view(), stream, cmr()); - ASSERT_NE(status, nullptr); expect_status_values(*status, {ST_MISMATCH}); EXPECT_EQ(got->null_count(), 1); } @@ -2049,11 +2089,15 @@ TEST_F(ExtractVariantFieldStatusTest, MissingNestedPathStatus) auto col = make_apache_variant(avf::object_nested); auto stream = cudf::test::get_default_stream(); - std::unique_ptr status; - auto got = cudf::io::parquet::experimental::extract_variant_field( - col, "$.species.nope", cudf::data_type{cudf::type_id::STRING}, &status, stream, cmr()); + auto status = make_status_buffer(cudf::column_view{col}.size()); + auto got = + cudf::io::parquet::experimental::extract_variant_field(col, + "$.species.nope", + cudf::data_type{cudf::type_id::STRING}, + status->mutable_view(), + stream, + cmr()); - ASSERT_NE(status, nullptr); expect_status_values(*status, {ST_MISSING}); EXPECT_EQ(got->null_count(), 1); } diff --git a/java/src/main/native/src/VariantUtilsJni.cpp b/java/src/main/native/src/VariantUtilsJni.cpp index 84c72bea1675..a9c0d97ca9bd 100644 --- a/java/src/main/native/src/VariantUtilsJni.cpp +++ b/java/src/main/native/src/VariantUtilsJni.cpp @@ -27,7 +27,7 @@ JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_VariantUtils_getVariantFieldValue( return cudf::jni::release_as_jlong( cudf::io::parquet::experimental::get_variant_field(variant_struct, path.get(), - nullptr, + std::nullopt, cudf::get_default_stream(), cudf::get_current_device_resource_ref())); } @@ -48,7 +48,7 @@ JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_VariantUtils_castVariantValue(JNIEnv value_bytes, cudf::data_type{static_cast(cudf_type_id)}, std::nullopt, - nullptr, + std::nullopt, cudf::get_default_stream(), cudf::get_current_device_resource_ref())); } @@ -69,7 +69,7 @@ JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_VariantUtils_extractVariantField( variant_struct, path.get(), cudf::data_type{static_cast(cudf_type_id)}, - nullptr, + std::nullopt, cudf::get_default_stream(), cudf::get_current_device_resource_ref())); }