diff --git a/cpp/include/cudf/copying.hpp b/cpp/include/cudf/copying.hpp index f1ab6e30f268..d5ed72451864 100644 --- a/cpp/include/cudf/copying.hpp +++ b/cpp/include/cudf/copying.hpp @@ -75,15 +75,14 @@ enum class negative_index_policy : bool { * better performance. If `policy` is set to `DONT_CHECK` and there are out-of-bounds indices * in the gather map, the behavior is undefined. Defaults to `DONT_CHECK`. * @param stream CUDA stream used for device memory operations and kernel launches - * @param mr Device memory resource used to allocate the returned table's device memory + * @param mr Memory resources used for temporary allocations and the returned table * @return Result of the gather */ -std::unique_ptr gather( - table_view const& source_table, - column_view const& gather_map, - out_of_bounds_policy bounds_policy = out_of_bounds_policy::DONT_CHECK, - cuda::stream_ref stream = cudf::get_default_stream(), - rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()); +std::unique_ptr
gather(table_view const& source_table, + column_view const& gather_map, + out_of_bounds_policy bounds_policy = out_of_bounds_policy::DONT_CHECK, + cuda::stream_ref stream = cudf::get_default_stream(), + cudf::memory_resources mr = cudf::get_current_device_resource_ref()); /** * @brief Gathers the specified rows of a set of columns according to a gather map. @@ -112,16 +111,15 @@ std::unique_ptr
gather( * @param bounds_policy Interpretation of out-of-bounds indices * @param neg_indices Interpretation of a negative index `i` in the `gather_map` * @param stream CUDA stream used for device memory operations and kernel launches. - * @param mr Device memory resource used to allocate the returned table's device memory + * @param mr Memory resources used for temporary allocations and the returned table * @return Result of the gather */ -std::unique_ptr
gather( - table_view const& source_table, - column_view const& gather_map, - out_of_bounds_policy bounds_policy, - negative_index_policy neg_indices, - cuda::stream_ref stream = cudf::get_default_stream(), - rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()); +std::unique_ptr
gather(table_view const& source_table, + column_view const& gather_map, + out_of_bounds_policy bounds_policy, + negative_index_policy neg_indices, + cuda::stream_ref stream = cudf::get_default_stream(), + cudf::memory_resources mr = cudf::get_current_device_resource_ref()); /** * @brief Reverses the rows within a table. diff --git a/cpp/include/cudf/detail/gather.cuh b/cpp/include/cudf/detail/gather.cuh index e5bb1f9ff575..faf3efcafc39 100644 --- a/cpp/include/cudf/detail/gather.cuh +++ b/cpp/include/cudf/detail/gather.cuh @@ -104,6 +104,7 @@ struct gather_bitmask_functor { * @param gather_map_end End of the gather map * @param nullify_out_of_bounds True if map values are checked against `source_size` * @param stream CUDA stream used for kernel launches. + * @param temp_mr Device memory resource used for temporary allocations */ template void gather_helper(InputItr source_itr, @@ -112,11 +113,12 @@ void gather_helper(InputItr source_itr, MapIterator gather_map_begin, MapIterator gather_map_end, bool nullify_out_of_bounds, - rmm::cuda_stream_view stream) + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref temp_mr = cudf::get_current_device_resource_ref()) { using map_type = typename std::iterator_traits::value_type; if (nullify_out_of_bounds) { - thrust::gather_if(rmm::exec_policy_nosync(stream, cudf::get_current_device_resource_ref()), + thrust::gather_if(rmm::exec_policy_nosync(stream, temp_mr), gather_map_begin, gather_map_end, gather_map_begin, @@ -124,7 +126,7 @@ void gather_helper(InputItr source_itr, target_itr, bounds_checker{0, source_size}); } else { - thrust::gather(rmm::exec_policy_nosync(stream, cudf::get_current_device_resource_ref()), + thrust::gather(rmm::exec_policy_nosync(stream, temp_mr), gather_map_begin, gather_map_end, source_itr, @@ -159,7 +161,7 @@ struct column_gatherer { * @param gather_map_end End of iterator range of integral values representing the gather map * @param nullify_out_of_bounds Nullify values in `gather_map` that are out of bounds * @param stream CUDA stream used for device memory operations and kernel launches. - * @param mr Device memory resource used to allocate the returned column's device memory + * @param mr Memory resources used for temporary allocations and the returned column */ template std::unique_ptr operator()(column_view const& source_column, @@ -167,7 +169,7 @@ struct column_gatherer { MapIterator gather_map_end, bool nullify_out_of_bounds, rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) + cudf::memory_resources mr) { column_gatherer_impl gatherer{}; @@ -199,7 +201,7 @@ struct column_gatherer_impl std::unique_ptr operator()(column_view const& source_column, @@ -207,11 +209,12 @@ struct column_gatherer_impl(), source_column.size(), @@ -219,7 +222,8 @@ struct column_gatherer_impl { * @param gather_map_end End of iterator range of integral values representing the gather map * @param nullify_out_of_bounds Nullify values in `gather_map` that are out of bounds * @param stream CUDA stream used for device memory operations and kernel launches. - * @param mr Device memory resource used to allocate the returned column's device memory + * @param mr Memory resources used for temporary allocations and the returned column */ template std::unique_ptr operator()(column_view const& source_column, @@ -252,7 +256,7 @@ struct column_gatherer_impl { MapItType gather_map_end, bool nullify_out_of_bounds, rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) + cudf::memory_resources mr) { if (true == nullify_out_of_bounds) { return cudf::strings::detail::gather( @@ -326,42 +330,46 @@ struct column_gatherer_impl { MapItRoot gather_map_end, bool nullify_out_of_bounds, rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) + cudf::memory_resources mr) { + auto const output_mr = mr.get_output_mr(); + lists_column_view list(column); auto gather_map_size = std::distance(gather_map_begin, gather_map_end); // if the gather map is empty, return an empty column if (gather_map_size == 0) { return empty_like(column); } // generate gather_data for the next level (N+1) - lists::detail::gather_data gd = nullify_out_of_bounds - ? lists::detail::make_gather_data( - column, gather_map_begin, gather_map_size, stream, mr) - : lists::detail::make_gather_data( - column, gather_map_begin, gather_map_size, stream, mr); + lists::detail::gather_data gd = + nullify_out_of_bounds ? lists::detail::make_gather_data( + column, gather_map_begin, gather_map_size, stream, output_mr) + : lists::detail::make_gather_data( + column, gather_map_begin, gather_map_size, stream, output_mr); // the nesting case. if (list.child().type() == cudf::data_type{type_id::LIST}) { // gather children - auto child = lists::detail::gather_list_nested(list.get_sliced_child(stream), gd, stream, mr); + auto child = + lists::detail::gather_list_nested(list.get_sliced_child(stream), gd, stream, output_mr); // return the final column return make_lists_column(gather_map_size, std::move(gd.offsets), std::move(child), 0, - rmm::device_buffer{0, stream, mr}); + rmm::device_buffer{0, stream, output_mr}); } // it's a leaf. do a regular gather - auto child = lists::detail::gather_list_leaf(list.get_sliced_child(stream), gd, stream, mr); + auto child = + lists::detail::gather_list_leaf(list.get_sliced_child(stream), gd, stream, output_mr); // assemble final column return make_lists_column(gather_map_size, std::move(gd.offsets), std::move(child), 0, - rmm::device_buffer{0, stream, mr}); + rmm::device_buffer{0, stream, output_mr}); } }; @@ -380,7 +388,7 @@ struct column_gatherer_impl { * @param gather_map_end End of iterator range of integral values representing the gather map * @param nullify_out_of_bounds Nullify values in `gather_map` that are out of bounds * @param stream CUDA stream used for device memory operations and kernel launches. - * @param mr Device memory resource used to allocate the returned column's device memory + * @param mr Memory resources used for temporary allocations and the returned column * @return New dictionary column with gathered rows. */ template @@ -389,8 +397,11 @@ struct column_gatherer_impl { MapItType gather_map_end, bool nullify_out_of_bounds, rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) + cudf::memory_resources mr) { + auto const output_mr = mr.get_output_mr(); + auto const temp_mr = mr.get_temporary_mr(); + dictionary_column_view dictionary(source_column); auto output_count = std::distance(gather_map_begin, gather_map_end); if (output_count == 0) return make_empty_column(type_id::DICTIONARY32); @@ -401,11 +412,11 @@ struct column_gatherer_impl { // be relatively smallish. // Also, there are scenarios where the keys are common with other dictionaries // and the original intention was to share the keys here. - auto keys_copy = std::make_unique(dictionary.keys(), stream, mr); + auto keys_copy = std::make_unique(dictionary.keys(), stream, output_mr); // Perform gather on just the indices column_view indices = dictionary.get_indices_annotated(); - auto new_indices = - cudf::allocate_like(indices, output_count, cudf::mask_allocation_policy::NEVER, stream, mr); + auto new_indices = cudf::allocate_like( + indices, output_count, cudf::mask_allocation_policy::NEVER, stream, output_mr); gather_helper( cudf::detail::indexalator_factory::make_input_iterator(indices), indices.size(), @@ -413,8 +424,9 @@ struct column_gatherer_impl { gather_map_begin, gather_map_end, nullify_out_of_bounds, - stream); - return make_dictionary_column(std::move(keys_copy), std::move(new_indices), stream, mr); + stream, + temp_mr); + return make_dictionary_column(std::move(keys_copy), std::move(new_indices), stream, output_mr); } }; @@ -426,8 +438,10 @@ struct column_gatherer_impl { MapItRoot gather_map_end, bool nullify_out_of_bounds, rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) + cudf::memory_resources mr) { + auto const output_mr = mr.get_output_mr(); + auto const gather_map_size = std::distance(gather_map_begin, gather_map_end); if (gather_map_size == 0) { return empty_like(column); } @@ -477,9 +491,9 @@ struct column_gatherer_impl { gather_map_size, std::move(output_struct_members), 0, - rmm::device_buffer{0, stream, mr}, // Null mask will be fixed up in cudf::gather(). + rmm::device_buffer{0, stream, output_mr}, // Null mask will be fixed up in cudf::gather(). stream, - mr); + output_mr); } }; @@ -532,10 +546,13 @@ void gather_bitmask(table_view const& source, std::vector>& target, gather_bitmask_op op, rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) + cudf::memory_resources mr) { if (target.empty()) { return; } + auto const output_mr = mr.get_output_mr(); + auto const temp_mr = mr.get_temporary_mr(); + // Validate that all target columns have the same size auto const target_rows = target.front()->size(); CUDF_EXPECTS(std::all_of(target.begin(), @@ -549,7 +566,7 @@ void gather_bitmask(table_view const& source, not target[i]->nullable()) { auto const state = op == gather_bitmask_op::PASSTHROUGH ? mask_state::ALL_VALID : mask_state::UNINITIALIZED; - auto mask = cudf::create_null_mask(target[i]->size(), state, stream, mr); + auto mask = cudf::create_null_mask(target[i]->size(), state, stream, output_mr); target[i]->set_null_mask(std::move(mask), 0); } } @@ -559,12 +576,10 @@ void gather_bitmask(table_view const& source, std::transform(target.begin(), target.end(), target_masks.begin(), [](auto const& col) { return col->mutable_view().null_mask(); }); - auto d_target_masks = - make_device_uvector_async(target_masks, stream, cudf::get_current_device_resource_ref()); + auto d_target_masks = make_device_uvector_async(target_masks, stream, temp_mr); - auto const device_source = table_device_view::create(source, stream); - auto d_valid_counts = make_zeroed_device_uvector_async( - target.size(), stream, cudf::get_current_device_resource_ref()); + auto const device_source = table_device_view::create(source, stream, temp_mr); + auto d_valid_counts = make_zeroed_device_uvector_async(target.size(), stream, temp_mr); // Dispatch operation enum to get implementation auto const impl = [op]() { @@ -621,7 +636,7 @@ void gather_bitmask(table_view const& source, * better performance. In case there are out-of-bound indices in the gather map, the behavior * is undefined. Defaults to `DONT_CHECK`. * @param[in] stream CUDA stream used for device memory operations and kernel launches. - * @param[in] mr Device memory resource used to allocate the returned table's device memory + * @param[in] mr Memory resources used for temporary allocations and the returned table * @return cudf::table Result of the gather */ template @@ -630,8 +645,10 @@ std::unique_ptr
gather(table_view const& source_table, MapIterator gather_map_end, out_of_bounds_policy bounds_policy, rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) + cudf::memory_resources mr) { + auto const output_mr = mr.get_output_mr(); + std::vector> destination_columns; // TODO: Could be beneficial to use streams internally here @@ -661,7 +678,8 @@ std::unique_ptr
gather(table_view const& source_table, gather_bitmask(source_table, gather_map_begin, destination_columns, op, stream, mr); } else { for (size_type i = 0; i < source_table.num_columns(); ++i) { - set_all_valid_null_masks(source_table.column(i), *destination_columns[i], stream, mr); + set_all_valid_null_masks( + source_table.column(i), *destination_columns[i], stream, output_mr); } } } diff --git a/cpp/include/cudf/detail/gather.hpp b/cpp/include/cudf/detail/gather.hpp index 9190b08e57f2..847f93586a5c 100644 --- a/cpp/include/cudf/detail/gather.hpp +++ b/cpp/include/cudf/detail/gather.hpp @@ -21,21 +21,21 @@ namespace cudf { namespace detail { /** - * @copydoc cudf::gather(table_view const&,column_view const&,table_view - * const&,cudf::out_of_bounds_policy,cudf::negative_index_policy,cuda::stream_ref, - * rmm::device_async_resource_ref) + * @copydoc cudf::gather(table_view const&,column_view const&,out_of_bounds_policy, + * negative_index_policy,cuda::stream_ref,rmm::device_async_resource_ref) + * + * @param mr Memory resources used for temporary allocations and the returned table */ std::unique_ptr
gather(table_view const& source_table, column_view const& gather_map, out_of_bounds_policy bounds_policy, negative_index_policy neg_indices, cuda::stream_ref stream, - rmm::device_async_resource_ref mr); + memory_resources mr); /** - * @copydoc cudf::detail::gather(table_view const&,column_view const&,table_view - * const&,cudf::out_of_bounds_policy,cudf::negative_index_policy,cuda::stream_ref, - * rmm::device_async_resource_ref) + * @copydoc cudf::detail::gather(table_view const&,column_view const&,out_of_bounds_policy, + * negative_index_policy,cuda::stream_ref,memory_resources) * * @throws cudf::logic_error if `gather_map` span size is larger than max of `size_type`. */ @@ -44,7 +44,7 @@ std::unique_ptr
gather(table_view const& source_table, out_of_bounds_policy bounds_policy, negative_index_policy neg_indices, cuda::stream_ref stream, - rmm::device_async_resource_ref mr); + memory_resources mr); } // namespace detail } // namespace cudf diff --git a/cpp/include/cudf/detail/row_operator/equality.cuh b/cpp/include/cudf/detail/row_operator/equality.cuh index f38acc8b7d78..9a5fd68b6ceb 100644 --- a/cpp/include/cudf/detail/row_operator/equality.cuh +++ b/cpp/include/cudf/detail/row_operator/equality.cuh @@ -407,9 +407,12 @@ class self_comparator { * @param t The table to compare * @param stream The stream to construct this object on. Not the stream that will be used for * comparisons using this object. + * @param temp_mr Device memory resource used for temporary allocations */ - self_comparator(table_view const& t, rmm::cuda_stream_view stream) - : d_t(preprocessed_table::create(t, stream)) + self_comparator(table_view const& t, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref temp_mr) + : d_t(preprocessed_table::create(t, stream, temp_mr)) { } @@ -515,10 +518,12 @@ class two_table_comparator { * @param right The right table to compare. * @param stream The stream to construct this object on. Not the stream that will be used for * comparisons using this object. + * @param temp_mr Device memory resource used for temporary allocations */ two_table_comparator(table_view const& left, table_view const& right, - rmm::cuda_stream_view stream); + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref temp_mr); /** * @brief Construct an owning object for performing equality comparisons between two rows from two diff --git a/cpp/include/cudf/detail/row_operator/hashing.cuh b/cpp/include/cudf/detail/row_operator/hashing.cuh index e71fc5213483..5bb77b125d7c 100644 --- a/cpp/include/cudf/detail/row_operator/hashing.cuh +++ b/cpp/include/cudf/detail/row_operator/hashing.cuh @@ -240,9 +240,12 @@ class row_hasher { * @param t The table containing rows to hash * @param stream The stream to construct this object on. Not the stream that will be used for * comparisons using this object. + * @param temp_mr Device memory resource used for temporary allocations */ - row_hasher(table_view const& t, rmm::cuda_stream_view stream) - : d_t(preprocessed_table::create(t, stream)) + row_hasher(table_view const& t, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref temp_mr) + : d_t(preprocessed_table::create(t, stream, temp_mr)) { } diff --git a/cpp/include/cudf/detail/row_operator/preprocessed_table.cuh b/cpp/include/cudf/detail/row_operator/preprocessed_table.cuh index 233294201ccd..08949be1f0b7 100644 --- a/cpp/include/cudf/detail/row_operator/preprocessed_table.cuh +++ b/cpp/include/cudf/detail/row_operator/preprocessed_table.cuh @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -47,10 +48,12 @@ struct preprocessed_table { * * @param table The table to preprocess * @param stream The cuda stream to use while preprocessing. + * @param temp_mr Device memory resource used for temporary allocations * @return A preprocessed table as shared pointer */ static std::shared_ptr create(table_view const& table, - rmm::cuda_stream_view stream); + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref temp_mr); /** * @brief Implicit conversion operator to a `table_device_view` of the preprocessed table. diff --git a/cpp/include/cudf/detail/sizes_to_offsets_iterator.cuh b/cpp/include/cudf/detail/sizes_to_offsets_iterator.cuh index ae79c6b683d2..05a643eca279 100644 --- a/cpp/include/cudf/detail/sizes_to_offsets_iterator.cuh +++ b/cpp/include/cudf/detail/sizes_to_offsets_iterator.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -249,27 +249,29 @@ static sizes_to_offsets_iterator make_sizes_to_offsets_i * @param result Output iterator for scan result * @param initial_offset Initial offset to add to scan * @param stream CUDA stream used for device memory operations and kernel launches + * @param temp_mr Device memory resource used for temporary allocations * @return The last element of the scan */ template -auto sizes_to_offsets(SizesIterator begin, - SizesIterator end, - OffsetsIterator result, - int64_t initial_offset, - rmm::cuda_stream_view stream) +auto sizes_to_offsets( + SizesIterator begin, + SizesIterator end, + OffsetsIterator result, + int64_t initial_offset, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref temp_mr = cudf::get_current_device_resource_ref()) { using SizeType = cuda::std::iter_value_t; static_assert(std::is_integral_v, "Only numeric types are supported by sizes_to_offsets"); - using LastType = std::conditional_t, int64_t, uint64_t>; - auto last_element = - cudf::detail::device_scalar(0, stream, cudf::get_current_device_resource_ref()); + using LastType = std::conditional_t, int64_t, uint64_t>; + auto last_element = cudf::detail::device_scalar(0, stream, temp_mr); auto output_itr = make_sizes_to_offsets_iterator(result, result + std::distance(begin, end), last_element.data()); // This function uses the type of the initialization parameter as the accumulator type // when computing the individual scan output elements. - thrust::exclusive_scan(rmm::exec_policy_nosync(stream, cudf::get_current_device_resource_ref()), + thrust::exclusive_scan(rmm::exec_policy_nosync(stream, temp_mr), begin, end, output_itr, @@ -293,21 +295,21 @@ auto sizes_to_offsets(SizesIterator begin, * @param begin The beginning of the input sequence * @param end The end of the input sequence * @param stream CUDA stream used for device memory operations and kernel launches - * @param mr Device memory resource used to allocate the returned column's device memory + * @param mr Memory resources used for temporary allocations and the returned column * @return Offsets column and total elements */ template std::pair, size_type> make_offsets_child_column( - InputIterator begin, - InputIterator end, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) + InputIterator begin, InputIterator end, rmm::cuda_stream_view stream, cudf::memory_resources mr) { auto count = static_cast(std::distance(begin, end)); - auto offsets_column = make_numeric_column( - data_type{type_to_id()}, count + 1, mask_state::UNALLOCATED, stream, mr); - auto offsets_view = offsets_column->mutable_view(); - auto d_offsets = offsets_view.template data(); + auto offsets_column = make_numeric_column(data_type{type_to_id()}, + count + 1, + mask_state::UNALLOCATED, + stream, + mr.get_output_mr()); + auto offsets_view = offsets_column->mutable_view(); + auto d_offsets = offsets_view.template data(); // The number of offsets is count+1 so to build the offsets from the sizes // using exclusive-scan technically requires count+1 input values even though @@ -320,7 +322,7 @@ std::pair, size_type> make_offsets_child_column( auto input_itr = cudf::detail::make_counting_transform_iterator(0, map_fn); // Use the sizes-to-offsets iterator to compute the total number of elements auto const total_elements = - sizes_to_offsets(input_itr, input_itr + count + 1, d_offsets, 0, stream); + sizes_to_offsets(input_itr, input_itr + count + 1, d_offsets, 0, stream, mr.get_temporary_mr()); CUDF_EXPECTS( total_elements <= static_cast(std::numeric_limits::max()), "Size of output exceeds the column size limit", diff --git a/cpp/include/cudf/dictionary/detail/encode.hpp b/cpp/include/cudf/dictionary/detail/encode.hpp index e728acc1e389..9eec766a80d1 100644 --- a/cpp/include/cudf/dictionary/detail/encode.hpp +++ b/cpp/include/cudf/dictionary/detail/encode.hpp @@ -37,13 +37,13 @@ namespace dictionary::detail { * @param column The column to dictionary encode. * @param indices_type The integer type to use for the indices. * @param stream CUDA stream used for device memory operations and kernel launches. - * @param mr Device memory resource used to allocate the returned column's device memory. + * @param mr Memory resources used for temporary allocations and the returned column. * @return Returns a dictionary column. */ std::unique_ptr encode(column_view const& column, data_type indices_type, cuda::stream_ref stream, - rmm::device_async_resource_ref mr); + cudf::memory_resources mr); /** * @brief Create a column by gathering the keys from the provided @@ -57,12 +57,12 @@ std::unique_ptr encode(column_view const& column, * * @param dictionary_column Existing dictionary column. * @param stream CUDA stream used for device memory operations and kernel launches. - * @param mr Device memory resource used to allocate the returned column's device memory. + * @param mr Memory resources used for temporary allocations and the returned column. * @return New column with type matching the dictionary_column's keys. */ std::unique_ptr decode(dictionary_column_view const& dictionary_column, cuda::stream_ref stream, - rmm::device_async_resource_ref mr); + cudf::memory_resources mr); /** * @brief Return minimal integer type for the given number of elements. diff --git a/cpp/include/cudf/dictionary/encode.hpp b/cpp/include/cudf/dictionary/encode.hpp index d915927e7c0b..514782812e22 100644 --- a/cpp/include/cudf/dictionary/encode.hpp +++ b/cpp/include/cudf/dictionary/encode.hpp @@ -47,14 +47,13 @@ namespace dictionary { * @param column The column to dictionary encode * @param indices_type The integer type to use for the indices * @param stream CUDA stream used for device memory operations and kernel launches - * @param mr Device memory resource used to allocate the returned column's device memory + * @param mr Memory resources used for temporary allocations and the returned column * @return Returns a dictionary column */ -std::unique_ptr encode( - column_view const& column, - data_type indices_type = data_type{type_id::INT32}, - cuda::stream_ref stream = cudf::get_default_stream(), - rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()); +std::unique_ptr encode(column_view const& column, + data_type indices_type = data_type{type_id::INT32}, + cuda::stream_ref stream = cudf::get_default_stream(), + cudf::memory_resources mr = cudf::get_current_device_resource_ref()); /** * @brief Create a column by gathering the keys from the provided @@ -68,13 +67,12 @@ std::unique_ptr encode( * * @param dictionary_column Existing dictionary column * @param stream CUDA stream used for device memory operations and kernel launches - * @param mr Device memory resource used to allocate the returned column's device memory + * @param mr Memory resources used for temporary allocations and the returned column * @return New column with type matching the dictionary_column's keys */ -std::unique_ptr decode( - dictionary_column_view const& dictionary_column, - cuda::stream_ref stream = cudf::get_default_stream(), - rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()); +std::unique_ptr decode(dictionary_column_view const& dictionary_column, + cuda::stream_ref stream = cudf::get_default_stream(), + cudf::memory_resources mr = cudf::get_current_device_resource_ref()); /** @} */ // end of group } // namespace dictionary diff --git a/cpp/include/cudf/lists/detail/gather.cuh b/cpp/include/cudf/lists/detail/gather.cuh index 59ffba0c8367..56ef467cc96e 100644 --- a/cpp/include/cudf/lists/detail/gather.cuh +++ b/cpp/include/cudf/lists/detail/gather.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ #pragma once @@ -290,7 +290,8 @@ std::unique_ptr gather_list_leaf(column_view const& column, * @copydoc cudf::lists::segmented_gather(lists_column_view const& source_column, * lists_column_view const& gather_map_list, * out_of_bounds_policy bounds_policy, - * rmm::device_async_resource_ref mr) + * rmm::cuda_stream_view stream, + * cudf::memory_resources mr) * * @param stream CUDA stream on which to execute kernels */ @@ -298,7 +299,7 @@ std::unique_ptr segmented_gather(lists_column_view const& source_column, lists_column_view const& gather_map_list, out_of_bounds_policy bounds_policy, rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr); + cudf::memory_resources mr); } // namespace detail } // namespace lists diff --git a/cpp/include/cudf/lists/gather.hpp b/cpp/include/cudf/lists/gather.hpp index 21925eff38ee..4da1a948ee34 100644 --- a/cpp/include/cudf/lists/gather.hpp +++ b/cpp/include/cudf/lists/gather.hpp @@ -59,7 +59,7 @@ namespace lists { * output list row's element, when the gather index falls outside the range `[-n, n)`, * where `n` is the number of elements in list row corresponding to the gather-map row. * @param stream CUDA stream used for device memory operations and kernel launches. - * @param mr Device memory resource to allocate any returned objects + * @param mr Memory resources used for temporary allocations and the returned column * @return column with elements in list of rows gathered based on `gather_map_list` * */ @@ -68,7 +68,7 @@ std::unique_ptr segmented_gather( lists_column_view const& gather_map_list, out_of_bounds_policy bounds_policy = out_of_bounds_policy::DONT_CHECK, rmm::cuda_stream_view stream = cudf::get_default_stream(), - rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()); + cudf::memory_resources mr = cudf::get_current_device_resource_ref()); /** @} */ // end of group } // namespace lists diff --git a/cpp/include/cudf/strings/detail/gather.cuh b/cpp/include/cudf/strings/detail/gather.cuh index 759448ac58a7..3af885f9b4f2 100644 --- a/cpp/include/cudf/strings/detail/gather.cuh +++ b/cpp/include/cudf/strings/detail/gather.cuh @@ -215,7 +215,7 @@ CUDF_KERNEL void gather_chars_fn_char_parallel(StringIterator strings_begin, * @param begin Start of index iterator. * @param end End of index iterator. * @param stream CUDA stream used for device memory operations and kernel launches. - * @param mr Device memory resource used to allocate the returned column's device memory. + * @param mr Memory resources used for temporary allocations and the returned column. * @return New strings column containing the gathered strings. */ template @@ -223,13 +223,16 @@ std::unique_ptr gather(strings_column_view const& strings, MapIterator begin, MapIterator end, rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) + cudf::memory_resources mr) { + auto const output_mr = mr.get_output_mr(); + auto const temp_mr = mr.get_temporary_mr(); + auto const output_count = std::distance(begin, end); if (output_count == 0) return make_empty_column(type_id::STRING); // build offsets column - auto const d_strings = column_device_view::create(strings.parent(), stream); + auto const d_strings = column_device_view::create(strings.parent(), stream, temp_mr); auto const d_in_offsets = cudf::detail::offsetalator_factory::make_input_iterator( strings.is_empty() ? make_empty_column(type_id::INT32)->view() : strings.offsets(), strings.offset()); @@ -252,7 +255,7 @@ std::unique_ptr gather(strings_column_view const& strings, cudf::prefetch::detail::prefetch(strings.chars_begin(stream), strings.chars_size(stream), stream); // build output char column - auto out_chars_data = rmm::device_uvector(out_char_bytes, stream, mr); + auto out_chars_data = rmm::device_uvector(out_char_bytes, stream, output_mr); cudf::prefetch::detail::prefetch(out_chars_data, stream); auto d_out_chars = out_chars_data.data(); @@ -318,7 +321,7 @@ std::unique_ptr gather(strings_column_view const& strings, stream.value()); // Allocate temporary storage - auto d_temp_storage = rmm::device_buffer(temp_storage_bytes, stream, mr); + auto d_temp_storage = rmm::device_buffer(temp_storage_bytes, stream, temp_mr); // Run batched copy algorithm cub::DeviceMemcpy::Batched(d_temp_storage.data(), @@ -358,7 +361,7 @@ std::unique_ptr gather(strings_column_view const& strings, * @param end End of index iterator. * @param nullify_out_of_bounds If true, indices outside the column's range are nullified. * @param stream CUDA stream used for device memory operations and kernel launches. - * @param mr Device memory resource used to allocate the returned column's device memory. + * @param mr Memory resources used for temporary allocations and the returned column. * @return New strings column containing the gathered strings. */ template @@ -367,7 +370,7 @@ std::unique_ptr gather(strings_column_view const& strings, MapIterator end, bool nullify_out_of_bounds, rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) + cudf::memory_resources mr) { if (nullify_out_of_bounds) return gather(strings, begin, end, stream, mr); return gather(strings, begin, end, stream, mr); diff --git a/cpp/include/cudf/strings/detail/strings_children.cuh b/cpp/include/cudf/strings/detail/strings_children.cuh index d15e5fa199f1..59f0334be297 100644 --- a/cpp/include/cudf/strings/detail/strings_children.cuh +++ b/cpp/include/cudf/strings/detail/strings_children.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ #pragma once @@ -115,23 +115,25 @@ rmm::device_uvector make_chars_buffer(column_view const& offsets, * @param begin The beginning of the input sequence * @param end The end of the input sequence * @param stream CUDA stream used for device memory operations and kernel launches - * @param mr Device memory resource used to allocate the returned column's device memory + * @param mr Memory resources used for temporary allocations and the returned column * @return Offsets column and total elements */ template -std::pair, int64_t> make_offsets_child_column( - InputIterator begin, - InputIterator end, - rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) +std::pair, int64_t> make_offsets_child_column(InputIterator begin, + InputIterator end, + rmm::cuda_stream_view stream, + cudf::memory_resources mr) { + auto const output_mr = mr.get_output_mr(); + auto const temp_mr = mr.get_temporary_mr(); + auto constexpr size_type_max = static_cast(std::numeric_limits::max()); auto const lcount = static_cast(std::distance(begin, end)); CUDF_EXPECTS( lcount <= size_type_max, "Size of output exceeds the column size limit", std::overflow_error); auto const strings_count = static_cast(lcount); auto offsets_column = make_numeric_column( - data_type{type_id::INT32}, strings_count + 1, mask_state::UNALLOCATED, stream, mr); + data_type{type_id::INT32}, strings_count + 1, mask_state::UNALLOCATED, stream, output_mr); auto d_offsets = offsets_column->mutable_view().template data(); // The number of offsets is strings_count+1 so to build the offsets from the sizes @@ -141,8 +143,8 @@ std::pair, int64_t> make_offsets_child_column( auto input_itr = cudf::detail::make_counting_transform_iterator(0, string_offsets_fn{begin, strings_count}); // Use the sizes-to-offsets iterator to compute the total number of elements - auto const total_bytes = - cudf::detail::sizes_to_offsets(input_itr, input_itr + strings_count + 1, d_offsets, 0, stream); + auto const total_bytes = cudf::detail::sizes_to_offsets( + input_itr, input_itr + strings_count + 1, d_offsets, 0, stream, temp_mr); auto const threshold = cudf::strings::get_offset64_threshold(); CUDF_EXPECTS(cudf::strings::is_large_strings_enabled() || (total_bytes < threshold), @@ -151,10 +153,10 @@ std::pair, int64_t> make_offsets_child_column( if (total_bytes >= cudf::strings::get_offset64_threshold()) { // recompute as int64 offsets when above the threshold offsets_column = make_numeric_column( - data_type{type_id::INT64}, strings_count + 1, mask_state::UNALLOCATED, stream, mr); + data_type{type_id::INT64}, strings_count + 1, mask_state::UNALLOCATED, stream, output_mr); auto d_offsets64 = offsets_column->mutable_view().template data(); cudf::detail::sizes_to_offsets( - input_itr, input_itr + strings_count + 1, d_offsets64, 0, stream); + input_itr, input_itr + strings_count + 1, d_offsets64, 0, stream, temp_mr); } return std::pair(std::move(offsets_column), total_bytes); diff --git a/cpp/include/cudf_test/base_fixture.hpp b/cpp/include/cudf_test/base_fixture.hpp index 003955369001..326bf1ad3b89 100644 --- a/cpp/include/cudf_test/base_fixture.hpp +++ b/cpp/include/cudf_test/base_fixture.hpp @@ -45,8 +45,10 @@ class BaseFixture : public ::testing::Test { /** * @brief Base fixture that instruments tests with a memory-resource harness. * - * Each test instantiates a fresh harness. Tests should construct results with `resources()`. - * `TearDown` asserts that no output or temporary allocations remain live. + * Each test instantiates a fresh harness and installs a failing current-device resource for the + * duration of the test, so accidental fallback to the default MR fails. Tests should construct + * results with `resources()`. `TearDown` asserts that no output or temporary allocations remain + * live; the prior current resource is restored when `_fail_on_current` is destroyed. */ struct BaseFixtureWithHarness : public BaseFixture { /** @@ -68,6 +70,7 @@ struct BaseFixtureWithHarness : public BaseFixture { protected: memory_resource_test_harness _harness{mr()}; + scoped_current_device_resource _fail_on_current{_harness.fail_on_current_device_resource_use()}; }; /** diff --git a/cpp/include/cudf_test/column_wrapper.hpp b/cpp/include/cudf_test/column_wrapper.hpp index 54acc5c5e7a9..34d217196e16 100644 --- a/cpp/include/cudf_test/column_wrapper.hpp +++ b/cpp/include/cudf_test/column_wrapper.hpp @@ -36,9 +36,13 @@ #include #include +#include +#include #include #include #include +#include +#include namespace CUDF_EXPORT cudf { namespace test { @@ -340,8 +344,155 @@ auto make_chars_and_offsets(StringsIterator begin, StringsIterator end, Validity } return std::pair(std::move(chars), std::move(offsets)); }; + } // namespace detail +// Forward declaration for lists_column_initializer::build +template +class lists_column_wrapper; + +/** + * @brief Host-side recursive initializer tree for constructing list columns with an + * explicit stream and memory resources at every nesting level. + * + * Prefer this over brace-nested `lists_column_wrapper` constructions that pass + * `stream`/`mr` only at the outer level, which leave brace-constructed children on + * the default test resources. + * + * Example: + * @code{.cpp} + * using Init = cudf::test::lists_column_initializer; + * // List: [{1, 2}, {3}] + * lists_column_wrapper col{Init{{{1, 2}, {3}}}, stream, mr}; + * @endcode + * + * Leaf and nested constructors accept the existing validity iterators + * (`valids`, `null_at(...)`, etc.) and materialize them into owned storage. + * + * @tparam T Host leaf element type (e.g. `int32_t` or `std::string`) + */ +template +class lists_column_initializer { + public: + /** + * @brief Construct an empty leaf. Avoids ambiguity between the leaf and nested + * empty `initializer_list` constructors. + */ + lists_column_initializer() = default; + + /** + * @brief Construct a leaf from scalar values. + * + * @param values Leaf element values + */ + lists_column_initializer(std::initializer_list values) : values_{values} {} + + /** + * @brief Construct a leaf from scalar values and a validity iterator. + * + * @tparam ValidityIterator Iterator convertible to `bool` + * @param values Leaf element values + * @param v Validity iterator over `values.size()` elements + */ + template + lists_column_initializer(std::initializer_list values, ValidityIterator v) : values_{values} + { + value_validity_.reserve(values_.size()); + for (std::size_t i = 0; i < values_.size(); ++i) { + value_validity_.push_back(static_cast(*v++)); + } + } + + /** + * @brief Construct a nested node from child initializers. + * + * This constructor is a template so the non-template leaf + * `initializer_list` constructor is preferred for scalar lists such as + * `{1, 2, 3}`. Otherwise both overloads are non-templates and constructing + * `Init` from an `int` via the nested overload recurses until the stack + * overflows. + * + * @param children Child list initializers + */ + template + requires(std::is_same_v) + lists_column_initializer(std::initializer_list children) + : children_{children.begin(), children.end()}, nested_{true} + { + } + + /** + * @brief Construct a nested node from child initializers and a row-validity iterator. + * + * @tparam ValidityIterator Iterator convertible to `bool` + * @param children Child list initializers + * @param v Validity iterator over `children.size()` rows + */ + template + requires(std::is_same_v) + lists_column_initializer(std::initializer_list children, ValidityIterator v) + : nested_{true} + { + children_.reserve(children.size()); + for (auto const& child : children) { + if (static_cast(*v++)) { + children_.push_back(child); + } else { + children_.emplace_back(); + children_.back().valid_ = false; + } + } + } + + [[nodiscard]] bool nested() const { return nested_; } + [[nodiscard]] bool valid() const { return valid_; } + [[nodiscard]] auto const& values() const { return values_; } + [[nodiscard]] auto const& value_validity() const { return value_validity_; } + [[nodiscard]] auto const& children() const { return children_; } + + /** + * @brief Recursively build child list wrappers and row validity for a nested node. + * + * Each valid child is allocated with the provided `stream` and `mr`. Null children are + * represented as default-constructed wrappers (skipped during concatenate). + * + * @tparam ElementT List wrapper element type + * @tparam SourceElementT Source type used by the list wrapper + * @param stream CUDA stream used for device memory operations + * @param mr Memory resources used to allocate child columns + * @return Child wrappers and an empty validity vector when all rows are valid, + * otherwise a validity mask matching `children().size()` + */ + template + [[nodiscard]] std::pair>, + std::vector> + build(rmm::cuda_stream_view stream, cudf::memory_resources mr) const + { + std::vector> children; + std::vector validity; + children.reserve(children_.size()); + validity.reserve(children_.size()); + bool any_null = false; + for (auto const& child : children_) { + any_null = any_null || !child.valid(); + validity.push_back(child.valid()); + if (child.valid()) { + children.emplace_back(child, stream, mr); + } else { + children.emplace_back(); // null rows are skipped during concatenate + } + } + return {std::move(children), any_null ? std::move(validity) : std::vector{}}; + } + + private: + std::vector values_; + std::vector value_validity_; + std::vector children_; + bool nested_{false}; + bool valid_{true}; +}; + /** * @brief `column_wrapper` derived class for wrapping columns of fixed-width * elements. @@ -1076,7 +1227,7 @@ class dictionary_column_wrapper : public detail::column_wrapper { begin, end, stream, mr.get_temporary_mr()), cudf::data_type{type_id::INT32}, stream, - mr.get_output_mr()); + mr); } /** @@ -1118,7 +1269,7 @@ class dictionary_column_wrapper : public detail::column_wrapper { begin, end, v, stream, mr.get_temporary_mr()), cudf::data_type{type_id::INT32}, stream, - mr.get_output_mr()); + mr); } /** @@ -1308,7 +1459,7 @@ class dictionary_column_wrapper : public detail::column_wrapper { cudf::dictionary::encode(strings_column_wrapper(begin, end, stream, mr.get_temporary_mr()), cudf::data_type{type_id::INT32}, stream, - mr.get_output_mr()); + mr); } /** @@ -1353,7 +1504,7 @@ class dictionary_column_wrapper : public detail::column_wrapper { cudf::dictionary::encode(strings_column_wrapper(begin, end, v, stream, mr.get_temporary_mr()), cudf::data_type{type_id::INT32}, stream, - mr.get_output_mr()); + mr); } /** @@ -1476,192 +1627,178 @@ class lists_column_wrapper : public detail::column_wrapper { */ operator lists_column_view() const { return cudf::lists_column_view{wrapped->view()}; } + using host_element_t = + std::conditional_t, std::string, SourceElementT>; + using leaf_wrapper_t = std::conditional_t, + strings_column_wrapper, + fixed_width_column_wrapper>; + /** - * @brief Construct a lists column containing a single list of fixed-width - * type from an initializer list of values. + * @brief Construct a lists column containing a single list from an initializer + * list of values. * * Example: * @code{.cpp} - * Creates a LIST column with 1 list composed of 2 total integers - * [{0, 1}] + * // Creates a LIST column with 1 list composed of 2 total integers + * // [{0, 1}] * lists_column_wrapper l{0, 1}; * @endcode * + * These leaf constructors are templates (via `requires`) so that the non-template + * nested `initializer_list` constructor is preferred for + * ambiguous cases such as `lists_column_wrapper{{}, {}}`. + * * @param elements The list of elements * @param stream CUDA stream used for device memory operations * @param mr Memory resources used to allocate the returned column */ - template ()>* = nullptr> + template + requires(cudf::is_fixed_width()) lists_column_wrapper(std::initializer_list elements, rmm::cuda_stream_view stream = cudf::test::get_default_stream(), cudf::memory_resources mr = cudf::get_current_device_resource_ref()) : column_wrapper{} { build_from_non_nested( - cudf::test::fixed_width_column_wrapper(elements, stream, mr).release(), - stream, - mr); + fixed_width_column_wrapper(elements, stream, mr).release(), stream, mr); } /** - * @brief Construct a lists column containing a single list of fixed-width - * type from an iterator range. + * @brief Construct a lists column containing a single list of strings. * * Example: * @code{.cpp} - * // Creates a LIST column with 1 list composed of 5 total integers - * auto elements = make_counting_transform_iterator(0, [](auto i){return i*2;}); - * // [{0, 1, 2, 3, 4}] - * lists_column_wrapper l(elements, elements+5); + * // Creates a LIST column with 1 list composed of 2 total strings + * // [{"abc", "def"}] + * lists_column_wrapper s{"abc", "def"}; * @endcode * - * @param begin Beginning of the sequence - * @param end End of the sequence + * @param elements The list of strings * @param stream CUDA stream used for device memory operations * @param mr Memory resources used to allocate the returned column */ - template ()>* = nullptr> - lists_column_wrapper(InputIterator begin, - InputIterator end, + template + requires(std::is_same_v) + lists_column_wrapper(std::initializer_list elements, rmm::cuda_stream_view stream = cudf::test::get_default_stream(), cudf::memory_resources mr = cudf::get_current_device_resource_ref()) : column_wrapper{} { - build_from_non_nested( - cudf::test::fixed_width_column_wrapper(begin, end, stream, mr).release(), - stream, - mr); + build_from_non_nested(strings_column_wrapper(elements, stream, mr).release(), stream, mr); } /** - * @brief Construct a lists column containing a single list of fixed-width - * type from an initializer list of values and a validity iterator. + * @brief Construct a lists column containing a single list from an iterator range. * * Example: * @code{.cpp} - * // Creates a LIST column with 1 lists composed of 2 total integers - * auto validity = make_counting_transform_iterator(0, [](auto i){return i%2;}); - * // [{0, NULL}] - * lists_column_wrapper l{{0, 1}, validity}; + * // Creates a LIST column with 1 list composed of 5 total integers + * auto elements = make_counting_transform_iterator(0, [](auto i){return i*2;}); + * // [{0, 1, 2, 3, 4}] + * lists_column_wrapper l(elements, elements+5); * @endcode * - * @param elements The list of elements - * @param v The validity iterator + * @param begin Beginning of the sequence + * @param end End of the sequence * @param stream CUDA stream used for device memory operations * @param mr Memory resources used to allocate the returned column */ - template ()>* = nullptr> - lists_column_wrapper(std::initializer_list elements, - ValidityIterator v, + template + lists_column_wrapper(InputIterator begin, + InputIterator end, rmm::cuda_stream_view stream = cudf::test::get_default_stream(), cudf::memory_resources mr = cudf::get_current_device_resource_ref()) : column_wrapper{} { - build_from_non_nested( - cudf::test::fixed_width_column_wrapper(elements, v, stream, mr).release(), - stream, - mr); + build_from_non_nested(leaf_wrapper_t(begin, end, stream, mr).release(), stream, mr); } /** - * @brief Construct a lists column containing a single list of fixed-width - * type from an iterator range and a validity iterator. + * @brief Construct a lists column containing a single list from an initializer + * list of values and a validity iterator. * * Example: * @code{.cpp} - * // Creates a LIST column with 1 lists composed of 5 total integers - * auto elements = make_counting_transform_iterator(0, [](auto i){return i*2;}); + * // Creates a LIST column with 1 list composed of 2 total integers * auto validity = make_counting_transform_iterator(0, [](auto i){return i%2;}); - * // [{0, NULL, 2, NULL, 4}] - * lists_column_wrapper l(elements, elements+5, validity); + * // [{0, NULL}] + * lists_column_wrapper l{{0, 1}, validity}; * @endcode * - * @param begin Beginning of the sequence - * @param end End of the sequence + * @param elements The list of elements * @param v The validity iterator * @param stream CUDA stream used for device memory operations * @param mr Memory resources used to allocate the returned column */ - template ()>* = nullptr> - lists_column_wrapper(InputIterator begin, - InputIterator end, + template + requires(cudf::is_fixed_width()) + lists_column_wrapper(std::initializer_list elements, ValidityIterator v, rmm::cuda_stream_view stream = cudf::test::get_default_stream(), cudf::memory_resources mr = cudf::get_current_device_resource_ref()) : column_wrapper{} { build_from_non_nested( - cudf::test::fixed_width_column_wrapper(begin, end, v, stream, mr) - .release(), + fixed_width_column_wrapper(elements, v, stream, mr).release(), stream, mr); } /** - * @brief Construct a lists column containing a single list of strings - * from an initializer list of values. + * @brief Construct a lists column containing a single list of strings and a + * validity iterator. * * Example: * @code{.cpp} - * // Creates a LIST column with 1 list composed of 2 total strings - * // [{"abc", "def"}] - * lists_column_wrapper l{"abc", "def"}; + * auto validity = make_counting_transform_iterator(0, [](auto i){return i%2;}); + * // [{"abc", NULL}] + * lists_column_wrapper l{{"abc", "def"}, validity}; * @endcode * - * @param elements The list of elements + * @param elements The list of strings + * @param v The validity iterator * @param stream CUDA stream used for device memory operations * @param mr Memory resources used to allocate the returned column */ - template >* = nullptr> + template + requires(std::is_same_v) lists_column_wrapper(std::initializer_list elements, + ValidityIterator v, rmm::cuda_stream_view stream = cudf::test::get_default_stream(), cudf::memory_resources mr = cudf::get_current_device_resource_ref()) : column_wrapper{} { - build_from_non_nested( - cudf::test::strings_column_wrapper(elements.begin(), elements.end(), stream, mr).release(), - stream, - mr); + build_from_non_nested(strings_column_wrapper(elements, v, stream, mr).release(), stream, mr); } /** - * @brief Construct a lists column containing a single list of strings - * from an initializer list of values and a validity iterator. + * @brief Construct a lists column containing a single list from an iterator + * range and a validity iterator. * * Example: * @code{.cpp} - * // Creates a LIST column with 1 list composed of 2 total strings + * // Creates a LIST column with 1 list composed of 5 total integers + * auto elements = make_counting_transform_iterator(0, [](auto i){return i*2;}); * auto validity = make_counting_transform_iterator(0, [](auto i){return i%2;}); - * // [{"abc", NULL}] - * lists_column_wrapper l{{"abc", "def"}, validity}; + * // [{0, NULL, 2, NULL, 4}] + * lists_column_wrapper l(elements, elements+5, validity); * @endcode * - * @param elements The list of elements + * @param begin Beginning of the sequence + * @param end End of the sequence * @param v The validity iterator * @param stream CUDA stream used for device memory operations * @param mr Memory resources used to allocate the returned column */ - template >* = nullptr> - lists_column_wrapper(std::initializer_list elements, + template + lists_column_wrapper(InputIterator begin, + InputIterator end, ValidityIterator v, rmm::cuda_stream_view stream = cudf::test::get_default_stream(), cudf::memory_resources mr = cudf::get_current_device_resource_ref()) : column_wrapper{} { - build_from_non_nested( - cudf::test::strings_column_wrapper(elements.begin(), elements.end(), v, stream, mr).release(), - stream, - mr); + build_from_non_nested(leaf_wrapper_t(begin, end, v, stream, mr).release(), stream, mr); } /** @@ -1683,6 +1820,11 @@ class lists_column_wrapper : public detail::column_wrapper { * lists_column_wrapper l{ {{0, 1}, {2, 3}}, {{4, 5}, {6, 7}} }; * @endcode * + * For multi-row (and deeper) columns that should allocate with an explicit stream/mr, use + * `lists_column_initializer` so every nesting level receives those arguments: + * `using Init = cudf::test::lists_column_initializer;` + * `lists_column_wrapper l{Init{{{0, 1}, {2, 3}, {4, 5}}}, stream, mr};` + * * @param elements The list of elements * @param stream CUDA stream used for device memory operations * @param mr Memory resources used to allocate the returned column @@ -1758,6 +1900,46 @@ class lists_column_wrapper : public detail::column_wrapper { build_from_nested(elements, validity, stream, mr); } + /** + * @brief Construct a lists column from a recursive `lists_column_initializer` tree. + * + * Every nesting level is allocated with the provided `stream` and `mr`. Prefer this over + * brace-nested `lists_column_wrapper` constructions that pass resources only at the outer + * level. + * + * Example: + * @code{.cpp} + * using Init = cudf::test::lists_column_initializer; + * // List: [{0, 1}, {2, 3}, {4, 5}] + * lists_column_wrapper l{Init{{{0, 1}, {2, 3}, {4, 5}}}, stream, mr}; + * + * // List>: [{{0, 1}, {2}}, {{3}}] + * lists_column_wrapper nested{Init{{{{0, 1}, {2}}, {{3}}}}, stream, mr}; + * @endcode + * + * @param init Host-side nested values (and optional validity) + * @param stream CUDA stream used for device memory operations + * @param mr Memory resources used to allocate the returned column + */ + lists_column_wrapper(lists_column_initializer init, + rmm::cuda_stream_view stream, + cudf::memory_resources mr) + : column_wrapper{} + { + if (!init.nested()) { + if (init.value_validity().empty()) { + *this = lists_column_wrapper(init.values().begin(), init.values().end(), stream, mr); + } else { + *this = lists_column_wrapper( + init.values().begin(), init.values().end(), init.value_validity().begin(), stream, mr); + } + return; + } + + auto [children, validity] = init.template build(stream, mr); + build_from_nested(children, validity, stream, mr); + } + /** * @brief Construct a list column containing a single empty, optionally null row. * @@ -1821,7 +2003,8 @@ class lists_column_wrapper : public detail::column_wrapper { * @param mr Memory resources used to allocate the returned column * */ - void build_from_nested(std::initializer_list> elements, + template + void build_from_nested(ListsRange const& elements, std::vector const& v, rmm::cuda_stream_view stream, cudf::memory_resources mr) @@ -1981,8 +2164,9 @@ class lists_column_wrapper : public detail::column_wrapper { cudf::copy_bitmask(col, stream, temp_mr)); } + template std::pair, std::vector>> preprocess_columns( - std::initializer_list> const& elements, + ListsRange const& elements, column_view& expected_hierarchy, int expected_depth, rmm::cuda_stream_view stream, diff --git a/cpp/src/binaryop/compiled/struct_binary_ops.cuh b/cpp/src/binaryop/compiled/struct_binary_ops.cuh index 18f4a193b4cd..1126c12a8365 100644 --- a/cpp/src/binaryop/compiled/struct_binary_ops.cuh +++ b/cpp/src/binaryop/compiled/struct_binary_ops.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -146,7 +146,8 @@ void apply_struct_equality_op(mutable_column_view& out, auto tlhs = table_view{{lhs}}; auto trhs = table_view{{rhs}}; - auto table_comparator = cudf::detail::row::equality::two_table_comparator{tlhs, trhs, stream}; + auto table_comparator = cudf::detail::row::equality::two_table_comparator{ + tlhs, trhs, stream, cudf::get_current_device_resource_ref()}; auto outd = column_device_view::create(out, stream); auto optional_iter = diff --git a/cpp/src/copying/gather.cu b/cpp/src/copying/gather.cu index a43e662478a0..ff6eae1c6443 100644 --- a/cpp/src/copying/gather.cu +++ b/cpp/src/copying/gather.cu @@ -28,7 +28,7 @@ std::unique_ptr
gather(table_view const& source_table, out_of_bounds_policy bounds_policy, negative_index_policy neg_indices, cuda::stream_ref stream, - rmm::device_async_resource_ref mr) + cudf::memory_resources mr) { CUDF_EXPECTS(not gather_map.has_nulls(), "gather_map contains nulls", std::invalid_argument); @@ -55,7 +55,7 @@ std::unique_ptr
gather(table_view const& source_table, out_of_bounds_policy bounds_policy, negative_index_policy neg_indices, cuda::stream_ref stream, - rmm::device_async_resource_ref mr) + cudf::memory_resources mr) { CUDF_EXPECTS(gather_map.size() <= static_cast(std::numeric_limits::max()), "gather map size exceeds the column size limit", @@ -74,7 +74,7 @@ std::unique_ptr
gather(table_view const& source_table, column_view const& gather_map, out_of_bounds_policy bounds_policy, cuda::stream_ref stream, - rmm::device_async_resource_ref mr) + cudf::memory_resources mr) { CUDF_FUNC_RANGE(); @@ -89,7 +89,7 @@ std::unique_ptr
gather(table_view const& source_table, out_of_bounds_policy bounds_policy, negative_index_policy neg_indices, cuda::stream_ref stream, - rmm::device_async_resource_ref mr) + cudf::memory_resources mr) { CUDF_FUNC_RANGE(); return detail::gather(source_table, gather_map, bounds_policy, neg_indices, stream, mr); diff --git a/cpp/src/dictionary/decode.cu b/cpp/src/dictionary/decode.cu index 0338a939ed7d..0249d0bf66cc 100644 --- a/cpp/src/dictionary/decode.cu +++ b/cpp/src/dictionary/decode.cu @@ -36,13 +36,13 @@ struct indices_handler_fn { */ std::unique_ptr decode(dictionary_column_view const& source, cuda::stream_ref stream, - rmm::device_async_resource_ref mr) + cudf::memory_resources mr) { if (source.is_empty()) return make_empty_column(type_id::EMPTY); // annotated indices include the offset, size and bitmask from it's parent auto const indices = source.get_indices_annotated(); - auto const d_indices = column_device_view::create(indices, stream); + auto const d_indices = column_device_view::create(indices, stream, mr.get_temporary_mr()); auto const d_iterator = cudf::detail::indexalator_factory::make_input_iterator(indices); auto const indices_begin = cudf::detail::make_counting_transform_iterator( 0, indices_handler_fn{d_iterator, *d_indices, source.keys().size()}); @@ -57,8 +57,8 @@ std::unique_ptr decode(dictionary_column_view const& source, auto output_column = std::unique_ptr(std::move(table_column.front())); // apply any nulls to the output column - output_column->set_null_mask(cudf::detail::copy_bitmask(source.parent(), stream, mr), - source.null_count()); + output_column->set_null_mask( + cudf::detail::copy_bitmask(source.parent(), stream, mr.get_output_mr()), source.null_count()); return output_column; } @@ -67,7 +67,7 @@ std::unique_ptr decode(dictionary_column_view const& source, std::unique_ptr decode(dictionary_column_view const& source, cuda::stream_ref stream, - rmm::device_async_resource_ref mr) + cudf::memory_resources mr) { CUDF_FUNC_RANGE(); return detail::decode(source, stream, mr); diff --git a/cpp/src/dictionary/detail/concatenate.cu b/cpp/src/dictionary/detail/concatenate.cu index e3e428c14ace..1380dd0e01d1 100644 --- a/cpp/src/dictionary/detail/concatenate.cu +++ b/cpp/src/dictionary/detail/concatenate.cu @@ -187,8 +187,9 @@ std::unique_ptr concatenate(host_span columns, cudf::detail::row::hash::device_row_hasher>; auto const tv = cudf::table_view({all_keys->view()}); - auto const row_hash = cudf::detail::row::hash::row_hasher(tv, stream); - auto const row_equal = cudf::detail::row::equality::self_comparator(tv, stream); + auto const temp_mr = cudf::get_current_device_resource_ref(); + auto const row_hash = cudf::detail::row::hash::row_hasher(tv, stream, temp_mr); + auto const row_equal = cudf::detail::row::equality::self_comparator(tv, stream, temp_mr); auto const comparator = cudf::detail::row::equality::nan_equal_physical_equality_comparator{}; auto const d_equal = row_equal.equal_to(cudf::nullate::NO{}, null_equality::EQUAL, comparator); diff --git a/cpp/src/dictionary/encode.cu b/cpp/src/dictionary/encode.cu index 7f43eb337370..eaab38d2bea3 100644 --- a/cpp/src/dictionary/encode.cu +++ b/cpp/src/dictionary/encode.cu @@ -56,7 +56,7 @@ struct encode_fn { std::unique_ptr encode(column_view const& input, data_type indices_type, cuda::stream_ref stream, - rmm::device_async_resource_ref mr) + cudf::memory_resources mr) { CUDF_EXPECTS(is_signed(indices_type) && is_index_type(indices_type), "indices must be type signed integer", @@ -68,8 +68,11 @@ std::unique_ptr encode(column_view const& input, "encoding nested types not supported", std::invalid_argument); + auto const output_mr = mr.get_output_mr(); + auto const temp_mr = mr.get_temporary_mr(); + auto indices_column = cudf::make_numeric_column( - indices_type, input.size(), cudf::mask_state::UNALLOCATED, stream, mr); + indices_type, input.size(), cudf::mask_state::UNALLOCATED, stream, output_mr); if (input.is_empty()) { return make_dictionary_column( make_empty_column(input.type()), std::move(indices_column), rmm::device_buffer{}, 0); @@ -82,13 +85,13 @@ std::unique_ptr encode(column_view const& input, auto const has_nulls = nullate::DYNAMIC{input.has_nulls()}; auto const tv = cudf::table_view({input}); - auto const row_hash = cudf::detail::row::hash::row_hasher(tv, stream); - auto const row_equal = cudf::detail::row::equality::self_comparator(tv, stream); + auto const row_hash = cudf::detail::row::hash::row_hasher(tv, stream, temp_mr); + auto const row_equal = cudf::detail::row::equality::self_comparator(tv, stream, temp_mr); auto const comparator = cudf::detail::row::equality::nan_equal_physical_equality_comparator{}; auto const d_equal = row_equal.equal_to(has_nulls, null_equality::EQUAL, comparator); auto const empty_key = cuco::empty_key{cudf::detail::CUDF_SIZE_TYPE_SENTINEL}; auto probe = encode_probe_t{row_hash.device_hasher(has_nulls)}; - auto allocator = rmm::mr::polymorphic_allocator{}; + auto allocator = rmm::mr::polymorphic_allocator{temp_mr}; auto set = cuco::static_set{input.size(), 0.5, empty_key, d_equal, probe, {}, {}, allocator, stream.get()}; auto set_ref = set.ref(cuco::insert_and_find); @@ -96,22 +99,20 @@ std::unique_ptr encode(column_view const& input, // build a static_set of the input values // and keep track of the indices of the unique values - auto d_indices = rmm::device_uvector(input.size(), stream); - auto d_input = column_device_view::create(input, stream); - thrust::transform(rmm::exec_policy_nosync(stream, cudf::get_current_device_resource_ref()), + auto d_indices = rmm::device_uvector(input.size(), stream, temp_mr); + auto d_input = column_device_view::create(input, stream, temp_mr); + thrust::transform(rmm::exec_policy_nosync(stream, temp_mr), cuda::counting_iterator{0}, cuda::counting_iterator{input.size()}, d_indices.begin(), encode_fn{set_ref, *d_input}); - auto keys_indices = rmm::device_uvector(input.size(), stream); + auto keys_indices = rmm::device_uvector(input.size(), stream, temp_mr); auto keys_end = set.retrieve_all(keys_indices.begin(), stream.get()); keys_indices.resize(cuda::std::distance(keys_indices.begin(), keys_end), stream); // sort the keys_indices so we can use lower-bound on them - thrust::sort(rmm::exec_policy_nosync(stream, cudf::get_current_device_resource_ref()), - keys_indices.begin(), - keys_indices.end()); + thrust::sort(rmm::exec_policy_nosync(stream, temp_mr), keys_indices.begin(), keys_indices.end()); // use keys_indices to retrieve the keys auto const oob_policy = cudf::out_of_bounds_policy::DONT_CHECK; @@ -124,7 +125,7 @@ std::unique_ptr encode(column_view const& input, // call lower-bound with keys_indices and d_indices to get the output indices_column auto d_result = cudf::detail::indexalator_factory::make_output_iterator(indices_column->mutable_view()); - thrust::lower_bound(rmm::exec_policy_nosync(stream, cudf::get_current_device_resource_ref()), + thrust::lower_bound(rmm::exec_policy_nosync(stream, temp_mr), keys_indices.begin(), keys_indices.end(), d_indices.begin(), @@ -134,7 +135,7 @@ std::unique_ptr encode(column_view const& input, // create column with keys_column and indices_column return make_dictionary_column(std::move(keys_column), std::move(indices_column), - cudf::detail::copy_bitmask(input, stream, mr), + cudf::detail::copy_bitmask(input, stream, output_mr), input.null_count()); } @@ -155,7 +156,7 @@ data_type get_indices_type_for_size(size_type keys_size) std::unique_ptr encode(column_view const& input_column, data_type indices_type, cuda::stream_ref stream, - rmm::device_async_resource_ref mr) + cudf::memory_resources mr) { CUDF_FUNC_RANGE(); return detail::encode(input_column, indices_type, stream, mr); diff --git a/cpp/src/dictionary/match_keys.cu b/cpp/src/dictionary/match_keys.cu index 269cb403998c..327a80c4f1ee 100644 --- a/cpp/src/dictionary/match_keys.cu +++ b/cpp/src/dictionary/match_keys.cu @@ -50,8 +50,9 @@ struct unique_keys_dispatch_fn { auto const has_nulls = nullate::DYNAMIC{false}; auto const keys_tv = table_view({all_keys}); - auto const row_hash = cudf::detail::row::hash::row_hasher(keys_tv, stream); - auto const row_equal = cudf::detail::row::equality::self_comparator(keys_tv, stream); + auto const temp_mr = cudf::get_current_device_resource_ref(); + auto const row_hash = cudf::detail::row::hash::row_hasher(keys_tv, stream, temp_mr); + auto const row_equal = cudf::detail::row::equality::self_comparator(keys_tv, stream, temp_mr); auto const comparator = cudf::detail::row::equality::nan_equal_physical_equality_comparator{}; auto const d_equal = row_equal.equal_to(has_nulls, null_equality::EQUAL, comparator); auto const empty_key = cuco::empty_key{cudf::detail::CUDF_SIZE_TYPE_SENTINEL}; diff --git a/cpp/src/groupby/hash/groupby.cu b/cpp/src/groupby/hash/groupby.cu index 161b0384537b..f78669799a84 100644 --- a/cpp/src/groupby/hash/groupby.cu +++ b/cpp/src/groupby/hash/groupby.cu @@ -42,10 +42,11 @@ std::unique_ptr
dispatch_groupby(table_view const& keys, auto const has_null = nullate::DYNAMIC{cudf::has_nested_nulls(keys)}; auto const skip_rows_with_nulls = keys_have_nulls and include_null_keys == null_policy::EXCLUDE; - auto preprocessed_keys = cudf::detail::row::hash::preprocessed_table::create(keys, stream); - auto const comparator = cudf::detail::row::equality::self_comparator{preprocessed_keys}; - auto const row_hash = cudf::detail::row::hash::row_hasher{std::move(preprocessed_keys)}; - auto const d_row_hash = row_hash.device_hasher(has_null); + auto preprocessed_keys = cudf::detail::row::hash::preprocessed_table::create( + keys, stream, cudf::get_current_device_resource_ref()); + auto const comparator = cudf::detail::row::equality::self_comparator{preprocessed_keys}; + auto const row_hash = cudf::detail::row::hash::row_hasher{std::move(preprocessed_keys)}; + auto const d_row_hash = row_hash.device_hasher(has_null); if (cudf::detail::has_nested_columns(keys)) { auto const d_row_equal = comparator.equal_to(has_null, null_keys_are_equal); diff --git a/cpp/src/groupby/sort/group_nunique.cu b/cpp/src/groupby/sort/group_nunique.cu index 96aaa4cace1c..2df7c9cf3205 100644 --- a/cpp/src/groupby/sort/group_nunique.cu +++ b/cpp/src/groupby/sort/group_nunique.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -79,7 +79,8 @@ std::unique_ptr group_nunique(column_view const& values, if (num_groups == 0) { return result; } auto const values_view = table_view{{values}}; - auto const comparator = cudf::detail::row::equality::self_comparator{values_view, stream}; + auto const comparator = cudf::detail::row::equality::self_comparator{ + values_view, stream, cudf::get_current_device_resource_ref()}; auto const d_values_view = column_device_view::create(values, stream); diff --git a/cpp/src/groupby/sort/group_rank_scan.cu b/cpp/src/groupby/sort/group_rank_scan.cu index 31ed09ff8cd7..2c96e45fc79f 100644 --- a/cpp/src/groupby/sort/group_rank_scan.cu +++ b/cpp/src/groupby/sort/group_rank_scan.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -94,7 +94,9 @@ std::unique_ptr rank_generator(column_view const& grouped_values, rmm::device_async_resource_ref mr) { auto const grouped_values_view = table_view{{grouped_values}}; - auto const comparator = cudf::detail::row::equality::self_comparator{grouped_values_view, stream}; + auto const temp_mr = cudf::get_current_device_resource_ref(); + auto const comparator = + cudf::detail::row::equality::self_comparator{grouped_values_view, stream, temp_mr}; auto ranks = make_fixed_width_column( data_type{type_to_id()}, grouped_values.size(), mask_state::UNALLOCATED, stream, mr); @@ -104,7 +106,7 @@ std::unique_ptr rank_generator(column_view const& grouped_values, auto const permuted_equal = permuted_row_equality_comparator(d_equal, value_order.begin()); - thrust::transform(rmm::exec_policy_nosync(stream, cudf::get_current_device_resource_ref()), + thrust::transform(rmm::exec_policy_nosync(stream, temp_mr), cuda::counting_iterator(0), cuda::counting_iterator(grouped_values.size()), mutable_ranks.begin(), @@ -130,14 +132,13 @@ std::unique_ptr rank_generator(column_view const& grouped_values, cuda::std::reverse_iterator(mutable_ranks.end())}; } }(); - thrust::inclusive_scan_by_key( - rmm::exec_policy_nosync(stream, cudf::get_current_device_resource_ref()), - group_labels_begin, - group_labels_begin + group_labels.size(), - mutable_rank_begin, - mutable_rank_begin, - cuda::std::equal_to{}, - scan_op); + thrust::inclusive_scan_by_key(rmm::exec_policy_nosync(stream, temp_mr), + group_labels_begin, + group_labels_begin + group_labels.size(), + mutable_rank_begin, + mutable_rank_begin, + cuda::std::equal_to{}, + scan_op); return ranks; } } // namespace diff --git a/cpp/src/groupby/sort/sort_helper_group_offsets.cuh b/cpp/src/groupby/sort/sort_helper_group_offsets.cuh index 958d424e2bf4..ddd2084c4d9b 100644 --- a/cpp/src/groupby/sort/sort_helper_group_offsets.cuh +++ b/cpp/src/groupby/sort/sort_helper_group_offsets.cuh @@ -38,22 +38,19 @@ size_type compute_group_offsets(table_view const& keys, rmm::device_uvector& group_offsets, rmm::cuda_stream_view stream) { - auto const comparator = cudf::detail::row::equality::self_comparator{keys, stream}; + auto const temp_mr = cudf::get_current_device_resource_ref(); + auto const comparator = cudf::detail::row::equality::self_comparator{keys, stream, temp_mr}; auto const d_key_equal = comparator.equal_to( cudf::nullate::DYNAMIC{cudf::has_nested_nulls(keys)}, null_equality::EQUAL); // Using a temporary buffer for intermediate transform results from the iterator containing // the comparator speeds up compile-time significantly without much degradation in // runtime performance over using the comparator directly in thrust::unique_copy. - auto result = rmm::device_uvector(size, stream); + auto result = rmm::device_uvector(size, stream, temp_mr); auto const itr = cuda::counting_iterator{0}; auto const row_eq = permuted_row_equality_comparator(d_key_equal, sorted_order); auto const ufn = cudf::detail::unique_copy_fn{ itr, duplicate_keep_option::KEEP_FIRST, row_eq, size - 1}; - thrust::transform(rmm::exec_policy_nosync(stream, cudf::get_current_device_resource_ref()), - itr, - itr + size, - result.begin(), - ufn); + thrust::transform(rmm::exec_policy_nosync(stream, temp_mr), itr, itr + size, result.begin(), ufn); auto const result_end = cudf::detail::copy_if( itr, itr + size, result.begin(), group_offsets.begin(), cuda::std::identity{}, stream); return cuda::std::distance(group_offsets.begin(), result_end); diff --git a/cpp/src/groupby/streaming_groupby/insert.cuh b/cpp/src/groupby/streaming_groupby/insert.cuh index 39be36b4af7e..4c51f0d9eefd 100644 --- a/cpp/src/groupby/streaming_groupby/insert.cuh +++ b/cpp/src/groupby/streaming_groupby/insert.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -34,7 +34,8 @@ streaming_groupby::impl::batch_insert_result streaming_groupby::impl::probe_and_ auto const has_null = cudf::nullate::DYNAMIC{_has_nullable_keys}; // Preprocess batch for row operators. - auto preprocessed_batch = cudf::detail::row::hash::preprocessed_table::create(batch_keys, stream); + auto preprocessed_batch = + cudf::detail::row::hash::preprocessed_table::create(batch_keys, stream, temp_mr); auto const batch_hasher_obj = cudf::detail::row::hash::row_hasher{preprocessed_batch}; auto const d_batch_hash = batch_hasher_obj.device_hasher(has_null); @@ -110,7 +111,7 @@ streaming_groupby::impl::batch_insert_result streaming_groupby::impl::probe_and_ temp_mr); auto preprocessed_compacted = - cudf::detail::row::hash::preprocessed_table::create(compacted->view(), stream); + cudf::detail::row::hash::preprocessed_table::create(compacted->view(), stream, temp_mr); // Store the compacted batch. auto const new_batch_id = static_cast(_compacted_batches.size()); diff --git a/cpp/src/hash/murmurhash3_x86_32.cu b/cpp/src/hash/murmurhash3_x86_32.cu index f82d552456d7..4e1f5ced36d5 100644 --- a/cpp/src/hash/murmurhash3_x86_32.cu +++ b/cpp/src/hash/murmurhash3_x86_32.cu @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -66,8 +67,8 @@ std::unique_ptr murmurhash3_x86_32(table_view const& input, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr) { - auto const preprocessed_input = - cudf::detail::row::hash::preprocessed_table::create(input, stream); + auto const preprocessed_input = cudf::detail::row::hash::preprocessed_table::create( + input, stream, cudf::get_current_device_resource_ref()); return murmurhash3_x86_32_impl( preprocessed_input, input.num_rows(), seed, nullate::DYNAMIC{has_nulls(input)}, stream, mr); } diff --git a/cpp/src/hash/xxhash_32.cu b/cpp/src/hash/xxhash_32.cu index 759a491b193b..0efd6f02ce12 100644 --- a/cpp/src/hash/xxhash_32.cu +++ b/cpp/src/hash/xxhash_32.cu @@ -30,9 +30,10 @@ std::unique_ptr xxhash_32(table_view const& input, if (input.num_rows() == 0) { return output; } - bool const nullable = has_nulls(input); - auto const row_hasher = cudf::detail::row::hash::row_hasher(input, stream); - auto output_view = output->mutable_view(); + bool const nullable = has_nulls(input); + auto const row_hasher = + cudf::detail::row::hash::row_hasher(input, stream, cudf::get_current_device_resource_ref()); + auto output_view = output->mutable_view(); // Compute the hash value for each row auto const output_begin = output_view.begin(); diff --git a/cpp/src/hash/xxhash_64.cu b/cpp/src/hash/xxhash_64.cu index fcf7009bd128..69c97724bbd6 100644 --- a/cpp/src/hash/xxhash_64.cu +++ b/cpp/src/hash/xxhash_64.cu @@ -32,9 +32,10 @@ std::unique_ptr xxhash_64(table_view const& input, if (input.num_rows() == 0) { return output; } - bool const nullable = has_nulls(input); - auto const row_hasher = cudf::detail::row::hash::row_hasher(input, stream); - auto output_view = output->mutable_view(); + bool const nullable = has_nulls(input); + auto const row_hasher = + cudf::detail::row::hash::row_hasher(input, stream, cudf::get_current_device_resource_ref()); + auto output_view = output->mutable_view(); // Compute the hash value for each row auto const output_begin = output_view.begin(); diff --git a/cpp/src/join/distinct_hash_join.cu b/cpp/src/join/distinct_hash_join.cu index 8a5cf7b2a279..d08e3da7406c 100644 --- a/cpp/src/join/distinct_hash_join.cu +++ b/cpp/src/join/distinct_hash_join.cu @@ -128,9 +128,9 @@ void find_matches_in_hash_table(HashTableType const& hash_table, hash_table.find_async( iter, iter + left_table_num_rows, d_equal, hasher, found_begin, stream.value()); } else { - auto stencil = cuda::counting_iterator{0}; - auto const row_bitmask = - cudf::detail::bitmask_and(left, stream, cudf::get_current_device_resource_ref()).first; + auto const temp_mr = cudf::get_current_device_resource_ref(); + auto stencil = cuda::counting_iterator{0}; + auto const row_bitmask = cudf::detail::bitmask_and(left, stream, temp_mr).first; auto const pred = cudf::detail::row_is_valid{reinterpret_cast(row_bitmask.data())}; @@ -163,7 +163,8 @@ distinct_hash_join::distinct_hash_join(cudf::table_view const& right, : _has_nested_columns{cudf::has_nested_columns(right)}, _nulls_equal{compare_nulls}, _right{right}, - _preprocessed_right{cudf::detail::row::equality::preprocessed_table::create(_right, stream)}, + _preprocessed_right{cudf::detail::row::equality::preprocessed_table::create( + _right, stream, cudf::get_current_device_resource_ref())}, _hash_table{cuco::extent{static_cast(right.num_rows())}, checked_load_factor(load_factor), cuco::empty_key{cuco::pair{std::numeric_limits::max(), @@ -181,13 +182,14 @@ distinct_hash_join::distinct_hash_join(cudf::table_view const& right, if (right_table_num_rows == 0) { return; } + auto const temp_mr = cudf::get_current_device_resource_ref(); + auto const build_hash_table = [&](auto iter) { if (this->_nulls_equal == cudf::null_equality::EQUAL or (not cudf::nullable(right))) { this->_hash_table.insert_async(iter, iter + right_table_num_rows, stream.value()); } else { - auto stencil = cuda::counting_iterator{0}; - auto const row_bitmask = - cudf::detail::bitmask_and(_right, stream, cudf::get_current_device_resource_ref()).first; + auto stencil = cuda::counting_iterator{0}; + auto const row_bitmask = cudf::detail::bitmask_and(_right, stream, temp_mr).first; auto const pred = cudf::detail::row_is_valid{reinterpret_cast(row_bitmask.data())}; @@ -240,7 +242,9 @@ distinct_hash_join::inner_join(cudf::table_view const& left, auto found_indices = rmm::device_uvector(left_table_num_rows, stream); auto const found_begin = cuda::make_transform_output_iterator(found_indices.begin(), output_fn{}); - auto preprocessed_left = cudf::detail::row::equality::preprocessed_table::create(left, stream); + auto const temp_mr = cudf::get_current_device_resource_ref(); + auto preprocessed_left = + cudf::detail::row::equality::preprocessed_table::create(left, stream, temp_mr); if (cudf::detail::is_primitive_row_op_compatible(_right)) { auto const d_hasher = cudf::detail::row::primitive::row_hasher{nullate::DYNAMIC{has_nulls}, preprocessed_left}; @@ -333,7 +337,9 @@ std::unique_ptr> distinct_hash_join::left_join( auto const output_begin = cuda::make_transform_output_iterator(right_indices->begin(), output_fn{}); - auto preprocessed_left = cudf::detail::row::equality::preprocessed_table::create(left, stream); + auto const temp_mr = cudf::get_current_device_resource_ref(); + auto preprocessed_left = + cudf::detail::row::equality::preprocessed_table::create(left, stream, temp_mr); if (cudf::detail::is_primitive_row_op_compatible(_right)) { auto const d_hasher = @@ -355,7 +361,7 @@ std::unique_ptr> distinct_hash_join::left_join( } else { // If right table is empty, return left table if (this->_right.num_rows() == 0) { - thrust::fill(rmm::exec_policy_nosync(stream, cudf::get_current_device_resource_ref()), + thrust::fill(rmm::exec_policy_nosync(stream, temp_mr), right_indices->begin(), right_indices->end(), cudf::JoinNoMatch); diff --git a/cpp/src/join/filtered_join/filtered_join.cu b/cpp/src/join/filtered_join/filtered_join.cu index d1766f0ffab3..024d000be34f 100644 --- a/cpp/src/join/filtered_join/filtered_join.cu +++ b/cpp/src/join/filtered_join/filtered_join.cu @@ -103,11 +103,12 @@ filtered_join::filtered_join(cudf::table_view const& right, : _right_mode{select_row_operator_mode(right)}, _bucket_storage{cuco::extent{compute_bucket_storage_size( right.num_rows(), checked_load_factor(load_factor), _right_mode)}, - rmm::mr::polymorphic_allocator{std::move(mr)}, + rmm::mr::polymorphic_allocator{mr}, stream.value()}, _right{right}, _nulls_equal{compare_nulls}, - _preprocessed_right{cudf::detail::row::equality::preprocessed_table::create(_right, stream)} + _preprocessed_right{cudf::detail::row::equality::preprocessed_table::create( + _right, stream, cudf::get_current_device_resource_ref())} { cudf::scoped_range range{"filtered_join::filtered_join"}; if (_right.num_rows() == 0) return; @@ -129,9 +130,10 @@ std::unique_ptr> filtered_join::semi_anti_j { cudf::scoped_range range{"filtered_join::semi_anti_join"}; - auto const preprocessed_left = [&left, stream] { + auto const preprocessed_left = [&left, stream, mr] { cudf::scoped_range range{"filtered_join::semi_anti_join::preprocessed_left"}; - return cudf::detail::row::equality::preprocessed_table::create(left, stream); + return cudf::detail::row::equality::preprocessed_table::create( + left, stream, cudf::get_current_device_resource_ref()); }(); auto contains_map = rmm::device_uvector(left.num_rows(), stream); diff --git a/cpp/src/join/hash_join/hash_join.cu b/cpp/src/join/hash_join/hash_join.cu index b699e04fff42..e0f7b8d2d87b 100644 --- a/cpp/src/join/hash_join/hash_join.cu +++ b/cpp/src/join/hash_join/hash_join.cu @@ -127,10 +127,11 @@ hash_join::hash_join(cudf::table_view const& right, {}, {}, {}, - rmm::mr::polymorphic_allocator{std::move(mr)}, + rmm::mr::polymorphic_allocator{mr}, stream.value()}})}, _right{right}, - _preprocessed_right{cudf::detail::row::equality::preprocessed_table::create(_right, stream)} + _preprocessed_right{cudf::detail::row::equality::preprocessed_table::create( + _right, stream, cudf::get_current_device_resource_ref())} { CUDF_FUNC_RANGE(); CUDF_EXPECTS(0 != right.num_columns(), "Hash join right table is empty", std::invalid_argument); diff --git a/cpp/src/join/hash_join/match_context.cu b/cpp/src/join/hash_join/match_context.cu index 5fc2dd5ba9cf..dbe44e01649d 100644 --- a/cpp/src/join/hash_join/match_context.cu +++ b/cpp/src/join/hash_join/match_context.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -32,9 +32,10 @@ std::unique_ptr> make_join_match_counts( rmm::device_async_resource_ref mr) { auto match_counts = std::make_unique>(left.num_rows(), stream, mr); + auto const temp_mr = cudf::get_current_device_resource_ref(); if (is_empty) { - thrust::fill(rmm::exec_policy_nosync(stream, cudf::get_current_device_resource_ref()), + thrust::fill(rmm::exec_policy_nosync(stream, temp_mr), match_counts->begin(), match_counts->end(), join == join_kind::INNER_JOIN ? 0 : 1); @@ -46,14 +47,14 @@ std::unique_ptr> make_join_match_counts( std::invalid_argument); auto const preprocessed_left = - cudf::detail::row::equality::preprocessed_table::create(left, stream); + cudf::detail::row::equality::preprocessed_table::create(left, stream, temp_mr); auto const left_table_num_rows = left.num_rows(); auto count_matches = [&](auto equality, auto d_hasher) { // Precompute left keys: {hash(row_idx), row_idx} for each left row. auto const n = static_cast(left_table_num_rows); - rmm::device_uvector left_keys(n, stream); - thrust::transform(rmm::exec_policy_nosync(stream, cudf::get_current_device_resource_ref()), + rmm::device_uvector left_keys(n, stream, temp_mr); + thrust::transform(rmm::exec_policy_nosync(stream, temp_mr), cuda::counting_iterator(0), cuda::counting_iterator(left_table_num_rows), left_keys.begin(), diff --git a/cpp/src/join/hash_join/partitioned_join_retrieve.cu b/cpp/src/join/hash_join/partitioned_join_retrieve.cu index 77dcae1e9325..af66590c39a8 100644 --- a/cpp/src/join/hash_join/partitioned_join_retrieve.cu +++ b/cpp/src/join/hash_join/partitioned_join_retrieve.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -106,8 +106,9 @@ hash_join::partitioned_join_retrieve(join_kind join, validate_hash_join_probe(_right, left_partition_view, _has_nulls); + auto const temp_mr = cudf::get_current_device_resource_ref(); auto const preprocessed_left = - cudf::detail::row::equality::preprocessed_table::create(left_partition_view, stream); + cudf::detail::row::equality::preprocessed_table::create(left_partition_view, stream, temp_mr); // For FULL_JOIN, probe with LEFT_JOIN semantics (no complement here) bool const is_outer = (join != join_kind::INNER_JOIN); @@ -123,8 +124,8 @@ hash_join::partitioned_join_retrieve(join_kind join, auto retrieve_partition = [&](auto equality, auto d_hasher) { // Precompute left keys for this partition slice. - rmm::device_uvector left_keys(n, stream); - thrust::transform(rmm::exec_policy_nosync(stream, cudf::get_current_device_resource_ref()), + rmm::device_uvector left_keys(n, stream, temp_mr); + thrust::transform(rmm::exec_policy_nosync(stream, temp_mr), cuda::counting_iterator(0), cuda::counting_iterator(partition_size), left_keys.begin(), diff --git a/cpp/src/join/hash_join/retrieve_impl.cuh b/cpp/src/join/hash_join/retrieve_impl.cuh index 5efe69afe850..5c7fefeb1cc0 100644 --- a/cpp/src/join/hash_join/retrieve_impl.cuh +++ b/cpp/src/join/hash_join/retrieve_impl.cuh @@ -173,8 +173,8 @@ hash_join::join_retrieve(cudf::table_view const& left, } } - auto const preprocessed_left = - cudf::detail::row::equality::preprocessed_table::create(left, stream); + auto const preprocessed_left = cudf::detail::row::equality::preprocessed_table::create( + left, stream, cudf::get_current_device_resource_ref()); auto join_indices = cudf::detail::probe_join_hash_table(_right, left, diff --git a/cpp/src/join/hash_join/size_impl.cuh b/cpp/src/join/hash_join/size_impl.cuh index 3e20ebc7367e..48218accc3f0 100644 --- a/cpp/src/join/hash_join/size_impl.cuh +++ b/cpp/src/join/hash_join/size_impl.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ #pragma once @@ -80,8 +80,8 @@ std::size_t hash_join::join_size(cudf::table_view const& left, "Left table has nulls while right table was not hashed with null check.", std::invalid_argument); - auto const preprocessed_left = - cudf::detail::row::equality::preprocessed_table::create(left, stream); + auto const preprocessed_left = cudf::detail::row::equality::preprocessed_table::create( + left, stream, cudf::get_current_device_resource_ref()); return cudf::detail::compute_join_output_size(_right, left, @@ -109,8 +109,8 @@ std::size_t hash_join::join_size(cudf::table_view const& left, "Left table has nulls while right table was not hashed with null check.", std::invalid_argument); - auto const preprocessed_left = - cudf::detail::row::equality::preprocessed_table::create(left, stream); + auto const preprocessed_left = cudf::detail::row::equality::preprocessed_table::create( + left, stream, cudf::get_current_device_resource_ref()); return cudf::detail::get_full_join_size(_right, left, diff --git a/cpp/src/join/key_remapping.cu b/cpp/src/join/key_remapping.cu index ef8540c261f4..a1ea75b7f3f2 100644 --- a/cpp/src/join/key_remapping.cu +++ b/cpp/src/join/key_remapping.cu @@ -305,6 +305,8 @@ class key_remap_table : public key_remap_table_interface { cudf::size_type const right_num_rows{_right.num_rows()}; if (right_num_rows == 0) { return; } + auto const temp_mr = cudf::get_current_device_resource_ref(); + auto const key_iter = cudf::detail::make_counting_transform_iterator( 0, make_key_pair{row_hasher}); @@ -312,9 +314,7 @@ class key_remap_table : public key_remap_table_interface { (_compare_nulls == cudf::null_equality::UNEQUAL) && cudf::nullable(right); auto const row_bitmask = - skip_nulls - ? cudf::detail::bitmask_and(_right, stream, cudf::get_current_device_resource_ref()).first - : rmm::device_buffer{}; + skip_nulls ? cudf::detail::bitmask_and(_right, stream, temp_mr).first : rmm::device_buffer{}; auto const bitmask_ptr = skip_nulls ? reinterpret_cast(row_bitmask.data()) : nullptr; @@ -325,7 +325,7 @@ class key_remap_table : public key_remap_table_interface { // No metrics - simple insert auto set_ref = _hash_table.ref(cuco::op::insert); thrust::for_each_n( - rmm::exec_policy_nosync(stream, cudf::get_current_device_resource_ref()), + rmm::exec_policy_nosync(stream, temp_mr), cuda::counting_iterator{0}, right_num_rows, insert_only_fn{set_ref, key_iter, bitmask_ptr}); @@ -339,14 +339,11 @@ class key_remap_table : public key_remap_table_interface { cudf::bitmask_type const* bitmask_ptr, rmm::cuda_stream_view stream) { + auto const temp_mr = cudf::get_current_device_resource_ref(); rmm::device_uvector counts(right_num_rows, stream); - thrust::fill(rmm::exec_policy_nosync(stream, cudf::get_current_device_resource_ref()), - counts.begin(), - counts.end(), - 0); + thrust::fill(rmm::exec_policy_nosync(stream, temp_mr), counts.begin(), counts.end(), 0); - cudf::detail::device_scalar d_distinct_count{ - 0, stream, cudf::get_current_device_resource_ref()}; + cudf::detail::device_scalar d_distinct_count{0, stream, temp_mr}; auto set_ref = _hash_table.ref(cuco::op::insert_and_find); @@ -358,12 +355,11 @@ class key_remap_table : public key_remap_table_interface { _distinct_count = d_distinct_count.value(stream); - _max_duplicate_count = - thrust::reduce(rmm::exec_policy_nosync(stream, cudf::get_current_device_resource_ref()), - counts.begin(), - counts.end(), - cudf::size_type{0}, - cuda::maximum{}); + _max_duplicate_count = thrust::reduce(rmm::exec_policy_nosync(stream, temp_mr), + counts.begin(), + counts.end(), + cudf::size_type{0}, + cuda::maximum{}); } public: @@ -380,10 +376,12 @@ class key_remap_table : public key_remap_table_interface { return std::make_unique>(0, stream, mr); } + auto const temp_mr = cudf::get_current_device_resource_ref(); + if (this->_right.num_rows() == 0) { auto result = std::make_unique>(left_num_rows, stream, mr); - thrust::fill(rmm::exec_policy_nosync(stream, cudf::get_current_device_resource_ref()), + thrust::fill(rmm::exec_policy_nosync(stream, temp_mr), result->begin(), result->end(), cudf::JoinNoMatch); @@ -395,7 +393,7 @@ class key_remap_table : public key_remap_table_interface { cuda::make_transform_output_iterator(result->begin(), extract_index{}); auto preprocessed_left = - cudf::detail::row::equality::preprocessed_table::create(left_keys, stream); + cudf::detail::row::equality::preprocessed_table::create(left_keys, stream, temp_mr); if (cudf::detail::is_primitive_row_op_compatible(_right)) { auto const d_hasher = cudf::detail::row::primitive::row_hasher{ @@ -500,7 +498,8 @@ std::unique_ptr create_key_remap_table( if (right.num_rows() == 0 || right.num_columns() == 0) { return nullptr; } - auto preprocessed_right = cudf::detail::row::equality::preprocessed_table::create(right, stream); + auto preprocessed_right = cudf::detail::row::equality::preprocessed_table::create( + right, stream, cudf::get_current_device_resource_ref()); if (cudf::detail::is_primitive_row_op_compatible(right)) { auto const d_hasher = cudf::detail::row::primitive::row_hasher{ diff --git a/cpp/src/join/mark_join.cu b/cpp/src/join/mark_join.cu index c050170e3853..94d531a2fb50 100644 --- a/cpp/src/join/mark_join.cu +++ b/cpp/src/join/mark_join.cu @@ -597,7 +597,8 @@ mark_join::mark_join(cudf::table_view const& left, _left{left}, _nulls_equal{compare_nulls}, _prefilter{prefilter}, - _preprocessed_left{cudf::detail::row::equality::preprocessed_table::create(left, stream)}, + _preprocessed_left{cudf::detail::row::equality::preprocessed_table::create( + left, stream, cudf::get_current_device_resource_ref())}, _bucket_storage{ cuco::extent{compute_mark_join_capacity(left, checked_load_factor(load_factor))}, rmm::mr::polymorphic_allocator{mr}, @@ -742,9 +743,10 @@ std::unique_ptr> mark_join::semi_anti_join( { clear_marks(stream); - auto const preprocessed_right = [&right, stream] { + auto const preprocessed_right = [&right, stream, mr] { cudf::scoped_range range{"mark_join::semi_anti_join::preprocessed_right"}; - return cudf::detail::row::equality::preprocessed_table::create(right, stream); + return cudf::detail::row::equality::preprocessed_table::create( + right, stream, cudf::get_current_device_resource_ref()); }(); if (is_primitive_row_op_compatible(_left)) { diff --git a/cpp/src/join/mixed_join_semi.cu b/cpp/src/join/mixed_join_semi.cu index 8021cdab21bc..0b44b2b6c3ed 100644 --- a/cpp/src/join/mixed_join_semi.cu +++ b/cpp/src/join/mixed_join_semi.cu @@ -105,10 +105,11 @@ std::unique_ptr> mixed_join_semi( auto left_conditional_view = table_device_view::create(left_conditional, stream); auto right_conditional_view = table_device_view::create(right_conditional, stream); + auto const temp_mr = cudf::get_current_device_resource_ref(); auto const preprocessed_right = - cudf::detail::row::equality::preprocessed_table::create(right, stream); + cudf::detail::row::equality::preprocessed_table::create(right, stream, temp_mr); auto const preprocessed_left = - cudf::detail::row::equality::preprocessed_table::create(left, stream); + cudf::detail::row::equality::preprocessed_table::create(left, stream, temp_mr); auto const row_comparator = cudf::detail::row::equality::two_table_comparator{preprocessed_left, preprocessed_right}; auto const equality_left = row_comparator.equal_to(has_nulls, compare_nulls); @@ -134,7 +135,7 @@ std::unique_ptr> mixed_join_semi( auto const equality_right_equality = row_comparator_right.equal_to(right_nulls, compare_nulls); auto const preprocessed_right_condtional = - cudf::detail::row::equality::preprocessed_table::create(right_conditional, stream); + cudf::detail::row::equality::preprocessed_table::create(right_conditional, stream, temp_mr); auto const row_comparator_conditional_right = cudf::detail::row::equality::two_table_comparator{ preprocessed_right_condtional, preprocessed_right_condtional}; auto const equality_right_conditional = @@ -147,7 +148,7 @@ std::unique_ptr> mixed_join_semi( {row_hash_right.device_hasher(right_nulls)}, {}, {}, - rmm::mr::polymorphic_allocator{}, + rmm::mr::polymorphic_allocator{temp_mr}, {stream.value()}}; auto iter = cuda::counting_iterator{0}; @@ -157,8 +158,7 @@ std::unique_ptr> mixed_join_semi( row_set.insert_async(iter, iter + right_num_rows, stream.value()); } else { cuda::counting_iterator stencil(0); - auto const [row_bitmask, _] = - cudf::detail::bitmask_and(right, stream, cudf::get_current_device_resource_ref()); + auto const [row_bitmask, _] = cudf::detail::bitmask_and(right, stream, temp_mr); row_is_valid pred{static_cast(row_bitmask.data())}; // insert valid rows @@ -176,7 +176,7 @@ std::unique_ptr> mixed_join_semi( hash_set_ref_type const row_set_ref = row_set.ref(cuco::contains).rebind_hash_function(hash_left); // Vector used to indicate indices from the left table which are present in output - auto left_table_keep_mask = rmm::device_uvector(left.num_rows(), stream); + auto left_table_keep_mask = rmm::device_uvector(left.num_rows(), stream, temp_mr); launch_mixed_join_semi(has_nulls, *left_conditional_view, diff --git a/cpp/src/lists/contains.cu b/cpp/src/lists/contains.cu index 58251a141111..6dc114ac0468 100644 --- a/cpp/src/lists/contains.cu +++ b/cpp/src/lists/contains.cu @@ -210,8 +210,8 @@ std::unique_ptr dispatch_index_of(lists_column_view const& lists, auto const keys_tview = cudf::table_view{{search_keys}}; auto const child_tview = cudf::table_view{{child}}; auto const has_nulls = has_nested_nulls(child_tview) || has_nested_nulls(keys_tview); - auto const comparator = - cudf::detail::row::equality::two_table_comparator(child_tview, keys_tview, stream); + auto const comparator = cudf::detail::row::equality::two_table_comparator( + child_tview, keys_tview, stream, cudf::get_current_device_resource_ref()); if (cudf::is_nested(search_keys.type())) { auto const d_comp = comparator.equal_to(nullate::DYNAMIC{has_nulls}); index_of(input_it, num_rows, output_it, child, search_keys, find_option, d_comp, stream); diff --git a/cpp/src/lists/copying/segmented_gather.cu b/cpp/src/lists/copying/segmented_gather.cu index fdbf786e1f5e..d0827df6d87c 100644 --- a/cpp/src/lists/copying/segmented_gather.cu +++ b/cpp/src/lists/copying/segmented_gather.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ #include @@ -28,7 +28,7 @@ std::unique_ptr segmented_gather(lists_column_view const& value_column, lists_column_view const& gather_map, out_of_bounds_policy bounds_policy, rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) + cudf::memory_resources mr) { CUDF_EXPECTS(is_index_type(gather_map.child().type()), "Gather map should be list column of index type"); @@ -37,12 +37,15 @@ std::unique_ptr segmented_gather(lists_column_view const& value_column, "Gather map and list column should be same size"); if (value_column.is_empty()) { return empty_like(value_column.parent()); } + auto const output_mr = mr.get_output_mr(); + auto const temp_mr = mr.get_temporary_mr(); + auto const gather_map_sliced_child = gather_map.get_sliced_child(stream); auto const gather_map_size = gather_map_sliced_child.size(); auto const gather_index_begin = gather_map.offsets_begin() + 1; auto const gather_index_end = gather_map.offsets_end(); auto const value_offsets = value_column.offsets_begin(); - auto const value_device_view = column_device_view::create(value_column.parent(), stream); + auto const value_device_view = column_device_view::create(value_column.parent(), stream, temp_mr); auto const map_begin = cudf::detail::indexalator_factory::make_input_iterator(gather_map_sliced_child); auto const out_of_bounds = [] __device__(auto const index, auto const list_size) { @@ -88,7 +91,7 @@ std::unique_ptr segmented_gather(lists_column_view const& value_column, // Create list offsets from gather_map. auto output_offset = cudf::detail::allocate_like( - gather_map.offsets(), gather_map.size() + 1, mask_allocation_policy::RETAIN, stream, mr); + gather_map.offsets(), gather_map.size() + 1, mask_allocation_policy::RETAIN, stream, output_mr); auto output_offset_view = output_offset->mutable_view(); cudf::detail::copy_range_in_place(gather_map.offsets(), output_offset_view, @@ -97,7 +100,7 @@ std::unique_ptr segmented_gather(lists_column_view const& value_column, 0, stream); // Assemble list column & return - auto null_mask = cudf::detail::copy_bitmask(value_column.parent(), stream, mr); + auto null_mask = cudf::detail::copy_bitmask(value_column.parent(), stream, output_mr); size_type null_count = value_column.null_count(); return make_lists_column(gather_map.size(), std::move(output_offset), @@ -112,7 +115,7 @@ std::unique_ptr segmented_gather(lists_column_view const& source_column, lists_column_view const& gather_map_list, out_of_bounds_policy bounds_policy, rmm::cuda_stream_view stream, - rmm::device_async_resource_ref mr) + cudf::memory_resources mr) { CUDF_FUNC_RANGE(); return detail::segmented_gather(source_column, gather_map_list, bounds_policy, stream, mr); diff --git a/cpp/src/partitioning/partitioning.cu b/cpp/src/partitioning/partitioning.cu index 3becafb1b4a3..0dc1b38791c5 100644 --- a/cpp/src/partitioning/partitioning.cu +++ b/cpp/src/partitioning/partitioning.cu @@ -575,8 +575,9 @@ std::pair, std::vector> hash_partition_table( rmm::device_async_resource_ref mr) { auto const num_rows = table_to_hash.num_rows(); + auto const temp_mr = cudf::get_current_device_resource_ref(); - auto const row_hasher = detail::row::hash::row_hasher(table_to_hash, stream); + auto const row_hasher = detail::row::hash::row_hasher(table_to_hash, stream, temp_mr); auto const hasher = row_hasher.device_hasher(nullate::DYNAMIC{hash_has_nulls}, seed); @@ -600,7 +601,7 @@ std::pair, std::vector> hash_partition_table( std::size_t const grid_size = util::div_rounding_up_safe(num_rows, rows_per_block); // Allocate array to hold which partition each row belongs to - auto row_partition_numbers = rmm::device_uvector(num_rows, stream); + auto row_partition_numbers = rmm::device_uvector(num_rows, stream, temp_mr); // Array to hold the size of each partition computed by each block // i.e., { {block0 partition0 size, block1 partition0 size, ...}, @@ -608,17 +609,18 @@ std::pair, std::vector> hash_partition_table( // ... // {block0 partition(num_partitions-1) size, block1 // partition(num_partitions -1) size, ...} } - auto block_partition_sizes = rmm::device_uvector(grid_size * num_partitions, stream); + auto block_partition_sizes = + rmm::device_uvector(grid_size * num_partitions, stream, temp_mr); auto scanned_block_partition_sizes = - rmm::device_uvector(grid_size * num_partitions, stream); + rmm::device_uvector(grid_size * num_partitions, stream, temp_mr); // Holds the total number of rows in each partition - auto global_partition_sizes = cudf::detail::make_zeroed_device_uvector_async( - num_partitions, stream, cudf::get_current_device_resource_ref()); + auto global_partition_sizes = + cudf::detail::make_zeroed_device_uvector_async(num_partitions, stream, temp_mr); - auto row_partition_offset = cudf::detail::make_zeroed_device_uvector_async( - num_rows, stream, cudf::get_current_device_resource_ref()); + auto row_partition_offset = + cudf::detail::make_zeroed_device_uvector_async(num_rows, stream, temp_mr); // If the number of partitions is a power of two, we can compute the partition // number of each row more efficiently with bitwise operations @@ -668,7 +670,7 @@ std::pair, std::vector> hash_partition_table( // Compute exclusive scan of all blocks' partition sizes in-place to determine // the starting point for each blocks portion of each partition in the output - thrust::exclusive_scan(rmm::exec_policy_nosync(stream, cudf::get_current_device_resource_ref()), + thrust::exclusive_scan(rmm::exec_policy_nosync(stream, temp_mr), block_partition_sizes.begin(), block_partition_sizes.end(), scanned_block_partition_sizes.data()); @@ -676,7 +678,7 @@ std::pair, std::vector> hash_partition_table( // Compute exclusive scan of size of each partition to determine offset // location of each partition in final output. // TODO This can be done independently on a separate stream - thrust::exclusive_scan(rmm::exec_policy_nosync(stream, cudf::get_current_device_resource_ref()), + thrust::exclusive_scan(rmm::exec_policy_nosync(stream, temp_mr), global_partition_sizes.begin(), global_partition_sizes.end(), global_partition_sizes.begin()); diff --git a/cpp/src/reductions/approx_distinct_count.cu b/cpp/src/reductions/approx_distinct_count.cu index 70b1c46ff58b..064f2eaa1b17 100644 --- a/cpp/src/reductions/approx_distinct_count.cu +++ b/cpp/src/reductions/approx_distinct_count.cu @@ -222,8 +222,9 @@ void approx_distinct_count::add(table_view const& input, rmm::cuda_strea typename approx_distinct_count::hll_ref_type ref{sketch(), cuda::std::identity{}}; auto const has_nulls = nullate::DYNAMIC{cudf::has_nested_nulls(input)}; + auto const temp_mr = cudf::get_current_device_resource_ref(); auto const preprocessed_input = - cudf::detail::row::hash::preprocessed_table::create(input, stream); + cudf::detail::row::hash::preprocessed_table::create(input, stream, temp_mr); auto const row_hasher = cudf::detail::row::hash::row_hasher(preprocessed_input); auto const hash_key = row_hasher.device_hasher(has_nulls); @@ -245,9 +246,8 @@ void approx_distinct_count::add(table_view const& input, rmm::cuda_strea if (!has_nulls) { ref.add_async(hash_iter, hash_iter + num_rows, stream); } else { - auto const row_bitmask = - cudf::detail::bitmask_and(input, stream, cudf::get_current_device_resource_ref()).first; - auto const pred = row_is_valid{static_cast(row_bitmask.data())}; + auto const row_bitmask = cudf::detail::bitmask_and(input, stream, temp_mr).first; + auto const pred = row_is_valid{static_cast(row_bitmask.data())}; ref.add_if_async(hash_iter, hash_iter + num_rows, stencil, pred, stream); } } else { @@ -256,8 +256,7 @@ void approx_distinct_count::add(table_view const& input, rmm::cuda_strea auto const pred = check_nans_predicate{*d_table, nullptr}; ref.add_if_async(hash_iter, hash_iter + num_rows, stencil, pred, stream); } else { - auto const row_bitmask = - cudf::detail::bitmask_and(input, stream, cudf::get_current_device_resource_ref()).first; + auto const row_bitmask = cudf::detail::bitmask_and(input, stream, temp_mr).first; auto const bitmask_ptr = static_cast(row_bitmask.data()); auto const pred = check_nans_predicate{*d_table, bitmask_ptr}; ref.add_if_async(hash_iter, hash_iter + num_rows, stencil, pred, stream); diff --git a/cpp/src/reductions/distinct_count.cu b/cpp/src/reductions/distinct_count.cu index bd2571e2dd0b..0a91ea95c356 100644 --- a/cpp/src/reductions/distinct_count.cu +++ b/cpp/src/reductions/distinct_count.cu @@ -135,11 +135,13 @@ cudf::size_type distinct_count(table_view const& keys, auto const num_rows = keys.num_rows(); if (num_rows == 0) { return 0; } // early exit for empty input auto const has_nulls = nullate::DYNAMIC{cudf::has_nested_nulls(keys)}; + auto const temp_mr = cudf::get_current_device_resource_ref(); - auto const preprocessed_input = cudf::detail::row::hash::preprocessed_table::create(keys, stream); - auto const row_hasher = cudf::detail::row::hash::row_hasher(preprocessed_input); - auto const hash_key = row_hasher.device_hasher(has_nulls); - auto const row_comp = cudf::detail::row::equality::self_comparator(preprocessed_input); + auto const preprocessed_input = + cudf::detail::row::hash::preprocessed_table::create(keys, stream, temp_mr); + auto const row_hasher = cudf::detail::row::hash::row_hasher(preprocessed_input); + auto const hash_key = row_hasher.device_hasher(has_nulls); + auto const row_comp = cudf::detail::row::equality::self_comparator(preprocessed_input); auto const comparator_helper = [&](auto const row_equal) { using hasher_type = decltype(hash_key); @@ -160,8 +162,7 @@ cudf::size_type distinct_count(table_view const& keys, cuda::counting_iterator stencil(0); // We must consider a row if any of its column entries is valid, // hence OR together the validities of the columns. - auto const [row_bitmask, null_count] = - cudf::detail::bitmask_or(keys, stream, cudf::get_current_device_resource_ref()); + auto const [row_bitmask, null_count] = cudf::detail::bitmask_or(keys, stream, temp_mr); // Unless all columns have a null mask, row_bitmask will be // null, and null_count will be zero. Equally, unless there is diff --git a/cpp/src/reductions/histogram.cu b/cpp/src/reductions/histogram.cu index e0408d64ee4f..661b1aaf2e24 100644 --- a/cpp/src/reductions/histogram.cu +++ b/cpp/src/reductions/histogram.cu @@ -111,8 +111,9 @@ compute_row_frequencies(table_view const& input, "Nested types are not yet supported in histogram aggregation.", std::invalid_argument); + auto const temp_mr = cudf::get_current_device_resource_ref(); auto const preprocessed_input = - cudf::detail::row::hash::preprocessed_table::create(input, stream); + cudf::detail::row::hash::preprocessed_table::create(input, stream, temp_mr); auto const has_nulls = nullate::DYNAMIC{cudf::has_nested_nulls(input)}; auto const row_hasher = cudf::detail::row::hash::row_hasher(preprocessed_input); @@ -131,12 +132,11 @@ compute_row_frequencies(table_view const& input, std::size_t const num_rows = input.num_rows(); // Construct a vector to store reduced counts and init to zero - rmm::device_uvector reduction_results(num_rows, stream, mr); - thrust::uninitialized_fill( - rmm::exec_policy_nosync(stream, cudf::get_current_device_resource_ref()), - reduction_results.begin(), - reduction_results.end(), - histogram_count_type{0}); + rmm::device_uvector reduction_results(num_rows, stream, temp_mr); + thrust::uninitialized_fill(rmm::exec_policy_nosync(stream, temp_mr), + reduction_results.begin(), + reduction_results.end(), + histogram_count_type{0}); // Construct a hash set auto row_set = @@ -156,7 +156,7 @@ compute_row_frequencies(table_view const& input, // Compute frequencies (aka distinct counts) for the input rows. // Note that we consider null and NaNs as always equal. thrust::for_each( - rmm::exec_policy_nosync(stream, cudf::get_current_device_resource_ref()), + rmm::exec_policy_nosync(stream, temp_mr), cuda::counting_iterator{0}, cuda::counting_iterator{num_rows}, [set_ref = row_set_ref, diff --git a/cpp/src/reductions/scan/rank_scan.cu b/cpp/src/reductions/scan/rank_scan.cu index 4d99f4199224..3ed92dfd88a1 100644 --- a/cpp/src/reductions/scan/rank_scan.cu +++ b/cpp/src/reductions/scan/rank_scan.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -60,14 +60,15 @@ std::unique_ptr rank_generator(column_view const& order_by, rmm::device_async_resource_ref mr) { auto const order_by_tview = table_view{{order_by}}; - auto comp = cudf::detail::row::equality::self_comparator(order_by_tview, stream); + auto const temp_mr = cudf::get_current_device_resource_ref(); + auto comp = cudf::detail::row::equality::self_comparator(order_by_tview, stream, temp_mr); auto ranks = make_fixed_width_column( data_type{type_to_id()}, order_by.size(), mask_state::UNALLOCATED, stream, mr); auto mutable_ranks = ranks->mutable_view(); auto const comparator_helper = [&](auto const device_comparator) { - thrust::transform(rmm::exec_policy_nosync(stream, cudf::get_current_device_resource_ref()), + thrust::transform(rmm::exec_policy_nosync(stream, temp_mr), cuda::counting_iterator(0), cuda::counting_iterator(order_by.size()), mutable_ranks.begin(), @@ -85,7 +86,7 @@ std::unique_ptr rank_generator(column_view const& order_by, comparator_helper(device_comparator); } - thrust::inclusive_scan(rmm::exec_policy_nosync(stream, cudf::get_current_device_resource_ref()), + thrust::inclusive_scan(rmm::exec_policy_nosync(stream, temp_mr), mutable_ranks.begin(), mutable_ranks.end(), mutable_ranks.begin(), diff --git a/cpp/src/reductions/segmented/nunique.cu b/cpp/src/reductions/segmented/nunique.cu index 62730cbb78c2..74f6caa814cb 100644 --- a/cpp/src/reductions/segmented/nunique.cu +++ b/cpp/src/reductions/segmented/nunique.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -52,19 +52,21 @@ std::unique_ptr segmented_nunique(column_view const& col, // compute the unique identifiers within each segment auto const identifiers = [&] { - auto const d_col = column_device_view::create(col, stream); - auto const comparator = cudf::detail::row::equality::self_comparator{table_view({col}), stream}; + auto const temp_mr = cudf::get_current_device_resource_ref(); + auto const d_col = column_device_view::create(col, stream); + auto const comparator = + cudf::detail::row::equality::self_comparator{table_view({col}), stream, temp_mr}; auto const row_equal = comparator.equal_to(cudf::nullate::DYNAMIC{col.has_nulls()}, null_equality::EQUAL); - auto labels = rmm::device_uvector(col.size(), stream); + auto labels = rmm::device_uvector(col.size(), stream, temp_mr); cudf::detail::label_segments( offsets.begin(), offsets.end(), labels.begin(), labels.end(), stream); auto fn = is_unique_fn{ *d_col, row_equal, null_handling, offsets.data(), labels.data()}; - auto identifiers = rmm::device_uvector(col.size(), stream); - thrust::transform(rmm::exec_policy_nosync(stream, cudf::get_current_device_resource_ref()), + auto identifiers = rmm::device_uvector(col.size(), stream, temp_mr); + thrust::transform(rmm::exec_policy_nosync(stream, temp_mr), cuda::counting_iterator{0}, cuda::counting_iterator{col.size()}, identifiers.begin(), diff --git a/cpp/src/reductions/unique_count.cu b/cpp/src/reductions/unique_count.cu index 278a12391e26..443d67466489 100644 --- a/cpp/src/reductions/unique_count.cu +++ b/cpp/src/reductions/unique_count.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -25,32 +25,30 @@ cudf::size_type unique_count(table_view const& keys, null_equality nulls_equal, rmm::cuda_stream_view stream) { - auto const row_comp = cudf::detail::row::equality::self_comparator(keys, stream); + auto const temp_mr = cudf::get_current_device_resource_ref(); + auto const row_comp = cudf::detail::row::equality::self_comparator(keys, stream, temp_mr); if (cudf::detail::has_nested_columns(keys)) { auto const comp = row_comp.equal_to(nullate::DYNAMIC{has_nested_nulls(keys)}, nulls_equal); // Using a temporary buffer for intermediate transform results from the lambda containing // the comparator speeds up compile-time significantly without much degradation in // runtime performance over using the comparator directly in thrust::count_if. - auto d_results = rmm::device_uvector(keys.num_rows(), stream); - thrust::transform(rmm::exec_policy_nosync(stream, cudf::get_current_device_resource_ref()), + auto d_results = rmm::device_uvector(keys.num_rows(), stream, temp_mr); + thrust::transform(rmm::exec_policy_nosync(stream, temp_mr), cuda::counting_iterator{0}, cuda::counting_iterator{keys.num_rows()}, d_results.begin(), [comp] __device__(auto i) { return (i == 0 or not comp(i, i - 1)); }); - return static_cast( - thrust::count(rmm::exec_policy_nosync(stream, cudf::get_current_device_resource_ref()), - d_results.begin(), - d_results.end(), - true)); + return static_cast(thrust::count( + rmm::exec_policy_nosync(stream, temp_mr), d_results.begin(), d_results.end(), true)); } else { auto const comp = row_comp.equal_to(nullate::DYNAMIC{has_nested_nulls(keys)}, nulls_equal); // Using thrust::copy_if with the comparator directly will compile more slowly but // improves runtime by up to 2x over the transform/count approach above. return thrust::count_if( - rmm::exec_policy_nosync(stream, cudf::get_current_device_resource_ref()), + rmm::exec_policy_nosync(stream, temp_mr), cuda::counting_iterator{0}, cuda::counting_iterator{keys.num_rows()}, [comp] __device__(cudf::size_type i) { return (i == 0 or not comp(i, i - 1)); }); diff --git a/cpp/src/reductions/unique_count_column.cu b/cpp/src/reductions/unique_count_column.cu index effdd3a5a323..48e62a0f7010 100644 --- a/cpp/src/reductions/unique_count_column.cu +++ b/cpp/src/reductions/unique_count_column.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -61,8 +61,9 @@ cudf::size_type unique_count(column_view const& input, auto device_view = *input_device_view; auto input_table_view = table_view{{input}}; - auto const comparator = cudf::detail::row::equality::self_comparator{input_table_view, stream}; - auto const comp = comparator.equal_to( + auto const comparator = cudf::detail::row::equality::self_comparator{ + input_table_view, stream, cudf::get_current_device_resource_ref()}; + auto const comp = comparator.equal_to( nullate::DYNAMIC{cudf::has_nulls(input_table_view)}, null_equality::EQUAL, cudf::detail::row::equality::nan_equal_physical_equality_comparator{}); diff --git a/cpp/src/row_operator/row_operators.cu b/cpp/src/row_operator/row_operators.cu index 698b184abef8..d4652cbbb919 100644 --- a/cpp/src/row_operator/row_operators.cu +++ b/cpp/src/row_operator/row_operators.cu @@ -842,27 +842,27 @@ two_table_comparator::two_table_comparator(table_view const& left, namespace equality { -std::shared_ptr preprocessed_table::create(table_view const& t, - rmm::cuda_stream_view stream) +std::shared_ptr preprocessed_table::create( + table_view const& t, rmm::cuda_stream_view stream, rmm::device_async_resource_ref temp_mr) { check_eq_compatibility(t); - auto [null_pushed_table, nullable_data] = - structs::detail::push_down_nulls(t, stream, cudf::get_current_device_resource_ref()); - auto struct_offset_removed_table = remove_struct_child_offsets(null_pushed_table); + auto [null_pushed_table, nullable_data] = structs::detail::push_down_nulls(t, stream, temp_mr); + auto struct_offset_removed_table = remove_struct_child_offsets(null_pushed_table); auto verticalized_t = std::get<0>(decompose_structs(struct_offset_removed_table, decompose_lists_column::YES)); - auto d_t = table_device_view_owner(table_device_view::create(verticalized_t, stream)); + auto d_t = table_device_view_owner(table_device_view::create(verticalized_t, stream, temp_mr)); return std::shared_ptr(new preprocessed_table( std::move(d_t), std::move(nullable_data.new_null_masks), std::move(nullable_data.new_columns))); } two_table_comparator::two_table_comparator(table_view const& left, table_view const& right, - rmm::cuda_stream_view stream) - : d_left_table{preprocessed_table::create(left, stream)}, - d_right_table{preprocessed_table::create(right, stream)} + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref temp_mr) + : d_left_table{preprocessed_table::create(left, stream, temp_mr)}, + d_right_table{preprocessed_table::create(right, stream, temp_mr)} { check_shape_compatibility(left, right); } diff --git a/cpp/src/search/contains_scalar.cu b/cpp/src/search/contains_scalar.cu index c6f329e5e1f8..39be13c77876 100644 --- a/cpp/src/search/contains_scalar.cu +++ b/cpp/src/search/contains_scalar.cu @@ -93,30 +93,30 @@ struct contains_scalar_dispatch { // In addition, haystack and needle structure compatibility will be checked later on by // constructor of the table comparator. - auto const haystack_tv = table_view{{haystack}}; - auto const needle_as_col = - make_column_from_scalar(needle, 1, stream, cudf::get_current_device_resource_ref()); - auto const needle_tv = table_view{{needle_as_col->view()}}; - auto const has_nulls = has_nested_nulls(haystack_tv) || has_nested_nulls(needle_tv); + auto const haystack_tv = table_view{{haystack}}; + auto const temp_mr = cudf::get_current_device_resource_ref(); + auto const needle_as_col = make_column_from_scalar(needle, 1, stream, temp_mr); + auto const needle_tv = table_view{{needle_as_col->view()}}; + auto const has_nulls = has_nested_nulls(haystack_tv) || has_nested_nulls(needle_tv); auto const comparator = - cudf::detail::row::equality::two_table_comparator(haystack_tv, needle_tv, stream); + cudf::detail::row::equality::two_table_comparator(haystack_tv, needle_tv, stream, temp_mr); auto const begin = cudf::detail::row::lhs_iterator(0); auto const end = begin + haystack.size(); using cudf::detail::row::rhs_index_type; auto const check_nulls = haystack.has_nulls(); - auto const haystack_cdv_ptr = column_device_view::create(haystack, stream); + auto const haystack_cdv_ptr = column_device_view::create(haystack, stream, temp_mr); auto const d_comp = comparator.equal_to(nullate::DYNAMIC{has_nulls}); // Using a temporary buffer for intermediate transform results from the lambda containing // the comparator speeds up compile-time significantly without much degradation in // runtime performance over using the comparator in a transform iterator with thrust::count_if. - auto d_results = rmm::device_uvector(haystack.size(), stream); + auto d_results = rmm::device_uvector(haystack.size(), stream, temp_mr); thrust::transform( - rmm::exec_policy_nosync(stream, cudf::get_current_device_resource_ref()), + rmm::exec_policy_nosync(stream, temp_mr), begin, end, d_results.begin(), @@ -127,10 +127,9 @@ struct contains_scalar_dispatch { return d_comp(idx, rhs_index_type{0}); // compare haystack[idx] == needle[0]. }); - return thrust::count(rmm::exec_policy_nosync(stream, cudf::get_current_device_resource_ref()), - d_results.begin(), - d_results.end(), - true) > 0; + return thrust::count( + rmm::exec_policy_nosync(stream, temp_mr), d_results.begin(), d_results.end(), true) > + 0; } }; diff --git a/cpp/src/search/contains_table.cu b/cpp/src/search/contains_table.cu index 189199ef1cf6..b8aecca6da0d 100644 --- a/cpp/src/search/contains_table.cu +++ b/cpp/src/search/contains_table.cu @@ -35,10 +35,11 @@ rmm::device_uvector contains(table_view const& haystack, auto const needles_has_nulls = has_nested_nulls(needles); auto const has_any_nulls = haystack_has_nulls || needles_has_nulls; + auto const temp_mr = cudf::get_current_device_resource_ref(); auto const preprocessed_needles = - cudf::detail::row::equality::preprocessed_table::create(needles, stream); + cudf::detail::row::equality::preprocessed_table::create(needles, stream, temp_mr); auto const preprocessed_haystack = - cudf::detail::row::equality::preprocessed_table::create(haystack, stream); + cudf::detail::row::equality::preprocessed_table::create(haystack, stream, temp_mr); // The output vector. auto contained = rmm::device_uvector(needles.num_rows(), stream, mr); diff --git a/cpp/src/sort/rank.cu b/cpp/src/sort/rank.cu index 27e9f3596845..833b7d58205b 100644 --- a/cpp/src/sort/rank.cu +++ b/cpp/src/sort/rank.cu @@ -61,7 +61,8 @@ rmm::device_uvector sorted_dense_rank(column_view input_col, rmm::cuda_stream_view stream) { auto const t_input = table_view{{input_col}}; - auto const comparator = cudf::detail::row::equality::self_comparator{t_input, stream}; + auto const temp_mr = cudf::get_current_device_resource_ref(); + auto const comparator = cudf::detail::row::equality::self_comparator{t_input, stream, temp_mr}; auto const sorted_index_order = cuda::make_permutation_iterator( sorted_order_view.begin(), cuda::counting_iterator{0}); @@ -70,7 +71,7 @@ rmm::device_uvector sorted_dense_rank(column_view input_col, rmm::device_uvector dense_rank_sorted(input_size, stream); auto const comparator_helper = [&](auto const device_comparator) { - thrust::transform(rmm::exec_policy_nosync(stream, cudf::get_current_device_resource_ref()), + thrust::transform(rmm::exec_policy_nosync(stream, temp_mr), cuda::counting_iterator{0}, cuda::counting_iterator{input_size}, dense_rank_sorted.data(), @@ -88,7 +89,7 @@ rmm::device_uvector sorted_dense_rank(column_view input_col, comparator_helper(device_comparator); } - thrust::inclusive_scan(rmm::exec_policy_nosync(stream, cudf::get_current_device_resource_ref()), + thrust::inclusive_scan(rmm::exec_policy_nosync(stream, temp_mr), dense_rank_sorted.begin(), dense_rank_sorted.end(), dense_rank_sorted.data()); diff --git a/cpp/src/stream_compaction/distinct.cu b/cpp/src/stream_compaction/distinct.cu index 9db6bbd76c8c..25a01a1cb8dc 100644 --- a/cpp/src/stream_compaction/distinct.cu +++ b/cpp/src/stream_compaction/distinct.cu @@ -86,8 +86,8 @@ rmm::device_uvector distinct_indices(table_view const& input, return rmm::device_uvector(0, stream, mr); } - auto const preprocessed_input = - cudf::detail::row::hash::preprocessed_table::create(input, stream); + auto const preprocessed_input = cudf::detail::row::hash::preprocessed_table::create( + input, stream, cudf::get_current_device_resource_ref()); auto const has_nulls = nullate::DYNAMIC{cudf::has_nested_nulls(input)}; auto const has_nested_columns = cudf::detail::has_nested_columns(input); diff --git a/cpp/src/stream_compaction/unique.cu b/cpp/src/stream_compaction/unique.cu index 12ccfcb97129..6ea7e3d63c53 100644 --- a/cpp/src/stream_compaction/unique.cu +++ b/cpp/src/stream_compaction/unique.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -50,12 +50,13 @@ std::unique_ptr
unique(table_view const& input, auto const num_rows = input.num_rows(); if (num_rows == 0 or input.num_columns() == 0 or keys.empty()) { return empty_like(input); } + auto const temp_mr = cudf::get_current_device_resource_ref(); auto unique_indices = make_numeric_column( - data_type{type_to_id()}, num_rows, mask_state::UNALLOCATED, stream, mr); + data_type{type_to_id()}, num_rows, mask_state::UNALLOCATED, stream, temp_mr); auto mutable_view = mutable_column_device_view::create(*unique_indices, stream); auto keys_view = input.select(keys); - auto comp = cudf::detail::row::equality::self_comparator(keys_view, stream); + auto comp = cudf::detail::row::equality::self_comparator(keys_view, stream, temp_mr); size_type const unique_size = [&] { if (cudf::detail::has_nested_columns(keys_view)) { @@ -64,10 +65,10 @@ std::unique_ptr
unique(table_view const& input, // runtime performance over using the comparator directly in thrust::unique_copy. auto row_equal = comp.equal_to(nullate::DYNAMIC{has_nested_nulls(keys_view)}, nulls_equal); - auto d_results = rmm::device_uvector(num_rows, stream); + auto d_results = rmm::device_uvector(num_rows, stream, temp_mr); auto itr = cuda::counting_iterator{0}; thrust::transform( - rmm::exec_policy_nosync(stream, cudf::get_current_device_resource_ref()), + rmm::exec_policy_nosync(stream, temp_mr), itr, itr + num_rows, d_results.begin(), diff --git a/cpp/src/table/table_equal.cu b/cpp/src/table/table_equal.cu index 0cc97e0da260..8033d2d408a5 100644 --- a/cpp/src/table/table_equal.cu +++ b/cpp/src/table/table_equal.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -31,11 +31,11 @@ template null_equality nulls_equal, rmm::cuda_stream_view stream) { - auto const comparator = detail::row::equality::two_table_comparator{left, right, stream}; + auto const temp_mr = cudf::get_current_device_resource_ref(); + auto const comparator = detail::row::equality::two_table_comparator{left, right, stream, temp_mr}; auto const rows_equal = comparator.equal_to( nullate::DYNAMIC{has_nested_nulls(left) or has_nested_nulls(right)}, nulls_equal); - rmm::device_uvector eq_rows{ - static_cast(left.num_rows()), stream, cudf::get_current_device_resource_ref()}; + rmm::device_uvector eq_rows{static_cast(left.num_rows()), stream, temp_mr}; CUDF_CUDA_TRY(cub::DeviceTransform::Transform( cuda::counting_iterator{0}, eq_rows.begin(), diff --git a/cpp/src/transform/one_hot_encode.cu b/cpp/src/transform/one_hot_encode.cu index 098618c7d96c..d35f1f0eaaf7 100644 --- a/cpp/src/transform/one_hot_encode.cu +++ b/cpp/src/transform/one_hot_encode.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -68,12 +68,14 @@ std::pair, table_view> one_hot_encode(column_view const& auto all_encodings = make_numeric_column(data_type{type_id::BOOL8}, total_size, mask_state::UNALLOCATED, stream, mr); - auto const t_lhs = table_view{{input}}; - auto const t_rhs = table_view{{categories}}; - auto const comparator = cudf::detail::row::equality::two_table_comparator{t_lhs, t_rhs, stream}; + auto const temp_mr = cudf::get_current_device_resource_ref(); + auto const t_lhs = table_view{{input}}; + auto const t_rhs = table_view{{categories}}; + auto const comparator = + cudf::detail::row::equality::two_table_comparator{t_lhs, t_rhs, stream, temp_mr}; auto const comparator_helper = [&](auto const d_equal) { - thrust::transform(rmm::exec_policy_nosync(stream, cudf::get_current_device_resource_ref()), + thrust::transform(rmm::exec_policy_nosync(stream, temp_mr), cuda::counting_iterator{0}, cuda::counting_iterator{total_size}, all_encodings->mutable_view().begin(), diff --git a/cpp/tests/copying/gather_list_tests.cpp b/cpp/tests/copying/gather_list_tests.cpp index a07747a9403c..52b731769c2e 100644 --- a/cpp/tests/copying/gather_list_tests.cpp +++ b/cpp/tests/copying/gather_list_tests.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -17,7 +17,7 @@ #include template -class GatherTestListTyped : public cudf::test::BaseFixture {}; +class GatherTestListTyped : public cudf::test::BaseFixtureWithHarness {}; using FixedWidthTypesNotBool = cudf::test::Concat; TYPED_TEST_SUITE(GatherTestListTyped, FixedWidthTypesNotBool); -class GatherTestList : public cudf::test::BaseFixture {}; +class GatherTestList : public cudf::test::BaseFixtureWithHarness {}; // to disambiguate between {} == 0 and {} == List{0} // Also, see note about compiler issues when declaring nested @@ -33,46 +33,62 @@ class GatherTestList : public cudf::test::BaseFixture {}; template using LCW = cudf::test::lists_column_wrapper; +// Nested list values. Passing these to lists_column_wrapper builds every nesting level with the +// explicit stream and memory resources instead of the current device resource. +using Init = cudf::test::lists_column_initializer; + TYPED_TEST(GatherTestListTyped, Gather) { using T = TypeParam; + auto const st = this->stream(); + auto const mr = this->resources(); + // List - LCW list{{1, 2, 3, 4}, {5}, {6, 7}, {8, 9, 10}}; - cudf::test::fixed_width_column_wrapper gather_map{0, 2}; + LCW list{Init{{1, 2, 3, 4}, {5}, {6, 7}, {8, 9, 10}}, st, mr}; + cudf::test::fixed_width_column_wrapper gather_map{{0, 2}, st, mr}; cudf::table_view source_table({list}); - auto results = cudf::gather(source_table, gather_map); + auto results = + cudf::gather(source_table, gather_map, cudf::out_of_bounds_policy::DONT_CHECK, st, mr); - LCW expected{{1, 2, 3, 4}, {6, 7}}; + LCW expected{Init{{1, 2, 3, 4}, {6, 7}}, st, mr}; - CUDF_TEST_EXPECT_COLUMNS_EQUAL(results->view().column(0), expected); + CUDF_TEST_EXPECT_COLUMNS_EQUAL( + results->view().column(0), expected, cudf::test::debug_output_level::FIRST_ERROR, st, mr); } TYPED_TEST(GatherTestListTyped, GatherNothing) { using T = TypeParam; + auto const st = this->stream(); + auto const mr = this->resources(); + // List { - LCW list{{1, 2, 3, 4}, {5}, {6, 7}, {8, 9, 10}}; + LCW list{Init{{1, 2, 3, 4}, {5}, {6, 7}, {8, 9, 10}}, st, mr}; cudf::test::fixed_width_column_wrapper gather_map{}; cudf::table_view source_table({list}); - auto results = cudf::gather(source_table, gather_map); + auto results = + cudf::gather(source_table, gather_map, cudf::out_of_bounds_policy::DONT_CHECK, st, mr); LCW expected; - CUDF_TEST_EXPECT_COLUMNS_EQUAL(results->view().column(0), expected); + CUDF_TEST_EXPECT_COLUMNS_EQUAL( + results->view().column(0), expected, cudf::test::debug_output_level::FIRST_ERROR, st, mr); } // List { - cudf::test::lists_column_wrapper list{{{{1, 2, 3, 4}, {5}}}, {{{6, 7}, {8, 9, 10}}}}; + cudf::test::lists_column_wrapper list{ + Init{{{{1, 2, 3, 4}, {5}}}, {{{6, 7}, {8, 9, 10}}}}, st, mr}; cudf::test::fixed_width_column_wrapper gather_map{}; cudf::table_view source_table({list}); - auto result = cudf::gather(source_table, gather_map); + auto result = + cudf::gather(source_table, gather_map, cudf::out_of_bounds_policy::DONT_CHECK, st, mr); // the result should preserve the full List>> hierarchy // even though it is empty past the first level @@ -94,58 +110,77 @@ TYPED_TEST(GatherTestListTyped, GatherNulls) { using T = TypeParam; + auto const st = this->stream(); + auto const mr = this->resources(); + auto valids = cudf::test::iterators::valids_at_multiples_of(2); // List - LCW list{{{1, 2, 3, 4}, valids}, {5}, {{6, 7}, valids}, {{8, 9, 10}, valids}}; - cudf::test::fixed_width_column_wrapper gather_map{0, 2}; + LCW list{Init{{{1, 2, 3, 4}, valids}, {5}, {{6, 7}, valids}, {{8, 9, 10}, valids}}, st, mr}; + cudf::test::fixed_width_column_wrapper gather_map{{0, 2}, st, mr}; cudf::table_view source_table({list}); - auto results = cudf::gather(source_table, gather_map); + auto results = + cudf::gather(source_table, gather_map, cudf::out_of_bounds_policy::DONT_CHECK, st, mr); - LCW expected{{{1, 2, 3, 4}, valids}, {{6, 7}, valids}}; + LCW expected{Init{{{1, 2, 3, 4}, valids}, {{6, 7}, valids}}, st, mr}; - CUDF_TEST_EXPECT_COLUMNS_EQUAL(results->view().column(0), expected); + CUDF_TEST_EXPECT_COLUMNS_EQUAL( + results->view().column(0), expected, cudf::test::debug_output_level::FIRST_ERROR, st, mr); } TYPED_TEST(GatherTestListTyped, GatherNested) { using T = TypeParam; + auto const st = this->stream(); + auto const mr = this->resources(); + // List> { - LCW list{{{2, 3}, {4, 5}}, - {{6, 7, 8}, {9, 10, 11}, {12, 13, 14}}, - {{15, 16}, {17, 18}, {17, 18}, {17, 18}, {17, 18}}}; - cudf::test::fixed_width_column_wrapper gather_map{0, 2}; + LCW list{Init{{{2, 3}, {4, 5}}, + {{6, 7, 8}, {9, 10, 11}, {12, 13, 14}}, + {{15, 16}, {17, 18}, {17, 18}, {17, 18}, {17, 18}}}, + st, + mr}; + cudf::test::fixed_width_column_wrapper gather_map{{0, 2}, st, mr}; cudf::table_view source_table({list}); - auto results = cudf::gather(source_table, gather_map); + auto results = + cudf::gather(source_table, gather_map, cudf::out_of_bounds_policy::DONT_CHECK, st, mr); - LCW expected{{{2, 3}, {4, 5}}, {{15, 16}, {17, 18}, {17, 18}, {17, 18}, {17, 18}}}; + LCW expected{ + Init{{{2, 3}, {4, 5}}, {{15, 16}, {17, 18}, {17, 18}, {17, 18}, {17, 18}}}, st, mr}; - CUDF_TEST_EXPECT_COLUMNS_EQUAL(results->view().column(0), expected); + CUDF_TEST_EXPECT_COLUMNS_EQUAL( + results->view().column(0), expected, cudf::test::debug_output_level::FIRST_ERROR, st, mr); } // List>> { - LCW list{{{{2, 3}, {4, 5}}, {{6, 7, 8}, {9, 10, 11}, {12, 13, 14}}}, - {{{15, 16}, {17, 18}, {17, 18}, {17, 18}, {17, 18}}}, - {{LCW{0}}}, - {{{10}, {20, 30, 40, 50}, {60, 70, 80}}, - {{0, 1, 3}, {5}}, - {{11, 12, 13, 14, 15}, {16, 17}, {0}}}, - {{{10, 20}}, {LCW{30}}, {{40, 50}, {60, 70, 80}}}}; - cudf::test::fixed_width_column_wrapper gather_map{1, 2, 4}; + LCW list{Init{{{{2, 3}, {4, 5}}, {{6, 7, 8}, {9, 10, 11}, {12, 13, 14}}}, + {{{15, 16}, {17, 18}, {17, 18}, {17, 18}, {17, 18}}}, + {{Init{0}}}, + {{{10}, {20, 30, 40, 50}, {60, 70, 80}}, + {{0, 1, 3}, {5}}, + {{11, 12, 13, 14, 15}, {16, 17}, {0}}}, + {{{10, 20}}, {Init{30}}, {{40, 50}, {60, 70, 80}}}}, + st, + mr}; + cudf::test::fixed_width_column_wrapper gather_map{{1, 2, 4}, st, mr}; cudf::table_view source_table({list}); - auto results = cudf::gather(source_table, gather_map); + auto results = + cudf::gather(source_table, gather_map, cudf::out_of_bounds_policy::DONT_CHECK, st, mr); - LCW expected{{{{15, 16}, {17, 18}, {17, 18}, {17, 18}, {17, 18}}}, - {{LCW{0}}}, - {{{10, 20}}, {LCW{30}}, {{40, 50}, {60, 70, 80}}}}; + LCW expected{Init{{{{15, 16}, {17, 18}, {17, 18}, {17, 18}, {17, 18}}}, + {{Init{0}}}, + {{{10, 20}}, {Init{30}}, {{40, 50}, {60, 70, 80}}}}, + st, + mr}; - CUDF_TEST_EXPECT_COLUMNS_EQUAL(results->view().column(0), expected); + CUDF_TEST_EXPECT_COLUMNS_EQUAL( + results->view().column(0), expected, cudf::test::debug_output_level::FIRST_ERROR, st, mr); } } @@ -153,21 +188,30 @@ TYPED_TEST(GatherTestListTyped, GatherOutOfOrder) { using T = TypeParam; + auto const st = this->stream(); + auto const mr = this->resources(); + // List> { - LCW list{{{2, 3}, {4, 5}}, - {{6, 7, 8}, {9, 10, 11}, {12, 13, 14}}, - {{15, 16}, {17, 18}, {17, 18}, {17, 18}, {17, 18}}}; - cudf::test::fixed_width_column_wrapper gather_map{1, 2, 0}; + LCW list{Init{{{2, 3}, {4, 5}}, + {{6, 7, 8}, {9, 10, 11}, {12, 13, 14}}, + {{15, 16}, {17, 18}, {17, 18}, {17, 18}, {17, 18}}}, + st, + mr}; + cudf::test::fixed_width_column_wrapper gather_map{{1, 2, 0}, st, mr}; cudf::table_view source_table({list}); - auto results = cudf::gather(source_table, gather_map); + auto results = + cudf::gather(source_table, gather_map, cudf::out_of_bounds_policy::DONT_CHECK, st, mr); - LCW expected{{{6, 7, 8}, {9, 10, 11}, {12, 13, 14}}, - {{15, 16}, {17, 18}, {17, 18}, {17, 18}, {17, 18}}, - {{2, 3}, {4, 5}}}; + LCW expected{Init{{{6, 7, 8}, {9, 10, 11}, {12, 13, 14}}, + {{15, 16}, {17, 18}, {17, 18}, {17, 18}, {17, 18}}, + {{2, 3}, {4, 5}}}, + st, + mr}; - CUDF_TEST_EXPECT_COLUMNS_EQUAL(results->view().column(0), expected); + CUDF_TEST_EXPECT_COLUMNS_EQUAL( + results->view().column(0), expected, cudf::test::debug_output_level::FIRST_ERROR, st, mr); } } @@ -175,48 +219,64 @@ TYPED_TEST(GatherTestListTyped, GatherNestedNulls) { using T = TypeParam; + auto const st = this->stream(); + auto const mr = this->resources(); + auto valids = cudf::test::iterators::valids_at_multiples_of(2); // List> { - LCW list{{{{2, 3}, valids}, {4, 5}}, - {{{6, 7, 8}, {9, 10, 11}, {12, 13, 14}}, valids}, - {{15, 16}, {17, 18}, {17, 18}, {17, 18}, {17, 18}}, - {{{{25, 26}, valids}, {27, 28}, {{29, 30}, valids}, {31, 32}, {33, 34}}, valids}}; + LCW list{ + Init{{{{2, 3}, valids}, {4, 5}}, + {{{6, 7, 8}, {9, 10, 11}, {12, 13, 14}}, valids}, + {{15, 16}, {17, 18}, {17, 18}, {17, 18}, {17, 18}}, + {{{{25, 26}, valids}, {27, 28}, {{29, 30}, valids}, {31, 32}, {33, 34}}, valids}}, + st, + mr}; - cudf::test::fixed_width_column_wrapper gather_map{0, 1, 3}; + cudf::test::fixed_width_column_wrapper gather_map{{0, 1, 3}, st, mr}; cudf::table_view source_table({list}); - auto results = cudf::gather(source_table, gather_map); + auto results = + cudf::gather(source_table, gather_map, cudf::out_of_bounds_policy::DONT_CHECK, st, mr); LCW expected{ - {{{2, 3}, valids}, {4, 5}}, - {{{6, 7, 8}, {9, 10, 11}, {12, 13, 14}}, valids}, - {{{{25, 26}, valids}, {27, 28}, {{29, 30}, valids}, {31, 32}, {33, 34}}, valids}}; - - CUDF_TEST_EXPECT_COLUMNS_EQUAL(results->view().column(0), expected); + Init{{{{2, 3}, valids}, {4, 5}}, + {{{6, 7, 8}, {9, 10, 11}, {12, 13, 14}}, valids}, + {{{{25, 26}, valids}, {27, 28}, {{29, 30}, valids}, {31, 32}, {33, 34}}, valids}}, + st, + mr}; + + CUDF_TEST_EXPECT_COLUMNS_EQUAL( + results->view().column(0), expected, cudf::test::debug_output_level::FIRST_ERROR, st, mr); } // List>> { - LCW list{{{{2, 3}, {4, 5}}, {{6, 7, 8}, {9, 10, 11}, {12, 13, 14}}}, - {{{15, 16}, {{27, 28}, valids}, {{37, 38}, valids}, {47, 48}, {57, 58}}}, - {{LCW{0}}}, - {{{10}, {20, 30, 40, 50}, {60, 70, 80}}, - {{0, 1, 3}, {5}}, - {{11, 12, 13, 14, 15}, {16, 17}, {0}}}, - {{{{{10, 20}, valids}}, {LCW{30}}, {{40, 50}, {60, 70, 80}}}, valids}}; - - cudf::test::fixed_width_column_wrapper gather_map{1, 2, 4}; + LCW list{Init{{{{2, 3}, {4, 5}}, {{6, 7, 8}, {9, 10, 11}, {12, 13, 14}}}, + {{{15, 16}, {{27, 28}, valids}, {{37, 38}, valids}, {47, 48}, {57, 58}}}, + {{Init{0}}}, + {{{10}, {20, 30, 40, 50}, {60, 70, 80}}, + {{0, 1, 3}, {5}}, + {{11, 12, 13, 14, 15}, {16, 17}, {0}}}, + {{{{{10, 20}, valids}}, {Init{30}}, {{40, 50}, {60, 70, 80}}}, valids}}, + st, + mr}; + + cudf::test::fixed_width_column_wrapper gather_map{{1, 2, 4}, st, mr}; cudf::table_view source_table({list}); - auto results = cudf::gather(source_table, gather_map); + auto results = + cudf::gather(source_table, gather_map, cudf::out_of_bounds_policy::DONT_CHECK, st, mr); - LCW expected{{{{15, 16}, {{27, 28}, valids}, {{37, 38}, valids}, {47, 48}, {57, 58}}}, - {{LCW{0}}}, - {{{{{10, 20}, valids}}, {LCW{30}}, {{40, 50}, {60, 70, 80}}}, valids}}; + LCW expected{Init{{{{15, 16}, {{27, 28}, valids}, {{37, 38}, valids}, {47, 48}, {57, 58}}}, + {{Init{0}}}, + {{{{{10, 20}, valids}}, {Init{30}}, {{40, 50}, {60, 70, 80}}}, valids}}, + st, + mr}; - CUDF_TEST_EXPECT_COLUMNS_EQUAL(results->view().column(0), expected); + CUDF_TEST_EXPECT_COLUMNS_EQUAL( + results->view().column(0), expected, cudf::test::debug_output_level::FIRST_ERROR, st, mr); } } @@ -224,55 +284,70 @@ TYPED_TEST(GatherTestListTyped, GatherNestedWithEmpties) { using T = TypeParam; - LCW list{{{2, 3}, LCW{}}, {{6, 7, 8}, {9, 10, 11}, {12, 13, 14}}, {LCW{}}}; - cudf::test::fixed_width_column_wrapper gather_map{0, 2}; + auto const st = this->stream(); + auto const mr = this->resources(); + + LCW list{Init{{{2, 3}, Init{}}, {{6, 7, 8}, {9, 10, 11}, {12, 13, 14}}, {Init{}}}, st, mr}; + cudf::test::fixed_width_column_wrapper gather_map{{0, 2}, st, mr}; cudf::table_view source_table({list}); - auto results = cudf::gather(source_table, gather_map); + auto results = + cudf::gather(source_table, gather_map, cudf::out_of_bounds_policy::DONT_CHECK, st, mr); - LCW expected{{{2, 3}, LCW{}}, {LCW{}}}; + LCW expected{Init{{{2, 3}, Init{}}, {Init{}}}, st, mr}; - CUDF_TEST_EXPECT_COLUMNS_EQUAL(results->view().column(0), expected); + CUDF_TEST_EXPECT_COLUMNS_EQUAL( + results->view().column(0), expected, cudf::test::debug_output_level::FIRST_ERROR, st, mr); } TYPED_TEST(GatherTestListTyped, GatherDetailInvalidIndex) { using T = TypeParam; + auto const st = this->stream(); + auto const mr = this->resources(); + // List> { - LCW list{{{2, 3}, {4, 5}}, - {{6, 7, 8}, {9, 10, 11}, {12, 13, 14}}, - {{15, 16}, {17, 18}, {17, 18}, {17, 18}, {17, 18}}}; - cudf::test::fixed_width_column_wrapper gather_map{0, 15, 16, 2}; + LCW list{Init{{{2, 3}, {4, 5}}, + {{6, 7, 8}, {9, 10, 11}, {12, 13, 14}}, + {{15, 16}, {17, 18}, {17, 18}, {17, 18}, {17, 18}}}, + st, + mr}; + cudf::test::fixed_width_column_wrapper gather_map{{0, 15, 16, 2}, st, mr}; cudf::table_view source_table({list}); - auto results = cudf::gather(source_table, gather_map, cudf::out_of_bounds_policy::NULLIFY); + auto results = + cudf::gather(source_table, gather_map, cudf::out_of_bounds_policy::NULLIFY, st, mr); std::vector expected_validity{1, 0, 0, 1}; - LCW expected{{{{2, 3}, {4, 5}}, - {LCW{}}, - {LCW{}}, - {{15, 16}, {17, 18}, {17, 18}, {17, 18}, {17, 18}}}, - expected_validity.begin()}; - - CUDF_TEST_EXPECT_COLUMNS_EQUAL(results->view().column(0), expected); + LCW expected{ + Init{ + {{{2, 3}, {4, 5}}, {Init{}}, {Init{}}, {{15, 16}, {17, 18}, {17, 18}, {17, 18}, {17, 18}}}, + expected_validity.begin()}, + st, + mr}; + + CUDF_TEST_EXPECT_COLUMNS_EQUAL( + results->view().column(0), expected, cudf::test::debug_output_level::FIRST_ERROR, st, mr); } } TEST_F(GatherTestList, GatherIncompleteHierarchies) { - using LCW = cudf::test::lists_column_wrapper; + auto const st = this->stream(); + auto const mr = this->resources(); { // List, but rows 1 and 2 are empty at the very top. // We expect to get back a "full" hierarchy of type List> anyway. - cudf::test::lists_column_wrapper list{{{{1, 2}}}, LCW{}, LCW{}}; + cudf::test::lists_column_wrapper list{Init{{{{1, 2}}}, Init{}, Init{}}, st, mr}; cudf::table_view source_table({list}); - cudf::test::fixed_width_column_wrapper row1_map{1}; - auto result = cudf::gather(source_table, row1_map); + cudf::test::fixed_width_column_wrapper row1_map{{1}, st, mr}; + auto result = + cudf::gather(source_table, row1_map, cudf::out_of_bounds_policy::DONT_CHECK, st, mr); // the result should preserve the full List>> hierarchy // even though it is empty past the first level @@ -292,12 +367,13 @@ TEST_F(GatherTestList, GatherIncompleteHierarchies) { // List, gathering nothing. // We expect to get back a "full" hierarchy of type List> anyway. - cudf::test::lists_column_wrapper list{{{{1, 2}}}, LCW{}}; + cudf::test::lists_column_wrapper list{Init{{{{1, 2}}}, Init{}}, st, mr}; cudf::table_view source_table({list}); cudf::test::fixed_width_column_wrapper empty_map{}; - auto result = cudf::gather(source_table, empty_map); + auto result = + cudf::gather(source_table, empty_map, cudf::out_of_bounds_policy::DONT_CHECK, st, mr); // the result should preserve the full List>> hierarchy // even though it is empty past the first level @@ -318,34 +394,54 @@ TEST_F(GatherTestList, GatherIncompleteHierarchies) TYPED_TEST(GatherTestListTyped, GatherSliced) { using T = TypeParam; + + auto const st = this->stream(); + auto const mr = this->resources(); + { - LCW a{ - {{1, 1, 1}, {2, 2}, {3, 3}}, - {{4, 4, 4}, {5, 5}, {6, 6}}, - {{7, 7, 7}, {8, 8}, {9, 9}}, - {{10, 10, 10}, {11, 11}, {12, 12}}, - {{20, 20, 20, 20}, {25}}, - {{30, 30, 30, 30}, {40}}, - {{50, 50, 50, 50}, {6, 13}}, - {{70, 70, 70, 70}, {80}}, - }; - auto split_a = cudf::split(a, {3}); + LCW a{Init{ + {{1, 1, 1}, {2, 2}, {3, 3}}, + {{4, 4, 4}, {5, 5}, {6, 6}}, + {{7, 7, 7}, {8, 8}, {9, 9}}, + {{10, 10, 10}, {11, 11}, {12, 12}}, + {{20, 20, 20, 20}, {25}}, + {{30, 30, 30, 30}, {40}}, + {{50, 50, 50, 50}, {6, 13}}, + {{70, 70, 70, 70}, {80}}, + }, + st, + mr}; + auto split_a = cudf::split(a, {3}, st); cudf::table_view tbl0({split_a[0]}); cudf::table_view tbl1({split_a[1]}); - auto result0 = cudf::gather(tbl0, cudf::test::fixed_width_column_wrapper{1, 2}); - LCW expected0{ - {{4, 4, 4}, {5, 5}, {6, 6}}, - {{7, 7, 7}, {8, 8}, {9, 9}}, - }; - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected0, result0->get_column(0).view()); - - auto result1 = cudf::gather(tbl1, cudf::test::fixed_width_column_wrapper{0, 3}); - LCW expected1{ - {{10, 10, 10}, {11, 11}, {12, 12}}, - {{50, 50, 50, 50}, {6, 13}}, - }; - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected1, result1->get_column(0).view()); + cudf::test::fixed_width_column_wrapper map0{{1, 2}, st, mr}; + auto result0 = cudf::gather(tbl0, map0, cudf::out_of_bounds_policy::DONT_CHECK, st, mr); + LCW expected0{Init{ + {{4, 4, 4}, {5, 5}, {6, 6}}, + {{7, 7, 7}, {8, 8}, {9, 9}}, + }, + st, + mr}; + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected0, + result0->get_column(0).view(), + cudf::test::debug_output_level::FIRST_ERROR, + st, + mr); + + cudf::test::fixed_width_column_wrapper map1{{0, 3}, st, mr}; + auto result1 = cudf::gather(tbl1, map1, cudf::out_of_bounds_policy::DONT_CHECK, st, mr); + LCW expected1{Init{ + {{10, 10, 10}, {11, 11}, {12, 12}}, + {{50, 50, 50, 50}, {6, 13}}, + }, + st, + mr}; + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected1, + result1->get_column(0).view(), + cudf::test::debug_output_level::FIRST_ERROR, + st, + mr); } auto valids = cudf::test::iterators::valids_at_multiples_of(2); @@ -353,70 +449,93 @@ TYPED_TEST(GatherTestListTyped, GatherSliced) // List>> { LCW list{ - // slice 0 - {{{2, 3}, {4, 5}}, {{6, 7, 8}, {9, 10, 11}, {12, 13, 14}}}, + Init{// slice 0 + {{{2, 3}, {4, 5}}, {{6, 7, 8}, {9, 10, 11}, {12, 13, 14}}}, - {{{15, 16}, {{27, 28}, valids}, {{37, 38}, valids}, {47, 48}, {57, 58}}, - {{11, 12}, {{42, 43, 44}, valids}, {{77, 78}, valids}}}, + {{{15, 16}, {{27, 28}, valids}, {{37, 38}, valids}, {47, 48}, {57, 58}}, + {{11, 12}, {{42, 43, 44}, valids}, {{77, 78}, valids}}}, - // slice 1 - {{LCW{0}}}, - {{{10}, {20, 30, 40, 50}, {60, 70, 80}}, - {{0, 1, 3}, {5}}, - {{11, 12, 13, 14, 15}, {16, 17}, {0}}}, - {{{{1, 6}, {60, 70, 80, 100}}, {{10, 11, 13}, {15}}, {{11, 12, 13, 14, 15}}}, valids}, + // slice 1 + {{Init{0}}}, + {{{10}, {20, 30, 40, 50}, {60, 70, 80}}, + {{0, 1, 3}, {5}}, + {{11, 12, 13, 14, 15}, {16, 17}, {0}}}, + {{{{1, 6}, {60, 70, 80, 100}}, {{10, 11, 13}, {15}}, {{11, 12, 13, 14, 15}}}, valids}, - // slice 2 - {{{{{10, 20}, valids}}, {LCW{30}}, {{40, 50}, {60, 70, 80}}}, valids}, - {{{{10, 20, 30}}, {LCW{30}}, {{{20, 30}, valids}, {62, 72, 82}}}, valids}}; + // slice 2 + {{{{{10, 20}, valids}}, {Init{30}}, {{40, 50}, {60, 70, 80}}}, valids}, + {{{{10, 20, 30}}, {Init{30}}, {{{20, 30}, valids}, {62, 72, 82}}}, valids}}, + st, + mr}; - auto sliced = cudf::slice(list, {0, 1, 2, 5, 5, 7}); + auto sliced = cudf::slice(list, {0, 1, 2, 5, 5, 7}, st); // gather from slice 0 { cudf::table_view tbl({sliced[0]}); - cudf::test::fixed_width_column_wrapper map{0}; - auto result = cudf::gather(tbl, map); - LCW expected{{{{2, 3}, {4, 5}}, {{6, 7, 8}, {9, 10, 11}, {12, 13, 14}}}}; - CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(expected, result->get_column(0).view()); + cudf::test::fixed_width_column_wrapper map{{0}, st, mr}; + auto result = cudf::gather(tbl, map, cudf::out_of_bounds_policy::DONT_CHECK, st, mr); + LCW expected{Init{{{{2, 3}, {4, 5}}, {{6, 7, 8}, {9, 10, 11}, {12, 13, 14}}}}, st, mr}; + CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(expected, + result->get_column(0).view(), + cudf::test::debug_output_level::FIRST_ERROR, + cudf::test::default_ulp, + st, + mr); } // gather from slice 1 { cudf::table_view tbl({sliced[1]}); - cudf::test::fixed_width_column_wrapper map{1, 2, 0, 1}; - auto result = cudf::gather(tbl, map); + cudf::test::fixed_width_column_wrapper map{{1, 2, 0, 1}, st, mr}; + auto result = cudf::gather(tbl, map, cudf::out_of_bounds_policy::DONT_CHECK, st, mr); LCW expected{ - {{{10}, {20, 30, 40, 50}, {60, 70, 80}}, - {{0, 1, 3}, {5}}, - {{11, 12, 13, 14, 15}, {16, 17}, {0}}}, - - {{{{1, 6}, {60, 70, 80, 100}}, {{10, 11, 13}, {15}}, {{11, 12, 13, 14, 15}}}, valids}, - - {{LCW{0}}}, - - {{{10}, {20, 30, 40, 50}, {60, 70, 80}}, - {{0, 1, 3}, {5}}, - {{11, 12, 13, 14, 15}, {16, 17}, {0}}}, - }; - CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(expected, result->get_column(0).view()); + Init{ + {{{10}, {20, 30, 40, 50}, {60, 70, 80}}, + {{0, 1, 3}, {5}}, + {{11, 12, 13, 14, 15}, {16, 17}, {0}}}, + + {{{{1, 6}, {60, 70, 80, 100}}, {{10, 11, 13}, {15}}, {{11, 12, 13, 14, 15}}}, valids}, + + {{Init{0}}}, + + {{{10}, {20, 30, 40, 50}, {60, 70, 80}}, + {{0, 1, 3}, {5}}, + {{11, 12, 13, 14, 15}, {16, 17}, {0}}}, + }, + st, + mr}; + CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(expected, + result->get_column(0).view(), + cudf::test::debug_output_level::FIRST_ERROR, + cudf::test::default_ulp, + st, + mr); } // gather from slice 2 { cudf::table_view tbl({sliced[2]}); - cudf::test::fixed_width_column_wrapper map{1, 0, 0, 1, 1, 0}; - auto result = cudf::gather(tbl, map); - LCW expected{{{{{10, 20, 30}}, {LCW{30}}, {{{20, 30}, valids}, {62, 72, 82}}}, valids}, - {{{{{10, 20}, valids}}, {LCW{30}}, {{40, 50}, {60, 70, 80}}}, valids}, - {{{{{10, 20}, valids}}, {LCW{30}}, {{40, 50}, {60, 70, 80}}}, valids}, - {{{{10, 20, 30}}, {LCW{30}}, {{{20, 30}, valids}, {62, 72, 82}}}, valids}, - {{{{10, 20, 30}}, {LCW{30}}, {{{20, 30}, valids}, {62, 72, 82}}}, valids}, - {{{{{10, 20}, valids}}, {LCW{30}}, {{40, 50}, {60, 70, 80}}}, valids}}; - CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(expected, result->get_column(0).view()); + cudf::test::fixed_width_column_wrapper map{{1, 0, 0, 1, 1, 0}, st, mr}; + auto result = cudf::gather(tbl, map, cudf::out_of_bounds_policy::DONT_CHECK, st, mr); + LCW expected{ + Init{{{{{10, 20, 30}}, {Init{30}}, {{{20, 30}, valids}, {62, 72, 82}}}, valids}, + {{{{{10, 20}, valids}}, {Init{30}}, {{40, 50}, {60, 70, 80}}}, valids}, + {{{{{10, 20}, valids}}, {Init{30}}, {{40, 50}, {60, 70, 80}}}, valids}, + {{{{10, 20, 30}}, {Init{30}}, {{{20, 30}, valids}, {62, 72, 82}}}, valids}, + {{{{10, 20, 30}}, {Init{30}}, {{{20, 30}, valids}, {62, 72, 82}}}, valids}, + {{{{{10, 20}, valids}}, {Init{30}}, {{40, 50}, {60, 70, 80}}}, valids}}, + st, + mr}; + CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(expected, + result->get_column(0).view(), + cudf::test::debug_output_level::FIRST_ERROR, + cudf::test::default_ulp, + st, + mr); } } } diff --git a/cpp/tests/copying/gather_str_tests.cpp b/cpp/tests/copying/gather_str_tests.cpp index c023803b7ef8..7fc8ce6ba241 100644 --- a/cpp/tests/copying/gather_str_tests.cpp +++ b/cpp/tests/copying/gather_str_tests.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ #include @@ -15,69 +15,96 @@ #include -class GatherTestStr : public cudf::test::BaseFixture {}; +class GatherTestStr : public cudf::test::BaseFixtureWithHarness {}; TEST_F(GatherTestStr, StringColumn) { - cudf::test::fixed_width_column_wrapper col1{{1, 2, 3, 4, 5, 6}, - {true, true, false, true, false, true}}; - cudf::test::strings_column_wrapper col2{{"This", "is", "not", "a", "string", "type"}, - {true, true, true, true, true, false}}; + auto const st = this->stream(); + auto const mr = this->resources(); + + cudf::test::fixed_width_column_wrapper col1{ + {1, 2, 3, 4, 5, 6}, {true, true, false, true, false, true}, st, mr}; + cudf::test::strings_column_wrapper col2{ + {"This", "is", "not", "a", "string", "type"}, {true, true, true, true, true, false}, st, mr}; cudf::table_view source_table{{col1, col2}}; - cudf::test::fixed_width_column_wrapper gather_map{{0, 1, 3, 4}}; + cudf::test::fixed_width_column_wrapper gather_map{{0, 1, 3, 4}, st, mr}; - cudf::test::fixed_width_column_wrapper exp_col1{{1, 2, 4, 5}, {true, true, true, false}}; - cudf::test::strings_column_wrapper exp_col2{{"This", "is", "a", "string"}, - {true, true, true, true}}; + cudf::test::fixed_width_column_wrapper exp_col1{ + {1, 2, 4, 5}, {true, true, true, false}, st, mr}; + cudf::test::strings_column_wrapper exp_col2{ + {"This", "is", "a", "string"}, {true, true, true, true}, st, mr}; cudf::table_view expected{{exp_col1, exp_col2}}; - auto got = cudf::gather(source_table, gather_map); + auto got = cudf::gather(source_table, gather_map, cudf::out_of_bounds_policy::DONT_CHECK, st, mr); - CUDF_TEST_EXPECT_TABLES_EQUAL(expected, got->view()); + CUDF_TEST_EXPECT_TABLES_EQUAL(expected, got->view(), st, mr); } TEST_F(GatherTestStr, GatherSlicedStringsColumn) { - cudf::test::strings_column_wrapper strings{{"This", "is", "not", "a", "string", "type"}, - {true, true, true, true, true, false}}; + auto const st = this->stream(); + auto const mr = this->resources(); + + cudf::test::strings_column_wrapper strings{ + {"This", "is", "not", "a", "string", "type"}, {true, true, true, true, true, false}, st, mr}; std::vector slice_indices{0, 2, 2, 3, 3, 6}; auto sliced_strings = cudf::slice(strings, slice_indices); { - cudf::test::fixed_width_column_wrapper gather_map{{1, 0, 1}}; - cudf::test::strings_column_wrapper expected_strings{{"is", "This", "is"}, {true, true, true}}; + cudf::test::fixed_width_column_wrapper gather_map{{1, 0, 1}, st, mr}; + cudf::test::strings_column_wrapper expected_strings{ + {"is", "This", "is"}, {true, true, true}, st, mr}; cudf::table_view expected{{expected_strings}}; - auto result = cudf::gather(cudf::table_view{{sliced_strings[0]}}, gather_map); - CUDF_TEST_EXPECT_TABLES_EQUAL(expected, result->view()); + auto result = cudf::gather(cudf::table_view{{sliced_strings[0]}}, + gather_map, + cudf::out_of_bounds_policy::DONT_CHECK, + st, + mr); + CUDF_TEST_EXPECT_TABLES_EQUAL(expected, result->view(), st, mr); } { - cudf::test::fixed_width_column_wrapper gather_map{{0, 0, 0}}; - cudf::test::strings_column_wrapper expected_strings{{"not", "not", "not"}, {true, true, true}}; + cudf::test::fixed_width_column_wrapper gather_map{{0, 0, 0}, st, mr}; + cudf::test::strings_column_wrapper expected_strings{ + {"not", "not", "not"}, {true, true, true}, st, mr}; cudf::table_view expected{{expected_strings}}; - auto result = cudf::gather(cudf::table_view{{sliced_strings[1]}}, gather_map); - CUDF_TEST_EXPECT_TABLES_EQUAL(expected, result->view()); + auto result = cudf::gather(cudf::table_view{{sliced_strings[1]}}, + gather_map, + cudf::out_of_bounds_policy::DONT_CHECK, + st, + mr); + CUDF_TEST_EXPECT_TABLES_EQUAL(expected, result->view(), st, mr); } { - cudf::test::fixed_width_column_wrapper gather_map{{2, 1, 0}}; - cudf::test::strings_column_wrapper expected_strings{{"", "string", "a"}, {false, true, true}}; + cudf::test::fixed_width_column_wrapper gather_map{{2, 1, 0}, st, mr}; + cudf::test::strings_column_wrapper expected_strings{ + {"", "string", "a"}, {false, true, true}, st, mr}; cudf::table_view expected{{expected_strings}}; - auto result = cudf::gather(cudf::table_view{{sliced_strings[2]}}, gather_map); - CUDF_TEST_EXPECT_TABLES_EQUAL(expected, result->view()); + auto result = cudf::gather(cudf::table_view{{sliced_strings[2]}}, + gather_map, + cudf::out_of_bounds_policy::DONT_CHECK, + st, + mr); + CUDF_TEST_EXPECT_TABLES_EQUAL(expected, result->view(), st, mr); } } TEST_F(GatherTestStr, Gather) { + auto const st = this->stream(); + auto const mr = this->resources(); + std::vector h_strings{"eee", "bb", "", "aa", "bbb", "ééé"}; - cudf::test::strings_column_wrapper strings(h_strings.begin(), h_strings.end()); + cudf::test::strings_column_wrapper strings(h_strings.begin(), h_strings.end(), st, mr); cudf::table_view source_table({strings}); std::vector h_map{4, 1, 5, 2, 7}; - cudf::test::fixed_width_column_wrapper gather_map(h_map.begin(), h_map.end()); + cudf::test::fixed_width_column_wrapper gather_map(h_map.begin(), h_map.end(), st, mr); auto results = cudf::gather(source_table, gather_map, cudf::out_of_bounds_policy::NULLIFY, - cudf::negative_index_policy::NOT_ALLOWED); + cudf::negative_index_policy::NOT_ALLOWED, + st, + mr); std::vector h_expected; std::vector expected_validity; @@ -91,56 +118,77 @@ TEST_F(GatherTestStr, Gather) } } cudf::test::strings_column_wrapper expected( - h_expected.begin(), h_expected.end(), expected_validity.begin()); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(results->view().column(0), expected); + h_expected.begin(), h_expected.end(), expected_validity.begin(), st, mr); + CUDF_TEST_EXPECT_COLUMNS_EQUAL( + results->view().column(0), expected, cudf::test::debug_output_level::FIRST_ERROR, st, mr); } TEST_F(GatherTestStr, GatherDontCheckOutOfBounds) { + auto const st = this->stream(); + auto const mr = this->resources(); + std::vector h_strings{"eee", "bb", "", "aa", "bbb", "ééé"}; - cudf::test::strings_column_wrapper strings(h_strings.begin(), h_strings.end()); + cudf::test::strings_column_wrapper strings(h_strings.begin(), h_strings.end(), st, mr); cudf::table_view source_table({strings}); std::vector h_map{3, 4, 0, 0}; - cudf::test::fixed_width_column_wrapper gather_map(h_map.begin(), h_map.end()); + cudf::test::fixed_width_column_wrapper gather_map(h_map.begin(), h_map.end(), st, mr); auto results = cudf::gather(source_table, gather_map, cudf::out_of_bounds_policy::DONT_CHECK, - cudf::negative_index_policy::NOT_ALLOWED); + cudf::negative_index_policy::NOT_ALLOWED, + st, + mr); std::vector h_expected; for (int itr : h_map) { h_expected.push_back(h_strings[itr]); } - cudf::test::strings_column_wrapper expected(h_expected.begin(), h_expected.end()); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(results->view().column(0), expected); + cudf::test::strings_column_wrapper expected(h_expected.begin(), h_expected.end(), st, mr); + CUDF_TEST_EXPECT_COLUMNS_EQUAL( + results->view().column(0), expected, cudf::test::debug_output_level::FIRST_ERROR, st, mr); } TEST_F(GatherTestStr, GatherEmptyMapStringsColumn) { + auto const st = this->stream(); + auto const mr = this->resources(); + auto const zero_size_strings_column = cudf::make_empty_column(cudf::type_id::STRING); - cudf::test::fixed_width_column_wrapper gather_map; + cudf::test::fixed_width_column_wrapper gather_map{}; auto results = cudf::gather(cudf::table_view({zero_size_strings_column->view()}), gather_map, cudf::out_of_bounds_policy::NULLIFY, - cudf::negative_index_policy::NOT_ALLOWED); + cudf::negative_index_policy::NOT_ALLOWED, + st, + mr); cudf::test::expect_column_empty(results->get_column(0).view()); } TEST_F(GatherTestStr, GatherZeroSizeStringsColumn) { + auto const st = this->stream(); + auto const mr = this->resources(); + auto const zero_size_strings_column = cudf::make_empty_column(cudf::type_id::STRING); - cudf::test::fixed_width_column_wrapper gather_map({0}); - cudf::test::strings_column_wrapper expected{std::pair{"", false}}; + cudf::test::fixed_width_column_wrapper gather_map{{0}, st, mr}; + cudf::test::strings_column_wrapper expected{{std::pair{"", false}}, st, mr}; auto results = cudf::gather(cudf::table_view({zero_size_strings_column->view()}), gather_map, cudf::out_of_bounds_policy::NULLIFY, - cudf::negative_index_policy::NOT_ALLOWED); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, results->get_column(0).view()); + cudf::negative_index_policy::NOT_ALLOWED, + st, + mr); + CUDF_TEST_EXPECT_COLUMNS_EQUAL( + expected, results->get_column(0).view(), cudf::test::debug_output_level::FIRST_ERROR, st, mr); } TEST_F(GatherTestStr, GatherRandomStringsColumn) { + auto const st = this->stream(); + auto const mr = this->resources(); + constexpr int num_total_strings = 512; constexpr int num_gathered_strings = 128; @@ -167,7 +215,7 @@ TEST_F(GatherTestStr, GatherRandomStringsColumn) h_ptrs.push_back(s.c_str()); } - cudf::test::strings_column_wrapper strings(h_ptrs.begin(), h_ptrs.end()); + cudf::test::strings_column_wrapper strings(h_ptrs.begin(), h_ptrs.end(), st, mr); cudf::table_view source_table({strings}); // Generate random string indices to gather @@ -179,15 +227,17 @@ TEST_F(GatherTestStr, GatherRandomStringsColumn) } // Gather strings - cudf::test::fixed_width_column_wrapper gather_map(h_map.begin(), h_map.end()); - auto result = cudf::gather(source_table, gather_map); + cudf::test::fixed_width_column_wrapper gather_map(h_map.begin(), h_map.end(), st, mr); + auto result = + cudf::gather(source_table, gather_map, cudf::out_of_bounds_policy::DONT_CHECK, st, mr); std::vector h_expected; h_expected.reserve(num_gathered_strings); for (auto idx : h_map) { h_expected.push_back(h_ptrs[static_cast(idx)]); } - cudf::test::strings_column_wrapper expected(h_expected.begin(), h_expected.end()); + cudf::test::strings_column_wrapper expected(h_expected.begin(), h_expected.end(), st, mr); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(result->view().column(0), expected); + CUDF_TEST_EXPECT_COLUMNS_EQUAL( + result->view().column(0), expected, cudf::test::debug_output_level::FIRST_ERROR, st, mr); } diff --git a/cpp/tests/copying/gather_struct_tests.cpp b/cpp/tests/copying/gather_struct_tests.cpp index e3b0dd9bee07..884aa2da79dc 100644 --- a/cpp/tests/copying/gather_struct_tests.cpp +++ b/cpp/tests/copying/gather_struct_tests.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2020-2024, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -40,10 +40,10 @@ using lists = cudf::test::lists_column_wrapper; auto constexpr null_index = std::numeric_limits::max(); -struct StructGatherTest : public cudf::test::BaseFixture {}; +struct StructGatherTest : public cudf::test::BaseFixtureWithHarness {}; template -struct TypedStructGatherTest : public cudf::test::BaseFixture {}; +struct TypedStructGatherTest : public cudf::test::BaseFixtureWithHarness {}; TYPED_TEST_SUITE(TypedStructGatherTest, cudf::test::FixedWidthTypes); @@ -51,10 +51,14 @@ namespace { template struct column_wrapper_constructor { template - auto operator()(ValueIter begin, ValueIter end, ValidityIter validity_begin) const + auto operator()(ValueIter begin, + ValueIter end, + ValidityIter validity_begin, + rmm::cuda_stream_view st, + cudf::memory_resources mr) const { return cudf::test::fixed_width_column_wrapper{ - begin, end, validity_begin}; + begin, end, validity_begin, st, mr}; } }; @@ -63,9 +67,11 @@ struct column_wrapper_constructor { template cudf::test::strings_column_wrapper operator()(ValueIter begin, ValueIter end, - ValidityIter validity_begin) const + ValidityIter validity_begin, + rmm::cuda_stream_view st, + cudf::memory_resources mr) const { - return cudf::test::strings_column_wrapper{begin, end, validity_begin}; + return cudf::test::strings_column_wrapper{begin, end, validity_begin, st, mr}; } }; @@ -76,7 +82,9 @@ template const& input_values, InputValidityIter input_validity, StructValidityIter struct_validity, - std::vector const& gather_map) + std::vector const& gather_map, + rmm::cuda_stream_view st, + cudf::memory_resources mr) { auto is_valid = // Validity predicate. [&input_values, &input_validity, &struct_validity, &gather_map](auto gather_index) { @@ -99,20 +107,30 @@ auto get_expected_column(std::vector const& input_values, return column_wrapper_constructor()( gather_iter, gather_iter + expected_row_count, - cudf::detail::make_counting_transform_iterator(0, is_valid)); + cudf::detail::make_counting_transform_iterator(0, is_valid), + st, + mr); } -auto do_gather(cudf::column_view const& input, gather_map_t const& gather_map) +auto do_gather(cudf::column_view const& input, + gather_map_t const& gather_map, + rmm::cuda_stream_view st, + cudf::memory_resources mr) { auto result = cudf::gather(cudf::table_view{{input}}, - offsets(gather_map.begin(), gather_map.end()), - cudf::out_of_bounds_policy::NULLIFY); + offsets(gather_map.begin(), gather_map.end(), st, mr), + cudf::out_of_bounds_policy::NULLIFY, + st, + mr); return std::move(result->release()[0]); } } // namespace TYPED_TEST(TypedStructGatherTest, TestSimpleStructGather) { + auto const st = this->stream(); + auto const mr = this->resources(); + // Testing gather() on struct. // 1. String "names" column. @@ -131,63 +149,80 @@ TYPED_TEST(TypedStructGatherTest, TestSimpleStructGather) // Assemble struct column. auto const struct_validity = null_at(5); auto const struct_column = [&] { - auto names_member = ::strings(names.begin(), names.end(), names_validity); - auto ages_member = ::numerics(ages.begin(), ages.end(), ages_validity); - auto is_human_member = ::bools(is_human.begin(), is_human.end(), is_human_validity); - return structs{{names_member, ages_member, is_human_member}, struct_validity}; + auto names_member = ::strings(names.begin(), names.end(), names_validity, st, mr); + auto ages_member = ::numerics(ages.begin(), ages.end(), ages_validity, st, mr); + auto is_human_member = ::bools(is_human.begin(), is_human.end(), is_human_validity, st, mr); + return structs{{names_member, ages_member, is_human_member}, struct_validity, st, mr}; }(); // Gather to new struct column. auto const gather_map = gather_map_t{null_index, 4, 3, 2, 1}; - auto const output = do_gather(struct_column, gather_map); + auto const output = do_gather(struct_column, gather_map, st, mr); auto const expected_output = [&] { auto names_member = - get_expected_column(names, names_validity, struct_validity, gather_map); - auto ages_member = - get_expected_column(ages, ages_validity, struct_validity, gather_map); + get_expected_column(names, names_validity, struct_validity, gather_map, st, mr); + auto ages_member = get_expected_column( + ages, ages_validity, struct_validity, gather_map, st, mr); auto is_human_member = get_expected_column(std::vector(is_human.begin(), is_human.end()), is_human_validity, struct_validity, - gather_map); - return structs{{names_member, ages_member, is_human_member}, null_at(0)}; + gather_map, + st, + mr); + return structs{{names_member, ages_member, is_human_member}, null_at(0), st, mr}; }(); - CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(output->view(), expected_output); + CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(output->view(), + expected_output, + cudf::test::debug_output_level::FIRST_ERROR, + cudf::test::default_ulp, + st, + mr); } TYPED_TEST(TypedStructGatherTest, TestSlicedStructsColumnGatherNoNulls) { - auto const structs_original = [] { - auto child1 = - cudf::test::fixed_width_column_wrapper{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + auto const st = this->stream(); + auto const mr = this->resources(); + + auto const structs_original = [&] { + auto child1 = cudf::test::fixed_width_column_wrapper{ + {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, st, mr}; auto child2 = cudf::test::strings_column_wrapper{ - "One", "Two", "Three", "Four", "Five", "Six", "Seven", "Eight", "Nine", "Ten"}; - return cudf::test::structs_column_wrapper{{child1, child2}}; + {"One", "Two", "Three", "Four", "Five", "Six", "Seven", "Eight", "Nine", "Ten"}, st, mr}; + return cudf::test::structs_column_wrapper{{child1, child2}, {}, st, mr}; }(); - auto const expected = [] { - auto child1 = cudf::test::fixed_width_column_wrapper{6, 10, 8}; - auto child2 = cudf::test::strings_column_wrapper{"Six", "Ten", "Eight"}; - return cudf::test::structs_column_wrapper{{child1, child2}}; + auto const expected = [&] { + auto child1 = cudf::test::fixed_width_column_wrapper{{6, 10, 8}, st, mr}; + auto child2 = cudf::test::strings_column_wrapper{{"Six", "Ten", "Eight"}, st, mr}; + return cudf::test::structs_column_wrapper{{child1, child2}, {}, st, mr}; }(); - auto const structs = cudf::slice(structs_original, {4, 10})[0]; - auto const gather_map = cudf::test::fixed_width_column_wrapper{1, 5, 3}; - auto const result = cudf::gather(cudf::table_view{{structs}}, gather_map)->get_column(0); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(result.view(), expected); + auto const structs = cudf::slice(structs_original, {4, 10}, st)[0]; + auto const gather_map = cudf::test::fixed_width_column_wrapper{{1, 5, 3}, st, mr}; + auto const result = + cudf::gather( + cudf::table_view{{structs}}, gather_map, cudf::out_of_bounds_policy::NULLIFY, st, mr) + ->get_column(0); + CUDF_TEST_EXPECT_COLUMNS_EQUAL( + result.view(), expected, cudf::test::debug_output_level::FIRST_ERROR, st, mr); } TYPED_TEST(TypedStructGatherTest, TestSlicedStructsColumnGatherWithNulls) { + auto const st = this->stream(); + auto const mr = this->resources(); + auto constexpr null = int32_t{0}; // null at child auto constexpr XXX = int32_t{0}; // null at parent - auto const structs_original = [] { + auto const structs_original = [&] { auto child1 = cudf::test::fixed_width_column_wrapper{ - {null, XXX, 3, null, null, 6, XXX, null, null, 10}, nulls_at({0, 3, 4, 7, 8})}; + {null, XXX, 3, null, null, 6, XXX, null, null, 10}, nulls_at({0, 3, 4, 7, 8}), st, mr}; auto child2 = cudf::test::strings_column_wrapper{{"One", "" /*NULL at both parent and child*/, "Three", @@ -198,29 +233,35 @@ TYPED_TEST(TypedStructGatherTest, TestSlicedStructsColumnGatherWithNulls) "" /*NULL*/, "Nine", "" /*NULL*/}, - nulls_at({1, 3, 5, 7, 9})}; - return cudf::test::structs_column_wrapper{{child1, child2}, nulls_at({1, 6})}; + nulls_at({1, 3, 5, 7, 9}), + st, + mr}; + return cudf::test::structs_column_wrapper{{child1, child2}, nulls_at({1, 6}), st, mr}; }(); - auto const expected = [] { - auto child1 = - cudf::test::fixed_width_column_wrapper{{6, 10, null, XXX}, null_at(2)}; - auto child2 = - cudf::test::strings_column_wrapper{{ - "" /*NULL*/, "" /*NULL*/, "Nine", "" /*NULL at parent*/ - }, - nulls_at({0, 1})}; - return cudf::test::structs_column_wrapper{{child1, child2}, null_at(3)}; + auto const expected = [&] { + auto child1 = cudf::test::fixed_width_column_wrapper{ + {6, 10, null, XXX}, null_at(2), st, mr}; + auto child2 = cudf::test::strings_column_wrapper{ + {"" /*NULL*/, "" /*NULL*/, "Nine", "" /*NULL at parent*/}, nulls_at({0, 1}), st, mr}; + return cudf::test::structs_column_wrapper{{child1, child2}, null_at(3), st, mr}; }(); - auto const structs = cudf::slice(structs_original, {4, 10})[0]; - auto const gather_map = cudf::test::fixed_width_column_wrapper{1, 5, 4, 2}; - auto const result = cudf::gather(cudf::table_view{{structs}}, gather_map)->get_column(0); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(result.view(), expected); + auto const structs = cudf::slice(structs_original, {4, 10}, st)[0]; + auto const gather_map = cudf::test::fixed_width_column_wrapper{{1, 5, 4, 2}, st, mr}; + auto const result = + cudf::gather( + cudf::table_view{{structs}}, gather_map, cudf::out_of_bounds_policy::NULLIFY, st, mr) + ->get_column(0); + CUDF_TEST_EXPECT_COLUMNS_EQUAL( + result.view(), expected, cudf::test::debug_output_level::FIRST_ERROR, st, mr); } TYPED_TEST(TypedStructGatherTest, TestNullifyOnNonNullInput) { + auto const st = this->stream(); + auto const mr = this->resources(); + // Test that the null masks of the struct output (and its children) are set correctly, // for an input struct column whose members are not nullable. @@ -236,68 +277,91 @@ TYPED_TEST(TypedStructGatherTest, TestNullifyOnNonNullInput) // Assemble struct column. auto const struct_column = [&] { - auto names_member = ::strings(names.begin(), names.end()); - auto ages_member = ::numerics(ages.begin(), ages.end()); - auto is_human_member = ::bools(is_human.begin(), is_human.end()); - return structs({names_member, ages_member, is_human_member}); + auto names_member = ::strings(names.begin(), names.end(), st, mr); + auto ages_member = ::numerics(ages.begin(), ages.end(), st, mr); + auto is_human_member = ::bools(is_human.begin(), is_human.end(), st, mr); + return structs({names_member, ages_member, is_human_member}, {}, st, mr); }(); // Gather to new struct column. auto const gather_map = gather_map_t{null_index, 4, 3, 2, 1}; - auto const output = do_gather(struct_column, gather_map); + auto const output = do_gather(struct_column, gather_map, st, mr); auto const expected_output = [&] { - auto names_member = get_expected_column(names, no_nulls(), no_nulls(), gather_map); + auto names_member = + get_expected_column(names, no_nulls(), no_nulls(), gather_map, st, mr); auto ages_member = - get_expected_column(ages, no_nulls(), no_nulls(), gather_map); - auto is_human_member = get_expected_column( - std::vector(is_human.begin(), is_human.end()), no_nulls(), no_nulls(), gather_map); - return cudf::test::structs_column_wrapper{{names_member, ages_member, is_human_member}, - null_at(0)}; + get_expected_column(ages, no_nulls(), no_nulls(), gather_map, st, mr); + auto is_human_member = + get_expected_column(std::vector(is_human.begin(), is_human.end()), + no_nulls(), + no_nulls(), + gather_map, + st, + mr); + return cudf::test::structs_column_wrapper{ + {names_member, ages_member, is_human_member}, null_at(0), st, mr}; }(); - CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(output->view(), expected_output); + CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(output->view(), + expected_output, + cudf::test::debug_output_level::FIRST_ERROR, + cudf::test::default_ulp, + st, + mr); } TYPED_TEST(TypedStructGatherTest, TestGatherStructOfLists) { + auto const st = this->stream(); + auto const mr = this->resources(); + // Testing gather() on struct> - auto lists_column_exemplar = [] { + auto lists_column_exemplar = [&] { return lists{ {{5}, {10, 15}, {20, 25, 30}, {35, 40, 45, 50}, {55, 60, 65}, {70, 75}, {80}, {}, {}}, - nulls_at({0, 3, 6, 9})}; + nulls_at({0, 3, 6, 9}), + st, + mr}; }; // Assemble struct column. auto const structs_column = [&] { auto lists_column = lists_column_exemplar(); - return cudf::test::structs_column_wrapper{{lists_column}}; + return cudf::test::structs_column_wrapper{{lists_column}, {}, st, mr}; }(); // Gather to new struct column. auto const gather_map = gather_map_t{null_index, 4, 3, 2, 1, 7, 3}; - auto const gathered_structs = do_gather(structs_column, gather_map); + auto const gathered_structs = do_gather(structs_column, gather_map, st, mr); // Verify that the gathered struct column's list member presents as if // it had itself been gathered individually. auto const expected_gathered_list_column = [&] { auto const list_column_before_gathering = lists_column_exemplar(); - return do_gather(list_column_before_gathering, gather_map); + return do_gather(list_column_before_gathering, gather_map, st, mr); }(); CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(expected_gathered_list_column->view(), - gathered_structs->view().child(0)); + gathered_structs->view().child(0), + cudf::test::debug_output_level::FIRST_ERROR, + cudf::test::default_ulp, + st, + mr); } TYPED_TEST(TypedStructGatherTest, TestGatherStructOfListsOfLists) { + auto const st = this->stream(); + auto const mr = this->resources(); + // Testing gather() on struct>> - auto const lists_column_exemplar = []() { + auto const lists_column_exemplar = [&]() { return lists{{{{5, 5}}, {{10, 15}}, {{20, 25}, {30}}, @@ -305,108 +369,136 @@ TYPED_TEST(TypedStructGatherTest, TestGatherStructOfListsOfLists) {{55}, {60, 65}}, {{70, 75}}, {{80, 80}}, - {}, - {}}, - nulls_at({0, 3, 6, 9})}; + {{}}, + {{}}}, + nulls_at({0, 3, 6, 9}), + st, + mr}; }; auto const structs_column = [&] { auto lists_column = lists_column_exemplar(); - return cudf::test::structs_column_wrapper{{lists_column}}; + return cudf::test::structs_column_wrapper{{lists_column}, {}, st, mr}; }(); // Gather to new struct column. auto const gather_map = gather_map_t{null_index, 4, 3, 2, 1, 7, 3}; - auto const gathered_structs = do_gather(structs_column, gather_map); + auto const gathered_structs = do_gather(structs_column, gather_map, st, mr); // Verify that the gathered struct column's list member presents as if // it had itself been gathered individually. auto const expected_gathered_list_column = [&] { auto const list_column_before_gathering = lists_column_exemplar(); - return do_gather(list_column_before_gathering, gather_map); + return do_gather(list_column_before_gathering, gather_map, st, mr); }(); CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(expected_gathered_list_column->view(), - gathered_structs->view().child(0)); + gathered_structs->view().child(0), + cudf::test::debug_output_level::FIRST_ERROR, + cudf::test::default_ulp, + st, + mr); } TYPED_TEST(TypedStructGatherTest, TestGatherStructOfStructs) { + auto const st = this->stream(); + auto const mr = this->resources(); + // Testing gather() on struct> - auto const numeric_column_exemplar = []() { + auto const numeric_column_exemplar = [&]() { return numerics{{5, 10, 15, 20, 25, 30, 35, 45, 50, 55, 60, 65, 70, 75}, - nulls_at({0, 3, 6, 9, 12, 15})}; + nulls_at({0, 3, 6, 9, 12, 15}), + st, + mr}; }; auto const struct_of_structs_column = [&] { auto numeric_column = numeric_column_exemplar(); - auto structs_column = cudf::test::structs_column_wrapper{{numeric_column}}; - return cudf::test::structs_column_wrapper{{structs_column}}; + auto structs_column = cudf::test::structs_column_wrapper{{numeric_column}, {}, st, mr}; + return cudf::test::structs_column_wrapper{{structs_column}, {}, st, mr}; }(); // Gather to new struct column. auto const gather_map = gather_map_t{null_index, 4, 3, 2, 1, 7, 3}; - auto const gathered_structs = do_gather(struct_of_structs_column, gather_map); + auto const gathered_structs = do_gather(struct_of_structs_column, gather_map, st, mr); // Verify that the underlying numeric column presents as if // it had itself been gathered individually. auto const expected_gathered_column = [&] { auto const numeric_column_before_gathering = numeric_column_exemplar(); - return do_gather(numeric_column_before_gathering, gather_map); + return do_gather(numeric_column_before_gathering, gather_map, st, mr); }(); CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(expected_gathered_column->view(), - gathered_structs->view().child(0).child(0)); + gathered_structs->view().child(0).child(0), + cudf::test::debug_output_level::FIRST_ERROR, + cudf::test::default_ulp, + st, + mr); } TYPED_TEST(TypedStructGatherTest, TestGatherStructOfListOfStructs) { + auto const st = this->stream(); + auto const mr = this->resources(); + // Testing gather() on struct>> auto const struct_of_list_of_structs = [&] { auto numeric_column = - numerics{{5, 10, 15, 20, 25, 30, 35, 45, 50, 55, 60, 65, 70, 75}}; - auto structs_column = structs{{numeric_column}}.release(); + numerics{{5, 10, 15, 20, 25, 30, 35, 45, 50, 55, 60, 65, 70, 75}, st, mr}; + auto structs_column = structs{{numeric_column}, {}, st, mr}.release(); auto list_of_structs_column = cudf::make_lists_column( - 7, offsets{0, 2, 4, 6, 8, 10, 12, 14}.release(), std::move(structs_column), 0, {}); + 7, offsets{{0, 2, 4, 6, 8, 10, 12, 14}, st, mr}.release(), std::move(structs_column), 0, {}); std::vector> vector_of_columns; vector_of_columns.push_back(std::move(list_of_structs_column)); - return structs{std::move(vector_of_columns)}; + return structs{std::move(vector_of_columns), {}, st, mr}; }(); // Gather to new struct column. auto const gather_map = gather_map_t{null_index, 4, 3, 2, 1}; - auto const gathered_structs = do_gather(struct_of_list_of_structs, gather_map); + auto const gathered_structs = do_gather(struct_of_list_of_structs, gather_map, st, mr); // Construct expected gather result. auto expected_gather_result = [&] { - auto expected_numeric_col = numerics{{70, 75, 50, 55, 35, 45, 25, 30, 15, 20}}; - auto expected_struct_col = structs{{expected_numeric_col}}.release(); + auto expected_numeric_col = + numerics{{70, 75, 50, 55, 35, 45, 25, 30, 15, 20}, st, mr}; + auto expected_struct_col = structs{{expected_numeric_col}, {}, st, mr}.release(); auto expected_list_of_structs_column = cudf::make_lists_column( - 5, offsets{0, 2, 4, 6, 8, 10}.release(), std::move(expected_struct_col), 0, {}); + 5, offsets{{0, 2, 4, 6, 8, 10}, st, mr}.release(), std::move(expected_struct_col), 0, {}); std::vector> expected_vector_of_columns; expected_vector_of_columns.push_back(std::move(expected_list_of_structs_column)); - return structs{std::move(expected_vector_of_columns), {false, true, true, true, true}}; + return structs{std::move(expected_vector_of_columns), {false, true, true, true, true}, st, mr}; }(); - CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(expected_gather_result, gathered_structs->view()); + CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(expected_gather_result, + gathered_structs->view(), + cudf::test::debug_output_level::FIRST_ERROR, + cudf::test::default_ulp, + st, + mr); } TYPED_TEST(TypedStructGatherTest, TestGatherStructOfStructsWithValidity) { + auto const st = this->stream(); + auto const mr = this->resources(); + // Testing gather() on struct> using validity_iter_t = decltype(nulls_at({0})); // Factory to construct numeric column with configurable null-mask. - auto const numeric_column_exemplar = [](validity_iter_t validity) { - return numerics{{5, 10, 15, 20, 25, 30, 35, 45, 50, 55, 60, 65, 70, 75}, validity}; + auto const numeric_column_exemplar = [&](validity_iter_t validity) { + return numerics{ + {5, 10, 15, 20, 25, 30, 35, 45, 50, 55, 60, 65, 70, 75}, validity, st, mr}; }; // Construct struct-of-struct-of-numerics. @@ -414,13 +506,14 @@ TYPED_TEST(TypedStructGatherTest, TestGatherStructOfStructsWithValidity) // Every 3rd element is null. auto numeric_column = numeric_column_exemplar(nulls_at({0, 3, 6, 9, 12, 15})); // 12th element is null. - auto structs_column = cudf::test::structs_column_wrapper{{numeric_column}, nulls_at({11})}; - return cudf::test::structs_column_wrapper{{structs_column}}; + auto structs_column = + cudf::test::structs_column_wrapper{{numeric_column}, nulls_at({11}), st, mr}; + return cudf::test::structs_column_wrapper{{structs_column}, {}, st, mr}; }(); // Gather to new struct column. auto const gather_map = gather_map_t{null_index, 4, 3, 2, 1, 7, 3}; - auto const gathered_structs = do_gather(struct_of_structs_column, gather_map); + auto const gathered_structs = do_gather(struct_of_structs_column, gather_map, st, mr); // Verify that the underlying numeric column presents as if // it had itself been gathered individually. @@ -429,28 +522,42 @@ TYPED_TEST(TypedStructGatherTest, TestGatherStructOfStructsWithValidity) // Every 3rd element *and* the 12th element are null. auto const final_validity = nulls_at({0, 3, 6, 9, 11, 12, 15}); auto const numeric_column_before_gathering = numeric_column_exemplar(final_validity); - return do_gather(numeric_column_before_gathering, gather_map); + return do_gather(numeric_column_before_gathering, gather_map, st, mr); }(); CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(expected_gathered_column->view(), - gathered_structs->view().child(0).child(0)); + gathered_structs->view().child(0).child(0), + cudf::test::debug_output_level::FIRST_ERROR, + cudf::test::default_ulp, + st, + mr); } TYPED_TEST(TypedStructGatherTest, TestEmptyGather) { + auto const st = this->stream(); + auto const mr = this->resources(); + auto const struct_column = [&] { - auto ages = numerics{{5, 10, 15, 20, 25, 30}, null_at(4)}; - return cudf::test::structs_column_wrapper{{ages}, null_at(5)}; + auto ages = numerics{{5, 10, 15, 20, 25, 30}, null_at(4), st, mr}; + return cudf::test::structs_column_wrapper{{ages}, null_at(5), st, mr}; }(); auto const empty_gather_map = gather_map_t{}; - auto const gathered_structs = do_gather(struct_column, empty_gather_map); + auto const gathered_structs = do_gather(struct_column, empty_gather_map, st, mr); // Expect empty struct column gathered. auto const expected_empty_column = [&] { - auto expected_empty_numerics = numerics{}; - return cudf::test::structs_column_wrapper{{expected_empty_numerics}}; + std::vector empty_values; + auto expected_empty_numerics = + numerics(empty_values.begin(), empty_values.end(), st, mr); + return cudf::test::structs_column_wrapper{{expected_empty_numerics}, {}, st, mr}; }(); - CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(expected_empty_column, gathered_structs->view()); + CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(expected_empty_column, + gathered_structs->view(), + cudf::test::debug_output_level::FIRST_ERROR, + cudf::test::default_ulp, + st, + mr); } diff --git a/cpp/tests/copying/gather_tests.cpp b/cpp/tests/copying/gather_tests.cpp index c5648c2ff586..6c99a551d9bd 100644 --- a/cpp/tests/copying/gather_tests.cpp +++ b/cpp/tests/copying/gather_tests.cpp @@ -26,61 +26,76 @@ #include template -class GatherTest : public cudf::test::BaseFixture {}; +class GatherTest : public cudf::test::BaseFixtureWithHarness {}; TYPED_TEST_SUITE(GatherTest, cudf::test::NumericTypes); -struct GatherZeroColumnTest : public cudf::test::BaseFixture {}; +struct GatherZeroColumnTest : public cudf::test::BaseFixtureWithHarness {}; TEST_F(GatherZeroColumnTest, PreservesRowCount) { + auto const st = this->stream(); + auto const mr = this->resources(); + cudf::table_view source{std::vector{}, 5}; - cudf::test::fixed_width_column_wrapper gather_map{{0, 2, 4, 1}}; - auto result = cudf::gather(source, gather_map); + cudf::test::fixed_width_column_wrapper gather_map{{0, 2, 4, 1}, st, mr}; + auto result = cudf::gather(source, gather_map, cudf::out_of_bounds_policy::DONT_CHECK, st, mr); EXPECT_EQ(result->num_columns(), 0); EXPECT_EQ(result->num_rows(), 4); } TYPED_TEST(GatherTest, IdentityTest) { + auto const st = this->stream(); + auto const mr = this->resources(); + constexpr cudf::size_type source_size{1000}; auto data = cuda::counting_iterator{0}; - cudf::test::fixed_width_column_wrapper source_column(data, data + source_size); - cudf::test::fixed_width_column_wrapper gather_map(data, data + source_size); + cudf::test::fixed_width_column_wrapper source_column(data, data + source_size, st, mr); + cudf::test::fixed_width_column_wrapper gather_map(data, data + source_size, st, mr); cudf::table_view source_table({source_column}); - std::unique_ptr result = cudf::gather(source_table, gather_map); + std::unique_ptr result = + cudf::gather(source_table, gather_map, cudf::out_of_bounds_policy::DONT_CHECK, st, mr); - CUDF_TEST_EXPECT_TABLES_EQUAL(source_table, result->view()); + CUDF_TEST_EXPECT_TABLES_EQUAL(source_table, result->view(), st, mr); } TYPED_TEST(GatherTest, ReverseIdentityTest) { + auto const st = this->stream(); + auto const mr = this->resources(); + constexpr cudf::size_type source_size{1000}; auto data = cuda::counting_iterator{0}; auto reversed_data = cudf::detail::make_counting_transform_iterator(0, [](auto i) { return source_size - 1 - i; }); - cudf::test::fixed_width_column_wrapper source_column(data, data + source_size); - cudf::test::fixed_width_column_wrapper gather_map(reversed_data, - reversed_data + source_size); + cudf::test::fixed_width_column_wrapper source_column(data, data + source_size, st, mr); + cudf::test::fixed_width_column_wrapper gather_map( + reversed_data, reversed_data + source_size, st, mr); cudf::table_view source_table({source_column}); - std::unique_ptr result = cudf::gather(source_table, gather_map); - cudf::test::fixed_width_column_wrapper expect_column(reversed_data, - reversed_data + source_size); + std::unique_ptr result = + cudf::gather(source_table, gather_map, cudf::out_of_bounds_policy::DONT_CHECK, st, mr); + cudf::test::fixed_width_column_wrapper expect_column( + reversed_data, reversed_data + source_size, st, mr); for (auto i = 0; i < source_table.num_columns(); ++i) { - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expect_column, result->view().column(i)); + CUDF_TEST_EXPECT_COLUMNS_EQUAL( + expect_column, result->view().column(i), cudf::test::debug_output_level::FIRST_ERROR, st, mr); } } TYPED_TEST(GatherTest, EveryOtherNullOdds) { + auto const st = this->stream(); + auto const mr = this->resources(); + constexpr cudf::size_type source_size{1000}; // Every other element is valid @@ -88,30 +103,35 @@ TYPED_TEST(GatherTest, EveryOtherNullOdds) auto validity = cudf::test::iterators::nulls_at_multiples_of(2); cudf::test::fixed_width_column_wrapper source_column( - data, data + source_size, validity); + data, data + source_size, validity, st, mr); // Gather odd-valued indices auto map_data = cudf::detail::make_counting_transform_iterator(0, [](auto i) { return i * 2; }); - cudf::test::fixed_width_column_wrapper gather_map(map_data, - map_data + (source_size / 2)); + cudf::test::fixed_width_column_wrapper gather_map( + map_data, map_data + (source_size / 2), st, mr); cudf::table_view source_table({source_column}); - std::unique_ptr result = cudf::gather(source_table, gather_map); + std::unique_ptr result = + cudf::gather(source_table, gather_map, cudf::out_of_bounds_policy::DONT_CHECK, st, mr); auto expect_data = cuda::constant_iterator{0}; auto expect_valid = cudf::test::iterators::all_nulls(); cudf::test::fixed_width_column_wrapper expect_column( - expect_data, expect_data + source_size / 2, expect_valid); + expect_data, expect_data + source_size / 2, expect_valid, st, mr); for (auto i = 0; i < source_table.num_columns(); ++i) { - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expect_column, result->view().column(i)); + CUDF_TEST_EXPECT_COLUMNS_EQUAL( + expect_column, result->view().column(i), cudf::test::debug_output_level::FIRST_ERROR, st, mr); } } TYPED_TEST(GatherTest, EveryOtherNullEvens) { + auto const st = this->stream(); + auto const mr = this->resources(); + constexpr cudf::size_type source_size{1000}; // Every other element is valid @@ -119,32 +139,37 @@ TYPED_TEST(GatherTest, EveryOtherNullEvens) auto validity = cudf::test::iterators::nulls_at_multiples_of(2); cudf::test::fixed_width_column_wrapper source_column( - data, data + source_size, validity); + data, data + source_size, validity, st, mr); // Gather even-valued indices auto map_data = cudf::detail::make_counting_transform_iterator(0, [](auto i) { return i * 2 + 1; }); - cudf::test::fixed_width_column_wrapper gather_map(map_data, - map_data + (source_size / 2)); + cudf::test::fixed_width_column_wrapper gather_map( + map_data, map_data + (source_size / 2), st, mr); cudf::table_view source_table({source_column}); - std::unique_ptr result = cudf::gather(source_table, gather_map); + std::unique_ptr result = + cudf::gather(source_table, gather_map, cudf::out_of_bounds_policy::DONT_CHECK, st, mr); auto expect_data = cudf::detail::make_counting_transform_iterator(0, [](auto i) { return i * 2 + 1; }); auto expect_valid = cudf::test::iterators::no_nulls(); cudf::test::fixed_width_column_wrapper expect_column( - expect_data, expect_data + source_size / 2, expect_valid); + expect_data, expect_data + source_size / 2, expect_valid, st, mr); for (auto i = 0; i < source_table.num_columns(); ++i) { - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expect_column, result->view().column(i)); + CUDF_TEST_EXPECT_COLUMNS_EQUAL( + expect_column, result->view().column(i), cudf::test::debug_output_level::FIRST_ERROR, st, mr); } } TYPED_TEST(GatherTest, AllNull) { + auto const st = this->stream(); + auto const mr = this->resources(); + constexpr cudf::size_type source_size{1000}; // Every element is invalid @@ -158,20 +183,24 @@ TYPED_TEST(GatherTest, AllNull) std::shuffle(host_map_data.begin(), host_map_data.end(), g); cudf::test::fixed_width_column_wrapper source_column{ - data, data + source_size, validity}; - cudf::test::fixed_width_column_wrapper gather_map(host_map_data.begin(), - host_map_data.end()); + data, data + source_size, validity, st, mr}; + cudf::test::fixed_width_column_wrapper gather_map( + host_map_data.begin(), host_map_data.end(), st, mr); cudf::table_view source_table({source_column}); - std::unique_ptr result = cudf::gather(source_table, gather_map); + std::unique_ptr result = + cudf::gather(source_table, gather_map, cudf::out_of_bounds_policy::DONT_CHECK, st, mr); // Check that the result is also all invalid - CUDF_TEST_EXPECT_TABLES_EQUAL(source_table, result->view()); + CUDF_TEST_EXPECT_TABLES_EQUAL(source_table, result->view(), st, mr); } TYPED_TEST(GatherTest, MultiColReverseIdentityTest) { + auto const st = this->stream(); + auto const mr = this->resources(); + constexpr cudf::size_type source_size{1000}; constexpr cudf::size_type n_cols = 3; @@ -185,27 +214,32 @@ TYPED_TEST(GatherTest, MultiColReverseIdentityTest) for (int i = 0; i < n_cols; ++i) { source_column_wrappers.push_back( - cudf::test::fixed_width_column_wrapper(data, data + source_size)); + cudf::test::fixed_width_column_wrapper(data, data + source_size, st, mr)); source_columns.push_back(source_column_wrappers[i]); } - cudf::test::fixed_width_column_wrapper gather_map(reversed_data, - reversed_data + source_size); + cudf::test::fixed_width_column_wrapper gather_map( + reversed_data, reversed_data + source_size, st, mr); cudf::table_view source_table{source_columns}; - std::unique_ptr result = cudf::gather(source_table, gather_map); + std::unique_ptr result = + cudf::gather(source_table, gather_map, cudf::out_of_bounds_policy::DONT_CHECK, st, mr); - cudf::test::fixed_width_column_wrapper expect_column(reversed_data, - reversed_data + source_size); + cudf::test::fixed_width_column_wrapper expect_column( + reversed_data, reversed_data + source_size, st, mr); for (auto i = 0; i < source_table.num_columns(); ++i) { - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expect_column, result->view().column(i)); + CUDF_TEST_EXPECT_COLUMNS_EQUAL( + expect_column, result->view().column(i), cudf::test::debug_output_level::FIRST_ERROR, st, mr); } } TYPED_TEST(GatherTest, MultiColNulls) { + auto const st = this->stream(); + auto const mr = this->resources(); + constexpr cudf::size_type source_size{1000}; static_assert(0 == source_size % 2, "Size of source data must be a multiple of 2."); @@ -219,20 +253,21 @@ TYPED_TEST(GatherTest, MultiColNulls) std::vector source_columns; for (int i = 0; i < n_cols; ++i) { - source_column_wrappers.push_back( - cudf::test::fixed_width_column_wrapper(data, data + source_size, validity)); + source_column_wrappers.push_back(cudf::test::fixed_width_column_wrapper( + data, data + source_size, validity, st, mr)); source_columns.push_back(source_column_wrappers[i]); } auto reversed_data = cudf::detail::make_counting_transform_iterator(0, [](auto i) { return source_size - 1 - i; }); - cudf::test::fixed_width_column_wrapper gather_map(reversed_data, - reversed_data + source_size); + cudf::test::fixed_width_column_wrapper gather_map( + reversed_data, reversed_data + source_size, st, mr); cudf::table_view source_table{source_columns}; - std::unique_ptr result = cudf::gather(source_table, gather_map); + std::unique_ptr result = + cudf::gather(source_table, gather_map, cudf::out_of_bounds_policy::DONT_CHECK, st, mr); // Expected data auto expect_data = @@ -240,30 +275,37 @@ TYPED_TEST(GatherTest, MultiColNulls) auto expect_valid = cudf::test::iterators::valids_at_multiples_of(2); cudf::test::fixed_width_column_wrapper expect_column( - expect_data, expect_data + source_size, expect_valid); + expect_data, expect_data + source_size, expect_valid, st, mr); for (auto i = 0; i < source_table.num_columns(); ++i) { - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expect_column, result->view().column(i)); + CUDF_TEST_EXPECT_COLUMNS_EQUAL( + expect_column, result->view().column(i), cudf::test::debug_output_level::FIRST_ERROR, st, mr); } } -class GatherNullableTest : public cudf::test::BaseFixture {}; +class GatherNullableTest : public cudf::test::BaseFixtureWithHarness {}; TEST_F(GatherNullableTest, NullableNoNulls) { + auto const st = this->stream(); + auto const mr = this->resources(); + constexpr cudf::size_type source_size{1000}; - auto source_zero = cudf::make_fixed_width_scalar(0); - std::unique_ptr source_column = cudf::sequence(source_size, *source_zero); + auto source_zero = cudf::make_fixed_width_scalar(0, st, mr.get_output_mr()); + std::unique_ptr source_column = + cudf::sequence(source_size, *source_zero, st, mr.get_output_mr()); - auto valid_mask = cudf::create_null_mask(source_size, cudf::mask_state::ALL_VALID); + auto valid_mask = + cudf::create_null_mask(source_size, cudf::mask_state::ALL_VALID, st, mr.get_output_mr()); source_column->set_null_mask(std::move(valid_mask), 0); cudf::table_view source_table({source_column->view(), source_column->view()}); - auto gather_zero = cudf::make_fixed_width_scalar(0); + auto gather_zero = cudf::make_fixed_width_scalar(0, st, mr.get_output_mr()); - std::unique_ptr gather_map = cudf::sequence(source_size, *gather_zero); + std::unique_ptr gather_map = + cudf::sequence(source_size, *gather_zero, st, mr.get_output_mr()); std::unique_ptr result = - cudf::gather(source_table, gather_map->view(), cudf::out_of_bounds_policy::DONT_CHECK); + cudf::gather(source_table, gather_map->view(), cudf::out_of_bounds_policy::DONT_CHECK, st, mr); - CUDF_TEST_EXPECT_TABLES_EQUAL(source_table, result->view()); + CUDF_TEST_EXPECT_TABLES_EQUAL(source_table, result->view(), st, mr); } diff --git a/cpp/tests/copying/segmented_gather_list_tests.cpp b/cpp/tests/copying/segmented_gather_list_tests.cpp index 1827275b328e..a915d9ac0b83 100644 --- a/cpp/tests/copying/segmented_gather_list_tests.cpp +++ b/cpp/tests/copying/segmented_gather_list_tests.cpp @@ -1,7 +1,15 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ + +// Nested `LCW{LCW...` constructions (including empty-list columns with explicit stream/mr) +// trip gcc14's -Wmaybe-uninitialized on column_view_base's copy constructor. Same diagnostic +// as lists/extract_tests.cpp; ignore for the whole file because the warning fires in headers. +#if defined(__GNUC__) && (__GNUC__ >= 14) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +#endif #include #include #include @@ -17,7 +25,7 @@ #include template -class SegmentedGatherTest : public cudf::test::BaseFixture {}; +class SegmentedGatherTest : public cudf::test::BaseFixtureWithHarness {}; using FixedWidthTypesNotBool = cudf::test::Concat; using namespace cudf::test::iterators; auto constexpr NULLIFY = cudf::out_of_bounds_policy::NULLIFY; +// Nested list values. Passing these to lists_column_wrapper builds every nesting level with the +// explicit stream and memory resources instead of the current device resource. +using Init = cudf::test::lists_column_initializer; +using I8Init = cudf::test::lists_column_initializer; +using StrInit = cudf::test::lists_column_initializer; + TYPED_TEST(SegmentedGatherTest, Gather) { using T = TypeParam; + auto const st = this->stream(); + auto const mr = this->resources(); + // List - LCW list{{1, 2, 3, 4}, {5}, {6, 7}, {8, 9, 10}}; + LCW list{Init{{1, 2, 3, 4}, {5}, {6, 7}, {8, 9, 10}}, st, mr}; { // Straight-line case. - auto const gather_map = LCW{{3, 2, 1, 0}, {0}, {0, 1}, {0, 2, 1}}; - auto const expected = LCW{{4, 3, 2, 1}, {5}, {6, 7}, {8, 10, 9}}; + auto const gather_map = LCW{Init{{3, 2, 1, 0}, {0}, {0, 1}, {0, 2, 1}}, st, mr}; + auto const expected = LCW{Init{{4, 3, 2, 1}, {5}, {6, 7}, {8, 10, 9}}, st, mr}; auto const results = cudf::lists::segmented_gather(cudf::lists_column_view{list}, - cudf::lists_column_view{gather_map}); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(results->view(), expected); + cudf::lists_column_view{gather_map}, + cudf::out_of_bounds_policy::DONT_CHECK, + st, + mr); + CUDF_TEST_EXPECT_COLUMNS_EQUAL( + results->view(), expected, cudf::test::debug_output_level::FIRST_ERROR, st, mr); } { // Nullify out-of-bounds values. - auto const gather_map = LCW{{3, 2, 4, 0}, {0}, {0, -3}, {0, 2, 1}}; - auto const expected = LCW{{{4, 3, 2, 1}, null_at(2)}, {5}, {{6, 7}, null_at(1)}, {8, 10, 9}}; - auto const results = cudf::lists::segmented_gather( - cudf::lists_column_view{list}, cudf::lists_column_view{gather_map}, NULLIFY); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(results->view(), expected); + auto const gather_map = LCW{Init{{3, 2, 4, 0}, {0}, {0, -3}, {0, 2, 1}}, st, mr}; + auto const expected = + LCW{Init{{{{4, 3, 2, 1}, null_at(2)}, {5}, {{6, 7}, null_at(1)}, {8, 10, 9}}}, st, mr}; + auto const results = cudf::lists::segmented_gather( + cudf::lists_column_view{list}, cudf::lists_column_view{gather_map}, NULLIFY, st, mr); + CUDF_TEST_EXPECT_COLUMNS_EQUAL( + results->view(), expected, cudf::test::debug_output_level::FIRST_ERROR, st, mr); } } @@ -62,37 +85,54 @@ TYPED_TEST(SegmentedGatherTest, GatherNothing) { using T = TypeParam; + auto const st = this->stream(); + auto const mr = this->resources(); + // List { - auto const list = LCW{{1, 2, 3, 4}, {5}, {6, 7}, {8, 9, 10}}; - auto const gather_map = LCW{LCW{}, LCW{}, LCW{}, LCW{}}; + auto const list = LCW{Init{{1, 2, 3, 4}, {5}, {6, 7}, {8, 9, 10}}, st, mr}; + auto const gather_map = LCW{{LCW{}, LCW{}, LCW{}, LCW{}}, st, mr}; auto const results = cudf::lists::segmented_gather(cudf::lists_column_view{list}, - cudf::lists_column_view{gather_map}); - auto const expected = LCW{LCW{}, LCW{}, LCW{}, LCW{}}; - CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected); + cudf::lists_column_view{gather_map}, + cudf::out_of_bounds_policy::DONT_CHECK, + st, + mr); + auto const expected = LCW{{LCW{}, LCW{}, LCW{}, LCW{}}, st, mr}; + CUDF_TEST_EXPECT_COLUMNS_EQUAL( + *results, expected, cudf::test::debug_output_level::FIRST_ERROR, st, mr); } // List> { - auto const list = LCW{{{1, 2, 3, 4}, {5}}, {{6, 7}}, {{}, {8, 9, 10}}}; - auto const gather_map = LCW{LCW{}, LCW{}, LCW{}}; + auto const list = LCW{Init{{{1, 2, 3, 4}, {5}}, {{6, 7}}, {Init{}, {8, 9, 10}}}, st, mr}; + auto const gather_map = LCW{{LCW{}, LCW{}, LCW{}}, st, mr}; auto const results = cudf::lists::segmented_gather(cudf::lists_column_view{list}, - cudf::lists_column_view{gather_map}); + cudf::lists_column_view{gather_map}, + cudf::out_of_bounds_policy::DONT_CHECK, + st, + mr); // hack to get column of empty list of list - auto const expected_dummy = LCW{{{1, 2, 3, 4}, {5}}, LCW{}, LCW{}, LCW{}}; - auto const expected = cudf::split(expected_dummy, {1})[1]; - CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected); + auto const expected_dummy = + LCW{{LCW{Init{{{1, 2, 3, 4}, {5}}}, st, mr}, LCW{}, LCW{}, LCW{}}, st, mr}; + auto const expected = cudf::split(expected_dummy, {1}, st)[1]; + CUDF_TEST_EXPECT_COLUMNS_EQUAL( + *results, expected, cudf::test::debug_output_level::FIRST_ERROR, st, mr); } // List>> { - auto const list = LCW{{{{1, 2, 3, 4}, {5}}}, {{{6, 7}, {8, 9, 10}}}}; - auto const gather_map = LCW{LCW{}, LCW{}}; + auto const list = LCW{Init{{{{1, 2, 3, 4}, {5}}}, {{{6, 7}, {8, 9, 10}}}}, st, mr}; + auto const gather_map = LCW{{LCW{}, LCW{}}, st, mr}; auto const results = cudf::lists::segmented_gather(cudf::lists_column_view{list}, - cudf::lists_column_view{gather_map}); + cudf::lists_column_view{gather_map}, + cudf::out_of_bounds_policy::DONT_CHECK, + st, + mr); // hack to get column of empty list of list of list - auto const expected_dummy = LCW{{{{1, 2, 3, 4}}}, LCW{}, LCW{}}; - auto const expected = cudf::split(expected_dummy, {1})[1]; - CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected); + auto const expected_dummy = + LCW{{LCW{Init{{{{1, 2, 3, 4}}}}, st, mr}, LCW{}, LCW{}}, st, mr}; + auto const expected = cudf::split(expected_dummy, {1}, st)[1]; + CUDF_TEST_EXPECT_COLUMNS_EQUAL( + *results, expected, cudf::test::debug_output_level::FIRST_ERROR, st, mr); // the result should preserve the full List>> hierarchy // even though it is empty past the first level @@ -113,40 +153,58 @@ TYPED_TEST(SegmentedGatherTest, GatherNothing) using SegmentedGatherTestSingle = SegmentedGatherTest; TEST_F(SegmentedGatherTestSingle, GatherEmpty) { + auto const st = this->stream(); + auto const mr = this->resources(); + auto const list = LCW{}; auto const gather_map = LCW{}; auto const expected = LCW{}; auto const results = cudf::lists::segmented_gather(cudf::lists_column_view{list}, - cudf::lists_column_view{gather_map}); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected); + cudf::lists_column_view{gather_map}, + cudf::out_of_bounds_policy::DONT_CHECK, + st, + mr); + CUDF_TEST_EXPECT_COLUMNS_EQUAL( + *results, expected, cudf::test::debug_output_level::FIRST_ERROR, st, mr); } TYPED_TEST(SegmentedGatherTest, GatherNulls) { using T = TypeParam; + auto const st = this->stream(); + auto const mr = this->resources(); + auto valids = cudf::test::iterators::valids_at_multiples_of(2); // List - auto const list = LCW{{{1, 2, 3, 4}, valids}, {5}, {{6, 7}, valids}, {{8, 9, 10}, valids}}; + auto const list = + LCW{Init{{{1, 2, 3, 4}, valids}, {5}, {{6, 7}, valids}, {{8, 9, 10}, valids}}, st, mr}; { // Test gathering on lists that contain nulls. - auto const gather_map = LCW{{0, 1}, LCW{}, {1}, {2, 1, 0}}; + auto const gather_map = LCW{Init{{0, 1}, Init{}, {1}, {2, 1, 0}}, st, mr}; auto const results = cudf::lists::segmented_gather(cudf::lists_column_view{list}, - cudf::lists_column_view{gather_map}); + cudf::lists_column_view{gather_map}, + cudf::out_of_bounds_policy::DONT_CHECK, + st, + mr); auto const expected = - LCW{{{1, 2}, valids}, LCW{}, {{7}, valids + 1}, {{10, 9, 8}, valids}}; - CUDF_TEST_EXPECT_COLUMNS_EQUAL(results->view(), expected); + LCW{Init{{{{1, 2}, valids}, Init{}, {{7}, valids + 1}, {{10, 9, 8}, valids}}}, st, mr}; + CUDF_TEST_EXPECT_COLUMNS_EQUAL( + results->view(), expected, cudf::test::debug_output_level::FIRST_ERROR, st, mr); } { // Test gathering on lists that contain nulls, with out-of-bounds indices. - auto const gather_map = LCW{{10, -10}, LCW{}, {1}, {2, -10, 0}}; + auto const gather_map = LCW{Init{{10, -10}, Init{}, {1}, {2, -10, 0}}, st, mr}; auto const results = cudf::lists::segmented_gather( - cudf::lists_column_view{list}, cudf::lists_column_view{gather_map}, NULLIFY); - auto const expected = - LCW{{{0, 0}, nulls_at({0, 1})}, LCW{}, {{7}, valids + 1}, {{10, 0, 8}, null_at(1)}}; - CUDF_TEST_EXPECT_COLUMNS_EQUAL(results->view(), expected); + cudf::lists_column_view{list}, cudf::lists_column_view{gather_map}, NULLIFY, st, mr); + auto const expected = LCW{ + Init{{{{0, 0}, nulls_at({0, 1})}, Init{}, {{7}, valids + 1}, {{10, 0, 8}, null_at(1)}}}, + st, + mr}; + CUDF_TEST_EXPECT_COLUMNS_EQUAL( + results->view(), expected, cudf::test::debug_output_level::FIRST_ERROR, st, mr); } } @@ -154,78 +212,85 @@ TYPED_TEST(SegmentedGatherTest, GatherNested) { using T = TypeParam; + auto const st = this->stream(); + auto const mr = this->resources(); + // List> { // clang-format off - auto const list = LCW{{{2, 3}, {4, 5}}, + auto const list = LCW{Init{{{2, 3}, {4, 5}}, {{6, 7, 8}, {9, 10, 11}, {12, 13, 14}}, - {{15, 16}, {17, 18}, {17, 18}, {17, 18}, {-17, -18}}}; - auto const gather_map = LCW{{0, -2, -2}, {1}, {1, 0, -1, -5}}; - auto const results = cudf::lists::segmented_gather(cudf::lists_column_view{list}, cudf::lists_column_view{gather_map}); - auto const expected = LCW{{{2, 3}, {2, 3}, {2, 3}}, + {{15, 16}, {17, 18}, {17, 18}, {17, 18}, {-17, -18}}}, st, mr}; + auto const gather_map = LCW{Init{{0, -2, -2}, {1}, {1, 0, -1, -5}}, st, mr}; + auto const results = cudf::lists::segmented_gather(cudf::lists_column_view{list}, cudf::lists_column_view{gather_map}, cudf::out_of_bounds_policy::DONT_CHECK, st, mr); + auto const expected = LCW{Init{{{2, 3}, {2, 3}, {2, 3}}, {{9, 10, 11}}, - {{17, 18}, {15, 16}, {-17, -18}, {15, 16}}}; - CUDF_TEST_EXPECT_COLUMNS_EQUAL(results->view(), expected); + {{17, 18}, {15, 16}, {-17, -18}, {15, 16}}}, st, mr}; + CUDF_TEST_EXPECT_COLUMNS_EQUAL(results->view(), expected, cudf::test::debug_output_level::FIRST_ERROR, st, mr); // clang-format on } // List>, with out-of-bounds gather indices. { // clang-format off - auto const list = LCW{{{2, 3}, {4, 5}}, + auto const list = LCW{Init{{{2, 3}, {4, 5}}, {{6, 7, 8}, {9, 10, 11}, {12, 13, 14}}, - {{15, 16}, {17, 18}, {17, 18}, {17, 18}, {-17, -18}}}; - auto const gather_map = LCW{{0, 2, -2}, {1}, {1, 0, -1, -6}}; + {{15, 16}, {17, 18}, {17, 18}, {17, 18}, {-17, -18}}}, st, mr}; + auto const gather_map = LCW{Init{{0, 2, -2}, {1}, {1, 0, -1, -6}}, st, mr}; auto const results = - cudf::lists::segmented_gather(cudf::lists_column_view{list}, cudf::lists_column_view{gather_map}, NULLIFY); - auto const expected = LCW{{{{2, 3}, LCW{}, {2, 3}}, null_at(1)}, + cudf::lists::segmented_gather(cudf::lists_column_view{list}, cudf::lists_column_view{gather_map}, NULLIFY, st, mr); + auto const expected = LCW{Init{{{{{2, 3}, Init{}, {2, 3}}, null_at(1)}, {{9, 10, 11}}, - {{{17, 18}, {15, 16}, {-17, -18}, LCW{}}, null_at(3)}}; - CUDF_TEST_EXPECT_COLUMNS_EQUAL(results->view(), expected); + {{{17, 18}, {15, 16}, {-17, -18}, Init{}}, null_at(3)}}}, st, mr}; + CUDF_TEST_EXPECT_COLUMNS_EQUAL(results->view(), expected, cudf::test::debug_output_level::FIRST_ERROR, st, mr); // clang-format on } // List>> { // clang-format off - auto const list = LCW{{{{2, 3}, {4, 5}}, {{6, 7, 8}, {9, 10, 11}, {12, 13, 14}}}, + auto const list = LCW{Init{{{{2, 3}, {4, 5}}, {{6, 7, 8}, {9, 10, 11}, {12, 13, 14}}}, {{{15, 16}, {17, 18}, {17, 18}, {17, 18}, {17, 18}}}, - {{LCW{0}}}, + {{Init{0}}}, {{{10}, {20, 30, 40, 50}, {60, 70, 80}}, {{0, 1, 3}, {5}}, {{11, 12, 13, 14, 15}, {16, 17}, {0}}}, - {{{10, 20}}, {LCW{30}}, {{40, 50}, {60, 70, 80}}}}; - auto const gather_map = LCW{{1}, LCW{}, {0}, {1}, {0, -1, 1}}; - auto const results = cudf::lists::segmented_gather(cudf::lists_column_view{list}, cudf::lists_column_view{gather_map}); - auto const expected = LCW{{{{6, 7, 8}, {9, 10, 11}, {12, 13, 14}}}, - LCW{}, - {{LCW{0}}}, + {{{10, 20}}, {Init{30}}, {{40, 50}, {60, 70, 80}}}}, st, mr}; + auto const gather_map = LCW{Init{{1}, Init{}, {0}, {1}, {0, -1, 1}}, st, mr}; + auto const results = cudf::lists::segmented_gather(cudf::lists_column_view{list}, cudf::lists_column_view{gather_map}, cudf::out_of_bounds_policy::DONT_CHECK, st, mr); + auto const expected = LCW{Init{{{{{6, 7, 8}, {9, 10, 11}, {12, 13, 14}}}, + Init{}, + {{Init{0}}}, {{{0, 1, 3}, {5}}}, - {{{10, 20}}, {{40, 50}, {60, 70, 80}}, {LCW{30}}}}; - CUDF_TEST_EXPECT_COLUMNS_EQUAL(results->view(), expected); + {{{10, 20}}, {{40, 50}, {60, 70, 80}}, {Init{30}}}}}, st, mr}; + CUDF_TEST_EXPECT_COLUMNS_EQUAL(results->view(), expected, cudf::test::debug_output_level::FIRST_ERROR, st, mr); // clang-format on } // List>>, with out-of-bounds gather indices. { - auto const list = LCW{{{{2, 3}, {4, 5}}, {{6, 7, 8}, {9, 10, 11}, {12, 13, 14}}}, - {{{15, 16}, {17, 18}, {17, 18}, {17, 18}, {17, 18}}}, - {{LCW{0}}}, - {{{10}, {20, 30, 40, 50}, {60, 70, 80}}, - {{0, 1, 3}, {5}}, - {{11, 12, 13, 14, 15}, {16, 17}, {0}}}, - {{{10, 20}}, {LCW{30}}, {{40, 50}, {60, 70, 80}}}}; - auto const gather_map = LCW{{1}, LCW{}, {0}, {1}, {0, -1, 3, -4}}; + auto const list = LCW{Init{{{{2, 3}, {4, 5}}, {{6, 7, 8}, {9, 10, 11}, {12, 13, 14}}}, + {{{15, 16}, {17, 18}, {17, 18}, {17, 18}, {17, 18}}}, + {{Init{0}}}, + {{{10}, {20, 30, 40, 50}, {60, 70, 80}}, + {{0, 1, 3}, {5}}, + {{11, 12, 13, 14, 15}, {16, 17}, {0}}}, + {{{10, 20}}, {Init{30}}, {{40, 50}, {60, 70, 80}}}}, + st, + mr}; + auto const gather_map = LCW{Init{{1}, Init{}, {0}, {1}, {0, -1, 3, -4}}, st, mr}; auto const results = cudf::lists::segmented_gather( - cudf::lists_column_view{list}, cudf::lists_column_view{gather_map}, NULLIFY); + cudf::lists_column_view{list}, cudf::lists_column_view{gather_map}, NULLIFY, st, mr); auto const expected = - LCW{{{{6, 7, 8}, {9, 10, 11}, {12, 13, 14}}}, - LCW{}, - {{LCW{0}}}, - {{{0, 1, 3}, {5}}}, - {{{{10, 20}}, {{40, 50}, {60, 70, 80}}, LCW{}, LCW{}}, nulls_at({2, 3})}}; - CUDF_TEST_EXPECT_COLUMNS_EQUAL(results->view(), expected); - // clang-format on + LCW{Init{{{{{6, 7, 8}, {9, 10, 11}, {12, 13, 14}}}, + Init{}, + {{Init{0}}}, + {{{0, 1, 3}, {5}}}, + {{{{10, 20}}, {{40, 50}, {60, 70, 80}}, Init{}, Init{}}, nulls_at({2, 3})}}}, + st, + mr}; + CUDF_TEST_EXPECT_COLUMNS_EQUAL( + results->view(), expected, cudf::test::debug_output_level::FIRST_ERROR, st, mr); } } @@ -233,33 +298,36 @@ TYPED_TEST(SegmentedGatherTest, GatherOutOfOrder) { using T = TypeParam; + auto const st = this->stream(); + auto const mr = this->resources(); + // List> { // clang-format off - auto const list = LCW{{{2, 3}, {4, 5}}, + auto const list = LCW{Init{{{2, 3}, {4, 5}}, {{6, 7, 8}, {9, 10, 11}, {12, 13, 14}}, - {{15, 16}, {17, 18}, {17, 18}, {17, 18}, {17, 18}}}; - auto const gather_map = LCW{{1, 0}, {1, 2, 0}, {4, 3, 2, 1, 0}}; - auto const results = cudf::lists::segmented_gather(cudf::lists_column_view{list}, cudf::lists_column_view{gather_map}); - auto const expected = LCW{{{4, 5}, {2, 3}}, + {{15, 16}, {17, 18}, {17, 18}, {17, 18}, {17, 18}}}, st, mr}; + auto const gather_map = LCW{Init{{1, 0}, {1, 2, 0}, {4, 3, 2, 1, 0}}, st, mr}; + auto const results = cudf::lists::segmented_gather(cudf::lists_column_view{list}, cudf::lists_column_view{gather_map}, cudf::out_of_bounds_policy::DONT_CHECK, st, mr); + auto const expected = LCW{Init{{{4, 5}, {2, 3}}, {{9, 10, 11}, {12, 13, 14}, {6, 7, 8}}, - {{17, 18}, {17, 18}, {17, 18}, {17, 18}, {15, 16}}}; - CUDF_TEST_EXPECT_COLUMNS_EQUAL(results->view(), expected); + {{17, 18}, {17, 18}, {17, 18}, {17, 18}, {15, 16}}}, st, mr}; + CUDF_TEST_EXPECT_COLUMNS_EQUAL(results->view(), expected, cudf::test::debug_output_level::FIRST_ERROR, st, mr); // clang-format on } // List>, with out-of-bounds gather indices. { // clang-format off - auto const list = LCW{{{2, 3}, {4, 5}}, + auto const list = LCW{Init{{{2, 3}, {4, 5}}, {{6, 7, 8}, {9, 10, 11}, {12, 13, 14}}, - {{15, 16}, {17, 18}, {17, 18}, {17, 18}, {17, 18}}}; - auto const gather_map = LCW{{1, 0}, {3, -1, -4}, {5, 4, 3, 2, 1, 0}}; - auto const results = cudf::lists::segmented_gather(cudf::lists_column_view{list}, cudf::lists_column_view{gather_map}, NULLIFY); - auto const expected = LCW{{{4, 5}, {2, 3}}, - {{LCW{}, {12, 13, 14}, LCW{}}, nulls_at({0, 2})}, - {{LCW{}, {17, 18}, {17, 18}, {17, 18}, {17, 18}, {15, 16}}, null_at(0)}}; - CUDF_TEST_EXPECT_COLUMNS_EQUAL(results->view(), expected); + {{15, 16}, {17, 18}, {17, 18}, {17, 18}, {17, 18}}}, st, mr}; + auto const gather_map = LCW{Init{{1, 0}, {3, -1, -4}, {5, 4, 3, 2, 1, 0}}, st, mr}; + auto const results = cudf::lists::segmented_gather(cudf::lists_column_view{list}, cudf::lists_column_view{gather_map}, NULLIFY, st, mr); + auto const expected = LCW{Init{{{4, 5}, {2, 3}}, + {{Init{}, {12, 13, 14}, Init{}}, nulls_at({0, 2})}, + {{Init{}, {17, 18}, {17, 18}, {17, 18}, {17, 18}, {15, 16}}, null_at(0)}}, st, mr}; + CUDF_TEST_EXPECT_COLUMNS_EQUAL(results->view(), expected, cudf::test::debug_output_level::FIRST_ERROR, st, mr); // clang-format on } } @@ -268,33 +336,36 @@ TYPED_TEST(SegmentedGatherTest, GatherNegatives) { using T = TypeParam; + auto const st = this->stream(); + auto const mr = this->resources(); + // List> { // clang-format off - auto const list = LCW{{{2, 3}, {4, 5}}, + auto const list = LCW{Init{{{2, 3}, {4, 5}}, {{6, 7, 8}, {9, 10, 11}, {12, 13, 14}}, - {{15, 16}, {17, 18}, {17, 18}, {17, 18}, {17, 18}}}; - auto const gather_map = LCW{{-1, 0}, {-2, -1, 0}, {-5, -4, -3, -2, -1, 0}}; - auto const results = cudf::lists::segmented_gather(cudf::lists_column_view{list}, cudf::lists_column_view{gather_map}); - auto const expected = LCW{{{4, 5}, {2, 3}}, + {{15, 16}, {17, 18}, {17, 18}, {17, 18}, {17, 18}}}, st, mr}; + auto const gather_map = LCW{Init{{-1, 0}, {-2, -1, 0}, {-5, -4, -3, -2, -1, 0}}, st, mr}; + auto const results = cudf::lists::segmented_gather(cudf::lists_column_view{list}, cudf::lists_column_view{gather_map}, cudf::out_of_bounds_policy::DONT_CHECK, st, mr); + auto const expected = LCW{Init{{{4, 5}, {2, 3}}, {{9, 10, 11}, {12, 13, 14}, {6, 7, 8}}, - {{15, 16}, {17, 18}, {17, 18}, {17, 18}, {17, 18}, {15, 16}}}; - CUDF_TEST_EXPECT_COLUMNS_EQUAL(results->view(), expected); + {{15, 16}, {17, 18}, {17, 18}, {17, 18}, {17, 18}, {15, 16}}}, st, mr}; + CUDF_TEST_EXPECT_COLUMNS_EQUAL(results->view(), expected, cudf::test::debug_output_level::FIRST_ERROR, st, mr); // clang-format on } // List>, with out-of-bounds gather indices. { // clang-format off - auto const list = LCW{{{2, 3}, {4, 5}}, + auto const list = LCW{Init{{{2, 3}, {4, 5}}, {{6, 7, 8}, {9, 10, 11}, {12, 13, 14}}, - {{15, 16}, {17, 18}, {17, 18}, {17, 18}, {17, 18}}}; - auto const gather_map = LCW{{-1, 0}, {-2, -1, -4}, {-6, -4, -3, -2, -1, 0}}; + {{15, 16}, {17, 18}, {17, 18}, {17, 18}, {17, 18}}}, st, mr}; + auto const gather_map = LCW{Init{{-1, 0}, {-2, -1, -4}, {-6, -4, -3, -2, -1, 0}}, st, mr}; auto const results = - cudf::lists::segmented_gather(cudf::lists_column_view{list}, cudf::lists_column_view{gather_map}, NULLIFY); - auto const expected = LCW{{{4, 5}, {2, 3}}, - {{{9, 10, 11}, {12, 13, 14}, LCW{}}, null_at(2)}, - {{LCW{}, {17, 18}, {17, 18}, {17, 18}, {17, 18}, {15, 16}}, null_at(0)}}; - CUDF_TEST_EXPECT_COLUMNS_EQUAL(results->view(), expected); + cudf::lists::segmented_gather(cudf::lists_column_view{list}, cudf::lists_column_view{gather_map}, NULLIFY, st, mr); + auto const expected = LCW{Init{{{4, 5}, {2, 3}}, + {{{9, 10, 11}, {12, 13, 14}, Init{}}, null_at(2)}, + {{Init{}, {17, 18}, {17, 18}, {17, 18}, {17, 18}, {15, 16}}, null_at(0)}}, st, mr}; + CUDF_TEST_EXPECT_COLUMNS_EQUAL(results->view(), expected, cudf::test::debug_output_level::FIRST_ERROR, st, mr); // clang-format on } } @@ -303,41 +374,44 @@ TYPED_TEST(SegmentedGatherTest, GatherNestedNulls) { using T = TypeParam; + auto const st = this->stream(); + auto const mr = this->resources(); + auto valids = cudf::test::iterators::valids_at_multiples_of(2); // List> { // clang-format off - auto const list = LCW{{{{2, 3}, valids}, {4, 5}}, + auto const list = LCW{Init{{{{2, 3}, valids}, {4, 5}}, {{{6, 7, 8}, {9, 10, 11}, {12, 13, 14}}, valids}, {{15, 16}, {17, 18}, {17, 18}, {17, 18}, {17, 18}}, - {{{{25, 26}, valids}, {27, 28}, {{29, 30}, valids}, {31, 32}, {33, 34}}, valids}}; - auto const gather_map = LCW{{0, 1}, {0, 2}, LCW{}, {0, 1, 4}}; - auto const results = cudf::lists::segmented_gather(cudf::lists_column_view{list}, cudf::lists_column_view{gather_map}); - auto const expected = LCW{{{{2, 3}, valids}, {4, 5}}, + {{{{25, 26}, valids}, {27, 28}, {{29, 30}, valids}, {31, 32}, {33, 34}}, valids}}, st, mr}; + auto const gather_map = LCW{Init{{0, 1}, {0, 2}, Init{}, {0, 1, 4}}, st, mr}; + auto const results = cudf::lists::segmented_gather(cudf::lists_column_view{list}, cudf::lists_column_view{gather_map}, cudf::out_of_bounds_policy::DONT_CHECK, st, mr); + auto const expected = LCW{Init{{{{2, 3}, valids}, {4, 5}}, {{{6, 7, 8}, {12, 13, 14}}, no_nulls()}, - LCW{}, - {{{{25, 26}, valids}, {27, 28}, {33, 34}}, valids}}; - CUDF_TEST_EXPECT_COLUMNS_EQUAL(results->view(), expected); + Init{}, + {{{{25, 26}, valids}, {27, 28}, {33, 34}}, valids}}, st, mr}; + CUDF_TEST_EXPECT_COLUMNS_EQUAL(results->view(), expected, cudf::test::debug_output_level::FIRST_ERROR, st, mr); // clang-format on } // List>>> { // clang-format off - auto const list = LCW{{{{{2, 3}, {4, 5}}, {{6, 7, 8}, {9, 10, 11}, {12, 13, 14}}}, + auto const list = LCW{Init{{{{{2, 3}, {4, 5}}, {{6, 7, 8}, {9, 10, 11}, {12, 13, 14}}}, {{{15, 16}, {{27, 28}, valids}, {{37, 38}, valids}, {47, 48}, {57, 58}}}, - {{LCW{0}}}, + {{Init{0}}}, {{{10}, {20, 30, 40, 50}, {60, 70, 80}}, {{0, 1, 3}, {5}}, {{11, 12, 13, 14, 15}, {16, 17}, {0}}}, - {{{{{10, 20}, valids}}, {LCW{30}}, {{40, 50}, {60, 70, 80}}}, valids}}}; - auto const gather_map = LCW{{1, 2, 4}}; - auto const results = cudf::lists::segmented_gather(cudf::lists_column_view{list}, cudf::lists_column_view{gather_map}); - auto const expected = LCW{{{{{15, 16}, {{27, 28}, valids}, {{37, 38}, valids}, {47, 48}, {57, 58}}}, - {{LCW{0}}}, - {{{{{10, 20}, valids}}, {LCW{30}}, {{40, 50}, {60, 70, 80}}}, valids}}}; - CUDF_TEST_EXPECT_COLUMNS_EQUAL(results->view(), expected); + {{{{{10, 20}, valids}}, {Init{30}}, {{40, 50}, {60, 70, 80}}}, valids}}}, st, mr}; + auto const gather_map = LCW{Init{{1, 2, 4}}, st, mr}; + auto const results = cudf::lists::segmented_gather(cudf::lists_column_view{list}, cudf::lists_column_view{gather_map}, cudf::out_of_bounds_policy::DONT_CHECK, st, mr); + auto const expected = LCW{Init{{{{{15, 16}, {{27, 28}, valids}, {{37, 38}, valids}, {47, 48}, {57, 58}}}, + {{Init{0}}}, + {{{{{10, 20}, valids}}, {Init{30}}, {{40, 50}, {60, 70, 80}}}, valids}}}, st, mr}; + CUDF_TEST_EXPECT_COLUMNS_EQUAL(results->view(), expected, cudf::test::debug_output_level::FIRST_ERROR, st, mr); // clang-format on } } @@ -346,52 +420,78 @@ TYPED_TEST(SegmentedGatherTest, GatherNestedWithEmpties) { using T = TypeParam; - auto const list = LCW{{{2, 3}, LCW{}}, {{6, 7, 8}, {9, 10, 11}, {12, 13, 14}}, {LCW{}}}; - auto const gather_map = LCW{LCW{0}, LCW{0}, LCW{0}}; + auto const st = this->stream(); + auto const mr = this->resources(); + + auto const list = + LCW{Init{{{2, 3}, Init{}}, {{6, 7, 8}, {9, 10, 11}, {12, 13, 14}}, {Init{}}}, st, mr}; + auto const gather_map = LCW{Init{{0}, {0}, {0}}, st, mr}; auto results = cudf::lists::segmented_gather(cudf::lists_column_view{list}, - cudf::lists_column_view{gather_map}); + cudf::lists_column_view{gather_map}, + cudf::out_of_bounds_policy::DONT_CHECK, + st, + mr); auto const expected = - LCW{{{2, 3}}, {{6, 7, 8}}, {LCW{}}}; // skip one null, gather one null. - CUDF_TEST_EXPECT_COLUMNS_EQUAL(results->view(), expected); + LCW{Init{{{2, 3}}, {{6, 7, 8}}, {Init{}}}, st, mr}; // skip one null, gather one null. + CUDF_TEST_EXPECT_COLUMNS_EQUAL( + results->view(), expected, cudf::test::debug_output_level::FIRST_ERROR, st, mr); } TYPED_TEST(SegmentedGatherTest, GatherSliced) { using T = TypeParam; + + auto const st = this->stream(); + auto const mr = this->resources(); + { - auto const a = LCW{ - {{1, 1, 1}, {2, 2}, {3, 3}}, - {{4, 4, 4}, {5, 5}, {6, 6}}, - {{7, 7, 7}, {8, 8}, {9, 9}}, - {{10, 10, 10}, {11, 11}, {12, 12}}, - {{20, 20, 20, 20}, {25}}, - {{30, 30, 30, 30}, {40}}, - {{50, 50, 50, 50}, {6, 13}}, - {{70, 70, 70, 70}, {80}}, - }; - auto const split_a = cudf::split(a, {3}); + auto const a = LCW{Init{ + {{1, 1, 1}, {2, 2}, {3, 3}}, + {{4, 4, 4}, {5, 5}, {6, 6}}, + {{7, 7, 7}, {8, 8}, {9, 9}}, + {{10, 10, 10}, {11, 11}, {12, 12}}, + {{20, 20, 20, 20}, {25}}, + {{30, 30, 30, 30}, {40}}, + {{50, 50, 50, 50}, {6, 13}}, + {{70, 70, 70, 70}, {80}}, + }, + st, + mr}; + auto const split_a = cudf::split(a, {3}, st); { - auto const list = LCW{{1, 2}, {0, 2}, {0, 1}}; + auto const list = LCW{Init{{1, 2}, {0, 2}, {0, 1}}, st, mr}; auto const gather_map = cudf::lists_column_view{list}; - auto const result = - cudf::lists::segmented_gather(cudf::lists_column_view{split_a[0]}, gather_map); - auto const expected = LCW{ - {{2, 2}, {3, 3}}, - {{4, 4, 4}, {6, 6}}, - {{7, 7, 7}, {8, 8}}, - }; - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view()); + auto const result = cudf::lists::segmented_gather(cudf::lists_column_view{split_a[0]}, + gather_map, + cudf::out_of_bounds_policy::DONT_CHECK, + st, + mr); + auto const expected = LCW{Init{ + {{2, 2}, {3, 3}}, + {{4, 4, 4}, {6, 6}}, + {{7, 7, 7}, {8, 8}}, + }, + st, + mr}; + CUDF_TEST_EXPECT_COLUMNS_EQUAL( + expected, result->view(), cudf::test::debug_output_level::FIRST_ERROR, st, mr); } { - auto const list = LCW{{0, 1}, LCW{}, LCW{}, {0, 1}, LCW{}}; + auto const list = LCW{Init{{0, 1}, Init{}, Init{}, {0, 1}, Init{}}, st, mr}; auto const gather_map = cudf::lists_column_view{list}; - auto const result = - cudf::lists::segmented_gather(cudf::lists_column_view{split_a[1]}, gather_map); + auto const result = cudf::lists::segmented_gather(cudf::lists_column_view{split_a[1]}, + gather_map, + cudf::out_of_bounds_policy::DONT_CHECK, + st, + mr); auto const expected = - LCW{{{10, 10, 10}, {11, 11}}, LCW{}, LCW{}, {{50, 50, 50, 50}, {6, 13}}, LCW{}}; - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view()); + LCW{Init{{{10, 10, 10}, {11, 11}}, Init{}, Init{}, {{50, 50, 50, 50}, {6, 13}}, Init{}}, + st, + mr}; + CUDF_TEST_EXPECT_COLUMNS_EQUAL( + expected, result->view(), cudf::test::debug_output_level::FIRST_ERROR, st, mr); } } @@ -400,74 +500,105 @@ TYPED_TEST(SegmentedGatherTest, GatherSliced) // List>> { LCW list{ - // slice 0 - {{{2, 3}, {4, 5}}, {{6, 7, 8}, {9, 10, 11}, {12, 13, 14}}}, + Init{// slice 0 + {{{2, 3}, {4, 5}}, {{6, 7, 8}, {9, 10, 11}, {12, 13, 14}}}, - {{{15, 16}, {{27, 28}, valids}, {{37, 38}, valids}, {47, 48}, {57, 58}}, - {{11, 12}, {{42, 43, 44}, valids}, {{77, 78}, valids}}}, + {{{15, 16}, {{27, 28}, valids}, {{37, 38}, valids}, {47, 48}, {57, 58}}, + {{11, 12}, {{42, 43, 44}, valids}, {{77, 78}, valids}}}, - // slice 1 - {{LCW{0}}}, - {{{10}, {20, 30, 40, 50}, {60, 70, 80}}, - {{0, 1, 3}, {5}}, - {{11, 12, 13, 14, 15}, {16, 17}, {0}}}, - {{{{1, 6}, {60, 70, 80, 100}}, {{10, 11, 13}, {15}}, {{11, 12, 13, 14, 15}}}, valids}, + // slice 1 + {{Init{0}}}, + {{{10}, {20, 30, 40, 50}, {60, 70, 80}}, + {{0, 1, 3}, {5}}, + {{11, 12, 13, 14, 15}, {16, 17}, {0}}}, + {{{{1, 6}, {60, 70, 80, 100}}, {{10, 11, 13}, {15}}, {{11, 12, 13, 14, 15}}}, valids}, - // slice 2 - {{{{{10, 20}, valids}}, {LCW{30}}, {{40, 50}, {60, 70, 80}}}, valids}, - {{{{10, 20, 30}}, {LCW{30}}, {{{20, 30}, valids}, {62, 72, 82}}}, valids}}; + // slice 2 + {{{{{10, 20}, valids}}, {Init{30}}, {{40, 50}, {60, 70, 80}}}, valids}, + {{{{10, 20, 30}}, {Init{30}}, {{{20, 30}, valids}, {62, 72, 82}}}, valids}}, + st, + mr}; - auto sliced = cudf::slice(list, {0, 1, 2, 5, 5, 7}); + auto sliced = cudf::slice(list, {0, 1, 2, 5, 5, 7}, st); // gather from slice 0 { - LCW map{{0, 1}}; + LCW map{Init{{0, 1}}, st, mr}; auto result = cudf::lists::segmented_gather(cudf::lists_column_view{sliced[0]}, - cudf::lists_column_view{map}); - LCW expected{{{{2, 3}, {4, 5}}, {{6, 7, 8}, {9, 10, 11}, {12, 13, 14}}}}; - CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(expected, result->view()); + cudf::lists_column_view{map}, + cudf::out_of_bounds_policy::DONT_CHECK, + st, + mr); + LCW expected{Init{{{{2, 3}, {4, 5}}, {{6, 7, 8}, {9, 10, 11}, {12, 13, 14}}}}, st, mr}; + CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(expected, + result->view(), + cudf::test::debug_output_level::FIRST_ERROR, + cudf::test::default_ulp, + st, + mr); } // gather from slice 1 { - LCW map{{0}, {1, 2, 0, 1}, {0, 1, 2}}; + LCW map{Init{{0}, {1, 2, 0, 1}, {0, 1, 2}}, st, mr}; auto result = cudf::lists::segmented_gather(cudf::lists_column_view{sliced[1]}, - cudf::lists_column_view{map}); + cudf::lists_column_view{map}, + cudf::out_of_bounds_policy::DONT_CHECK, + st, + mr); LCW expected{ - {{LCW{0}}}, - - {{{0, 1, 3}, {5}}, - {{11, 12, 13, 14, 15}, {16, 17}, {0}}, - {{10}, {20, 30, 40, 50}, {60, 70, 80}}, - {{0, 1, 3}, {5}}}, - - {{{{1, 6}, {60, 70, 80, 100}}, {{10, 11, 13}, {15}}, {{11, 12, 13, 14, 15}}}, valids}, - }; - CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(expected, result->view()); + Init{ + {{Init{0}}}, + + {{{0, 1, 3}, {5}}, + {{11, 12, 13, 14, 15}, {16, 17}, {0}}, + {{10}, {20, 30, 40, 50}, {60, 70, 80}}, + {{0, 1, 3}, {5}}}, + + {{{{1, 6}, {60, 70, 80, 100}}, {{10, 11, 13}, {15}}, {{11, 12, 13, 14, 15}}}, valids}, + }, + st, + mr}; + CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(expected, + result->view(), + cudf::test::debug_output_level::FIRST_ERROR, + cudf::test::default_ulp, + st, + mr); } // gather from slice 2 { - LCW map{{1, 0, 0, 1, 1, 0}, {1, 0, 0, 1, 1, 2}}; + LCW map{Init{{1, 0, 0, 1, 1, 0}, {1, 0, 0, 1, 1, 2}}, st, mr}; auto result = cudf::lists::segmented_gather(cudf::lists_column_view{sliced[2]}, - cudf::lists_column_view{map}); + cudf::lists_column_view{map}, + cudf::out_of_bounds_policy::DONT_CHECK, + st, + mr); std::vector expected_valids = {false, true, true, false, false, true}; - LCW expected{{{{LCW{30}}, - {{{10, 20}, valids}}, - {{{10, 20}, valids}}, - {LCW{30}}, - {LCW{30}}, - {{{10, 20}, valids}}}, - expected_valids.begin()}, - {{{LCW{30}}, - {{10, 20, 30}}, - {{10, 20, 30}}, - {LCW{30}}, - {LCW{30}}, - {{{20, 30}, valids}, {62, 72, 82}}}, - expected_valids.begin()}}; - CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(expected, result->view()); + LCW expected{Init{{{{Init{30}}, + {{{10, 20}, valids}}, + {{{10, 20}, valids}}, + {Init{30}}, + {Init{30}}, + {{{10, 20}, valids}}}, + expected_valids.begin()}, + {{{Init{30}}, + {{10, 20, 30}}, + {{10, 20, 30}}, + {Init{30}}, + {Init{30}}, + {{{20, 30}, valids}, {62, 72, 82}}}, + expected_valids.begin()}}, + st, + mr}; + CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(expected, + result->view(), + cudf::test::debug_output_level::FIRST_ERROR, + cudf::test::default_ulp, + st, + mr); } } } @@ -476,26 +607,43 @@ using SegmentedGatherTestString = SegmentedGatherTest; TEST_F(SegmentedGatherTestString, StringGather) { using T = cudf::string_view; + + auto const st = this->stream(); + auto const mr = this->resources(); + // List { - auto const list = LCW{{"a", "b", "c", "d"}, {"1", "22", "333", "4"}, {"x", "y", "z"}}; - auto const gather_map = LCW{{0, 1, 3, 2}, {1, 0, 3, 2}, LCW{}}; - auto const expected = LCW{{"a", "b", "d", "c"}, {"22", "1", "4", "333"}, LCW{}}; - auto const result = cudf::lists::segmented_gather(cudf::lists_column_view{list}, - cudf::lists_column_view{gather_map}); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view()); + auto const list = + LCW{StrInit{{"a", "b", "c", "d"}, {"1", "22", "333", "4"}, {"x", "y", "z"}}, st, mr}; + auto const gather_map = cudf::test::lists_column_wrapper{ + I8Init{{0, 1, 3, 2}, {1, 0, 3, 2}, I8Init{}}, st, mr}; + auto const expected = + LCW{StrInit{{"a", "b", "d", "c"}, {"22", "1", "4", "333"}, StrInit{}}, st, mr}; + auto const result = cudf::lists::segmented_gather(cudf::lists_column_view{list}, + cudf::lists_column_view{gather_map}, + cudf::out_of_bounds_policy::DONT_CHECK, + st, + mr); + CUDF_TEST_EXPECT_COLUMNS_EQUAL( + expected, result->view(), cudf::test::debug_output_level::FIRST_ERROR, st, mr); } // List, with out-of-order gather indices. { - auto const list = LCW{{"a", "b", "c", "d"}, {"1", "22", "333", "4"}, {"x", "y", "z"}}; - auto const gather_map = LCW{{0, 1, 3, 4}, {1, -5, 3, 2}, LCW{}}; - auto const expected = LCW{{{"a", "b", "d", "c"}, cudf::test::iterators::null_at(3)}, - {{"22", "1", "4", "333"}, cudf::test::iterators::null_at(1)}, - LCW{}}; - auto result = cudf::lists::segmented_gather( - cudf::lists_column_view{list}, cudf::lists_column_view{gather_map}, NULLIFY); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, result->view()); + auto const list = + LCW{StrInit{{"a", "b", "c", "d"}, {"1", "22", "333", "4"}, {"x", "y", "z"}}, st, mr}; + auto const gather_map = cudf::test::lists_column_wrapper{ + I8Init{{0, 1, 3, 4}, {1, -5, 3, 2}, I8Init{}}, st, mr}; + auto const expected = + LCW{StrInit{{{{"a", "b", "d", "c"}, cudf::test::iterators::null_at(3)}, + {{"22", "1", "4", "333"}, cudf::test::iterators::null_at(1)}, + StrInit{}}}, + st, + mr}; + auto result = cudf::lists::segmented_gather( + cudf::lists_column_view{list}, cudf::lists_column_view{gather_map}, NULLIFY, st, mr); + CUDF_TEST_EXPECT_COLUMNS_EQUAL( + expected, result->view(), cudf::test::debug_output_level::FIRST_ERROR, st, mr); } } @@ -504,92 +652,168 @@ TEST_F(SegmentedGatherTestFloat, GatherMapSliced) { using T = float; + auto const st = this->stream(); + auto const mr = this->resources(); + // List { - auto const list = LCW{{1, 2, 3, 4}, {5}, {6, 7}, {8, 9, 10}, {11, 12}, {13, 14, 15, 16}}; - auto const gather_map = LCW{{3, 2, 1, 0}, {0}, {0, 1}, {0, 2, 1}, {0}, {1}}; + auto const list = + LCW{Init{{1, 2, 3, 4}, {5}, {6, 7}, {8, 9, 10}, {11, 12}, {13, 14, 15, 16}}, st, mr}; + auto const gather_map = LCW{Init{{3, 2, 1, 0}, {0}, {0, 1}, {0, 2, 1}, {0}, {1}}, st, mr}; // gather_map.offset: 0, 4, 5, 7, 10, 11, 12 - auto const expected = LCW{{4, 3, 2, 1}, {5}, {6, 7}, {8, 10, 9}, {11}, {14}}; + auto const expected = LCW{Init{{4, 3, 2, 1}, {5}, {6, 7}, {8, 10, 9}, {11}, {14}}, st, mr}; auto const results = cudf::lists::segmented_gather(cudf::lists_column_view{list}, - cudf::lists_column_view{gather_map}); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(results->view(), expected); + cudf::lists_column_view{gather_map}, + cudf::out_of_bounds_policy::DONT_CHECK, + st, + mr); + CUDF_TEST_EXPECT_COLUMNS_EQUAL( + results->view(), expected, cudf::test::debug_output_level::FIRST_ERROR, st, mr); - auto const sliced = cudf::split(list, {1, 4}); - auto const split_m = cudf::split(gather_map, {1, 4}); - auto const split_e = cudf::split(expected, {1, 4}); + auto const sliced = cudf::split(list, {1, 4}, st); + auto const split_m = cudf::split(gather_map, {1, 4}, st); + auto const split_e = cudf::split(expected, {1, 4}, st); auto result0 = cudf::lists::segmented_gather(cudf::lists_column_view{sliced[0]}, - cudf::lists_column_view{split_m[0]}); - CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(split_e[0], result0->view()); + cudf::lists_column_view{split_m[0]}, + cudf::out_of_bounds_policy::DONT_CHECK, + st, + mr); + CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(split_e[0], + result0->view(), + cudf::test::debug_output_level::FIRST_ERROR, + cudf::test::default_ulp, + st, + mr); auto result1 = cudf::lists::segmented_gather(cudf::lists_column_view{sliced[1]}, - cudf::lists_column_view{split_m[1]}); - CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(split_e[1], result1->view()); + cudf::lists_column_view{split_m[1]}, + cudf::out_of_bounds_policy::DONT_CHECK, + st, + mr); + CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(split_e[1], + result1->view(), + cudf::test::debug_output_level::FIRST_ERROR, + cudf::test::default_ulp, + st, + mr); auto result2 = cudf::lists::segmented_gather(cudf::lists_column_view{sliced[2]}, - cudf::lists_column_view{split_m[2]}); - CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(split_e[2], result2->view()); + cudf::lists_column_view{split_m[2]}, + cudf::out_of_bounds_policy::DONT_CHECK, + st, + mr); + CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(split_e[2], + result2->view(), + cudf::test::debug_output_level::FIRST_ERROR, + cudf::test::default_ulp, + st, + mr); } // List, with out-of-bounds gather indices. { - auto const list = LCW{{1, 2, 3, 4}, {5}, {6, 7}, {8, 9, 10}, {11, 12}, {13, 14, 15, 16}}; - auto const gather_map = LCW{{3, -5, 1, 0}, {0}, {0, 1}, {0, 2, 3}, {0}, {1}}; + auto const list = + LCW{Init{{1, 2, 3, 4}, {5}, {6, 7}, {8, 9, 10}, {11, 12}, {13, 14, 15, 16}}, st, mr}; + auto const gather_map = LCW{Init{{3, -5, 1, 0}, {0}, {0, 1}, {0, 2, 3}, {0}, {1}}, st, mr}; // gather_map.offset: 0, 4, 5, 7, 10, 11, 12 auto const expected = - LCW{{{4, 0, 2, 1}, null_at(1)}, {5}, {6, 7}, {{8, 10, 9}, null_at(2)}, {11}, {14}}; + LCW{Init{{{{4, 0, 2, 1}, null_at(1)}, {5}, {6, 7}, {{8, 10, 9}, null_at(2)}, {11}, {14}}}, + st, + mr}; auto results = cudf::lists::segmented_gather( - cudf::lists_column_view{list}, cudf::lists_column_view{gather_map}, NULLIFY); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(results->view(), expected); + cudf::lists_column_view{list}, cudf::lists_column_view{gather_map}, NULLIFY, st, mr); + CUDF_TEST_EXPECT_COLUMNS_EQUAL( + results->view(), expected, cudf::test::debug_output_level::FIRST_ERROR, st, mr); - auto const sliced = cudf::split(list, {1, 4}); - auto const split_m = cudf::split(gather_map, {1, 4}); - auto const split_e = cudf::split(expected, {1, 4}); + auto const sliced = cudf::split(list, {1, 4}, st); + auto const split_m = cudf::split(gather_map, {1, 4}, st); + auto const split_e = cudf::split(expected, {1, 4}, st); auto const result0 = cudf::lists::segmented_gather( - cudf::lists_column_view{sliced[0]}, cudf::lists_column_view{split_m[0]}, NULLIFY); - CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(split_e[0], result0->view()); + cudf::lists_column_view{sliced[0]}, cudf::lists_column_view{split_m[0]}, NULLIFY, st, mr); + CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(split_e[0], + result0->view(), + cudf::test::debug_output_level::FIRST_ERROR, + cudf::test::default_ulp, + st, + mr); auto const result1 = cudf::lists::segmented_gather( - cudf::lists_column_view{sliced[1]}, cudf::lists_column_view{split_m[1]}, NULLIFY); - CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(split_e[1], result1->view()); + cudf::lists_column_view{sliced[1]}, cudf::lists_column_view{split_m[1]}, NULLIFY, st, mr); + CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(split_e[1], + result1->view(), + cudf::test::debug_output_level::FIRST_ERROR, + cudf::test::default_ulp, + st, + mr); auto const result2 = cudf::lists::segmented_gather( - cudf::lists_column_view{sliced[2]}, cudf::lists_column_view{split_m[2]}, NULLIFY); - CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(split_e[2], result2->view()); + cudf::lists_column_view{sliced[2]}, cudf::lists_column_view{split_m[2]}, NULLIFY, st, mr); + CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(split_e[2], + result2->view(), + cudf::test::debug_output_level::FIRST_ERROR, + cudf::test::default_ulp, + st, + mr); } } TEST_F(SegmentedGatherTestFloat, Fails) { using T = float; + + auto const st = this->stream(); + auto const mr = this->resources(); + // List - LCW list{{1, 2, 3, 4}, {5}, {6, 7}, {8, 9, 10}}; - LCW size_mismatch_map{{3, 2, 1, 0}, {0}, {0, 1}}; - cudf::test::fixed_width_column_wrapper nonlist_map0{1, 2, 0, 1}; - cudf::test::strings_column_wrapper nonlist_map1{"1", "2", "0", "1"}; - LCW nonlist_map2{{"1", "2", "0", "1"}}; + LCW list{Init{{1, 2, 3, 4}, {5}, {6, 7}, {8, 9, 10}}, st, mr}; + cudf::test::lists_column_wrapper size_mismatch_map{ + I8Init{{3, 2, 1, 0}, {0}, {0, 1}}, st, mr}; + cudf::test::fixed_width_column_wrapper nonlist_map0{{1, 2, 0, 1}, st, mr}; + cudf::test::strings_column_wrapper nonlist_map1{{"1", "2", "0", "1"}, st, mr}; + LCW nonlist_map2{StrInit{{"1", "2", "0", "1"}}, st, mr}; // Input must be a list of integer indices. It should fail for integers, // strings, or lists containing anything other than integers. EXPECT_THROW(cudf::lists::segmented_gather(cudf::lists_column_view{list}, - cudf::lists_column_view{nonlist_map0}), + cudf::lists_column_view{nonlist_map0}, + cudf::out_of_bounds_policy::DONT_CHECK, + st, + mr), cudf::logic_error); EXPECT_THROW(cudf::lists::segmented_gather(cudf::lists_column_view{list}, - cudf::lists_column_view{nonlist_map1}), + cudf::lists_column_view{nonlist_map1}, + cudf::out_of_bounds_policy::DONT_CHECK, + st, + mr), cudf::logic_error); EXPECT_THROW(cudf::lists::segmented_gather(cudf::lists_column_view{list}, - cudf::lists_column_view{nonlist_map2}), + cudf::lists_column_view{nonlist_map2}, + cudf::out_of_bounds_policy::DONT_CHECK, + st, + mr), cudf::logic_error); auto valids = cudf::test::iterators::valids_at_multiples_of(2); - LCW nulls_map{{{3, 2, 1, 0}, {0}, {0}, {0, 1}}, valids}; + cudf::test::lists_column_wrapper nulls_map{ + I8Init{{{{3, 2, 1, 0}, {0}, {0}, {0, 1}}, valids}}, st, mr}; // Nulls are not supported in the gather map. EXPECT_THROW(cudf::lists::segmented_gather(cudf::lists_column_view{list}, - cudf::lists_column_view{nulls_map}), + cudf::lists_column_view{nulls_map}, + cudf::out_of_bounds_policy::DONT_CHECK, + st, + mr), std::invalid_argument); // Gather map and list column sizes must be the same. EXPECT_THROW(cudf::lists::segmented_gather(cudf::lists_column_view{list}, - cudf::lists_column_view{size_mismatch_map}), + cudf::lists_column_view{size_mismatch_map}, + cudf::out_of_bounds_policy::DONT_CHECK, + st, + mr), cudf::logic_error); } + +#if defined(__GNUC__) && (__GNUC__ >= 14) +#pragma GCC diagnostic pop +#endif diff --git a/cpp/tests/dictionary/decode_test.cpp b/cpp/tests/dictionary/decode_test.cpp index 778affa05de8..8c46c9a77df5 100644 --- a/cpp/tests/dictionary/decode_test.cpp +++ b/cpp/tests/dictionary/decode_test.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2020-2024, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -12,46 +12,72 @@ #include -struct DictionaryDecodeTest : public cudf::test::BaseFixture {}; +struct DictionaryDecodeTest : public cudf::test::BaseFixtureWithHarness {}; TEST_F(DictionaryDecodeTest, StringColumn) { + auto const stream = this->stream(); + auto const mr = this->resources(); + std::vector h_strings{"eee", "aaa", "ddd", "bbb", "ccc", "ccc", "ccc", "eee", "aaa"}; - cudf::test::strings_column_wrapper strings(h_strings.begin(), h_strings.end()); + cudf::test::strings_column_wrapper strings(h_strings.begin(), h_strings.end(), stream, mr); - auto dictionary = cudf::dictionary::encode(strings); - auto output = cudf::dictionary::decode(cudf::dictionary_column_view(dictionary->view())); + auto dictionary = + cudf::dictionary::encode(strings, cudf::data_type{cudf::type_id::INT32}, stream, mr); + auto output = + cudf::dictionary::decode(cudf::dictionary_column_view(dictionary->view()), stream, mr); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(strings, *output); + CUDF_TEST_EXPECT_COLUMNS_EQUAL( + strings, *output, cudf::test::debug_output_level::FIRST_ERROR, stream, mr); } TEST_F(DictionaryDecodeTest, FloatColumn) { - cudf::test::fixed_width_column_wrapper input{4.25, 7.125, 0.5, -11.75, 7.125, 0.5}; + auto const stream = this->stream(); + auto const mr = this->resources(); + + cudf::test::fixed_width_column_wrapper input{ + {4.25, 7.125, 0.5, -11.75, 7.125, 0.5}, stream, mr}; - auto dictionary = cudf::dictionary::encode(input); - auto output = cudf::dictionary::decode(cudf::dictionary_column_view(dictionary->view())); + auto dictionary = + cudf::dictionary::encode(input, cudf::data_type{cudf::type_id::INT32}, stream, mr); + auto output = + cudf::dictionary::decode(cudf::dictionary_column_view(dictionary->view()), stream, mr); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(input, *output); + CUDF_TEST_EXPECT_COLUMNS_EQUAL( + input, *output, cudf::test::debug_output_level::FIRST_ERROR, stream, mr); } TEST_F(DictionaryDecodeTest, ColumnWithNull) { + auto const stream = this->stream(); + auto const mr = this->resources(); + cudf::test::fixed_width_column_wrapper input{ {444, 0, 333, 111, 222, 222, 222, 444, 000}, - {true, true, true, true, true, false, true, true, true}}; + {true, true, true, true, true, false, true, true, true}, + stream, + mr}; - auto dictionary = cudf::dictionary::encode(input); - auto output = cudf::dictionary::decode(cudf::dictionary_column_view(dictionary->view())); + auto dictionary = + cudf::dictionary::encode(input, cudf::data_type{cudf::type_id::INT32}, stream, mr); + auto output = + cudf::dictionary::decode(cudf::dictionary_column_view(dictionary->view()), stream, mr); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(input, *output); + CUDF_TEST_EXPECT_COLUMNS_EQUAL( + input, *output, cudf::test::debug_output_level::FIRST_ERROR, stream, mr); } TEST_F(DictionaryDecodeTest, EmptyColumn) { - cudf::test::fixed_width_column_wrapper input; - auto dictionary = cudf::dictionary::encode(input); - auto output = cudf::dictionary::decode(cudf::dictionary_column_view(dictionary->view())); + auto const stream = this->stream(); + auto const mr = this->resources(); + + cudf::test::fixed_width_column_wrapper input{}; + auto dictionary = + cudf::dictionary::encode(input, cudf::data_type{cudf::type_id::INT32}, stream, mr); + auto output = + cudf::dictionary::decode(cudf::dictionary_column_view(dictionary->view()), stream, mr); // check empty EXPECT_EQ(output->size(), 0); diff --git a/cpp/tests/dictionary/encode_test.cpp b/cpp/tests/dictionary/encode_test.cpp index ee82e3cf7175..90cdb71517af 100644 --- a/cpp/tests/dictionary/encode_test.cpp +++ b/cpp/tests/dictionary/encode_test.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -12,18 +12,23 @@ #include #include -struct DictionaryEncodeTest : public cudf::test::BaseFixture {}; +struct DictionaryEncodeTest : public cudf::test::BaseFixtureWithHarness {}; TEST_F(DictionaryEncodeTest, EncodeStringColumn) { + auto const stream = this->stream(); + auto const mr = this->resources(); + cudf::test::strings_column_wrapper input( - {"eee", "aaa", "ddd", "bbb", "ccc", "ccc", "ccc", "eee", "aaa"}); + {"eee", "aaa", "ddd", "bbb", "ccc", "ccc", "ccc", "eee", "aaa"}, stream, mr); - auto dictionary = cudf::dictionary::encode(input); - auto view = cudf::dictionary_column_view(dictionary->view()); + auto dictionary = + cudf::dictionary::encode(input, cudf::data_type{cudf::type_id::INT32}, stream, mr); + auto view = cudf::dictionary_column_view(dictionary->view()); - auto decoded = cudf::dictionary::decode(view); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(decoded->view(), input); + auto decoded = cudf::dictionary::decode(view, stream, mr); + CUDF_TEST_EXPECT_COLUMNS_EQUAL( + decoded->view(), input, cudf::test::debug_output_level::FIRST_ERROR, stream, mr); } template @@ -34,26 +39,39 @@ TYPED_TEST_SUITE(DictionaryEncodeNumericTest, NumericTypes); TYPED_TEST(DictionaryEncodeNumericTest, Encode) { - auto input = cudf::test::fixed_width_column_wrapper{4, 7, 0, -11, 7, 0}; + auto const stream = this->stream(); + auto const mr = this->resources(); - auto dictionary = cudf::dictionary::encode(input); - auto view = cudf::dictionary_column_view(dictionary->view()); + auto input = + cudf::test::fixed_width_column_wrapper{{4, 7, 0, -11, 7, 0}, stream, mr}; + + auto dictionary = + cudf::dictionary::encode(input, cudf::data_type{cudf::type_id::INT32}, stream, mr); + auto view = cudf::dictionary_column_view(dictionary->view()); - auto decoded = cudf::dictionary::decode(view); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(decoded->view(), input); + auto decoded = cudf::dictionary::decode(view, stream, mr); + CUDF_TEST_EXPECT_COLUMNS_EQUAL( + decoded->view(), input, cudf::test::debug_output_level::FIRST_ERROR, stream, mr); } TEST_F(DictionaryEncodeTest, EncodeWithNull) { + auto const stream = this->stream(); + auto const mr = this->resources(); + cudf::test::fixed_width_column_wrapper input{ {444, 0, 333, 111, 222, 222, 222, 444, 000}, - {true, true, true, true, true, false, true, true, true}}; + {true, true, true, true, true, false, true, true, true}, + stream, + mr}; - auto dictionary = cudf::dictionary::encode(input); - auto view = cudf::dictionary_column_view(dictionary->view()); + auto dictionary = + cudf::dictionary::encode(input, cudf::data_type{cudf::type_id::INT32}, stream, mr); + auto view = cudf::dictionary_column_view(dictionary->view()); - auto decoded = cudf::dictionary::decode(view); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(decoded->view(), input); + auto decoded = cudf::dictionary::decode(view, stream, mr); + CUDF_TEST_EXPECT_COLUMNS_EQUAL( + decoded->view(), input, cudf::test::debug_output_level::FIRST_ERROR, stream, mr); } template @@ -63,20 +81,28 @@ TYPED_TEST_SUITE(DictionaryEncodeIndicesTest, IndexTypes); TYPED_TEST(DictionaryEncodeIndicesTest, IndexType) { - auto input = cudf::test::strings_column_wrapper({"aaa", "bbb", "bbb", "cccc"}); + auto const stream = this->stream(); + auto const mr = this->resources(); + + auto input = cudf::test::strings_column_wrapper({"aaa", "bbb", "bbb", "cccc"}, stream, mr); auto data_type = cudf::data_type{cudf::type_to_id()}; - auto dictionary = cudf::dictionary::encode(input, data_type); + auto dictionary = cudf::dictionary::encode(input, data_type, stream, mr); auto view = cudf::dictionary_column_view(dictionary->view()); EXPECT_EQ(view.indices().type(), data_type); } TEST_F(DictionaryEncodeTest, Errors) { - cudf::test::fixed_width_column_wrapper input{0, 1, 2, 3, -1, -2, -3}; + auto const stream = this->stream(); + auto const mr = this->resources(); + + cudf::test::fixed_width_column_wrapper input{{0, 1, 2, 3, -1, -2, -3}, stream, mr}; - EXPECT_THROW(cudf::dictionary::encode(input, cudf::data_type{cudf::type_id::UINT16}), + EXPECT_THROW(cudf::dictionary::encode(input, cudf::data_type{cudf::type_id::UINT16}, stream, mr), cudf::data_type_error); - auto encoded = cudf::dictionary::encode(input); - EXPECT_THROW(cudf::dictionary::encode(encoded->view()), std::invalid_argument); + auto encoded = cudf::dictionary::encode(input, cudf::data_type{cudf::type_id::INT32}, stream, mr); + EXPECT_THROW( + cudf::dictionary::encode(encoded->view(), cudf::data_type{cudf::type_id::INT32}, stream, mr), + std::invalid_argument); } diff --git a/cpp/tests/row_operator/row_operator_tests.cu b/cpp/tests/row_operator/row_operator_tests.cu index 08046c5e3128..e56eb5a0ea8c 100644 --- a/cpp/tests/row_operator/row_operator_tests.cu +++ b/cpp/tests/row_operator/row_operator_tests.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -24,7 +24,7 @@ #include template -struct TypedTableViewTest : public cudf::test::BaseFixture {}; +struct TypedTableViewTest : public cudf::test::BaseFixtureWithHarness {}; using NumericTypesNotBool = cudf::test::Concat; @@ -33,66 +33,99 @@ TYPED_TEST_SUITE(TypedTableViewTest, NumericTypesNotBool); template std::unique_ptr self_comparison(cudf::table_view input, std::vector const& column_order, - PhysicalElementComparator comparator); + PhysicalElementComparator comparator, + rmm::cuda_stream_view stream, + cudf::memory_resources mr); template std::unique_ptr two_table_comparison(cudf::table_view lhs, cudf::table_view rhs, std::vector const& column_order, - PhysicalElementComparator comparator); + PhysicalElementComparator comparator, + rmm::cuda_stream_view stream, + cudf::memory_resources mr); template std::unique_ptr two_table_equality(cudf::table_view lhs, cudf::table_view rhs, std::vector const& column_order, - PhysicalElementComparator comparator); + PhysicalElementComparator comparator, + rmm::cuda_stream_view stream, + cudf::memory_resources mr); template std::unique_ptr sorted_order( std::shared_ptr preprocessed_input, cudf::size_type num_rows, bool has_nested, PhysicalElementComparator comparator, - rmm::cuda_stream_view stream); + rmm::cuda_stream_view stream, + cudf::memory_resources mr); TYPED_TEST(TypedTableViewTest, TestLexicographicalComparatorTwoTables) { using T = TypeParam; - auto const col1 = cudf::test::fixed_width_column_wrapper{{1, 2, 3, 4}}; - auto const col2 = cudf::test::fixed_width_column_wrapper{{0, 1, 4, 3}}; + // TODO: lexicographic row operators still allocate from the current device resource. + + auto const stream = this->stream(); + auto const mr = this->resources(); + + auto const col1 = cudf::test::fixed_width_column_wrapper{{1, 2, 3, 4}, stream, mr}; + auto const col2 = cudf::test::fixed_width_column_wrapper{{0, 1, 4, 3}, stream, mr}; auto const column_order = std::vector{cudf::order::DESCENDING}; auto const lhs = cudf::table_view{{col1}}; auto const rhs = cudf::table_view{{col2}}; - auto const expected = cudf::test::fixed_width_column_wrapper{{1, 1, 0, 1}}; - auto const got = two_table_comparison( - lhs, rhs, column_order, cudf::detail::row::lexicographic::physical_element_comparator{}); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, got->view()); + auto const expected = cudf::test::fixed_width_column_wrapper{{1, 1, 0, 1}, stream, mr}; + auto const got = + two_table_comparison(lhs, + rhs, + column_order, + cudf::detail::row::lexicographic::physical_element_comparator{}, + stream, + mr); + CUDF_TEST_EXPECT_COLUMNS_EQUAL( + expected, got->view(), cudf::test::debug_output_level::FIRST_ERROR, stream, mr); auto const sorting_got = two_table_comparison(lhs, rhs, column_order, - cudf::detail::row::lexicographic::sorting_physical_element_comparator{}); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, sorting_got->view()); + cudf::detail::row::lexicographic::sorting_physical_element_comparator{}, + stream, + mr); + CUDF_TEST_EXPECT_COLUMNS_EQUAL( + expected, sorting_got->view(), cudf::test::debug_output_level::FIRST_ERROR, stream, mr); } TYPED_TEST(TypedTableViewTest, TestLexicographicalComparatorSameTable) { using T = TypeParam; - auto const col1 = cudf::test::fixed_width_column_wrapper{{1, 2, 3, 4}}; + // TODO: lexicographic row operators still allocate from the current device resource. + + auto const stream = this->stream(); + auto const mr = this->resources(); + + auto const col1 = cudf::test::fixed_width_column_wrapper{{1, 2, 3, 4}, stream, mr}; auto const column_order = std::vector{cudf::order::DESCENDING}; auto const input_table = cudf::table_view{{col1}}; - auto const expected = cudf::test::fixed_width_column_wrapper{{0, 0, 0, 0}}; - auto const got = self_comparison( - input_table, column_order, cudf::detail::row::lexicographic::physical_element_comparator{}); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, got->view()); + auto const expected = cudf::test::fixed_width_column_wrapper{{0, 0, 0, 0}, stream, mr}; + auto const got = self_comparison(input_table, + column_order, + cudf::detail::row::lexicographic::physical_element_comparator{}, + stream, + mr); + CUDF_TEST_EXPECT_COLUMNS_EQUAL( + expected, got->view(), cudf::test::debug_output_level::FIRST_ERROR, stream, mr); auto const sorting_got = self_comparison(input_table, column_order, - cudf::detail::row::lexicographic::sorting_physical_element_comparator{}); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, sorting_got->view()); + cudf::detail::row::lexicographic::sorting_physical_element_comparator{}, + stream, + mr); + CUDF_TEST_EXPECT_COLUMNS_EQUAL( + expected, sorting_got->view(), cudf::test::debug_output_level::FIRST_ERROR, stream, mr); } TYPED_TEST(TypedTableViewTest, TestSortSameTableFromTwoTables) @@ -100,23 +133,28 @@ TYPED_TEST(TypedTableViewTest, TestSortSameTableFromTwoTables) using data_col = cudf::test::fixed_width_column_wrapper; using int32s_col = cudf::test::fixed_width_column_wrapper; - auto const col1 = data_col{5, 2, 7, 1, 3}; + // TODO: lexicographic row operators still allocate from the current device resource. + + auto const stream = this->stream(); + auto const mr = this->resources(); + + auto const col1 = data_col{{5, 2, 7, 1, 3}, stream, mr}; auto const col2 = data_col{}; // empty auto const lhs = cudf::table_view{{col1}}; auto const empty_rhs = cudf::table_view{{col2}}; - auto const stream = cudf::get_default_stream(); auto const test_sort = - [stream]( + [stream, mr]( auto const& preprocessed, auto const& input, auto const& comparator, auto const& expected) { auto const order = sorted_order( - preprocessed, input.num_rows(), cudf::has_nested_columns(input), comparator, stream); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, order->view()); + preprocessed, input.num_rows(), cudf::has_nested_columns(input), comparator, stream, mr); + CUDF_TEST_EXPECT_COLUMNS_EQUAL( + expected, order->view(), cudf::test::debug_output_level::FIRST_ERROR, stream, mr); }; auto const test_sort_two_tables = [&](auto const& preprocessed_lhs, auto const& preprocessed_empty_rhs) { - auto const expected_lhs = int32s_col{3, 1, 4, 0, 2}; + auto const expected_lhs = int32s_col{{3, 1, 4, 0, 2}, stream, mr}; test_sort(preprocessed_lhs, lhs, cudf::detail::row::lexicographic::physical_element_comparator{}, @@ -161,17 +199,22 @@ TYPED_TEST(TypedTableViewTest, TestSortSameTableFromTwoTablesWithListsOfStructs) using strings_col = cudf::test::strings_column_wrapper; using structs_col = cudf::test::structs_column_wrapper; - auto const col1 = [] { - auto const get_structs = [] { - auto child0 = data_col{0, 3, 0, 2}; - auto child1 = strings_col{"a", "c", "a", "b"}; - return structs_col{{child0, child1}}; + // TODO: lexicographic row operators still allocate from the current device resource. + + auto const stream = this->stream(); + auto const mr = this->resources(); + + auto const col1 = [&] { + auto const get_structs = [&] { + auto child0 = data_col{{0, 3, 0, 2}, stream, mr}; + auto child1 = strings_col{{"a", "c", "a", "b"}, stream, mr}; + return structs_col{{child0, child1}, {}, stream, mr}; }; return cudf::make_lists_column( - 2, int32s_col{0, 2, 4}.release(), get_structs().release(), 0, {}); + 2, int32s_col{{0, 2, 4}, stream, mr}.release(), get_structs().release(), 0, {}); }(); - auto const col2 = [] { - auto const get_structs = [] { + auto const col2 = [&] { + auto const get_structs = [&] { auto child0 = data_col{}; auto child1 = strings_col{}; return structs_col{{child0, child1}}; @@ -183,18 +226,18 @@ TYPED_TEST(TypedTableViewTest, TestSortSameTableFromTwoTablesWithListsOfStructs) auto const lhs = cudf::table_view{{*col1}}; auto const empty_rhs = cudf::table_view{{*col2}}; - auto const stream = cudf::get_default_stream(); auto const test_sort = - [stream]( + [stream, mr]( auto const& preprocessed, auto const& input, auto const& comparator, auto const& expected) { auto const order = sorted_order( - preprocessed, input.num_rows(), cudf::has_nested_columns(input), comparator, stream); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, order->view()); + preprocessed, input.num_rows(), cudf::has_nested_columns(input), comparator, stream, mr); + CUDF_TEST_EXPECT_COLUMNS_EQUAL( + expected, order->view(), cudf::test::debug_output_level::FIRST_ERROR, stream, mr); }; auto const test_sort_two_tables = [&](auto const& preprocessed_lhs, auto const& preprocessed_empty_rhs) { - auto const expected_lhs = int32s_col{1, 0}; + auto const expected_lhs = int32s_col{{1, 0}, stream, mr}; test_sort(preprocessed_lhs, lhs, cudf::detail::row::lexicographic::sorting_physical_element_comparator{}, @@ -236,7 +279,7 @@ TYPED_TEST(TypedTableViewTest, TestSortSameTableFromTwoTablesWithListsOfStructs) } template -struct NaNTableViewTest : public cudf::test::BaseFixture {}; +struct NaNTableViewTest : public cudf::test::BaseFixtureWithHarness {}; TYPED_TEST_SUITE(NaNTableViewTest, cudf::test::FloatingPointTypes); @@ -244,65 +287,103 @@ TYPED_TEST(NaNTableViewTest, TestLexicographicalComparatorTwoTableNaNCase) { using T = TypeParam; - auto const col1 = cudf::test::fixed_width_column_wrapper{{T(NAN), T(NAN), T(1), T(1)}}; - auto const col2 = cudf::test::fixed_width_column_wrapper{{T(NAN), T(1), T(NAN), T(1)}}; + // TODO: lexicographic row operators still allocate from the current device resource. + + auto const stream = this->stream(); + auto const mr = this->resources(); + + auto const col1 = + cudf::test::fixed_width_column_wrapper{{T(NAN), T(NAN), T(1), T(1)}, stream, mr}; + auto const col2 = + cudf::test::fixed_width_column_wrapper{{T(NAN), T(1), T(NAN), T(1)}, stream, mr}; auto const column_order = std::vector{cudf::order::DESCENDING}; auto const lhs = cudf::table_view{{col1}}; auto const rhs = cudf::table_view{{col2}}; - auto const expected = cudf::test::fixed_width_column_wrapper{{0, 0, 0, 0}}; - auto const got = two_table_comparison( - lhs, rhs, column_order, cudf::detail::row::lexicographic::physical_element_comparator{}); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, got->view()); - - auto const sorting_expected = cudf::test::fixed_width_column_wrapper{{0, 1, 0, 0}}; + auto const expected = cudf::test::fixed_width_column_wrapper{{0, 0, 0, 0}, stream, mr}; + auto const got = + two_table_comparison(lhs, + rhs, + column_order, + cudf::detail::row::lexicographic::physical_element_comparator{}, + stream, + mr); + CUDF_TEST_EXPECT_COLUMNS_EQUAL( + expected, got->view(), cudf::test::debug_output_level::FIRST_ERROR, stream, mr); + + auto const sorting_expected = + cudf::test::fixed_width_column_wrapper{{0, 1, 0, 0}, stream, mr}; auto const sorting_got = two_table_comparison(lhs, rhs, column_order, - cudf::detail::row::lexicographic::sorting_physical_element_comparator{}); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(sorting_expected, sorting_got->view()); + cudf::detail::row::lexicographic::sorting_physical_element_comparator{}, + stream, + mr); + CUDF_TEST_EXPECT_COLUMNS_EQUAL( + sorting_expected, sorting_got->view(), cudf::test::debug_output_level::FIRST_ERROR, stream, mr); } TYPED_TEST(NaNTableViewTest, TestEqualityComparatorTwoTableNaNCase) { using T = TypeParam; - auto const col1 = cudf::test::fixed_width_column_wrapper{{T(NAN), T(NAN), T(1), T(1)}}; - auto const col2 = cudf::test::fixed_width_column_wrapper{{T(NAN), T(1), T(NAN), T(1)}}; + auto const stream = this->stream(); + auto const mr = this->resources(); + + auto const col1 = + cudf::test::fixed_width_column_wrapper{{T(NAN), T(NAN), T(1), T(1)}, stream, mr}; + auto const col2 = + cudf::test::fixed_width_column_wrapper{{T(NAN), T(1), T(NAN), T(1)}, stream, mr}; auto const column_order = std::vector{cudf::order::DESCENDING}; auto const lhs = cudf::table_view{{col1}}; auto const rhs = cudf::table_view{{col2}}; - auto const expected = cudf::test::fixed_width_column_wrapper{{0, 0, 0, 1}}; - auto const got = two_table_equality( - lhs, rhs, column_order, cudf::detail::row::equality::physical_equality_comparator{}); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, got->view()); - - auto const nan_equal_expected = cudf::test::fixed_width_column_wrapper{{1, 0, 0, 1}}; - auto const nan_equal_got = two_table_equality( - lhs, rhs, column_order, cudf::detail::row::equality::nan_equal_physical_equality_comparator{}); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(nan_equal_expected, nan_equal_got->view()); + auto const expected = cudf::test::fixed_width_column_wrapper{{0, 0, 0, 1}, stream, mr}; + auto const got = two_table_equality(lhs, + rhs, + column_order, + cudf::detail::row::equality::physical_equality_comparator{}, + stream, + mr); + CUDF_TEST_EXPECT_COLUMNS_EQUAL( + expected, got->view(), cudf::test::debug_output_level::FIRST_ERROR, stream, mr); + + auto const nan_equal_expected = + cudf::test::fixed_width_column_wrapper{{1, 0, 0, 1}, stream, mr}; + auto const nan_equal_got = + two_table_equality(lhs, + rhs, + column_order, + cudf::detail::row::equality::nan_equal_physical_equality_comparator{}, + stream, + mr); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(nan_equal_expected, + nan_equal_got->view(), + cudf::test::debug_output_level::FIRST_ERROR, + stream, + mr); } -struct RowOperatorTest : public cudf::test::BaseFixture {}; +struct RowOperatorTest : public cudf::test::BaseFixtureWithHarness {}; TEST_F(RowOperatorTest, TestTwoTableComparatorColumnCountCheck) { - rmm::cuda_stream_view stream{cudf::get_default_stream()}; + auto const stream = this->stream(); + auto const mr = this->resources(); - auto left_col1 = cudf::test::fixed_width_column_wrapper{{1, 2}}; - auto left_col2 = cudf::test::fixed_width_column_wrapper{{3, 4}}; + auto left_col1 = cudf::test::fixed_width_column_wrapper{{1, 2}, stream, mr}; + auto left_col2 = cudf::test::fixed_width_column_wrapper{{3, 4}, stream, mr}; auto const left_table = cudf::table_view{{left_col1, left_col2}}; - auto right_col = cudf::test::fixed_width_column_wrapper{{1, 2}}; + auto right_col = cudf::test::fixed_width_column_wrapper{{1, 2}, stream, mr}; auto const right_table = cudf::table_view{{right_col}}; - auto left_preprocessed = - cudf::detail::row::equality::preprocessed_table::create(left_table, stream); - auto right_preprocessed = - cudf::detail::row::equality::preprocessed_table::create(right_table, stream); + auto left_preprocessed = cudf::detail::row::equality::preprocessed_table::create( + left_table, stream, mr.get_temporary_mr()); + auto right_preprocessed = cudf::detail::row::equality::preprocessed_table::create( + right_table, stream, mr.get_temporary_mr()); EXPECT_THROW( cudf::detail::row::equality::two_table_comparator(left_preprocessed, right_preprocessed), @@ -311,48 +392,54 @@ TEST_F(RowOperatorTest, TestTwoTableComparatorColumnCountCheck) TEST_F(RowOperatorTest, TestCheckShapeCompatibility) { - rmm::cuda_stream_view stream{cudf::get_default_stream()}; + auto const stream = this->stream(); + auto const mr = this->resources(); - auto left_col1_2 = cudf::test::fixed_width_column_wrapper{{1, 2}}; - auto left_col2_2 = cudf::test::fixed_width_column_wrapper{{3, 4}}; + auto left_col1_2 = cudf::test::fixed_width_column_wrapper{{1, 2}, stream, mr}; + auto left_col2_2 = cudf::test::fixed_width_column_wrapper{{3, 4}, stream, mr}; auto const left_table = cudf::table_view{{left_col1_2, left_col2_2}}; - auto right_col_2 = cudf::test::fixed_width_column_wrapper{{1, 2}}; + auto right_col_2 = cudf::test::fixed_width_column_wrapper{{1, 2}, stream, mr}; auto const right_table = cudf::table_view{{right_col_2}}; - EXPECT_THROW(cudf::detail::row::equality::two_table_comparator(left_table, right_table, stream), + EXPECT_THROW(cudf::detail::row::equality::two_table_comparator( + left_table, right_table, stream, mr.get_temporary_mr()), std::invalid_argument); - auto int_col = cudf::test::fixed_width_column_wrapper{{1, 2}}; + auto int_col = cudf::test::fixed_width_column_wrapper{{1, 2}, stream, mr}; auto const int_table = cudf::table_view{{int_col}}; - auto float_col = cudf::test::fixed_width_column_wrapper{{1.0f, 2.0f}}; + auto float_col = cudf::test::fixed_width_column_wrapper{{1.0f, 2.0f}, stream, mr}; auto const float_table = cudf::table_view{{float_col}}; - EXPECT_THROW(cudf::detail::row::equality::two_table_comparator(int_table, float_table, stream), + EXPECT_THROW(cudf::detail::row::equality::two_table_comparator( + int_table, float_table, stream, mr.get_temporary_mr()), std::invalid_argument); - auto str_col = cudf::test::strings_column_wrapper({"hello", "world"}); + auto str_col = cudf::test::strings_column_wrapper({"hello", "world"}, stream, mr); auto const string_table = cudf::table_view{{str_col}}; - auto num_col = cudf::test::fixed_width_column_wrapper({1, 2}); + auto num_col = cudf::test::fixed_width_column_wrapper({1, 2}, stream, mr); auto const numeric_table = cudf::table_view{{num_col}}; - EXPECT_THROW( - cudf::detail::row::equality::two_table_comparator(string_table, numeric_table, stream), - std::invalid_argument); + EXPECT_THROW(cudf::detail::row::equality::two_table_comparator( + string_table, numeric_table, stream, mr.get_temporary_mr()), + std::invalid_argument); } TEST_F(RowOperatorTest, TestRowHasher64BitHash) { - auto const col = cudf::test::fixed_width_column_wrapper{{0, 42, 123456789}}; + auto const stream = this->stream(); + auto const mr = this->resources(); + + auto const col = cudf::test::fixed_width_column_wrapper{{0, 42, 123456789}, stream, mr}; auto const input = cudf::table_view{{col}}; - auto const stream = cudf::get_default_stream(); - auto const preprocessed = cudf::detail::row::hash::preprocessed_table::create(input, stream); - auto const row_hasher = cudf::detail::row::hash::row_hasher{preprocessed}; + auto const preprocessed = + cudf::detail::row::hash::preprocessed_table::create(input, stream, mr.get_temporary_mr()); + auto const row_hasher = cudf::detail::row::hash::row_hasher{preprocessed}; auto const hasher = row_hasher.device_hasher(cudf::nullate::DYNAMIC{false}); - auto results = cudf::test::fixed_width_column_wrapper{{0, 0, 0}}; - thrust::transform(rmm::exec_policy_nosync(stream), + auto results = cudf::test::fixed_width_column_wrapper{{0, 0, 0}, stream, mr}; + thrust::transform(rmm::exec_policy_nosync(stream, mr.get_temporary_mr()), cuda::counting_iterator{0}, cuda::counting_iterator{3}, cudf::mutable_column_view{results}.begin(), @@ -361,24 +448,27 @@ TEST_F(RowOperatorTest, TestRowHasher64BitHash) // Expected values match cuCollections xxhash_64 reference implementation // https://github.com/NVIDIA/cuCollections/blob/4f03dcccb3a944594c693aa8cebc89302bbd8e20/tests/utility/hash_test.cu#L134-L137 auto const expected = cudf::test::fixed_width_column_wrapper{ - {4246796580750024372ul, 15516826743637085169ul, 9462334144942111946ul}}; - CUDF_TEST_EXPECT_COLUMNS_EQUAL(results, expected); + {4246796580750024372ul, 15516826743637085169ul, 9462334144942111946ul}, stream, mr}; + CUDF_TEST_EXPECT_COLUMNS_EQUAL( + results, expected, cudf::test::debug_output_level::FIRST_ERROR, stream, mr); } TEST_F(RowOperatorTest, TestPrimitiveRowHasher64BitHash) { - auto const col = cudf::test::fixed_width_column_wrapper{{0, 42, 123456789}}; + auto const stream = this->stream(); + auto const mr = this->resources(); + + auto const col = cudf::test::fixed_width_column_wrapper{{0, 42, 123456789}, stream, mr}; auto const input = cudf::table_view{{col}}; - auto const stream = cudf::get_default_stream(); - auto const d_input = cudf::table_device_view::create(input, stream); + auto const d_input = cudf::table_device_view::create(input, stream, mr.get_temporary_mr()); auto const hasher = cudf::detail::row::primitive::row_hasher( cudf::nullate::DYNAMIC{false}, *d_input, static_cast(cudf::DEFAULT_HASH_SEED)); - auto results = cudf::test::fixed_width_column_wrapper{{0, 0, 0}}; + auto results = cudf::test::fixed_width_column_wrapper{{0, 0, 0}, stream, mr}; - thrust::transform(rmm::exec_policy_nosync(stream), + thrust::transform(rmm::exec_policy_nosync(stream, mr.get_temporary_mr()), cuda::counting_iterator{0}, cuda::counting_iterator{3}, cudf::mutable_column_view{results}.begin(), @@ -387,77 +477,89 @@ TEST_F(RowOperatorTest, TestPrimitiveRowHasher64BitHash) // Expected values match cuCollections xxhash_64 reference implementation // https://github.com/NVIDIA/cuCollections/blob/4f03dcccb3a944594c693aa8cebc89302bbd8e20/tests/utility/hash_test.cu#L134-L137 auto const expected = cudf::test::fixed_width_column_wrapper{ - {4246796580750024372ul, 15516826743637085169ul, 9462334144942111946ul}}; - CUDF_TEST_EXPECT_COLUMNS_EQUAL(results, expected); + {4246796580750024372ul, 15516826743637085169ul, 9462334144942111946ul}, stream, mr}; + CUDF_TEST_EXPECT_COLUMNS_EQUAL( + results, expected, cudf::test::debug_output_level::FIRST_ERROR, stream, mr); } TEST_F(RowOperatorTest, TestRowHasherDictionaryColumn) { + auto const stream = this->stream(); + auto const mr = this->resources(); + // Dictionary and equivalent string column should produce identical hashes. // This also verifies same logical values get same hashes (e.g., "baz" at rows 0 and 2). - auto const dict_col = - cudf::test::dictionary_column_wrapper({"baz", "foo", "baz", "bar", "foo"}); - auto const str_col = cudf::test::strings_column_wrapper({"baz", "foo", "baz", "bar", "foo"}); + auto const dict_col = cudf::test::dictionary_column_wrapper( + {"baz", "foo", "baz", "bar", "foo"}, stream, mr); + auto const str_col = + cudf::test::strings_column_wrapper({"baz", "foo", "baz", "bar", "foo"}, stream, mr); - auto const stream = cudf::get_default_stream(); - auto const dict_row_hasher = - cudf::detail::row::hash::row_hasher(cudf::table_view{{dict_col}}, stream); + auto const dict_row_hasher = cudf::detail::row::hash::row_hasher( + cudf::table_view{{dict_col}}, stream, mr.get_temporary_mr()); auto const str_row_hasher = - cudf::detail::row::hash::row_hasher(cudf::table_view{{str_col}}, stream); + cudf::detail::row::hash::row_hasher(cudf::table_view{{str_col}}, stream, mr.get_temporary_mr()); auto const dict_hasher = dict_row_hasher.device_hasher(cudf::nullate::DYNAMIC{false}); auto const str_hasher = str_row_hasher.device_hasher(cudf::nullate::DYNAMIC{false}); - auto dict_results = cudf::test::fixed_width_column_wrapper{{0, 0, 0, 0, 0}}; - auto str_results = cudf::test::fixed_width_column_wrapper{{0, 0, 0, 0, 0}}; + auto dict_results = + cudf::test::fixed_width_column_wrapper{{0, 0, 0, 0, 0}, stream, mr}; + auto str_results = + cudf::test::fixed_width_column_wrapper{{0, 0, 0, 0, 0}, stream, mr}; - thrust::transform(rmm::exec_policy_nosync(stream), + thrust::transform(rmm::exec_policy_nosync(stream, mr.get_temporary_mr()), cuda::counting_iterator{0}, cuda::counting_iterator{5}, cudf::mutable_column_view{dict_results}.begin(), dict_hasher); - thrust::transform(rmm::exec_policy_nosync(stream), + thrust::transform(rmm::exec_policy_nosync(stream, mr.get_temporary_mr()), cuda::counting_iterator{0}, cuda::counting_iterator{5}, cudf::mutable_column_view{str_results}.begin(), str_hasher); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(dict_results, str_results); + CUDF_TEST_EXPECT_COLUMNS_EQUAL( + dict_results, str_results, cudf::test::debug_output_level::FIRST_ERROR, stream, mr); } TEST_F(RowOperatorTest, TestRowHasherDictionaryColumnWithNulls) { - auto const dict_col = - cudf::test::dictionary_column_wrapper({100, 200, 300, 100, 200}, {1, 0, 1, 0, 1}); - auto const int_col = - cudf::test::fixed_width_column_wrapper({100, 200, 300, 100, 200}, {1, 0, 1, 0, 1}); - - auto const stream = cudf::get_default_stream(); - auto const dict_row_hasher = - cudf::detail::row::hash::row_hasher(cudf::table_view{{dict_col}}, stream); + auto const stream = this->stream(); + auto const mr = this->resources(); + + auto const dict_col = cudf::test::dictionary_column_wrapper( + {100, 200, 300, 100, 200}, {1, 0, 1, 0, 1}, stream, mr); + auto const int_col = cudf::test::fixed_width_column_wrapper( + {100, 200, 300, 100, 200}, {1, 0, 1, 0, 1}, stream, mr); + + auto const dict_row_hasher = cudf::detail::row::hash::row_hasher( + cudf::table_view{{dict_col}}, stream, mr.get_temporary_mr()); auto const int_row_hasher = - cudf::detail::row::hash::row_hasher(cudf::table_view{{int_col}}, stream); + cudf::detail::row::hash::row_hasher(cudf::table_view{{int_col}}, stream, mr.get_temporary_mr()); auto const dict_hasher = dict_row_hasher.device_hasher(cudf::nullate::DYNAMIC{true}); auto const int_hasher = int_row_hasher.device_hasher(cudf::nullate::DYNAMIC{true}); - auto dict_results = cudf::test::fixed_width_column_wrapper{{0, 0, 0, 0, 0}}; - auto int_results = cudf::test::fixed_width_column_wrapper{{0, 0, 0, 0, 0}}; + auto dict_results = + cudf::test::fixed_width_column_wrapper{{0, 0, 0, 0, 0}, stream, mr}; + auto int_results = + cudf::test::fixed_width_column_wrapper{{0, 0, 0, 0, 0}, stream, mr}; - thrust::transform(rmm::exec_policy_nosync(stream), + thrust::transform(rmm::exec_policy_nosync(stream, mr.get_temporary_mr()), cuda::counting_iterator{0}, cuda::counting_iterator{5}, cudf::mutable_column_view{dict_results}.begin(), dict_hasher); - thrust::transform(rmm::exec_policy_nosync(stream), + thrust::transform(rmm::exec_policy_nosync(stream, mr.get_temporary_mr()), cuda::counting_iterator{0}, cuda::counting_iterator{5}, cudf::mutable_column_view{int_results}.begin(), int_hasher); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(dict_results, int_results); + CUDF_TEST_EXPECT_COLUMNS_EQUAL( + dict_results, int_results, cudf::test::debug_output_level::FIRST_ERROR, stream, mr); } diff --git a/cpp/tests/row_operator/row_operator_tests_utilities.hpp b/cpp/tests/row_operator/row_operator_tests_utilities.hpp index 3841187d14d2..5b8f35558b40 100644 --- a/cpp/tests/row_operator/row_operator_tests_utilities.hpp +++ b/cpp/tests/row_operator/row_operator_tests_utilities.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023-2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ #pragma once @@ -7,6 +7,7 @@ #include #include #include +#include #include @@ -20,21 +21,28 @@ using nan_equality_t = cudf::detail::row::equality::nan_equal_physical_eq template std::unique_ptr self_comparison(cudf::table_view input, std::vector const& column_order, - PhysicalElementComparator comparator); + PhysicalElementComparator comparator, + rmm::cuda_stream_view stream, + cudf::memory_resources mr); template std::unique_ptr two_table_comparison(cudf::table_view lhs, cudf::table_view rhs, std::vector const& column_order, - PhysicalElementComparator comparator); + PhysicalElementComparator comparator, + rmm::cuda_stream_view stream, + cudf::memory_resources mr); template std::unique_ptr two_table_equality(cudf::table_view lhs, cudf::table_view rhs, std::vector const& column_order, - PhysicalElementComparator comparator); + PhysicalElementComparator comparator, + rmm::cuda_stream_view stream, + cudf::memory_resources mr); template std::unique_ptr sorted_order( std::shared_ptr preprocessed_input, cudf::size_type num_rows, bool has_nested, PhysicalElementComparator comparator, - rmm::cuda_stream_view stream); + rmm::cuda_stream_view stream, + cudf::memory_resources mr); diff --git a/cpp/tests/row_operator/self_comparison_utilities.cu b/cpp/tests/row_operator/self_comparison_utilities.cu index 6e43af6dc7bd..898601ef62ea 100644 --- a/cpp/tests/row_operator/self_comparison_utilities.cu +++ b/cpp/tests/row_operator/self_comparison_utilities.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -20,25 +20,28 @@ template std::unique_ptr self_comparison(cudf::table_view input, std::vector const& column_order, - PhysicalElementComparator comparator) + PhysicalElementComparator comparator, + rmm::cuda_stream_view stream, + cudf::memory_resources mr) { - rmm::cuda_stream_view stream{cudf::get_default_stream()}; - auto const table_comparator = cudf::detail::row::lexicographic::self_comparator{input, column_order, {}, stream}; - auto output = cudf::make_numeric_column( - cudf::data_type(cudf::type_id::BOOL8), input.num_rows(), cudf::mask_state::UNALLOCATED); + auto output = cudf::make_numeric_column(cudf::data_type(cudf::type_id::BOOL8), + input.num_rows(), + cudf::mask_state::UNALLOCATED, + stream, + mr.get_output_mr()); if (cudf::has_nested_columns(input)) { - thrust::transform(rmm::exec_policy_nosync(stream), + thrust::transform(rmm::exec_policy_nosync(stream, mr.get_temporary_mr()), cuda::counting_iterator{0}, cuda::counting_iterator{input.num_rows()}, cuda::counting_iterator{0}, output->mutable_view().data(), table_comparator.less(cudf::nullate::NO{}, comparator)); } else { - thrust::transform(rmm::exec_policy_nosync(stream), + thrust::transform(rmm::exec_policy_nosync(stream, mr.get_temporary_mr()), cuda::counting_iterator{0}, cuda::counting_iterator{input.num_rows()}, cuda::counting_iterator{0}, @@ -51,8 +54,12 @@ std::unique_ptr self_comparison(cudf::table_view input, template std::unique_ptr self_comparison( cudf::table_view input, std::vector const& column_order, - physical_comparator_t comparator); + physical_comparator_t comparator, + rmm::cuda_stream_view stream, + cudf::memory_resources mr); template std::unique_ptr self_comparison( cudf::table_view input, std::vector const& column_order, - sorting_comparator_t comparator); + sorting_comparator_t comparator, + rmm::cuda_stream_view stream, + cudf::memory_resources mr); diff --git a/cpp/tests/row_operator/two_table_comparison_utilities.cu b/cpp/tests/row_operator/two_table_comparison_utilities.cu index f42bec2a4f9a..55eb526eb809 100644 --- a/cpp/tests/row_operator/two_table_comparison_utilities.cu +++ b/cpp/tests/row_operator/two_table_comparison_utilities.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -21,27 +21,31 @@ template std::unique_ptr two_table_comparison(cudf::table_view lhs, cudf::table_view rhs, std::vector const& column_order, - PhysicalElementComparator comparator) + PhysicalElementComparator comparator, + rmm::cuda_stream_view stream, + cudf::memory_resources mr) { - rmm::cuda_stream_view stream{cudf::get_default_stream()}; - + // TODO: lexicographic::two_table_comparator still allocates from the current device resource. auto const table_comparator = cudf::detail::row::lexicographic::two_table_comparator{lhs, rhs, column_order, {}, stream}; auto const lhs_it = cudf::detail::row::lhs_iterator(0); auto const rhs_it = cudf::detail::row::rhs_iterator(0); - auto output = cudf::make_numeric_column( - cudf::data_type(cudf::type_id::BOOL8), lhs.num_rows(), cudf::mask_state::UNALLOCATED); + auto output = cudf::make_numeric_column(cudf::data_type(cudf::type_id::BOOL8), + lhs.num_rows(), + cudf::mask_state::UNALLOCATED, + stream, + mr.get_output_mr()); if (cudf::has_nested_columns(lhs) || cudf::has_nested_columns(rhs)) { - thrust::transform(rmm::exec_policy_nosync(stream), + thrust::transform(rmm::exec_policy_nosync(stream, mr.get_temporary_mr()), lhs_it, lhs_it + lhs.num_rows(), rhs_it, output->mutable_view().data(), table_comparator.less(cudf::nullate::NO{}, comparator)); } else { - thrust::transform(rmm::exec_policy_nosync(stream), + thrust::transform(rmm::exec_policy_nosync(stream, mr.get_temporary_mr()), lhs_it, lhs_it + lhs.num_rows(), rhs_it, @@ -55,12 +59,16 @@ template std::unique_ptr two_table_comparison const& column_order, - physical_comparator_t comparator); + physical_comparator_t comparator, + rmm::cuda_stream_view stream, + cudf::memory_resources mr); template std::unique_ptr two_table_comparison( cudf::table_view lhs, cudf::table_view rhs, std::vector const& column_order, - sorting_comparator_t comparator); + sorting_comparator_t comparator, + rmm::cuda_stream_view stream, + cudf::memory_resources mr); template std::unique_ptr sorted_order( @@ -68,23 +76,32 @@ std::unique_ptr sorted_order( cudf::size_type num_rows, bool has_nested, PhysicalElementComparator comparator, - rmm::cuda_stream_view stream) + rmm::cuda_stream_view stream, + cudf::memory_resources mr) { auto output = cudf::make_numeric_column(cudf::data_type(cudf::type_to_id()), num_rows, cudf::mask_state::UNALLOCATED, - stream); + stream, + mr.get_output_mr()); auto const out_begin = output->mutable_view().begin(); - thrust::sequence(rmm::exec_policy_nosync(stream), out_begin, out_begin + num_rows, 0); + thrust::sequence( + rmm::exec_policy_nosync(stream, mr.get_temporary_mr()), out_begin, out_begin + num_rows, 0); auto const table_comparator = cudf::detail::row::lexicographic::self_comparator{preprocessed_input}; if (has_nested) { auto const comp = table_comparator.less(cudf::nullate::NO{}, comparator); - thrust::stable_sort(rmm::exec_policy_nosync(stream), out_begin, out_begin + num_rows, comp); + thrust::stable_sort(rmm::exec_policy_nosync(stream, mr.get_temporary_mr()), + out_begin, + out_begin + num_rows, + comp); } else { auto const comp = table_comparator.less(cudf::nullate::NO{}, comparator); - thrust::stable_sort(rmm::exec_policy_nosync(stream), out_begin, out_begin + num_rows, comp); + thrust::stable_sort(rmm::exec_policy_nosync(stream, mr.get_temporary_mr()), + out_begin, + out_begin + num_rows, + comp); } return output; @@ -95,10 +112,12 @@ template std::unique_ptr sorted_order( cudf::size_type num_rows, bool has_nested, physical_comparator_t comparator, - rmm::cuda_stream_view stream); + rmm::cuda_stream_view stream, + cudf::memory_resources mr); template std::unique_ptr sorted_order( std::shared_ptr preprocessed_input, cudf::size_type num_rows, bool has_nested, sorting_comparator_t comparator, - rmm::cuda_stream_view stream); + rmm::cuda_stream_view stream, + cudf::memory_resources mr); diff --git a/cpp/tests/row_operator/two_table_equality_utilities.cu b/cpp/tests/row_operator/two_table_equality_utilities.cu index 4d167f911b0d..94deaf7e19e7 100644 --- a/cpp/tests/row_operator/two_table_equality_utilities.cu +++ b/cpp/tests/row_operator/two_table_equality_utilities.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -17,23 +17,27 @@ template std::unique_ptr two_table_equality(cudf::table_view lhs, cudf::table_view rhs, std::vector const& column_order, - PhysicalElementComparator comparator) + PhysicalElementComparator comparator, + rmm::cuda_stream_view stream, + cudf::memory_resources mr) { - rmm::cuda_stream_view stream{cudf::get_default_stream()}; - - auto const table_comparator = cudf::detail::row::equality::two_table_comparator{lhs, rhs, stream}; + auto const table_comparator = + cudf::detail::row::equality::two_table_comparator{lhs, rhs, stream, mr.get_temporary_mr()}; auto const lhs_it = cudf::detail::row::lhs_iterator(0); auto const rhs_it = cudf::detail::row::rhs_iterator(0); - auto output = cudf::make_numeric_column( - cudf::data_type(cudf::type_id::BOOL8), lhs.num_rows(), cudf::mask_state::UNALLOCATED); + auto output = cudf::make_numeric_column(cudf::data_type(cudf::type_id::BOOL8), + lhs.num_rows(), + cudf::mask_state::UNALLOCATED, + stream, + mr.get_output_mr()); if (cudf::has_nested_columns(lhs) or cudf::has_nested_columns(rhs)) { auto const equal_comparator = table_comparator.equal_to(cudf::nullate::NO{}, cudf::null_equality::EQUAL, comparator); - thrust::transform(rmm::exec_policy_nosync(stream), + thrust::transform(rmm::exec_policy_nosync(stream, mr.get_temporary_mr()), lhs_it, lhs_it + lhs.num_rows(), rhs_it, @@ -43,7 +47,7 @@ std::unique_ptr two_table_equality(cudf::table_view lhs, auto const equal_comparator = table_comparator.equal_to(cudf::nullate::NO{}, cudf::null_equality::EQUAL, comparator); - thrust::transform(rmm::exec_policy_nosync(stream), + thrust::transform(rmm::exec_policy_nosync(stream, mr.get_temporary_mr()), lhs_it, lhs_it + lhs.num_rows(), rhs_it, @@ -57,9 +61,13 @@ template std::unique_ptr two_table_equality( cudf::table_view lhs, cudf::table_view rhs, std::vector const& column_order, - physical_equality_t comparator); + physical_equality_t comparator, + rmm::cuda_stream_view stream, + cudf::memory_resources mr); template std::unique_ptr two_table_equality( cudf::table_view lhs, cudf::table_view rhs, std::vector const& column_order, - nan_equality_t comparator); + nan_equality_t comparator, + rmm::cuda_stream_view stream, + cudf::memory_resources mr); diff --git a/cpp/tests/utilities/column_utilities.cu b/cpp/tests/utilities/column_utilities.cu index 5d6b2664a651..5a481a88aab8 100644 --- a/cpp/tests/utilities/column_utilities.cu +++ b/cpp/tests/utilities/column_utilities.cu @@ -542,10 +542,8 @@ struct column_comparator_impl { auto lhs_tview = table_view{{lhs}}; auto rhs_tview = table_view{{rhs}}; - // TODO: Pass `mr` once two_table_comparator / equality preprocessed_table::create accept - // memory_resources instead of allocating from the current device resource. - auto const comparator = - cudf::detail::row::equality::two_table_comparator{lhs_tview, rhs_tview, stream}; + auto const comparator = cudf::detail::row::equality::two_table_comparator{ + lhs_tview, rhs_tview, stream, mr.get_temporary_mr()}; auto const has_nulls = cudf::has_nulls(lhs_tview) or cudf::has_nulls(rhs_tview); auto const device_comparator = comparator.equal_to(cudf::nullate::DYNAMIC{has_nulls}); diff --git a/cpp/tests/utilities/memory_resource_utilities.cpp b/cpp/tests/utilities/memory_resource_utilities.cpp index 117758d670d3..95017db866ca 100644 --- a/cpp/tests/utilities/memory_resource_utilities.cpp +++ b/cpp/tests/utilities/memory_resource_utilities.cpp @@ -11,7 +11,11 @@ #include +#include +#include + #include +#include #include namespace cudf::test { @@ -27,11 +31,27 @@ scoped_current_device_resource::~scoped_current_device_resource() std::ignore = cudf::set_current_device_resource(std::move(_previous)); } +namespace { + +void print_allocation_stacktrace() +{ + constexpr int max_frames = 64; + void* frames[max_frames]; + int const nframes = ::backtrace(frames, max_frames); + std::cerr << "Unexpected allocation from the current device resource. Callstack (" << nframes + << " frames):\n"; + if (nframes > 0) { ::backtrace_symbols_fd(frames, nframes, STDERR_FILENO); } + std::cerr.flush(); +} + +} // namespace + memory_resource_test_harness::memory_resource_test_harness(rmm::device_async_resource_ref upstream) : _setup_mr{upstream}, _output_mr{upstream}, _temporary_mr{upstream}, _failing_mr{[](std::size_t, cuda::stream_ref, void*) -> void* { + print_allocation_stacktrace(); throw rmm::bad_alloc{"Unexpected allocation from the current device resource"}; }, [](void*, std::size_t, cuda::stream_ref, void*) {}} diff --git a/cpp/tests/utilities_tests/column_wrapper_tests.cpp b/cpp/tests/utilities_tests/column_wrapper_tests.cpp index dfc5638e2653..d5b5a6bcf1c7 100644 --- a/cpp/tests/utilities_tests/column_wrapper_tests.cpp +++ b/cpp/tests/utilities_tests/column_wrapper_tests.cpp @@ -1,4 +1,6 @@ /* + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -410,8 +412,6 @@ TYPED_TEST(FixedWidthColumnWrapperTest, NullablePairListConstructorAllNullMatch) this->resources()); cudf::column_view view = col; - // TODO: Check the harness with a failing current resource once equality row preprocessing uses - // the supplied memory resources. CUDF_TEST_EXPECT_COLUMNS_EQUAL(view, match_view, cudf::test::debug_output_level::FIRST_ERROR, @@ -517,8 +517,8 @@ TYPED_TEST(StringsColumnWrapperTest, NullablePairListConstructorAllNullMatch) this->resources()); cudf::column_view view = col; - // TODO: Check the harness with a failing current resource once equality row preprocessing uses - // the supplied memory resources. + // TODO: has_nonempty_nulls (via count_if/transform_reduce) still allocates temporaries from the + // current device resource for strings columns. CUDF_TEST_EXPECT_COLUMNS_EQUAL(view, match_view, cudf::test::debug_output_level::FIRST_ERROR,