From 760bc2eac639251981d96955c3f902f2945eed99 Mon Sep 17 00:00:00 2001 From: achirkin Date: Thu, 6 Aug 2026 17:25:06 +0200 Subject: [PATCH 1/4] Implement a kernel dispatcher raft::launch_kernel --- cpp/include/raft/common/detail/scatter.cuh | 7 +- cpp/include/raft/core/bitset.cuh | 7 +- cpp/include/raft/core/detail/copy.hpp | 6 +- cpp/include/raft/label/detail/classlabels.cuh | 13 +- .../raft/label/detail/merge_labels.cuh | 13 +- cpp/include/raft/linalg/detail/add.cuh | 7 +- .../linalg/detail/coalesced_reduction-inl.cuh | 52 ++-- cpp/include/raft/linalg/detail/map.cuh | 6 +- .../raft/linalg/detail/map_then_reduce.cuh | 15 +- cpp/include/raft/linalg/detail/normalize.cuh | 13 +- .../raft/linalg/detail/reduce_cols_by_key.cuh | 11 +- .../raft/linalg/detail/reduce_rows_by_key.cuh | 24 +- .../raft/linalg/detail/strided_reduction.cuh | 11 +- cpp/include/raft/linalg/detail/subtract.cuh | 8 +- cpp/include/raft/linalg/detail/transpose.cuh | 23 +- .../raft/matrix/detail/columnWiseSort.cuh | 19 +- cpp/include/raft/matrix/detail/gather.cuh | 15 +- .../raft/matrix/detail/linewise_op.cuh | 90 ++++--- cpp/include/raft/matrix/detail/math.cuh | 19 +- cpp/include/raft/matrix/detail/matrix.cuh | 20 +- .../raft/matrix/detail/select_radix.cuh | 78 +++--- .../raft/matrix/detail/select_warpsort.cuh | 29 ++- cpp/include/raft/matrix/detail/shift.cuh | 39 ++- cpp/include/raft/random/detail/make_blobs.cuh | 38 ++- .../raft/random/detail/make_regression.cuh | 18 +- .../random/detail/multi_variable_gaussian.cuh | 9 +- cpp/include/raft/random/detail/permute.cuh | 13 +- .../detail/rmat_rectangular_generator.cuh | 11 +- cpp/include/raft/random/detail/rng_impl.cuh | 9 +- .../random/detail/rng_impl_deprecated.cuh | 24 +- .../raft/solver/detail/lap_functions.cuh | 225 ++++++++---------- .../raft/sparse/convert/detail/adj_to_csr.cuh | 8 +- .../sparse/convert/detail/bitmap_to_csr.cuh | 26 +- .../sparse/convert/detail/bitset_to_csr.cuh | 12 +- .../raft/sparse/convert/detail/coo.cuh | 8 +- .../raft/sparse/convert/detail/dense.cuh | 7 +- cpp/include/raft/sparse/detail/csr.cuh | 23 +- cpp/include/raft/sparse/detail/utils.h | 5 +- cpp/include/raft/sparse/linalg/detail/add.cuh | 32 ++- .../raft/sparse/linalg/detail/degree.cuh | 14 +- .../raft/sparse/linalg/detail/laplacian.cuh | 8 +- .../raft/sparse/linalg/detail/norm.cuh | 11 +- .../raft/sparse/linalg/detail/symmetrize.cuh | 63 +++-- .../raft/sparse/linalg/detail/utils.cuh | 51 +++- cpp/include/raft/sparse/op/detail/filter.cuh | 28 +-- cpp/include/raft/sparse/op/detail/reduce.cuh | 19 +- cpp/include/raft/sparse/op/detail/row_op.cuh | 8 +- .../raft/sparse/solver/detail/lanczos.cuh | 39 ++- .../sparse/solver/detail/mst_solver_inl.cuh | 104 ++++---- .../solver/detail/svds_sign_correction.cuh | 7 +- .../raft/spectral/detail/matrix_wrappers.hpp | 7 +- .../stats/detail/batched/silhouette_score.cuh | 20 +- .../raft/stats/detail/contingencyMatrix.cuh | 24 +- cpp/include/raft/stats/detail/dispersion.cuh | 13 +- cpp/include/raft/stats/detail/histogram.cuh | 31 ++- cpp/include/raft/stats/detail/meanvar.cuh | 14 +- cpp/include/raft/stats/detail/minmax.cuh | 34 +-- .../raft/stats/detail/mutual_info_score.cuh | 13 +- .../raft/stats/detail/neighborhood_recall.cuh | 13 +- cpp/include/raft/stats/detail/rand_index.cuh | 12 +- cpp/include/raft/stats/detail/scores.cuh | 12 +- .../raft/stats/detail/silhouette_score.cuh | 19 +- .../stats/detail/trustworthiness_score.cuh | 11 +- cpp/include/raft/util/cache.cuh | 52 ++-- cpp/include/raft/util/detail/scatter.cuh | 7 +- cpp/include/raft/util/kernel_launch.hpp | 206 ++++++++++++++++ cpp/tests/CMakeLists.txt | 1 + cpp/tests/core/interruptible.cu | 7 +- cpp/tests/core/math_device.cu | 6 +- cpp/tests/core/operators_device.cu | 6 +- cpp/tests/core/span.cu | 13 +- cpp/tests/linalg/add.cuh | 6 +- cpp/tests/linalg/axpy.cu | 7 +- cpp/tests/linalg/binary_op.cuh | 7 +- cpp/tests/linalg/divide.cu | 6 +- cpp/tests/linalg/dot.cu | 7 +- cpp/tests/linalg/eltwise.cu | 9 +- cpp/tests/linalg/gemm_layout.cu | 15 +- cpp/tests/linalg/gemv.cu | 13 +- cpp/tests/linalg/map_then_reduce.cu | 7 +- cpp/tests/linalg/matrix_vector_op.cuh | 12 +- cpp/tests/linalg/mean_squared_error.cu | 7 +- cpp/tests/linalg/norm.cu | 13 +- cpp/tests/linalg/power.cu | 9 +- cpp/tests/linalg/reduce.cuh | 13 +- cpp/tests/linalg/reduce_rows_by_key.cu | 16 +- cpp/tests/linalg/sqrt.cu | 6 +- cpp/tests/linalg/subtract.cu | 9 +- cpp/tests/linalg/unary_op.cuh | 7 +- cpp/tests/matrix/math.cu | 12 +- cpp/tests/random/make_blobs.cu | 30 +-- cpp/tests/random/multi_variable_gaussian.cu | 25 +- .../random/rmat_rectangular_generator.cu | 21 +- cpp/tests/random/rng.cu | 19 +- cpp/tests/random/rng_int.cu | 11 +- cpp/tests/random/rng_pcg_host_api.cu | 13 +- cpp/tests/sparse/convert_csr.cu | 7 +- cpp/tests/sparse/spmm.cu | 36 +-- cpp/tests/sparse/symmetrize.cu | 7 +- cpp/tests/stats/histogram.cu | 6 +- cpp/tests/stats/minmax.cu | 16 +- cpp/tests/util/bitonic_sort.cu | 8 +- cpp/tests/util/device_atomics.cu | 7 +- cpp/tests/util/integer_utils.cu | 16 +- cpp/tests/util/kernel_launch.cu | 81 +++++++ cpp/tests/util/reduction.cu | 18 +- 106 files changed, 1530 insertions(+), 838 deletions(-) create mode 100644 cpp/include/raft/util/kernel_launch.hpp create mode 100644 cpp/tests/util/kernel_launch.cu diff --git a/cpp/include/raft/common/detail/scatter.cuh b/cpp/include/raft/common/detail/scatter.cuh index f7eec52985..3b17e9f0a3 100644 --- a/cpp/include/raft/common/detail/scatter.cuh +++ b/cpp/include/raft/common/detail/scatter.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -7,6 +7,7 @@ #include #include +#include #include namespace raft { @@ -36,8 +37,8 @@ void scatterImpl( DataT* out, const DataT* in, const IdxT* idx, IdxT len, Lambda op, cudaStream_t stream) { const IdxT nblks = raft::ceildiv(VecLen ? len / VecLen : len, (IdxT)TPB); - scatterKernel<<>>(out, in, idx, len, op); - RAFT_CUDA_TRY(cudaGetLastError()); + raft::launch_kernel(stream, nblks, TPB)( + scatterKernel, out, in, idx, len, op); } } // namespace detail diff --git a/cpp/include/raft/core/bitset.cuh b/cpp/include/raft/core/bitset.cuh index 5616a9019c..80ce7c9a4e 100644 --- a/cpp/include/raft/core/bitset.cuh +++ b/cpp/include/raft/core/bitset.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -155,8 +156,8 @@ void bitset_repeat(raft::resources const& handle, int threadsPerBlock = 128; int blocksPerGrid = (output_size + threadsPerBlock - 1) / threadsPerBlock; - bitset_repeat_kernel<<>>( - d_src, d_output, src_bit_len, repeat_times); + raft::launch_kernel(stream, blocksPerGrid, threadsPerBlock)( + bitset_repeat_kernel, d_src, d_output, src_bit_len, repeat_times); return; } diff --git a/cpp/include/raft/core/detail/copy.hpp b/cpp/include/raft/core/detail/copy.hpp index 785665a99a..c437bf835e 100644 --- a/cpp/include/raft/core/detail/copy.hpp +++ b/cpp/include/raft/core/detail/copy.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -24,6 +24,7 @@ #ifdef __CUDACC__ #include #include +#include #endif #endif @@ -503,7 +504,8 @@ mdspan_copyable_t copy(resources const& res, DstType&& dst, Sr raft::ceildiv(typename config::index_type(dst.size()), typename config::index_type(mdspan_copy_tile_elems))); auto constexpr const threads = dim3{mdspan_copy_tile_dim, mdspan_copy_tile_dim, 1}; - mdspan_copy_kernel<<>>(dst, src); + raft::launch_kernel(res, blocks, threads)( + mdspan_copy_kernel, dst, src); #else // Should never actually reach this because of enable_ifs. Included for // safety. diff --git a/cpp/include/raft/label/detail/classlabels.cuh b/cpp/include/raft/label/detail/classlabels.cuh index f0e9a14f69..2a29be01f4 100644 --- a/cpp/include/raft/label/detail/classlabels.cuh +++ b/cpp/include/raft/label/detail/classlabels.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -163,8 +164,14 @@ void make_monotonic( rmm::device_uvector map_ids(0, stream); int num_clusters = getUniquelabels(map_ids, in, N, stream); - map_label_kernel<<>>( - map_ids.data(), num_clusters, in, out, N, filter_op, zero_based); + raft::launch_kernel(stream, blocks, threads)(map_label_kernel, + map_ids.data(), + num_clusters, + in, + out, + N, + filter_op, + zero_based); } /** diff --git a/cpp/include/raft/label/detail/merge_labels.cuh b/cpp/include/raft/label/detail/merge_labels.cuh index b58a0753a3..a0e5797d57 100644 --- a/cpp/include/raft/label/detail/merge_labels.cuh +++ b/cpp/include/raft/label/detail/merge_labels.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -9,6 +9,7 @@ #include #include #include +#include #include @@ -129,18 +130,16 @@ void merge_labels(value_idx* labels_a, do { RAFT_CUDA_TRY(cudaMemsetAsync(m, false, sizeof(bool), stream)); - propagate_label_kernel - <<>>(labels_a, labels_b, R, mask, m, N); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, blocks, threads)( + propagate_label_kernel, labels_a, labels_b, R, mask, m, N); raft::update_host(&host_m, m, 1, stream); RAFT_CUDA_TRY(cudaStreamSynchronize(stream)); } while (host_m); // Step 2: re-assign minimum equivalent label - reassign_label_kernel - <<>>(labels_a, labels_b, R, N, MAX_LABEL); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, blocks, threads)( + reassign_label_kernel, labels_a, labels_b, R, N, MAX_LABEL); } } // namespace detail diff --git a/cpp/include/raft/linalg/detail/add.cuh b/cpp/include/raft/linalg/detail/add.cuh index 905b59938c..5fc394d497 100644 --- a/cpp/include/raft/linalg/detail/add.cuh +++ b/cpp/include/raft/linalg/detail/add.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -10,6 +10,7 @@ #include #include #include +#include namespace raft { namespace linalg { @@ -44,8 +45,8 @@ void addDevScalar( // TODO: block dimension has not been tuned dim3 block(256); dim3 grid(raft::ceildiv(len, (IdxType)block.x)); - add_dev_scalar_kernel<<>>(outDev, inDev, singleScalarDev, len); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, grid, block)( + add_dev_scalar_kernel, outDev, inDev, singleScalarDev, len); } } // namespace detail diff --git a/cpp/include/raft/linalg/detail/coalesced_reduction-inl.cuh b/cpp/include/raft/linalg/detail/coalesced_reduction-inl.cuh index 4cc549f79e..9e9b0a13ad 100644 --- a/cpp/include/raft/linalg/detail/coalesced_reduction-inl.cuh +++ b/cpp/include/raft/linalg/detail/coalesced_reduction-inl.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -9,6 +9,7 @@ #include #include #include +#include #include @@ -252,13 +253,20 @@ void coalescedReductionThin(OutType* dots, dim3 threads(Policy::LogicalWarpSize, Policy::NumLogicalWarps, 1); dim3 blocks(ceildiv(N, Policy::RowsPerBlock), 1, 1); if constexpr (std::is_same_v) { - coalescedSumThinKernel - <<>>(dots, data, D, N, init, main_op, final_op, inplace); + raft::launch_kernel(stream, blocks, threads)( + coalescedSumThinKernel, dots, data, D, N, init, main_op, final_op, inplace); } else { - coalescedReductionThinKernel<<>>( - dots, data, D, N, init, main_op, reduce_op, final_op, inplace); + raft::launch_kernel(stream, blocks, threads)(coalescedReductionThinKernel, + dots, + data, + D, + N, + init, + main_op, + reduce_op, + final_op, + inplace); } - RAFT_CUDA_TRY(cudaPeekAtLastError()); } template fun_scope("coalescedReductionMedium<%d>", TPB); if constexpr (std::is_same_v) { - coalescedSumMediumKernel - <<>>(dots, data, D, N, init, main_op, final_op, inplace); + raft::launch_kernel(stream, N, TPB)( + coalescedSumMediumKernel, dots, data, D, N, init, main_op, final_op, inplace); } else { - coalescedReductionMediumKernel - <<>>(dots, data, D, N, init, main_op, reduce_op, final_op, inplace); + raft::launch_kernel(stream, N, TPB)(coalescedReductionMediumKernel, + dots, + data, + D, + N, + init, + main_op, + reduce_op, + final_op, + inplace); } - RAFT_CUDA_TRY(cudaPeekAtLastError()); } template ) { - coalescedSumThickKernel - <<>>(buffer.data(), data, D, N, init, main_op); + raft::launch_kernel(stream, blocks, threads)( + coalescedSumThickKernel, buffer.data(), data, D, N, init, main_op); } else { - coalescedReductionThickKernel - <<>>(buffer.data(), data, D, N, init, main_op, reduce_op); + raft::launch_kernel(stream, blocks, threads)(coalescedReductionThickKernel, + buffer.data(), + data, + D, + N, + init, + main_op, + reduce_op); } - RAFT_CUDA_TRY(cudaPeekAtLastError()); coalescedReductionThin(dots, buffer.data(), diff --git a/cpp/include/raft/linalg/detail/map.cuh b/cpp/include/raft/linalg/detail/map.cuh index 714869aaa5..d91cce2ba2 100644 --- a/cpp/include/raft/linalg/detail/map.cuh +++ b/cpp/include/raft/linalg/detail/map.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -97,7 +98,8 @@ void map_call(rmm::cuda_stream_view stream, OutT* out_ptr, IdxT len, Func f, con const int threads = std::max(WarpSize, std::min(raft::bound_by_power_of_two(len_vectorized), 256)); const IdxT blocks = raft::div_rounding_up_unsafe(len_vectorized, threads); - map_kernel<<>>(out_ptr, len, f, in_ptrs...); + raft::launch_kernel(stream, blocks, threads)( + map_kernel, out_ptr, len, f, in_ptrs...); } constexpr int kCoalescedVectorSize = 16; diff --git a/cpp/include/raft/linalg/detail/map_then_reduce.cuh b/cpp/include/raft/linalg/detail/map_then_reduce.cuh index e495e2eae1..062b0cecdf 100644 --- a/cpp/include/raft/linalg/detail/map_then_reduce.cuh +++ b/cpp/include/raft/linalg/detail/map_then_reduce.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -79,9 +80,15 @@ void mapThenReduceImpl(OutType* out, { raft::update_device(out, &neutral, 1, stream); const int nblks = raft::ceildiv(len, IdxType(TPB)); - mapThenReduceKernel - <<>>(out, len, neutral, map, op, in, args...); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, nblks, TPB)( + mapThenReduceKernel, + out, + len, + neutral, + map, + op, + in, + args...); } }; // end namespace detail diff --git a/cpp/include/raft/linalg/detail/normalize.cuh b/cpp/include/raft/linalg/detail/normalize.cuh index f63533e885..44ed474c5c 100644 --- a/cpp/include/raft/linalg/detail/normalize.cuh +++ b/cpp/include/raft/linalg/detail/normalize.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -7,6 +7,7 @@ #include #include +#include #include @@ -73,9 +74,8 @@ inline void coalesced_normalize_thin(Type* out, { dim3 grid(ceildiv(N, (IdxType)Policy::RowsPerBlock), 1, 1); dim3 block(Policy::LogicalWarpSize, Policy::RowsPerBlock, 1); - coalesced_normalize_thin_kernel - <<>>(out, in, D, N, init, main_op, reduce_op, fin_op, eps); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, grid, block)( + coalesced_normalize_thin_kernel, out, in, D, N, init, main_op, reduce_op, fin_op, eps); } template - <<>>(out, in, D, N, init, main_op, reduce_op, fin_op, eps); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, N, TPB)( + coalesced_normalize_medium_kernel, out, in, D, N, init, main_op, reduce_op, fin_op, eps); } template #include +#include #include @@ -117,13 +118,13 @@ void reduce_cols_by_key(const T* data, int target_nblks = 4 * n_sm; int max_nblks = raft::ceildiv(nrows * ncols, TPB); int nblks = std::min(target_nblks, max_nblks); - reduce_cols_by_key_cached_kernel<<>>( - data, keys, out, nrows, ncols, nkeys); + raft::launch_kernel(stream, nblks, TPB, cache_size)( + reduce_cols_by_key_cached_kernel, data, keys, out, nrows, ncols, nkeys); } else { constexpr int TPB = 256; int nblks = raft::ceildiv(nrows * ncols, TPB); - reduce_cols_by_key_direct_kernel<<>>( - data, keys, out, nrows, ncols, nkeys); + raft::launch_kernel(stream, nblks, TPB)( + reduce_cols_by_key_direct_kernel, data, keys, out, nrows, ncols, nkeys); } RAFT_CUDA_TRY(cudaPeekAtLastError()); } diff --git a/cpp/include/raft/linalg/detail/reduce_rows_by_key.cuh b/cpp/include/raft/linalg/detail/reduce_rows_by_key.cuh index 29d714fbe1..0604e705e1 100644 --- a/cpp/include/raft/linalg/detail/reduce_rows_by_key.cuh +++ b/cpp/include/raft/linalg/detail/reduce_rows_by_key.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -7,6 +7,7 @@ #include #include +#include #include #include @@ -45,7 +46,7 @@ void convert_array(IteratorT1 dst, IteratorT2 src, int n, cudaStream_t st) grid.x = raft::ceildiv(n, (int)block.x); grid.x = std::min(grid.x, MAX_BLOCKS); - convert_array_kernel<<>>(dst, src, n); + raft::launch_kernel(st, grid, block)(convert_array_kernel, dst, src, n); } template @@ -167,8 +168,8 @@ void sum_rows_by_key_small_nkeys(const DataIteratorT d_A, grid.x = std::min(grid.x, 32u); grid.y = ncols; grid.y = std::min(grid.y, MAX_BLOCKS); - sum_rows_by_key_small_nkeys_kernel<<>>( - d_A, lda, d_keys, d_weights, nrows, ncols, nkeys, d_sums); + raft::launch_kernel(st, grid, block)( + sum_rows_by_key_small_nkeys_kernel, d_A, lda, d_keys, d_weights, nrows, ncols, nkeys, d_sums); } // @@ -251,8 +252,15 @@ void sum_rows_by_key_large_nkeys_colmajor(const DataIteratorT d_A, grid.x = std::min(grid.x, 32u); grid.y = ncols; grid.y = std::min(grid.y, MAX_BLOCKS); - sum_rows_by_key_large_nkeys_kernel_colmajor<<>>( - d_A, lda, d_keys, nrows, ncols, key_offset, nkeys, d_sums); + raft::launch_kernel(st, grid, block)(sum_rows_by_key_large_nkeys_kernel_colmajor, + d_A, + lda, + d_keys, + nrows, + ncols, + key_offset, + nkeys, + d_sums); } template (ceildiv(nrows * ncols, (IdxT)block_dim)); - sum_rows_by_key_large_nkeys_kernel_rowmajor<<>>( - d_A, lda, d_weights, d_keys, nrows, ncols, d_sums); + raft::launch_kernel(st, grid_dim, block_dim)( + sum_rows_by_key_large_nkeys_kernel_rowmajor, d_A, lda, d_weights, d_keys, nrows, ncols, d_sums); } /** diff --git a/cpp/include/raft/linalg/detail/strided_reduction.cuh b/cpp/include/raft/linalg/detail/strided_reduction.cuh index 85839cea1a..0c56f7ab4c 100644 --- a/cpp/include/raft/linalg/detail/strided_reduction.cuh +++ b/cpp/include/raft/linalg/detail/strided_reduction.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -9,6 +9,7 @@ #include #include #include +#include #include @@ -155,8 +156,8 @@ void stridedReduction(OutType* dots, raft::min((IdxType)MaxBlocksDimY, raft::ceildiv(N, (IdxType)MinRowsPerBlk))); const size_t shmemSize = sizeof(OutType) * Block.x * 2; - stridedSummationKernel - <<>>(dots, data, D, N, init, main_op); + raft::launch_kernel(stream, grid, Block, shmemSize)( + stridedSummationKernel, dots, data, D, N, init, main_op); } else { // Arbitrary numbers for now, probably need to tune const dim3 thrds(32, 16); @@ -166,8 +167,8 @@ void stridedReduction(OutType* dots, raft::ceildiv(N, (IdxType)thrds.y * elemsPerThread)); const size_t shmemSize = sizeof(OutType) * thrds.x * thrds.y; - stridedReductionKernel - <<>>(dots, data, D, N, init, main_op, reduce_op); + raft::launch_kernel(stream, nblks, thrds, shmemSize)( + stridedReductionKernel, dots, data, D, N, init, main_op, reduce_op); } ///@todo: this complication should go away once we have eliminated the need diff --git a/cpp/include/raft/linalg/detail/subtract.cuh b/cpp/include/raft/linalg/detail/subtract.cuh index 32eec06b1e..b8cc8680a3 100644 --- a/cpp/include/raft/linalg/detail/subtract.cuh +++ b/cpp/include/raft/linalg/detail/subtract.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -10,6 +10,7 @@ #include #include #include +#include namespace raft { namespace linalg { @@ -48,9 +49,8 @@ void subtractDevScalar(math_t* outDev, // Just for the note - there is no way to express such operation with cuBLAS in effective way // https://stackoverflow.com/questions/14051064/add-scalar-to-vector-in-blas-cublas-cuda const IdxType nblks = raft::ceildiv(len, (IdxType)TPB); - subtract_dev_scalar_kernel - <<>>(outDev, inDev, singleScalarDev, len); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, nblks, TPB)( + subtract_dev_scalar_kernel, outDev, inDev, singleScalarDev, len); } }; // end namespace detail diff --git a/cpp/include/raft/linalg/detail/transpose.cuh b/cpp/include/raft/linalg/detail/transpose.cuh index 82fdb1c6f7..cdc926d0e2 100644 --- a/cpp/include/raft/linalg/detail/transpose.cuh +++ b/cpp/include/raft/linalg/detail/transpose.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -12,6 +12,7 @@ #include #include #include +#include #include @@ -117,11 +118,23 @@ void transpose_half(raft::resources const& handle, dim3 grids(adjusted_grid_x, adjusted_grid_y); if (stride_in > 1 || stride_out > 1) { - transpose_half_kernel - <<>>(n_rows, n_cols, in, out, stride_in, stride_out); + raft::launch_kernel(stream, grids, blocks)( + transpose_half_kernel, + n_rows, + n_cols, + in, + out, + stride_in, + stride_out); } else { - transpose_half_kernel - <<>>(n_rows, n_cols, in, out, n_cols, n_rows); + raft::launch_kernel(stream, grids, blocks)( + transpose_half_kernel, + n_rows, + n_cols, + in, + out, + n_cols, + n_rows); } RAFT_CUDA_TRY(cudaPeekAtLastError()); diff --git a/cpp/include/raft/matrix/detail/columnWiseSort.cuh b/cpp/include/raft/matrix/detail/columnWiseSort.cuh index c8c8b9090d..11d5ccc825 100644 --- a/cpp/include/raft/matrix/detail/columnWiseSort.cuh +++ b/cpp/include/raft/matrix/detail/columnWiseSort.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -7,6 +7,7 @@ #include #include +#include #include #include @@ -18,9 +19,15 @@ #include #include -#define INST_BLOCK_SORT(keyIn, keyOut, valueInOut, rows, columns, blockSize, elemPT, stream) \ - devKeyValSortColumnPerRow<<>>( \ - keyIn, keyOut, valueInOut, rows, columns, std::numeric_limits::max()) +#define INST_BLOCK_SORT(keyIn, keyOut, valueInOut, rows, columns, blockSize, elemPT, stream) \ + raft::launch_kernel(stream, rows, blockSize)( \ + devKeyValSortColumnPerRow, \ + keyIn, \ + keyOut, \ + valueInOut, \ + rows, \ + columns, \ + std::numeric_limits::max()) namespace raft { namespace matrix { @@ -136,7 +143,7 @@ cudaError_t layoutIdx(OutType* in, int n_rows, int n_columns, cudaStream_t strea int totalElements = n_rows * n_columns; dim3 block(256); dim3 grid((totalElements + block.x - 1) / block.x); - devLayoutIdx<<>>(in, n_columns, totalElements); + raft::launch_kernel(stream, grid, block)(devLayoutIdx, in, n_columns, totalElements); return cudaGetLastError(); } @@ -146,7 +153,7 @@ cudaError_t layoutSortOffset(T* in, T value, int n_times, cudaStream_t stream) { dim3 block(128); dim3 grid((n_times + block.x - 1) / block.x); - devOffsetKernel<<>>(in, value, n_times); + raft::launch_kernel(stream, grid, block)(devOffsetKernel, in, value, n_times); return cudaGetLastError(); } diff --git a/cpp/include/raft/matrix/detail/gather.cuh b/cpp/include/raft/matrix/detail/gather.cuh index c1686b2f55..9b88c8a8e5 100644 --- a/cpp/include/raft/matrix/detail/gather.cuh +++ b/cpp/include/raft/matrix/detail/gather.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -17,6 +17,7 @@ #include #include #include +#include #include @@ -148,18 +149,18 @@ void gatherImpl(const InputIteratorT in, if (len < static_cast(32 * TPB * n_sm)) { using Policy = gather_policy; IndexT n_blocks = raft::ceildiv(map_length * D, static_cast(Policy::stride)); - gather_kernel<<>>( - in, ld, D, len, map, stencil, out, pred_op, transform_op); + raft::launch_kernel(stream, n_blocks, Policy::n_threads)( + gather_kernel, in, ld, D, len, map, stencil, out, pred_op, transform_op); } else if (len < static_cast(32 * 4 * TPB * n_sm)) { using Policy = gather_policy; IndexT n_blocks = raft::ceildiv(map_length * D, static_cast(Policy::stride)); - gather_kernel<<>>( - in, ld, D, len, map, stencil, out, pred_op, transform_op); + raft::launch_kernel(stream, n_blocks, Policy::n_threads)( + gather_kernel, in, ld, D, len, map, stencil, out, pred_op, transform_op); } else { using Policy = gather_policy; IndexT n_blocks = raft::ceildiv(map_length * D, static_cast(Policy::stride)); - gather_kernel<<>>( - in, ld, D, len, map, stencil, out, pred_op, transform_op); + raft::launch_kernel(stream, n_blocks, Policy::n_threads)( + gather_kernel, in, ld, D, len, map, stencil, out, pred_op, transform_op); } RAFT_CUDA_TRY(cudaPeekAtLastError()); } diff --git a/cpp/include/raft/matrix/detail/linewise_op.cuh b/cpp/include/raft/matrix/detail/linewise_op.cuh index a764bd6e17..dbea350c23 100644 --- a/cpp/include/raft/matrix/detail/linewise_op.cuh +++ b/cpp/include/raft/matrix/detail/linewise_op.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -531,17 +532,30 @@ void matrixLinewiseVecCols(Type* out, // The value below determines how many scalar elements are processed by on thread in total. const IdxType elemsPerThread = raft::ceildiv(alignedLen, gs.x * VecElems * BlockSize) * VecElems; - matrixLinewiseVecColsMainKernel - <<>>(out, in, alignedOff, rowLen, alignedLen, elemsPerThread, op, vecs...); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, gs, bs)( + matrixLinewiseVecColsMainKernel, + out, + in, + alignedOff, + rowLen, + alignedLen, + elemsPerThread, + op, + vecs...); } if (alignedLen < totalLen) { // should be not smaller than the warp size for better branching constexpr std::size_t MaxOffset = std::max(std::size_t(raft::WarpSize), VecBytes); - matrixLinewiseVecColsTailKernel - <<>>( - out, in, alignedOff, alignedEnd, rowLen, totalLen, op, vecs...); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, dim3(2, 1, 1), dim3(MaxOffset, 1, 1))( + matrixLinewiseVecColsTailKernel, + out, + in, + alignedOff, + alignedEnd, + rowLen, + totalLen, + op, + vecs...); } } @@ -585,16 +599,16 @@ void matrixLinewiseVecColsSpan( // The value below determines how many scalar elements are processed by on thread in total. const IdxType elemsPerThread = raft::ceildiv(alignedLen, gs.x * VecElems * BlockSize) * VecElems; - matrixLinewiseVecColsMainKernel - <<>>(out.data_handle(), - in.data_handle(), - 0, - paddedRowLen, - alignedLen, - elemsPerThread, - op, - vecs...); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, gs, bs)( + matrixLinewiseVecColsMainKernel, + out.data_handle(), + in.data_handle(), + 0, + paddedRowLen, + alignedLen, + elemsPerThread, + op, + vecs...); } } @@ -647,18 +661,29 @@ void matrixLinewiseVecRows(Type* out, 1, 1); - matrixLinewiseVecRowsMainKernel - <<>>( - out + alignedOff, alignedStart, alignedOff, rowLen, alignedLen, op, vecs...); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, gs, bs)( + matrixLinewiseVecRowsMainKernel, + out + alignedOff, + alignedStart, + alignedOff, + rowLen, + alignedLen, + op, + vecs...); } if (alignedLen < totalLen) { // should be not smaller than the warp size for better branching constexpr std::size_t MaxOffset = std::max(std::size_t(raft::WarpSize), VecBytes); - matrixLinewiseVecRowsTailKernel - <<>>( - out, in, alignedOff, alignedEnd, rowLen, totalLen, op, vecs...); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, dim3(2, 1, 1), dim3(MaxOffset, 1, 1))( + matrixLinewiseVecRowsTailKernel, + out, + in, + alignedOff, + alignedEnd, + rowLen, + totalLen, + op, + vecs...); } } @@ -719,10 +744,15 @@ void matrixLinewiseVecRowsSpan( 1, 1); - matrixLinewiseVecRowsSpanKernel - <<>>( - out.data_handle(), in.data_handle(), rowLen, paddedRowLen, alignedLen, op, vecs...); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, gs, bs)( + matrixLinewiseVecRowsSpanKernel, + out.data_handle(), + in.data_handle(), + rowLen, + paddedRowLen, + alignedLen, + op, + vecs...); } } diff --git a/cpp/include/raft/matrix/detail/math.cuh b/cpp/include/raft/matrix/detail/math.cuh index 14a7846704..785484bf42 100644 --- a/cpp/include/raft/matrix/detail/math.cuh +++ b/cpp/include/raft/matrix/detail/math.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -327,13 +328,13 @@ template inline void argReduce(const math_t* in, idx_t D, idx_t N, out_t* out, cudaStream_t stream) { if (D <= 32) { - argReduceKernel<<>>(in, D, N, out); + raft::launch_kernel(stream, N, 32)(argReduceKernel, in, D, N, out); } else if (D <= 64) { - argReduceKernel<<>>(in, D, N, out); + raft::launch_kernel(stream, N, 64)(argReduceKernel, in, D, N, out); } else if (D <= 128) { - argReduceKernel<<>>(in, D, N, out); + raft::launch_kernel(stream, N, 128)(argReduceKernel, in, D, N, out); } else { - argReduceKernel<<>>(in, D, N, out); + raft::launch_kernel(stream, N, 256)(argReduceKernel, in, D, N, out); } RAFT_CUDA_TRY(cudaPeekAtLastError()); } @@ -389,13 +390,13 @@ void signFlip(math_t* inout, int n_rows, int n_cols, cudaStream_t stream) int N = n_cols; auto data = inout; if (D <= 32) { - signFlipKernel<<>>(data, D, N); + raft::launch_kernel(stream, N, 32)(signFlipKernel, data, D, N); } else if (D <= 64) { - signFlipKernel<<>>(data, D, N); + raft::launch_kernel(stream, N, 64)(signFlipKernel, data, D, N); } else if (D <= 128) { - signFlipKernel<<>>(data, D, N); + raft::launch_kernel(stream, N, 128)(signFlipKernel, data, D, N); } else { - signFlipKernel<<>>(data, D, N); + raft::launch_kernel(stream, N, 256)(signFlipKernel, data, D, N); } RAFT_CUDA_TRY(cudaPeekAtLastError()); } diff --git a/cpp/include/raft/matrix/detail/matrix.cuh b/cpp/include/raft/matrix/detail/matrix.cuh index f3545fb103..e3e43d7efe 100644 --- a/cpp/include/raft/matrix/detail/matrix.cuh +++ b/cpp/include/raft/matrix/detail/matrix.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -12,6 +12,7 @@ #include #include #include +#include #include @@ -40,9 +41,8 @@ void copyRows(const m_t* in, { if (rowMajor) { const idx_t TPB = 256; - cache::get_vecs<<>>( - in, n_cols, indices, n_rows_indices, out); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, raft::ceildiv(n_rows_indices * n_cols, TPB), TPB)( + cache::get_vecs, in, n_cols, indices, n_rows_indices, out); return; } @@ -187,9 +187,9 @@ void sliceMatrix(const m_t* in, dim3 block(64); dim3 grid(((x2 - x1) * (y2 - y1) + block.x - 1) / block.x); if (row_major) - slice<<>>(in, lda, out, y1, x1, y2, x2); + raft::launch_kernel(stream, grid, block)(slice, in, lda, out, y1, x1, y2, x2); else - slice<<>>(in, lda, out, x1, y1, x2, y2); + raft::launch_kernel(stream, grid, block)(slice, in, lda, out, x1, y1, x2, y2); } /** @@ -218,7 +218,7 @@ void copyUpperTriangular(const m_t* src, m_t* dst, idx_t n_rows, idx_t n_cols, c idx_t k = std::min(m, n); dim3 block(64); dim3 grid((m * n + block.x - 1) / block.x); - getUpperTriangular<<>>(src, dst, m, n, k); + raft::launch_kernel(stream, grid, block)(getUpperTriangular, src, dst, m, n, k); } /** @@ -259,7 +259,7 @@ void initializeDiagonalMatrix( idx_t lda = row_major ? n_cols : n_rows; dim3 block(64); dim3 grid((k + block.x - 1) / block.x); - copyVectorToMatrixDiagonal<<>>(vec, matrix, lda, k); + raft::launch_kernel(stream, grid, block)(copyVectorToMatrixDiagonal, vec, matrix, lda, k); } template @@ -270,7 +270,7 @@ void getDiagonalMatrix( idx_t lda = row_major ? n_cols : n_rows; dim3 block(64); dim3 grid((k + block.x - 1) / block.x); - copyVectorFromMatrixDiagonal<<>>(vec, matrix, lda, k); + raft::launch_kernel(stream, grid, block)(copyVectorFromMatrixDiagonal, vec, matrix, lda, k); } /** @@ -291,7 +291,7 @@ void getDiagonalInverseMatrix(m_t* in, idx_t len, cudaStream_t stream) { dim3 block(64); dim3 grid((len + block.x - 1) / block.x); - matrixDiagonalInverse<<>>(in, len); + raft::launch_kernel(stream, grid, block)(matrixDiagonalInverse, in, len); } template diff --git a/cpp/include/raft/matrix/detail/select_radix.cuh b/cpp/include/raft/matrix/detail/select_radix.cuh index 718096c466..38bdf52467 100644 --- a/cpp/include/raft/matrix/detail/select_radix.cuh +++ b/cpp/include/raft/matrix/detail/select_radix.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -930,36 +931,36 @@ void radix_topk(const T* in, kernel = radix_kernel; } - kernel<<>>(in, - in_idx, - reinterpret_cast(bufs.data()), - offset, - chunk_out, - chunk_out_idx, - counters.data(), - histograms.data(), - len, - chunk_len_i, - k, - select_min, - pass); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, blocks, BlockSize)(kernel, + in, + in_idx, + reinterpret_cast(bufs.data()), + offset, + chunk_out, + chunk_out_idx, + counters.data(), + histograms.data(), + len, + chunk_len_i, + k, + select_min, + pass); } if (!fused_last_filter) { - last_filter_kernel - <<>>(in, - in_idx, - reinterpret_cast(bufs.data()), - offset, - chunk_out, - chunk_out_idx, - len, - chunk_len_i, - k, - counters.data(), - select_min); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, blocks, BlockSize)( + last_filter_kernel, + in, + in_idx, + reinterpret_cast(bufs.data()), + offset, + chunk_out, + chunk_out_idx, + len, + chunk_len_i, + k, + counters.data(), + select_min); } } } @@ -1177,16 +1178,17 @@ void radix_topk_one_block(const T* in, for (size_t offset = 0; offset < static_cast(batch_size); offset += max_chunk_size) { int chunk_size = std::min(max_chunk_size, batch_size - offset); const IdxT* chunk_len_i = len_i ? (len_i + offset) : nullptr; - kernel<<>>(in, - in_idx, - len, - chunk_len_i, - k, - out + offset * k, - out_idx + offset * k, - select_min, - reinterpret_cast(bufs.data()), - offset); + raft::launch_kernel(stream, chunk_size, BlockSize)(kernel, + in, + in_idx, + len, + chunk_len_i, + k, + out + offset * k, + out_idx + offset * k, + select_min, + reinterpret_cast(bufs.data()), + offset); } } diff --git a/cpp/include/raft/matrix/detail/select_warpsort.cuh b/cpp/include/raft/matrix/detail/select_warpsort.cuh index b517ef8c10..b6027e240a 100644 --- a/cpp/include/raft/matrix/detail/select_warpsort.cuh +++ b/cpp/include/raft/matrix/detail/select_warpsort.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -893,13 +894,27 @@ struct launch_setup { size_t batch_chunk = std::min(kMaxGridDimY, batch_size - offset); dim3 gs(num_blocks, batch_chunk, 1); if (select_min) { - block_kernel - <<>>( - in_key, in_idx, in_indptr, g_offset, IdxT(len), k, out_key, out_idx); + raft::launch_kernel(stream, gs, block_dim, smem_size)( + block_kernel, + in_key, + in_idx, + in_indptr, + g_offset, + IdxT(len), + k, + out_key, + out_idx); } else { - block_kernel - <<>>( - in_key, in_idx, in_indptr, g_offset, IdxT(len), k, out_key, out_idx); + raft::launch_kernel(stream, gs, block_dim, smem_size)( + block_kernel, + in_key, + in_idx, + in_indptr, + g_offset, + IdxT(len), + k, + out_key, + out_idx); } RAFT_CUDA_TRY(cudaPeekAtLastError()); out_key += batch_chunk * num_blocks * k; diff --git a/cpp/include/raft/matrix/detail/shift.cuh b/cpp/include/raft/matrix/detail/shift.cuh index f7e9f78a6f..a5d611f4ba 100644 --- a/cpp/include/raft/matrix/detail/shift.cuh +++ b/cpp/include/raft/matrix/detail/shift.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -9,6 +9,7 @@ #include #include #include +#include namespace raft { namespace matrix::detail { @@ -143,20 +144,40 @@ void shift_dispatch(raft::resources const& handle, if (shift_type == ShiftType::COL) { size_t num_blocks = static_cast((n_rows + TPB) / TPB); if (shift_direction == ShiftDirection::TOWARDS_BEGINNING) { - col_shift_towards_beginning - <<>>(in_out.data_handle(), n_rows, n_cols, k, value); + raft::launch_kernel(stream, num_blocks, TPB)( + col_shift_towards_beginning, + in_out.data_handle(), + n_rows, + n_cols, + k, + value); } else { // ShiftDirection::TOWARDS_END - col_shift_towards_end - <<>>(in_out.data_handle(), n_rows, n_cols, k, value); + raft::launch_kernel(stream, num_blocks, TPB)( + col_shift_towards_end, + in_out.data_handle(), + n_rows, + n_cols, + k, + value); } } else { // ShiftType::ROW size_t num_blocks = static_cast((n_cols + TPB) / TPB); if (shift_direction == ShiftDirection::TOWARDS_BEGINNING) { - row_shift_towards_beginning - <<>>(in_out.data_handle(), n_rows, n_cols, k, value); + raft::launch_kernel(stream, num_blocks, TPB)( + row_shift_towards_beginning, + in_out.data_handle(), + n_rows, + n_cols, + k, + value); } else { // ShiftDirection::TOWARDS_END - row_shift_towards_end - <<>>(in_out.data_handle(), n_rows, n_cols, k, value); + raft::launch_kernel(stream, num_blocks, TPB)( + row_shift_towards_end, + in_out.data_handle(), + n_rows, + n_cols, + k, + value); } } raft::resource::sync_stream(handle); diff --git a/cpp/include/raft/random/detail/make_blobs.cuh b/cpp/include/raft/random/detail/make_blobs.cuh index 3132bb0bae..9d1747bc52 100644 --- a/cpp/include/raft/random/detail/make_blobs.cuh +++ b/cpp/include/raft/random/detail/make_blobs.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -14,6 +14,7 @@ #include #include #include +#include #include @@ -137,6 +138,33 @@ RAFT_KERNEL generate_data_kernel(raft::random::DeviceState rng_state, } } +template +void call_generate_data_kernel(raft::random::DeviceState const& rng_state, + DataT* out, + const IdxT* labels, + IdxT n_rows, + IdxT n_cols, + IdxT n_clusters, + bool row_major, + const DataT* centers, + const DataT* cluster_std, + const DataT cluster_std_scalar, + int64_t n_blocks, + cudaStream_t stream) +{ + raft::launch_kernel(stream, n_blocks, 128)(generate_data_kernel, + rng_state, + out, + labels, + n_rows, + n_cols, + n_clusters, + row_major, + centers, + cluster_std, + cluster_std_scalar); +} + template void generate_data(DataT* out, const IdxT* labels, @@ -154,10 +182,8 @@ void generate_data(DataT* out, int64_t items = static_cast(n_rows) * n_cols; // Choose a grid size so that each thread can write two output values. int64_t nBlocks = ceildiv(items, 2 * block_size); - // parentheses needed here for kernel, otherwise macro interprets the arguments - // of triple chevron notation as macro arguments RAFT_CALL_RNG_FUNC(rng_state, - (generate_data_kernel<<>>), + call_generate_data_kernel, out, labels, n_rows, @@ -166,7 +192,9 @@ void generate_data(DataT* out, row_major, centers, cluster_std, - cluster_std_scalar); + cluster_std_scalar, + nBlocks, + stream); } /** diff --git a/cpp/include/raft/random/detail/make_regression.cuh b/cpp/include/raft/random/detail/make_regression.cuh index 773eae7b39..d445ef995b 100644 --- a/cpp/include/raft/random/detail/make_regression.cuh +++ b/cpp/include/raft/random/detail/make_regression.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -20,6 +20,7 @@ #include #include #include +#include #include @@ -67,9 +68,8 @@ static void _make_low_rank_matrix(raft::resources const& handle, // Build the singular profile by assembling signal and noise components rmm::device_uvector singular_vec(n, stream); - _singular_profile_kernel<<(n, 256), 256, 0, stream>>>( - singular_vec.data(), n, tail_strength, effective_rank); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, raft::ceildiv(n, 256), 256)( + _singular_profile_kernel, singular_vec.data(), n, tail_strength, effective_rank); rmm::device_uvector singular_mat(n * n, stream); RAFT_CUDA_TRY(cudaMemsetAsync(singular_mat.data(), 0, n * n * sizeof(DataT), stream)); @@ -250,9 +250,8 @@ void make_regression_caller(raft::resources const& handle, raft::random::permute( perms_samples.data(), tmp_out.data(), out, n_cols, n_rows, true, stream); IdxT nblks_rows = raft::ceildiv(n_rows, Nthreads); - _gather2d_kernel<<>>( - values, _values, perms_samples.data(), n_rows, n_targets); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, nblks_rows, Nthreads)( + _gather2d_kernel, values, _values, perms_samples.data(), n_rows, n_targets); // Shuffle the features from tmp_out to out raft::random::permute( @@ -261,9 +260,8 @@ void make_regression_caller(raft::resources const& handle, // Shuffle the coefficients accordingly if (coef != nullptr) { IdxT nblks_cols = raft::ceildiv(n_cols, Nthreads); - _gather2d_kernel<<>>( - coef, _coef, perms_features.data(), n_cols, n_targets); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, nblks_cols, Nthreads)( + _gather2d_kernel, coef, _coef, perms_features.data(), n_cols, n_targets); } } } diff --git a/cpp/include/raft/random/detail/multi_variable_gaussian.cuh b/cpp/include/raft/random/detail/multi_variable_gaussian.cuh index b4291b950a..4966e601f9 100644 --- a/cpp/include/raft/random/detail/multi_variable_gaussian.cuh +++ b/cpp/include/raft/random/detail/multi_variable_gaussian.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2018-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2018-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -221,8 +222,7 @@ class multi_variable_gaussian_impl { // upper part (0) being filled with 0.0 dim3 block(32, 32); dim3 grid(raft::ceildiv(dim, (int)block.x), raft::ceildiv(dim, (int)block.y)); - fill_uplo<<>>(dim, UPPER, (T)0.0, P); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(cudaStream, grid, block)(fill_uplo, dim, UPPER, (T)0.0, P); // P is lower triangular chol decomp mtrx raft::linalg::gemm( @@ -233,8 +233,7 @@ class multi_variable_gaussian_impl { dim3 grid(raft::ceildiv(dim, (int)block.x)); RAFT_CUDA_TRY(cudaMemsetAsync(info, 0, sizeof(int), cudaStream)); grid.x = raft::ceildiv(dim * dim, (int)block.x); - combined_dot_product<<>>(dim, dim, eig, P, info); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(cudaStream, grid, block)(combined_dot_product, dim, dim, eig, P, info); // checking if any eigen vals were negative raft::update_host(&info_h, info, 1, cudaStream); diff --git a/cpp/include/raft/random/detail/permute.cuh b/cpp/include/raft/random/detail/permute.cuh index d991d6fc50..edf265d805 100644 --- a/cpp/include/raft/random/detail/permute.cuh +++ b/cpp/include/raft/random/detail/permute.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -84,9 +85,8 @@ struct permute_impl_t { // check if we can execute at this vector length if (D % VLen == 0 && raft::is_aligned(vout, sizeof(VType)) && raft::is_aligned(vin, sizeof(VType))) { - permuteKernel - <<>>(perms, vout, vin, a, b, N, D / VLen); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, nblks, TPB)( + permuteKernel, perms, vout, vin, a, b, N, D / VLen); } else { // otherwise try the next lower vector length permute_impl_t::permuteImpl( perms, out, in, N, D, nblks, a, b, stream); @@ -107,9 +107,8 @@ struct permute_impl_t { IdxType b, cudaStream_t stream) { - permuteKernel - <<>>(perms, out, in, a, b, N, D); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, nblks, TPB)( + permuteKernel, perms, out, in, a, b, N, D); } }; diff --git a/cpp/include/raft/random/detail/rmat_rectangular_generator.cuh b/cpp/include/raft/random/detail/rmat_rectangular_generator.cuh index 2dc9754f63..3bff053b8a 100644 --- a/cpp/include/raft/random/detail/rmat_rectangular_generator.cuh +++ b/cpp/include/raft/random/detail/rmat_rectangular_generator.cuh @@ -14,6 +14,7 @@ #include #include #include +#include namespace raft { namespace random { @@ -117,9 +118,8 @@ void rmat_rectangular_gen_caller(IdxT* out, auto max_scale = max(r_scale, c_scale); size_t smem_size = sizeof(ProbT) * max_scale * 2 * 2; auto n_blks = raft::ceildiv(n_edges, N_THREADS); - rmat_gen_kernel<<>>( - out, out_src, out_dst, theta, r_scale, c_scale, n_edges, max_scale, r); - RAFT_CUDA_TRY(cudaGetLastError()); + raft::launch_kernel(stream, n_blks, N_THREADS, smem_size)( + rmat_gen_kernel, out, out_src, out_dst, theta, r_scale, c_scale, n_edges, max_scale, r); r.advance(n_edges, max_scale); } @@ -171,9 +171,8 @@ void rmat_rectangular_gen_caller(IdxT* out, static constexpr int N_THREADS = 512; auto max_scale = max(r_scale, c_scale); auto n_blks = raft::ceildiv(n_edges, N_THREADS); - rmat_gen_kernel<<>>( - out, out_src, out_dst, a, b, c, r_scale, c_scale, n_edges, max_scale, r); - RAFT_CUDA_TRY(cudaGetLastError()); + raft::launch_kernel(stream, n_blks, N_THREADS)( + rmat_gen_kernel, out, out_src, out_dst, a, b, c, r_scale, c_scale, n_edges, max_scale, r); r.advance(n_edges, max_scale); } diff --git a/cpp/include/raft/random/detail/rng_impl.cuh b/cpp/include/raft/random/detail/rng_impl.cuh index 683114318a..a5d8a59c2a 100644 --- a/cpp/include/raft/random/detail/rng_impl.cuh +++ b/cpp/include/raft/random/detail/rng_impl.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2018-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2018-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -69,7 +70,7 @@ void call_rng_kernel(DeviceState const& dev_state, { auto n_threads = 256; auto n_blocks = 4 * getMultiProcessorCount(); - rngKernel<<>>(dev_state, args...); + raft::launch_kernel(stream, n_blocks, n_threads)(rngKernel, dev_state, args...); rng_state.advance(uint64_t(n_blocks) * n_threads, 16); } @@ -245,8 +246,8 @@ void call_sample_with_replacement_kernel(DeviceState const& dev_state, { IdxType n_threads = 256; IdxType n_blocks = raft::ceildiv(sampledLen, n_threads); - sample_with_replacement_kernel<<>>( - dev_state, out, weights_csum, sampledLen, len); + raft::launch_kernel(stream, n_blocks, n_threads)( + sample_with_replacement_kernel, dev_state, out, weights_csum, sampledLen, len); rng_state.advance(uint64_t(n_blocks) * n_threads, 1); } diff --git a/cpp/include/raft/random/detail/rng_impl_deprecated.cuh b/cpp/include/raft/random/detail/rng_impl_deprecated.cuh index 3284f6d9a8..782905b3ab 100644 --- a/cpp/include/raft/random/detail/rng_impl_deprecated.cuh +++ b/cpp/include/raft/random/detail/rng_impl_deprecated.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -235,13 +236,24 @@ class RngImpl { { switch (state.type) { case GenPhilox: - fillKernel - <<>>( - state.seed, state.base_subsequence, 0, ptr, len, params); + raft::launch_kernel(stream, nBlocks, nThreads)( + fillKernel, + state.seed, + state.base_subsequence, + 0, + ptr, + len, + params); break; case GenPC: - fillKernel<<>>( - state.seed, state.base_subsequence, 0, ptr, len, params); + raft::launch_kernel(stream, nBlocks, nThreads)( + fillKernel, + state.seed, + state.base_subsequence, + 0, + ptr, + len, + params); break; default: break; } diff --git a/cpp/include/raft/solver/detail/lap_functions.cuh b/cpp/include/raft/solver/detail/lap_functions.cuh index 38d1d4fed0..ba39d5d17c 100644 --- a/cpp/include/raft/solver/detail/lap_functions.cuh +++ b/cpp/include/raft/solver/detail/lap_functions.cuh @@ -33,7 +33,7 @@ #include #include #include -#include +#include #include #include @@ -117,16 +117,16 @@ inline void initialReduction(raft::resources const& handle, detail::calculateRectangularDims(blocks_per_grid, threads_per_block, total_blocks, N, SP); - kernel_rowReduction<<>>( - d_costs, d_vertices_dev.row_duals, SP, N); + raft::launch_kernel(handle, blocks_per_grid, threads_per_block)( + kernel_rowReduction, d_costs, d_vertices_dev.row_duals, SP, N); - RAFT_CHECK_CUDA(resource::get_cuda_stream(handle)); - kernel_columnReduction<<>>( - d_costs, d_vertices_dev.row_duals, d_vertices_dev.col_duals, SP, N); - RAFT_CHECK_CUDA(resource::get_cuda_stream(handle)); + raft::launch_kernel(handle, blocks_per_grid, threads_per_block)( + kernel_columnReduction, + d_costs, + d_vertices_dev.row_duals, + d_vertices_dev.col_duals, + SP, + N); } template @@ -153,10 +153,8 @@ inline void computeInitialAssignments(raft::resources const& handle, detail::calculateRectangularDims(blocks_per_grid, threads_per_block, total_blocks, N, SP); - kernel_computeInitialAssignments<<>>( + raft::launch_kernel(handle, blocks_per_grid, threads_per_block)( + kernel_computeInitialAssignments, d_costs, d_vertices.row_duals, d_vertices.col_duals, @@ -167,7 +165,6 @@ inline void computeInitialAssignments(raft::resources const& handle, SP, N, epsilon); - RAFT_CHECK_CUDA(resource::get_cuda_stream(handle)); } // Function for finding row cover on individual devices. @@ -196,13 +193,12 @@ inline int computeRowCovers(raft::resources const& handle, thrust::fill_n(thrust::device, d_col_data.children, size, vertex_t{-1}); detail::calculateRectangularDims(blocks_per_grid, threads_per_block, total_blocks, N, SP); - kernel_computeRowCovers<<>>( - d_vertices.row_assignments, d_vertices.row_covers, d_row_data.is_visited, SP, N); - - RAFT_CHECK_CUDA(resource::get_cuda_stream(handle)); + raft::launch_kernel(handle, blocks_per_grid, threads_per_block)(kernel_computeRowCovers, + d_vertices.row_assignments, + d_vertices.row_covers, + d_row_data.is_visited, + SP, + N); return thrust::reduce(thrust::device, d_vertices.row_covers, d_vertices.row_covers + size); } @@ -227,19 +223,18 @@ inline void coverZeroAndExpand(raft::resources const& handle, detail::calculateRectangularDims(blocks_per_grid, threads_per_block, total_blocks, N, SP); - kernel_coverAndExpand<<>>(d_flag, - d_rows_csr_ptrs, - d_rows_csr_neighbors, - d_costs_dev, - d_vertices_dev, - d_row_data_dev, - d_col_data_dev, - SP, - N, - epsilon); + raft::launch_kernel(handle, blocks_per_grid, threads_per_block)( + kernel_coverAndExpand, + d_flag, + d_rows_csr_ptrs, + d_rows_csr_neighbors, + d_costs_dev, + d_vertices_dev, + d_row_data_dev, + d_col_data_dev, + SP, + N, + epsilon); } template @@ -276,12 +271,13 @@ inline vertex_t zeroCoverIteration(raft::resources const& handle, detail::calculateRectangularDims(blocks_per_grid, threads_per_block, total_blocks, N, SP); // construct predicate matrix for edges. - kernel_rowPredicateConstructionCSR<<>>( - predicates_v.data(), addresses_v.data(), d_row_data_dev.is_visited, SP, N); - RAFT_CHECK_CUDA(resource::get_cuda_stream(handle)); + raft::launch_kernel(handle, blocks_per_grid, threads_per_block)( + kernel_rowPredicateConstructionCSR, + predicates_v.data(), + addresses_v.data(), + d_row_data_dev.is_visited, + SP, + N); M = thrust::reduce(thrust::device, addresses_v.begin(), addresses_v.end()); thrust::exclusive_scan( @@ -290,18 +286,15 @@ inline vertex_t zeroCoverIteration(raft::resources const& handle, if (M > 0) { csr_neighbors_v.resize(M, resource::get_cuda_stream(handle)); - kernel_rowScatterCSR<<>>(predicates_v.data(), - addresses_v.data(), - csr_neighbors_v.data(), - csr_ptrs_v.data(), - M, - SP, - N); - - RAFT_CHECK_CUDA(resource::get_cuda_stream(handle)); + raft::launch_kernel(handle, blocks_per_grid, threads_per_block)( + kernel_rowScatterCSR, + predicates_v.data(), + addresses_v.data(), + csr_neighbors_v.data(), + csr_ptrs_v.data(), + M, + SP, + N); } } @@ -365,13 +358,12 @@ inline void reversePass(raft::resources const& handle, thrust::fill_n(thrust::device, addresses_v.data(), size, vertex_t{0}); // compact the reverse pass row vertices. - kernel_augmentPredicateConstruction<<>>( - predicates_v.data(), addresses_v.data(), d_col_data_dev.is_visited, size); - - RAFT_CHECK_CUDA(resource::get_cuda_stream(handle)); + raft::launch_kernel(handle, blocks_per_grid, threads_per_block)( + kernel_augmentPredicateConstruction, + predicates_v.data(), + addresses_v.data(), + d_col_data_dev.is_visited, + size); // calculate total number of vertices. std::size_t csr_size = thrust::reduce(thrust::device, addresses_v.begin(), addresses_v.end()); @@ -387,20 +379,18 @@ inline void reversePass(raft::resources const& handle, rmm::device_uvector elements_v(csr_size, resource::get_cuda_stream(handle)); - kernel_augmentScatter<<>>( - elements_v.data(), predicates_v.data(), addresses_v.data(), size); - - RAFT_CHECK_CUDA(resource::get_cuda_stream(handle)); - - kernel_reverseTraversal<<>>( - elements_v.data(), d_row_data_dev, d_col_data_dev, csr_size); - RAFT_CHECK_CUDA(resource::get_cuda_stream(handle)); + raft::launch_kernel(handle, blocks_per_grid, threads_per_block)(kernel_augmentScatter, + elements_v.data(), + predicates_v.data(), + addresses_v.data(), + size); + + raft::launch_kernel(handle, blocks_per_grid_1, threads_per_block_1)( + kernel_reverseTraversal, + elements_v.data(), + d_row_data_dev, + d_col_data_dev, + csr_size); } } @@ -425,13 +415,12 @@ inline void augmentationPass(raft::resources const& handle, thrust::fill_n(thrust::device, addresses_v.data(), SP * N, vertex_t{0}); // compact the reverse pass row vertices. - kernel_augmentPredicateConstruction<<>>( - predicates_v.data(), addresses_v.data(), d_row_data_dev.is_visited, SP * N); - - RAFT_CHECK_CUDA(resource::get_cuda_stream(handle)); + raft::launch_kernel(handle, blocks_per_grid, threads_per_block)( + kernel_augmentPredicateConstruction, + predicates_v.data(), + addresses_v.data(), + d_row_data_dev.is_visited, + SP * N); // calculate total number of vertices. // TODO: should be vertex_t @@ -450,26 +439,21 @@ inline void augmentationPass(raft::resources const& handle, rmm::device_uvector elements_v(row_ids_csr_size, resource::get_cuda_stream(handle)); - kernel_augmentScatter<<>>( - elements_v.data(), predicates_v.data(), addresses_v.data(), vertex_t{SP * N}); - - RAFT_CHECK_CUDA(resource::get_cuda_stream(handle)); - - kernel_augmentation<<>>(d_vertices_dev.row_assignments, - d_vertices_dev.col_assignments, - elements_v.data(), - d_row_data_dev, - d_col_data_dev, - vertex_t{N}, - row_ids_csr_size); - - RAFT_CHECK_CUDA(resource::get_cuda_stream(handle)); + raft::launch_kernel(handle, blocks_per_grid, threads_per_block)(kernel_augmentScatter, + elements_v.data(), + predicates_v.data(), + addresses_v.data(), + vertex_t{SP * N}); + + raft::launch_kernel(handle, blocks_per_grid_1, threads_per_block_1)( + kernel_augmentation, + d_vertices_dev.row_assignments, + d_vertices_dev.col_assignments, + elements_v.data(), + d_row_data_dev, + d_col_data_dev, + vertex_t{N}, + row_ids_csr_size); } } @@ -489,13 +473,17 @@ inline void dualUpdate(raft::resources const& handle, rmm::device_uvector sp_min_v(SP, resource::get_cuda_stream(handle)); detail::calculateLinearDims(blocks_per_grid, threads_per_block, total_blocks, SP); - kernel_dualUpdate_1<<>>( - sp_min_v.data(), d_vertices_dev.col_slacks, d_vertices_dev.col_covers, SP, N); - - RAFT_CHECK_CUDA(resource::get_cuda_stream(handle)); + raft::launch_kernel(handle, blocks_per_grid, threads_per_block)( + kernel_dualUpdate_1, + sp_min_v.data(), + d_vertices_dev.col_slacks, + d_vertices_dev.col_covers, + SP, + N); detail::calculateRectangularDims(blocks_per_grid, threads_per_block, total_blocks, N, SP); - kernel_dualUpdate_2<<>>( + raft::launch_kernel(handle, blocks_per_grid, threads_per_block)( + kernel_dualUpdate_2, sp_min_v.data(), d_vertices_dev.row_duals, d_vertices_dev.col_duals, @@ -507,8 +495,6 @@ inline void dualUpdate(raft::resources const& handle, SP, N, epsilon); - - RAFT_CHECK_CUDA(resource::get_cuda_stream(handle)); } // Function for calculating optimal objective function value using dual variables. @@ -525,13 +511,13 @@ inline void calcObjValDual(raft::resources const& handle, detail::calculateLinearDims(blocks_per_grid, threads_per_block, total_blocks, SP); - kernel_calcObjValDual<<>>( - d_obj_val, d_vertices_dev.row_duals, d_vertices_dev.col_duals, SP, N); - - RAFT_CHECK_CUDA(resource::get_cuda_stream(handle)); + raft::launch_kernel(handle, blocks_per_grid, threads_per_block)( + kernel_calcObjValDual, + d_obj_val, + d_vertices_dev.row_duals, + d_vertices_dev.col_duals, + SP, + N); } // Function for calculating optimal objective function value using dual variables. @@ -549,13 +535,8 @@ inline void calcObjValPrimal(raft::resources const& handle, detail::calculateLinearDims(blocks_per_grid, threads_per_block, total_blocks, SP); - kernel_calcObjValPrimal<<>>( - d_obj_val, d_costs, d_row_assignments, SP, N); - - RAFT_CHECK_CUDA(resource::get_cuda_stream(handle)); + raft::launch_kernel(handle, blocks_per_grid, threads_per_block)( + kernel_calcObjValPrimal, d_obj_val, d_costs, d_row_assignments, SP, N); } } // namespace solver::detail diff --git a/cpp/include/raft/sparse/convert/detail/adj_to_csr.cuh b/cpp/include/raft/sparse/convert/detail/adj_to_csr.cuh index b937195ba9..c13a809112 100644 --- a/cpp/include/raft/sparse/convert/detail/adj_to_csr.cuh +++ b/cpp/include/raft/sparse/convert/detail/adj_to_csr.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -156,9 +157,8 @@ void adj_to_csr(raft::resources const& handle, dim3 block(adj_to_csr_tpb, 1); dim3 grid(blocks_per_row, grid_rows); - adj_to_csr_kernel - <<>>(adj, row_ind, num_rows, num_cols, tmp, out_col_ind); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, grid, block)( + adj_to_csr_kernel, adj, row_ind, num_rows, num_cols, tmp, out_col_ind); } }; // end NAMESPACE detail diff --git a/cpp/include/raft/sparse/convert/detail/bitmap_to_csr.cuh b/cpp/include/raft/sparse/convert/detail/bitmap_to_csr.cuh index 39c13412e6..d9775c5d64 100644 --- a/cpp/include/raft/sparse/convert/detail/bitmap_to_csr.cuh +++ b/cpp/include/raft/sparse/convert/detail/bitmap_to_csr.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -12,6 +12,7 @@ #include #include #include +#include #include @@ -116,9 +117,13 @@ void calc_nnz_by_rows(raft::resources const& handle, auto block = bitmap_to_csr_tpb; - calc_nnz_by_rows_kernel<<>>( - bitmap, num_rows, num_cols, bitmap_num, sub_col_nnz, bits_per_sub_col); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, grid, block)(calc_nnz_by_rows_kernel, + bitmap, + num_rows, + num_cols, + bitmap_num, + sub_col_nnz, + bits_per_sub_col); } template @@ -251,9 +256,16 @@ void fill_indices_by_rows(raft::resources const& handle, auto block = bitmap_to_csr_tpb; - fill_indices_by_rows_kernel<<>>( - bitmap, indptr, num_rows, num_cols, nnz, indices, sub_col_nnz, bits_per_sub_col); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, grid, block)( + fill_indices_by_rows_kernel, + bitmap, + indptr, + num_rows, + num_cols, + nnz, + indices, + sub_col_nnz, + bits_per_sub_col); } template #include #include +#include #include @@ -68,8 +69,13 @@ void gpu_repeat_csr(raft::resources const& handle, index_t repeat_csr_tpb = 256; index_t grid = (nnz + repeat_csr_tpb - 1) / (repeat_csr_tpb); - repeat_csr_kernel<<>>( - d_indptr, d_indices, d_repeated_indptr, d_repeated_indices, nnz, repeat_count); + raft::launch_kernel(stream, grid, repeat_csr_tpb)(repeat_csr_kernel, + d_indptr, + d_indices, + d_repeated_indptr, + d_repeated_indices, + nnz, + repeat_count); } template #include #include +#include #include #include @@ -58,9 +59,8 @@ void csr_to_coo( dim3 grid(raft::ceildiv(m, (value_idx)TPB_X), 1, 1); dim3 blk(TPB_X, 1, 1); - csr_to_coo_kernel<<>>(row_ind, m, coo_rows, nnz); - - RAFT_CUDA_TRY(cudaGetLastError()); + raft::launch_kernel(stream, grid, blk)( + csr_to_coo_kernel, row_ind, m, coo_rows, nnz); } }; // end NAMESPACE detail diff --git a/cpp/include/raft/sparse/convert/detail/dense.cuh b/cpp/include/raft/sparse/convert/detail/dense.cuh index 36aa71e533..57107c7a15 100644 --- a/cpp/include/raft/sparse/convert/detail/dense.cuh +++ b/cpp/include/raft/sparse/convert/detail/dense.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -10,6 +10,7 @@ #include #include #include +#include #include @@ -123,8 +124,8 @@ void csr_to_dense(cusparseHandle_t handle, } else { int blockdim = block_dim(ncols); RAFT_CUDA_TRY(cudaMemsetAsync(out, 0, nrows * ncols * sizeof(value_t), stream)); - csr_to_dense_warp_per_row_kernel<<>>( - ncols, csr_data, csr_indptr, csr_indices, out); + raft::launch_kernel(stream, nrows, blockdim)( + csr_to_dense_warp_per_row_kernel, ncols, csr_data, csr_indptr, csr_indices, out); } } diff --git a/cpp/include/raft/sparse/detail/csr.cuh b/cpp/include/raft/sparse/detail/csr.cuh index 3d834d0ecf..152649af1c 100644 --- a/cpp/include/raft/sparse/detail/csr.cuh +++ b/cpp/include/raft/sparse/detail/csr.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -136,18 +137,24 @@ void weak_cc_batched(Index_* labels, bool host_m; Index_ MAX_LABEL = std::numeric_limits::max(); - weak_cc_init_all_kernel - <<>>(labels, N, MAX_LABEL, filter_op); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, raft::ceildiv(N, Index_(TPB_X)), TPB_X)( + weak_cc_init_all_kernel, labels, N, MAX_LABEL, filter_op); int n_iters = 0; do { RAFT_CUDA_TRY(cudaMemsetAsync(state->m, false, sizeof(bool), stream)); - weak_cc_label_device - <<>>( - labels, row_ind, row_ind_ptr, nnz, state->m, start_vertex_id, batch_size, N, filter_op); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, raft::ceildiv(batch_size, Index_(TPB_X)), TPB_X)( + weak_cc_label_device, + labels, + row_ind, + row_ind_ptr, + nnz, + state->m, + start_vertex_id, + batch_size, + N, + filter_op); //** Updating m * raft::update_host(&host_m, state->m, 1, stream); diff --git a/cpp/include/raft/sparse/detail/utils.h b/cpp/include/raft/sparse/detail/utils.h index 0fc59e47e3..c51c9c6164 100644 --- a/cpp/include/raft/sparse/detail/utils.h +++ b/cpp/include/raft/sparse/detail/utils.h @@ -1,11 +1,12 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include +#include namespace raft { namespace sparse { @@ -90,7 +91,7 @@ void iota_fill(value_idx* indices, value_idx nrows, value_idx ncols, cudaStream_ { int blockdim = block_dim(ncols); - iota_fill_block_kernel<<>>(indices, ncols); + raft::launch_kernel(stream, nrows, blockdim)(iota_fill_block_kernel, indices, ncols); } template diff --git a/cpp/include/raft/sparse/linalg/detail/add.cuh b/cpp/include/raft/sparse/linalg/detail/add.cuh index d8416567a6..700c492d02 100644 --- a/cpp/include/raft/sparse/linalg/detail/add.cuh +++ b/cpp/include/raft/sparse/linalg/detail/add.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -180,8 +181,17 @@ size_t csr_add_calc_inds(const int* a_ind, rmm::device_uvector row_counts(m + 1, stream); RAFT_CUDA_TRY(cudaMemsetAsync(row_counts.data(), 0, (m + 1) * sizeof(int), stream)); - csr_add_calc_row_counts_kernel<<>>( - a_ind, a_indptr, a_val, nnz1, b_ind, b_indptr, b_val, nnz2, m, row_counts.data()); + raft::launch_kernel(stream, grid, blk)(csr_add_calc_row_counts_kernel, + a_ind, + a_indptr, + a_val, + nnz1, + b_ind, + b_indptr, + b_val, + nnz2, + m, + row_counts.data()); int cnnz = 0; raft::update_host(&cnnz, row_counts.data() + m, 1, stream); @@ -230,9 +240,19 @@ void csr_add_finalize(const int* a_ind, dim3 grid(raft::ceildiv(m, TPB_X), 1, 1); dim3 blk(TPB_X, 1, 1); - csr_add_kernel<<>>( - a_ind, a_indptr, a_val, nnz1, b_ind, b_indptr, b_val, nnz2, m, c_ind, c_indptr, c_val); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, grid, blk)(csr_add_kernel, + a_ind, + a_indptr, + a_val, + nnz1, + b_ind, + b_indptr, + b_val, + nnz2, + m, + c_ind, + c_indptr, + c_val); } }; // end NAMESPACE detail diff --git a/cpp/include/raft/sparse/linalg/detail/degree.cuh b/cpp/include/raft/sparse/linalg/detail/degree.cuh index 1e65721d20..8679814f5e 100644 --- a/cpp/include/raft/sparse/linalg/detail/degree.cuh +++ b/cpp/include/raft/sparse/linalg/detail/degree.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -10,6 +10,7 @@ #include #include #include +#include #include @@ -50,8 +51,7 @@ void coo_degree(const T* rows, nnz_t nnz, outT* results, cudaStream_t stream) dim3 grid_rc(raft::ceildiv((nnz_t)nnz, (nnz_t)TPB_X), 1, 1); dim3 blk_rc(TPB_X, 1, 1); - coo_degree_kernel<<>>(rows, nnz, results); - RAFT_CUDA_TRY(cudaGetLastError()); + raft::launch_kernel(stream, grid_rc, blk_rc)(coo_degree_kernel, rows, nnz, results); } template @@ -86,8 +86,8 @@ void coo_degree_scalar( { dim3 grid_rc(raft::ceildiv(nnz, static_cast(TPB_X)), 1, 1); dim3 blk_rc(TPB_X, 1, 1); - coo_degree_scalar_kernel - <<>>(rows, vals, nnz, scalar, results); + raft::launch_kernel(stream, grid_rc, blk_rc)( + coo_degree_scalar_kernel, rows, vals, nnz, scalar, results); } /** @@ -105,8 +105,8 @@ void coo_degree_nz(const idx_t* rows, const T* vals, nnz_t nnz, idx_t* results, { dim3 grid_rc(raft::ceildiv(nnz, TPB_X), 1, 1); dim3 blk_rc(TPB_X, 1, 1); - coo_degree_nz_kernel - <<>>(rows, vals, nnz, results); + raft::launch_kernel(stream, grid_rc, blk_rc)( + coo_degree_nz_kernel, rows, vals, nnz, results); } }; // end NAMESPACE detail diff --git a/cpp/include/raft/sparse/linalg/detail/laplacian.cuh b/cpp/include/raft/sparse/linalg/detail/laplacian.cuh index 40662a5484..fc356ce6a0 100644 --- a/cpp/include/raft/sparse/linalg/detail/laplacian.cuh +++ b/cpp/include/raft/sparse/linalg/detail/laplacian.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ #pragma once @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -105,7 +106,10 @@ auto compute_graph_laplacian( auto static constexpr const threads_per_block = 256; auto blocks = std::min(int((dim + threads_per_block - 1) / threads_per_block), 65535); auto stream = resource::get_cuda_stream(res); - detail::compute_graph_laplacian_kernel<<>>( + raft::launch_kernel(stream, blocks, threads_per_block)( + detail::compute_graph_laplacian_kernel, + std::remove_const_t, + std::remove_const_t>, result.get_elements().data(), result_structure.get_indices().data(), result_structure.get_indptr().data(), diff --git a/cpp/include/raft/sparse/linalg/detail/norm.cuh b/cpp/include/raft/sparse/linalg/detail/norm.cuh index 2c9d6710b2..c57767ee45 100644 --- a/cpp/include/raft/sparse/linalg/detail/norm.cuh +++ b/cpp/include/raft/sparse/linalg/detail/norm.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -92,8 +93,8 @@ void csr_row_normalize_l1(const indT* ia, // csr row ex_scan (sorted by row) dim3 grid(raft::ceildiv(m, TPB_X), 1, 1); dim3 blk(TPB_X, 1, 1); - csr_row_normalize_l1_kernel<<>>(ia, vals, nnz, m, result); - RAFT_CUDA_TRY(cudaGetLastError()); + raft::launch_kernel(stream, grid, blk)( + csr_row_normalize_l1_kernel, ia, vals, nnz, m, result); } template @@ -159,8 +160,8 @@ void csr_row_normalize_max(const int* ia, // csr row ind array (sorted by row) dim3 grid(raft::ceildiv(m, TPB_X), 1, 1); dim3 blk(TPB_X, 1, 1); - csr_row_normalize_max_kernel<<>>(ia, vals, nnz, m, result); - RAFT_CUDA_TRY(cudaGetLastError()); + raft::launch_kernel(stream, grid, blk)( + csr_row_normalize_max_kernel, ia, vals, nnz, m, result); } template #include #include +#include #include #include @@ -141,17 +142,17 @@ void coo_symmetrize(COO* in, out->allocate(in->nnz * 2, in->n_rows, in->n_cols, true, stream); - coo_symmetrize_kernel<<>>(in_row_ind.data(), - in->rows(), - in->cols(), - in->vals(), - out->rows(), - out->cols(), - out->vals(), - in->n_rows, - in->nnz, - reduction_op); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, grid, blk)(coo_symmetrize_kernel, + in_row_ind.data(), + in->rows(), + in->cols(), + in->vals(), + out->rows(), + out->cols(), + out->vals(), + in->n_rows, + in->nnz, + reduction_op); } /** @@ -207,18 +208,17 @@ void coo_symmetrize(raft::resources const& handle, handle, raft::make_device_vector_view(out_cols, out_nnz), static_cast(0)); raft::matrix::fill(handle, raft::make_device_vector_view(out_vals, out_nnz), static_cast(0.0)); - coo_symmetrize_kernel<<>>(in_row_ind.data(), - in_rows, - in_cols, - in_vals, - out_rows, - out_cols, - out_vals, - in_n_rows, - in_nnz, - reduction_op); - - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, grid, blk)(coo_symmetrize_kernel, + in_row_ind.data(), + in_rows, + in_cols, + in_vals, + out_rows, + out_cols, + out_vals, + in_n_rows, + in_nnz, + reduction_op); } /** @@ -349,13 +349,11 @@ void from_knn_symmetrize_matrix(const value_idx* __restrict__ knn_indices, rmm::device_uvector row_sizes2(n, stream); RAFT_CUDA_TRY(cudaMemsetAsync(row_sizes2.data(), 0, sizeof(value_idx) * n, stream)); - symmetric_find_size<<>>( - knn_dists, knn_indices, n, k, row_sizes.data(), row_sizes2.data()); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, numBlocks, threadsPerBlock)( + symmetric_find_size, knn_dists, knn_indices, n, k, row_sizes.data(), row_sizes2.data()); - reduce_find_size<<>>( - n, k, row_sizes.data(), row_sizes2.data()); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, raft::ceildiv(n, (value_idx)1024), 1024)( + reduce_find_size, n, k, row_sizes.data(), row_sizes2.data()); // (2) Compute final space needed (n*k + sum(row_sizes)) == 2*n*k // Notice we don't do any merging and leave the result as 2*NNZ @@ -376,9 +374,8 @@ void from_knn_symmetrize_matrix(const value_idx* __restrict__ knn_indices, thrust::exclusive_scan(rmm::exec_policy(stream), __row_sizes, __row_sizes + n, __edges); // (5) Perform final data + data.T operation in tandem with memcpying - symmetric_sum<<>>( - edges, knn_dists, knn_indices, out->vals(), out->cols(), out->rows(), n, k); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, numBlocks, threadsPerBlock)( + symmetric_sum, edges, knn_dists, knn_indices, out->vals(), out->cols(), out->rows(), n, k); } /** diff --git a/cpp/include/raft/sparse/linalg/detail/utils.cuh b/cpp/include/raft/sparse/linalg/detail/utils.cuh index f4f9385ea5..725f4f4a59 100644 --- a/cpp/include/raft/sparse/linalg/detail/utils.cuh +++ b/cpp/include/raft/sparse/linalg/detail/utils.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -7,6 +7,7 @@ #include #include +#include #include #include @@ -114,8 +115,16 @@ void faster_dot_on_csr(raft::resources const& handle, (std::min(value_idx(blocks_per_sm * sm_count * 16), nnz) + block_x - 1) / block_x; dim3 blocks(block_x, block_y, 1); - faster_dot_on_csr_kernel - <<>>(dot, indptr, cols, A, B, nnz, n_rows, dim); + raft::launch_kernel(stream, blocks, tpb, smem_size)( + faster_dot_on_csr_kernel, + dot, + indptr, + cols, + A, + B, + nnz, + n_rows, + dim); } else if (dim < 256) { constexpr int tpb = 128; @@ -126,8 +135,16 @@ void faster_dot_on_csr(raft::resources const& handle, (std::min(value_idx(blocks_per_sm * sm_count * 16), nnz) + block_x - 1) / block_x; dim3 blocks(block_x, block_y, 1); - faster_dot_on_csr_kernel - <<>>(dot, indptr, cols, A, B, nnz, n_rows, dim); + raft::launch_kernel(stream, blocks, tpb, smem_size)( + faster_dot_on_csr_kernel, + dot, + indptr, + cols, + A, + B, + nnz, + n_rows, + dim); } else if (dim < 512) { constexpr int tpb = 256; cudaOccupancyMaxActiveBlocksPerMultiprocessor( @@ -137,8 +154,16 @@ void faster_dot_on_csr(raft::resources const& handle, (std::min(value_idx(blocks_per_sm * sm_count * 16), nnz) + block_x - 1) / block_x; dim3 blocks(block_x, block_y, 1); - faster_dot_on_csr_kernel - <<>>(dot, indptr, cols, A, B, nnz, n_rows, dim); + raft::launch_kernel(stream, blocks, tpb, smem_size)( + faster_dot_on_csr_kernel, + dot, + indptr, + cols, + A, + B, + nnz, + n_rows, + dim); } else { constexpr int tpb = 512; cudaOccupancyMaxActiveBlocksPerMultiprocessor( @@ -148,8 +173,16 @@ void faster_dot_on_csr(raft::resources const& handle, (std::min(value_idx(blocks_per_sm * sm_count * 16), nnz) + block_x - 1) / block_x; dim3 blocks(block_x, block_y, 1); - faster_dot_on_csr_kernel - <<>>(dot, indptr, cols, A, B, nnz, n_rows, dim); + raft::launch_kernel(stream, blocks, tpb, smem_size)( + faster_dot_on_csr_kernel, + dot, + indptr, + cols, + A, + B, + nnz, + n_rows, + dim); } RAFT_CUDA_TRY(cudaPeekAtLastError()); diff --git a/cpp/include/raft/sparse/op/detail/filter.cuh b/cpp/include/raft/sparse/op/detail/filter.cuh index 631e17ec9b..f246cfb04a 100644 --- a/cpp/include/raft/sparse/op/detail/filter.cuh +++ b/cpp/include/raft/sparse/op/detail/filter.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -116,18 +117,18 @@ void coo_remove_scalar(const idx_t* rows, dim3 grid(raft::ceildiv(n, static_cast(TPB_X)), 1, 1); dim3 blk(TPB_X, 1, 1); - coo_remove_scalar_kernel<<>>(rows, - cols, - vals, - nnz, - crows, - ccols, - cvals, - dev_ex_scan.get(), - dev_cur_ex_scan.get(), - n, - scalar); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, grid, blk)(coo_remove_scalar_kernel, + rows, + cols, + vals, + nnz, + crows, + ccols, + cvals, + dev_ex_scan.get(), + dev_cur_ex_scan.get(), + n, + scalar); } /** @@ -153,7 +154,6 @@ void coo_remove_scalar(COO* in, cudaMemsetAsync(row_count.data(), 0, static_cast(in->n_rows) * sizeof(nnz_t), stream)); linalg::coo_degree(in->rows(), in->nnz, row_count.data(), stream); - RAFT_CUDA_TRY(cudaPeekAtLastError()); using nnz_cast_t = std::conditional_t, unsigned long long, nnz_t>; linalg::coo_degree_scalar(in->rows(), diff --git a/cpp/include/raft/sparse/op/detail/reduce.cuh b/cpp/include/raft/sparse/op/detail/reduce.cuh index cc1ebbfc3c..699c6238ea 100644 --- a/cpp/include/raft/sparse/op/detail/reduce.cuh +++ b/cpp/include/raft/sparse/op/detail/reduce.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -16,6 +16,7 @@ #include #include #include +#include #include @@ -94,8 +95,8 @@ void compute_duplicates_mask( { RAFT_CUDA_TRY(cudaMemsetAsync(mask, 0, nnz * sizeof(value_idx), stream)); - compute_duplicates_diffs_kernel<<>>( - rows, cols, mask, nnz); + raft::launch_kernel(stream, raft::ceildiv(nnz, (nnz_t)256), 256)( + compute_duplicates_diffs_kernel, rows, cols, mask, nnz); } /** @@ -143,8 +144,16 @@ void max_duplicates(raft::resources const& handle, out.allocate(size, m, n, true, stream); // perform reduce - max_duplicates_kernel<<>>( - rows, cols, vals, diff.data() + 1, out.rows(), out.cols(), out.vals(), nnz); + raft::launch_kernel(stream, raft::ceildiv(nnz, (nnz_t)256), 256)( + max_duplicates_kernel, + rows, + cols, + vals, + diff.data() + 1, + out.rows(), + out.cols(), + out.vals(), + nnz); } }; // END namespace detail diff --git a/cpp/include/raft/sparse/op/detail/row_op.cuh b/cpp/include/raft/sparse/op/detail/row_op.cuh index b28f0ca900..b81c385434 100644 --- a/cpp/include/raft/sparse/op/detail/row_op.cuh +++ b/cpp/include/raft/sparse/op/detail/row_op.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -53,9 +54,8 @@ void csr_row_op(const Index_* row_ind, Index_ n_rows, Index_ nnz, Lambda op, cud { dim3 grid(raft::ceildiv(n_rows, Index_(TPB_X)), 1, 1); dim3 blk(TPB_X, 1, 1); - csr_row_op_kernel<<>>(row_ind, n_rows, nnz, op); - - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, grid, blk)( + csr_row_op_kernel, row_ind, n_rows, nnz, op); } }; // namespace detail diff --git a/cpp/include/raft/sparse/solver/detail/lanczos.cuh b/cpp/include/raft/sparse/solver/detail/lanczos.cuh index 9aa4304c04..c03ce32383 100644 --- a/cpp/include/raft/sparse/solver/detail/lanczos.cuh +++ b/cpp/include/raft/sparse/solver/detail/lanczos.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -49,6 +49,7 @@ #include #include #include +#include #include #include @@ -158,14 +159,20 @@ void lanczos_solve_ritz( int blockSize = 256; int numBlocks = (ncv + blockSize - 1) / blockSize; - kernel_triangular_populate - <<>>(triangular_matrix.data_handle(), beta.data_handle(), ncv); + raft::launch_kernel(stream, blockSize, numBlocks)(kernel_triangular_populate, + triangular_matrix.data_handle(), + beta.data_handle(), + ncv); if (beta_k) { int threadsPerBlock = 256; int blocksPerGrid = (k + threadsPerBlock - 1) / threadsPerBlock; - kernel_triangular_beta_k<<>>( - triangular_matrix.data_handle(), beta_k.value().data_handle(), (int)k, ncv); + raft::launch_kernel(stream, blocksPerGrid, threadsPerBlock)( + kernel_triangular_beta_k, + triangular_matrix.data_handle(), + beta_k.value().data_handle(), + k, + ncv); } auto triangular_matrix_view = @@ -371,7 +378,8 @@ void lanczos_aux(raft::resources const& handle, auto uu_i = raft::make_device_scalar_view(uu.data_handle() + uu.stride(1) * i); // uu(0, i) raft::linalg::add(handle, make_const_mdspan(alpha_i), make_const_mdspan(uu_i), alpha_i); - kernel_clamp_down<<<1, 1, 0, stream>>>(alpha_i.data_handle(), static_cast(1e-9)); + raft::launch_kernel(stream, 1, 1)( + kernel_clamp_down, alpha_i.data_handle(), static_cast(1e-9)); auto output = raft::make_device_vector_view( beta.data_handle() + beta.stride(1) * i, 1); @@ -382,19 +390,26 @@ void lanczos_aux(raft::resources const& handle, int blockSize = 256; int numBlocks = (n + blockSize - 1) / blockSize; - kernel_clamp_down_vector<<>>( - u.data_handle(), static_cast(1e-7), n); + raft::launch_kernel(stream, numBlocks, blockSize)( + kernel_clamp_down_vector, u.data_handle(), static_cast(1e-7), n); - kernel_clamp_down<<<1, 1, 0, stream>>>(beta.data_handle() + beta.stride(1) * i, - static_cast(1e-6)); + raft::launch_kernel(stream, 1, 1)(kernel_clamp_down, + beta.data_handle() + beta.stride(1) * i, + static_cast(1e-6)); if (i >= end_idx - 1) { break; } int threadsPerBlock = 256; int blocksPerGrid = (n + threadsPerBlock - 1) / threadsPerBlock; - kernel_normalize<<>>( - u.data_handle(), beta.data_handle(), i, n, v.data_handle(), V.data_handle(), n); + raft::launch_kernel(stream, blocksPerGrid, threadsPerBlock)(kernel_normalize, + u.data_handle(), + beta.data_handle(), + i, + n, + v.data_handle(), + V.data_handle(), + n); } } diff --git a/cpp/include/raft/sparse/solver/detail/mst_solver_inl.cuh b/cpp/include/raft/sparse/solver/detail/mst_solver_inl.cuh index feaa6c0d21..e867c1f31a 100644 --- a/cpp/include/raft/sparse/solver/detail/mst_solver_inl.cuh +++ b/cpp/include/raft/sparse/solver/detail/mst_solver_inl.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -233,8 +234,15 @@ void MST_solver::alteration() RAFT_EXPECTS(curand_status == CURAND_STATUS_SUCCESS, "MST: CURAND cleanup failed"); // Alterate the weights, make all undirected edge weight unique while keeping Wuv == Wvu - detail::alteration_kernel<<>>( - v, e, offsets, indices, weights, max, rand_values.data(), altered_weights.data()); + raft::launch_kernel(stream, nblocks, nthreads)(detail::alteration_kernel, + v, + e, + offsets, + indices, + weights, + max, + rand_values.data(), + altered_weights.data()); } // updates colors of vertices by propagating the lower color to the higher @@ -262,16 +270,22 @@ void MST_solver::label_prop(vertex_t* while (!done.value(stream)) { done.set_value_async(true_val, stream); - detail::min_pair_colors<<>>( - v, indices, new_mst_edge_ptr, color_ptr, color_index, next_color_ptr); - - detail::update_colors<<>>( - v, color_ptr, color_index, next_color_ptr, done_ptr); + raft::launch_kernel(stream, min_pair_nblocks, min_pair_nthreads)( + detail::min_pair_colors, + v, + indices, + new_mst_edge_ptr, + color_ptr, + color_index, + next_color_ptr); + + raft::launch_kernel(stream, min_pair_nblocks, min_pair_nthreads)( + detail::update_colors, v, color_ptr, color_index, next_color_ptr, done_ptr); i++; } - detail::final_color_indices<<>>( - v, color_ptr, color_index); + raft::launch_kernel(stream, min_pair_nblocks, min_pair_nthreads)( + detail::final_color_indices, v, color_ptr, color_index); } // Finds the minimum edge from each vertex to the lowest color @@ -292,15 +306,17 @@ void MST_solver::min_edge_per_vertex() alteration_t* min_edge_color_ptr = min_edge_color.data(); alteration_t* altered_weights_ptr = altered_weights.data(); - detail::kernel_min_edge_per_vertex<<>>(offsets, - indices, - altered_weights_ptr, - color_ptr, - color_index, - new_mst_edge_ptr, - mst_edge_ptr, - min_edge_color_ptr, - v); + raft::launch_kernel(stream, v, n_threads)( + detail::kernel_min_edge_per_vertex, + offsets, + indices, + altered_weights_ptr, + color_ptr, + color_index, + new_mst_edge_ptr, + mst_edge_ptr, + min_edge_color_ptr, + v); } // Finds the minimum edge from each supervertex to the lowest color @@ -322,32 +338,36 @@ void MST_solver::min_edge_per_superver vertex_t* temp_dst_ptr = temp_dst.data(); weight_t* temp_weights_ptr = temp_weights.data(); - detail::min_edge_per_supervertex<<>>(color_ptr, - color_index, - new_mst_edge_ptr, - mst_edge_ptr, - indices, - weights, - altered_weights_ptr, - temp_src_ptr, - temp_dst_ptr, - temp_weights_ptr, - min_edge_color_ptr, - v, - symmetrize_output); + raft::launch_kernel(stream, nblocks, nthreads)( + detail::min_edge_per_supervertex, + color_ptr, + color_index, + new_mst_edge_ptr, + mst_edge_ptr, + indices, + weights, + altered_weights_ptr, + temp_src_ptr, + temp_dst_ptr, + temp_weights_ptr, + min_edge_color_ptr, + v, + symmetrize_output); // the above kernel only adds directed mst edges in the case where // a pair of vertices don't pick the same min edge between them // so, now we add the reverse edge to make it undirected if (symmetrize_output) { - detail::add_reverse_edge<<>>(new_mst_edge_ptr, - indices, - weights, - temp_src_ptr, - temp_dst_ptr, - temp_weights_ptr, - v, - symmetrize_output); + raft::launch_kernel(stream, nblocks, nthreads)( + detail::add_reverse_edge, + new_mst_edge_ptr, + indices, + weights, + temp_src_ptr, + temp_dst_ptr, + temp_weights_ptr, + v, + symmetrize_output); } } @@ -361,8 +381,8 @@ void MST_solver::check_termination() edge_t* mst_edge_count_ptr = mst_edge_count.data(); vertex_t* temp_src_ptr = temp_src.data(); - detail::kernel_count_new_mst_edges<<>>( - temp_src_ptr, mst_edge_count_ptr, 2 * v); + raft::launch_kernel(stream, nblocks, nthreads)( + detail::kernel_count_new_mst_edges, temp_src_ptr, mst_edge_count_ptr, 2 * v); } template diff --git a/cpp/include/raft/sparse/solver/detail/svds_sign_correction.cuh b/cpp/include/raft/sparse/solver/detail/svds_sign_correction.cuh index e35889d91d..cfa7993524 100644 --- a/cpp/include/raft/sparse/solver/detail/svds_sign_correction.cuh +++ b/cpp/include/raft/sparse/solver/detail/svds_sign_correction.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -148,8 +149,8 @@ void svd_sign_correction( ValueTypeT* U_ptr = U ? U->data_handle() : nullptr; ValueTypeT* Vt_ptr = Vt ? Vt->data_handle() : nullptr; - svd_sign_correction_kernel<<>>(U_ptr, Vt_ptr, m, n, k); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, k, threads_per_block, smem_size)( + svd_sign_correction_kernel, U_ptr, Vt_ptr, m, n, k); } } // namespace raft::sparse::solver::detail diff --git a/cpp/include/raft/spectral/detail/matrix_wrappers.hpp b/cpp/include/raft/spectral/detail/matrix_wrappers.hpp index 87e45cb3fa..16ac7e4192 100644 --- a/cpp/include/raft/spectral/detail/matrix_wrappers.hpp +++ b/cpp/include/raft/spectral/detail/matrix_wrappers.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -16,6 +16,7 @@ #include #include #include +#include #include @@ -384,8 +385,8 @@ struct laplacian_matrix_t : sparse_matrix_t { dim3 gridDim{std::min((n + BLOCK_SIZE - 1) / BLOCK_SIZE, 65535), 1, 1}; dim3 blockDim{BLOCK_SIZE, 1, 1}; - diagmv<<>>(n, alpha, diagonal_.raw(), x, y); - RAFT_CHECK_CUDA(stream); + raft::launch_kernel(stream, gridDim, blockDim)( + diagmv, n, alpha, diagonal_.raw(), x, y); // Apply adjacency matrix // diff --git a/cpp/include/raft/stats/detail/batched/silhouette_score.cuh b/cpp/include/raft/stats/detail/batched/silhouette_score.cuh index 17829f1cd1..2af5fcb4f1 100644 --- a/cpp/include/raft/stats/detail/batched/silhouette_score.cuh +++ b/cpp/include/raft/stats/detail/batched/silhouette_score.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -159,8 +160,17 @@ void compute_chunked_a_b(raft::resources const& handle, dim3 grid_size(raft::ceildiv(dist_rows, (value_idx)block_size.x), raft::ceildiv(dist_cols, (value_idx)block_size.y)); - detail::compute_chunked_a_b_kernel<<>>( - a, b, row_offset, col_offset, y, n_labels, cluster_counts, distances, dist_rows, dist_cols); + raft::launch_kernel(stream, grid_size, block_size)(detail::compute_chunked_a_b_kernel, + a, + b, + row_offset, + col_offset, + y, + n_labels, + cluster_counts, + distances, + dist_rows, + dist_cols); } template @@ -204,8 +214,8 @@ value_t silhouette_score( dim3 block_size(std::min(n_rows, 32), std::min(n_labels, 32)); dim3 grid_size(raft::ceildiv(n_rows, (value_idx)block_size.x), raft::ceildiv(n_labels, (label_idx)block_size.y)); - detail::fill_b_kernel<<>>( - b_ptr, y, n_rows, n_labels, cluster_counts.data()); + raft::launch_kernel(stream, grid_size, block_size)( + detail::fill_b_kernel, b_ptr, y, n_rows, n_labels, cluster_counts.data()); resource::wait_stream_pool_on_stream(handle); diff --git a/cpp/include/raft/stats/detail/contingencyMatrix.cuh b/cpp/include/raft/stats/detail/contingencyMatrix.cuh index c6b47ee47e..bb54aec935 100644 --- a/cpp/include/raft/stats/detail/contingencyMatrix.cuh +++ b/cpp/include/raft/stats/detail/contingencyMatrix.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -58,9 +59,13 @@ void computeCMatWAtomics(const T* groundTruth, cudaFuncSetCacheConfig(devConstructContingencyMatrix, cudaFuncCachePreferL1)); static const int block = 128; auto grid = raft::ceildiv(nSamples, block); - devConstructContingencyMatrix<<>>( - groundTruth, predictedLabel, nSamples, outMat, outIdxOffset, outDimN); - RAFT_CUDA_TRY(cudaGetLastError()); + raft::launch_kernel(stream, grid, block)(devConstructContingencyMatrix, + groundTruth, + predictedLabel, + nSamples, + outMat, + outIdxOffset, + outDimN); } template @@ -102,9 +107,14 @@ void computeCMatWSmemAtomics(const T* groundTruth, static const int block = 128; auto grid = raft::ceildiv(nSamples, block); size_t smemSizePerBlock = outDimN * outDimN * sizeof(OutT); - devConstructContingencyMatrixSmem<<>>( - groundTruth, predictedLabel, nSamples, outMat, outIdxOffset, outDimN); - RAFT_CUDA_TRY(cudaGetLastError()); + raft::launch_kernel(stream, grid, block, smemSizePerBlock)( + devConstructContingencyMatrixSmem, + groundTruth, + predictedLabel, + nSamples, + outMat, + outIdxOffset, + outDimN); } template diff --git a/cpp/include/raft/stats/detail/dispersion.cuh b/cpp/include/raft/stats/detail/dispersion.cuh index 17ea419b48..3fc21de6a3 100644 --- a/cpp/include/raft/stats/detail/dispersion.cuh +++ b/cpp/include/raft/stats/detail/dispersion.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -10,6 +10,7 @@ #include #include #include +#include #include @@ -109,17 +110,15 @@ DataT dispersion(const DataT* centroids, } RAFT_CUDA_TRY(cudaMemsetAsync(mu, 0, sizeof(DataT) * dim, stream)); RAFT_CUDA_TRY(cudaMemsetAsync(result.data(), 0, sizeof(DataT), stream)); - weightedMeanKernel - <<>>(mu, centroids, clusterSizes, dim, nClusters); - RAFT_CUDA_TRY(cudaGetLastError()); + raft::launch_kernel(stream, grid, TPB)( + weightedMeanKernel, mu, centroids, clusterSizes, dim, nClusters); DataT ratio = DataT(1) / DataT(nPoints); raft::linalg::scalarMultiply(mu, mu, ratio, dim, stream); // finally, compute the dispersion constexpr int ItemsPerThread = 4; int nblks = raft::ceildiv(dim * nClusters, TPB * ItemsPerThread); - dispersionKernel - <<>>(result.data(), centroids, clusterSizes, mu, dim, nClusters); - RAFT_CUDA_TRY(cudaGetLastError()); + raft::launch_kernel(stream, nblks, TPB)( + dispersionKernel, result.data(), centroids, clusterSizes, mu, dim, nClusters); DataT h_result; raft::update_host(&h_result, result.data(), 1, stream); raft::interruptible::synchronize(stream); diff --git a/cpp/include/raft/stats/detail/histogram.cuh b/cpp/include/raft/stats/detail/histogram.cuh index 7800843844..253d2d324f 100644 --- a/cpp/include/raft/stats/detail/histogram.cuh +++ b/cpp/include/raft/stats/detail/histogram.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -94,8 +95,8 @@ void gmemHist(int* bins, { auto blks = computeGridDim( nrows, ncols, (const void*)gmemHistKernel); - gmemHistKernel - <<>>(bins, data, nrows, nbins, binner); + raft::launch_kernel(stream, blks, ThreadsPerBlock)( + gmemHistKernel, bins, data, nrows, nbins, binner); } template @@ -145,8 +146,8 @@ void smemHist(int* bins, auto blks = computeGridDim( nrows, ncols, (const void*)smemHistKernel); size_t smemSize = nbins * sizeof(unsigned); - smemHistKernel - <<>>(bins, data, nrows, nbins, binner); + raft::launch_kernel(stream, blks, ThreadsPerBlock, smemSize)( + smemHistKernel, bins, data, nrows, nbins, binner); } template @@ -231,8 +232,13 @@ void smemBitsHist(int* bins, auto blks = computeGridDim( nrows, ncols, (const void*)smemBitsHistKernel); size_t smemSize = raft::ceildiv(nbins, Bits::WORD_BITS / Bits::BIN_BITS) * sizeof(int); - smemBitsHistKernel - <<>>(bins, data, nrows, nbins, binner); + raft::launch_kernel(stream, blks, ThreadsPerBlock, smemSize)( + smemBitsHistKernel, + bins, + data, + nrows, + nbins, + binner); } #define INVALID_KEY -1 @@ -349,8 +355,15 @@ void smemHashHist(int* bins, nrows, ncols, (const void*)smemHashHistKernel); int hashSize = computeHashTableSize(); size_t smemSize = hashSize * sizeof(int2) + sizeof(int); - smemHashHistKernel<<>>( - bins, data, nrows, nbins, binner, hashSize, flushThreshold); + raft::launch_kernel(stream, blks, ThreadsPerBlock, smemSize)( + smemHashHistKernel, + bins, + data, + nrows, + nbins, + binner, + hashSize, + flushThreshold); } template diff --git a/cpp/include/raft/stats/detail/meanvar.cuh b/cpp/include/raft/stats/detail/meanvar.cuh index 2ca64536b2..19e3ba421f 100644 --- a/cpp/include/raft/stats/detail/meanvar.cuh +++ b/cpp/include/raft/stats/detail/meanvar.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -8,6 +8,7 @@ #include #include #include +#include namespace raft { namespace stats::detail { @@ -208,12 +209,13 @@ void meanvar( const uint64_t len = uint64_t(D) * uint64_t(N); ASSERT(len <= uint64_t(std::numeric_limits::max()), "N * D does not fit the indexing type"); - meanvar_kernel_rowmajor<<>>(data, mvs, locks, len, D); - meanvar_kernel_fill - <<(D, BlockSize), BlockSize, 0, stream>>>(mean, var, mvs, D, sample); + raft::launch_kernel(stream, gs, bs)( + meanvar_kernel_rowmajor, data, mvs, locks, len, D); + raft::launch_kernel(stream, raft::ceildiv(D, BlockSize), BlockSize)( + meanvar_kernel_fill, mean, var, mvs, D, sample); } else { - meanvar_kernel_colmajor - <<>>(mean, var, data, D, N, sample); + raft::launch_kernel(stream, D, BlockSize)( + meanvar_kernel_colmajor, mean, var, data, D, N, sample); } RAFT_CHECK_CUDA(stream); } diff --git a/cpp/include/raft/stats/detail/minmax.cuh b/cpp/include/raft/stats/detail/minmax.cuh index a52640aa15..0e53c4a3e8 100644 --- a/cpp/include/raft/stats/detail/minmax.cuh +++ b/cpp/include/raft/stats/detail/minmax.cuh @@ -8,6 +8,7 @@ #include #include #include +#include #include @@ -186,8 +187,8 @@ void minmax(const T* data, using E = typename encode_traits::E; int nblks = raft::ceildiv(ncols, TPB); T init_val = std::numeric_limits::max(); - minmaxInitKernel<<>>(ncols, globalmin, globalmax, init_val); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, nblks, TPB)( + minmaxInitKernel, ncols, globalmin, globalmax, init_val); nblks = raft::ceildiv(nrows * ncols, TPB); nblks = min(nblks, 65536); size_t smemSize = sizeof(T) * 2 * ncols; @@ -199,21 +200,20 @@ void minmax(const T* data, int num_batches = raft::ceildiv(ncols, batch_ncols); smemSize = sizeof(T) * 2 * batch_ncols; - minmaxKernel<<>>(data, - rowids, - colids, - nrows, - ncols, - row_stride, - globalmin, - globalmax, - sampledcols, - init_val, - batch_ncols, - num_batches); - RAFT_CUDA_TRY(cudaPeekAtLastError()); - decodeKernel<<>>(globalmin, globalmax, ncols); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, nblks, TPB, smemSize)(minmaxKernel, + data, + rowids, + colids, + nrows, + ncols, + row_stride, + globalmin, + globalmax, + sampledcols, + init_val, + batch_ncols, + num_batches); + raft::launch_kernel(stream, nblks, TPB)(decodeKernel, globalmin, globalmax, ncols); } }; // end namespace detail diff --git a/cpp/include/raft/stats/detail/mutual_info_score.cuh b/cpp/include/raft/stats/detail/mutual_info_score.cuh index e4c72b36fd..0ec6600207 100644 --- a/cpp/include/raft/stats/detail/mutual_info_score.cuh +++ b/cpp/include/raft/stats/detail/mutual_info_score.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ /** @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -150,8 +151,14 @@ double mutual_info_score(const T* firstClusterArray, raft::ceildiv(numUniqueClasses, numThreadsPerBlock.y)); // calling the kernel - mutual_info_kernel<<>>( - dContingencyMatrix.data(), a.data(), b.data(), numUniqueClasses, size, d_MI.data()); + raft::launch_kernel(stream, numBlocks, numThreadsPerBlock)( + mutual_info_kernel, + dContingencyMatrix.data(), + a.data(), + b.data(), + numUniqueClasses, + size, + d_MI.data()); // updating in the host memory h_MI = d_MI.value(stream); diff --git a/cpp/include/raft/stats/detail/neighborhood_recall.cuh b/cpp/include/raft/stats/detail/neighborhood_recall.cuh index 347f77c094..f856261a3c 100644 --- a/cpp/include/raft/stats/detail/neighborhood_recall.cuh +++ b/cpp/include/raft/stats/detail/neighborhood_recall.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -99,8 +100,14 @@ void neighborhood_recall( auto constexpr kThreadsPerBlock = 32; auto const num_blocks = indices.extent(0); - neighborhood_recall<<>>( - indices, ref_indices, distances, ref_distances, recall_score, eps); + raft::launch_kernel(res, num_blocks, kThreadsPerBlock)( + neighborhood_recall, + indices, + ref_indices, + distances, + ref_distances, + recall_score, + eps); } } // namespace stats::detail diff --git a/cpp/include/raft/stats/detail/rand_index.cuh b/cpp/include/raft/stats/detail/rand_index.cuh index d3c9353926..0a3fc6b43c 100644 --- a/cpp/include/raft/stats/detail/rand_index.cuh +++ b/cpp/include/raft/stats/detail/rand_index.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -45,6 +45,7 @@ #include #include #include +#include #include @@ -139,8 +140,13 @@ double compute_rand_index(const T* firstClusterArray, raft::ceildiv(size, numThreadsPerBlock.y)); // calling the kernel - computeTheNumerator<<>>( - firstClusterArray, secondClusterArray, size, arr_buf.data(), arr_buf.data() + 1); + raft::launch_kernel(stream, numBlocks, numThreadsPerBlock)( + computeTheNumerator, + firstClusterArray, + secondClusterArray, + size, + arr_buf.data(), + arr_buf.data() + 1); // synchronizing and updating the calculated values of a and b from device to host uint64_t ab_host[2] = {0}; diff --git a/cpp/include/raft/stats/detail/scores.cuh b/cpp/include/raft/stats/detail/scores.cuh index 062a47f2e4..8a81deb969 100644 --- a/cpp/include/raft/stats/detail/scores.cuh +++ b/cpp/include/raft/stats/detail/scores.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -163,9 +164,12 @@ void regression_metrics(const T* predictions, rmm::device_uvector tmp_sums(2 * sizeof(double), stream); RAFT_CUDA_TRY(cudaMemsetAsync(tmp_sums.data(), 0, 2 * sizeof(double), stream)); - reg_metrics_kernel<<>>( - predictions, ref_predictions, n, abs_diffs_array.data(), tmp_sums.data()); - RAFT_CUDA_TRY(cudaGetLastError()); + raft::launch_kernel(stream, block_cnt, thread_cnt)(reg_metrics_kernel, + predictions, + ref_predictions, + n, + abs_diffs_array.data(), + tmp_sums.data()); raft::update_host(&mean_errors[0], tmp_sums.data(), 2, stream); raft::interruptible::synchronize(stream); diff --git a/cpp/include/raft/stats/detail/silhouette_score.cuh b/cpp/include/raft/stats/detail/silhouette_score.cuh index ded61f93c9..3d11cf4710 100644 --- a/cpp/include/raft/stats/detail/silhouette_score.cuh +++ b/cpp/include/raft/stats/detail/silhouette_score.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -17,6 +17,7 @@ #include #include #include +#include #include @@ -246,14 +247,14 @@ DataT silhouette_score( dim3 numBlocks(raft::ceildiv(nRows, numThreadsPerBlock.x), 1, 1); // calling the kernel - populateAKernel<<>>( - sampleToClusterSumOfDistances.data(), - binCountArray.data(), - d_aArray.data(), - labels, - nRows, - nLabels, - std::numeric_limits::max()); + raft::launch_kernel(stream, numBlocks, numThreadsPerBlock)(populateAKernel, + sampleToClusterSumOfDistances.data(), + binCountArray.data(), + d_aArray.data(), + labels, + nRows, + nLabels, + std::numeric_limits::max()); // elementwise dividing by bincounts rmm::device_uvector averageDistanceBetweenSampleAndCluster(nRows * nLabels, stream); diff --git a/cpp/include/raft/stats/detail/trustworthiness_score.cuh b/cpp/include/raft/stats/detail/trustworthiness_score.cuh index d92ecf355c..9bcd8045fe 100644 --- a/cpp/include/raft/stats/detail/trustworthiness_score.cuh +++ b/cpp/include/raft/stats/detail/trustworthiness_score.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -180,21 +181,21 @@ double trustworthiness_score(const raft::resources& h, int work = curBatchSize * n; int n_blocks = raft::ceildiv(work, N_THREADS); - build_lookup_table<<>>( - lookup_table.data(), X_ind.data(), n, work); + raft::launch_kernel(stream, n_blocks, N_THREADS)( + build_lookup_table, lookup_table.data(), X_ind.data(), n, work); RAFT_CUDA_TRY(cudaMemsetAsync(t_dbuf.data(), 0, sizeof(double), stream)); work = curBatchSize * (n_neighbors + 1); n_blocks = raft::ceildiv(work, N_THREADS); - compute_rank<<>>( + raft::launch_kernel(stream, n_blocks, N_THREADS)( + compute_rank, t_dbuf.data(), lookup_table.data(), &emb_ind.data()[(n - toDo) * (n_neighbors + 1)], n, n_neighbors + 1, work); - RAFT_CUDA_TRY(cudaPeekAtLastError()); t += t_dbuf.value(stream); diff --git a/cpp/include/raft/util/cache.cuh b/cpp/include/raft/util/cache.cuh index d67e1e6d3d..af75372339 100644 --- a/cpp/include/raft/util/cache.cuh +++ b/cpp/include/raft/util/cache.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -180,8 +181,8 @@ class Cache { void GetVecs(const int* idx, int n, math_t* out, cudaStream_t stream) { if (n > 0) { - get_vecs<<>>(cache.data(), n_vec, idx, n, out); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, raft::ceildiv(n * n_vec, TPB), TPB)( + get_vecs, cache.data(), n_vec, idx, n, out); } } @@ -215,9 +216,15 @@ class Cache { const int* tile_idx = nullptr) { if (n > 0) { - store_vecs<<>>( - tile, n_tile, n_vec, tile_idx, n, cache_idx, cache.data(), cache.size() / n_vec); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, raft::ceildiv(n * n_vec, TPB), TPB)(store_vecs, + tile, + n_tile, + n_vec, + tile_idx, + n, + cache_idx, + cache.data(), + cache.size() / n_vec); } } @@ -247,16 +254,16 @@ class Cache { { n_iter++; // we increase the iteration counter, that is used to time stamp // accessing entries from the cache - get_cache_idx<<>>(keys, - n, - cached_keys.data(), - n_cache_sets, - associativity, - cache_time.data(), - cache_idx, - is_cached, - n_iter); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, raft::ceildiv(n, TPB), TPB)(get_cache_idx, + keys, + n, + cached_keys.data(), + n_cache_sets, + associativity, + cache_time.data(), + cache_idx, + is_cached, + n_iter); } /** @brief Map a set of keys to cache indices. @@ -339,10 +346,15 @@ class Cache { RAFT_CUDA_TRY(cudaMemsetAsync(cidx, 255, n * sizeof(int), stream)); const int nthreads = associativity <= 32 ? associativity : 32; - assign_cache_idx<<>>( - keys, n, ws_tmp.data(), cached_keys.data(), n_cache_sets, cache_time.data(), n_iter, cidx); - - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, n_cache_sets, nthreads)(assign_cache_idx, + keys, + n, + ws_tmp.data(), + cached_keys.data(), + n_cache_sets, + cache_time.data(), + n_iter, + cidx); if (debug_mode) RAFT_CUDA_TRY(cudaDeviceSynchronize()); } diff --git a/cpp/include/raft/util/detail/scatter.cuh b/cpp/include/raft/util/detail/scatter.cuh index f7eec52985..3b17e9f0a3 100644 --- a/cpp/include/raft/util/detail/scatter.cuh +++ b/cpp/include/raft/util/detail/scatter.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -7,6 +7,7 @@ #include #include +#include #include namespace raft { @@ -36,8 +37,8 @@ void scatterImpl( DataT* out, const DataT* in, const IdxT* idx, IdxT len, Lambda op, cudaStream_t stream) { const IdxT nblks = raft::ceildiv(VecLen ? len / VecLen : len, (IdxT)TPB); - scatterKernel<<>>(out, in, idx, len, op); - RAFT_CUDA_TRY(cudaGetLastError()); + raft::launch_kernel(stream, nblks, TPB)( + scatterKernel, out, in, idx, len, op); } } // namespace detail diff --git a/cpp/include/raft/util/kernel_launch.hpp b/cpp/include/raft/util/kernel_launch.hpp new file mode 100644 index 0000000000..5c845aa3ca --- /dev/null +++ b/cpp/include/raft/util/kernel_launch.hpp @@ -0,0 +1,206 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include +#include + +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace raft { + +namespace detail { + +/** + * @brief Format a cuda_error message with an explicit call-site location. + * + * Mirrors SET_ERROR_MSG / RAFT_CUDA_TRY formatting but does not use those macros, so the reported + * location is the caller's rather than this header. The enclosing function is reported too, since + * it names the template instantiation that the file and line alone cannot. + */ +inline std::string format_cuda_launch_error(cudaError_t status, std::source_location location) +{ + char const* location_prefix = "CUDA error encountered at: "; + char const* location_fmt = "file=%s line=%d function=%s: "; + char const* fmt = "call='%s', Reason=%s:%s"; + char const* call = "cudaLaunchKernel"; + char const* file = location.file_name(); + auto line = static_cast(location.line()); + char const* function = location.function_name(); + + int size1 = std::snprintf(nullptr, 0, "%s", location_prefix); + int size2 = std::snprintf(nullptr, 0, location_fmt, file, line, function); + int size3 = + std::snprintf(nullptr, 0, fmt, call, cudaGetErrorName(status), cudaGetErrorString(status)); + if (size1 < 0 || size2 < 0 || size3 < 0) { + throw raft::exception("Error in snprintf, cannot handle raft exception."); + } + auto size = static_cast(size1 + size2 + size3 + 1); + std::vector buf(size); + std::snprintf(buf.data(), static_cast(size1) + 1, "%s", location_prefix); + std::snprintf( + buf.data() + size1, static_cast(size2) + 1, location_fmt, file, line, function); + std::snprintf(buf.data() + size1 + size2, + static_cast(size3) + 1, + fmt, + call, + cudaGetErrorName(status), + cudaGetErrorString(status)); + return std::string(buf.data(), buf.data() + size - 1); +} + +inline void throw_on_cuda_launch_error(cudaError_t status, std::source_location location) +{ + if (status == cudaSuccess) { return; } + cudaGetLastError(); // clear sticky error + throw raft::cuda_error(format_cuda_launch_error(status, location)); +} + +} // namespace detail + +/** + * @brief Temporary object that launches a CUDA kernel with call-site error reporting. + * + * Capture @c std::source_location on @ref launch_kernel, then launch via rvalue @c operator(). + * Prefer the one-liner form so diagnostics point at the launch expression: + * @code + * raft::launch_kernel(res, grid, block)(kernel, args...); + * raft::launch_kernel(stream, grid, block, smem)(kernel, args...); + * @endcode + * + * The kernel must resolve to a unique @c __global__ function pointer. Partially-specified function + * templates are OK when the remaining parameters can be deduced from the launch argument types. + * Overload sets that remain ambiguous after that conversion are not supported. + */ +class kernel_launcher { + public: + kernel_launcher(kernel_launcher const&) = delete; + kernel_launcher& operator=(kernel_launcher const&) = delete; + kernel_launcher(kernel_launcher&&) = default; + kernel_launcher& operator=(kernel_launcher&&) = delete; + + /** + * @brief Launch @p kernel with @p args, which already have the kernel parameter types. + * + * The function-pointer parameter type is a non-deduced context derived from @p args, so a + * partially specified function template (e.g. @c map_kernel) can still convert to + * a unique @c __global__ pointer by deducing its remaining template parameters from that type. + */ + template + void operator()(std::type_identity_t...)> kernel, + Args&&... args) && + { + dispatch_by_value...>(reinterpret_cast(kernel), + std::forward(args)...); + } + + /** + * @brief Launch @p kernel, converting @p args to the kernel parameter types. + * + * Handles call sites where an argument merely converts to its parameter (e.g. @c T* to + * @c const T*), so they do not need casts. @p kernel must name a single specialization here, + * because its parameter types are what the arguments are converted to. + */ + template + requires(sizeof...(Params) == sizeof...(Args) && + !(std::is_same_v, Params> && ...)) void + operator()(void (*kernel)(Params...), Args&&... args) && + { + static_assert((std::is_convertible_v && ...), + "Each launch argument must be convertible to the corresponding kernel parameter"); + + dispatch_by_value(reinterpret_cast(kernel), std::forward(args)...); + } + + private: + friend kernel_launcher launch_kernel( + resources const&, dim3, dim3, std::size_t, std::source_location); + friend kernel_launcher launch_kernel( + rmm::cuda_stream_view, dim3, dim3, std::size_t, std::source_location); + + /** + * @brief Copy the launch arguments into parameters and pass their addresses to @ref dispatch. + * + * Taking the address of a copy rather than of the caller's object means passing a constant (e.g. + * a + * @c static @c const data member) does not odr-use it, matching the @c <<<>>> launch syntax. + */ + template + void dispatch_by_value(void* kernel, Params... params) const + { + std::array arg_ptrs{ + {const_cast(static_cast(std::addressof(params)))...}}; + dispatch(kernel, arg_ptrs.data()); + } + + void dispatch(void* kernel, void** arg_ptrs) const + { + cudaError_t status = + cudaLaunchKernel(kernel, grid_, block_, arg_ptrs, shared_mem_bytes_, stream_.value()); + + if (status == cudaSuccess) { +#ifndef NDEBUG + status = cudaStreamSynchronize(stream_.value()); +#else + status = cudaPeekAtLastError(); +#endif + } + detail::throw_on_cuda_launch_error(status, location_); + } + + kernel_launcher(rmm::cuda_stream_view stream, + dim3 grid, + dim3 block, + std::size_t shared_mem_bytes, + std::source_location location) + : stream_{stream}, + grid_{grid}, + block_{block}, + shared_mem_bytes_{shared_mem_bytes}, + location_{location} + { + } + + rmm::cuda_stream_view stream_; + dim3 grid_{}; + dim3 block_{}; + std::size_t shared_mem_bytes_{0}; + std::source_location location_{}; +}; + +inline kernel_launcher launch_kernel( + resources const& res, + dim3 grid, + dim3 block, + std::size_t shared_mem_bytes = 0, + std::source_location location = std::source_location::current()) +{ + return kernel_launcher{resource::get_cuda_stream(res), grid, block, shared_mem_bytes, location}; +} + +inline kernel_launcher launch_kernel( + rmm::cuda_stream_view stream, + dim3 grid, + dim3 block, + std::size_t shared_mem_bytes = 0, + std::source_location location = std::source_location::current()) +{ + return kernel_launcher{stream, grid, block, shared_mem_bytes, location}; +} + +} // namespace raft diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index fce5a0ea47..22db9223fa 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -323,6 +323,7 @@ if(BUILD_TESTS) util/fast_int_div.cu util/integer_utils.cpp util/integer_utils.cu + util/kernel_launch.cu util/memory_type_dispatcher.cu util/popc.cu util/pow2_utils.cu diff --git a/cpp/tests/core/interruptible.cu b/cpp/tests/core/interruptible.cu index 89a6df4e29..2ee97e9310 100644 --- a/cpp/tests/core/interruptible.cu +++ b/cpp/tests/core/interruptible.cu @@ -1,11 +1,12 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ #include #include #include +#include #include @@ -108,14 +109,14 @@ TEST(Raft, InterruptibleOpenMP) auto i = omp_get_thread_num(); common::nvtx::range omp_scope("interruptible::thread-%d", i); rmm::cuda_stream stream; - gpu_wait<<<1, 1, 0, stream.value()>>>(1); + raft::launch_kernel(stream.value(), 1, 1)(gpu_wait, 1); interruptible::synchronize(stream); thread_tokens[i] = interruptible::get_token(); #pragma omp barrier try { common::nvtx::range wait_scope("interruptible::wait-%d", i); - gpu_wait<<<1, 1, 0, stream.value()>>>((1 + i) * thread_delay_millis); + raft::launch_kernel(stream.value(), 1, 1)(gpu_wait, (1 + i) * thread_delay_millis); interruptible::synchronize(stream); n_finished = 1; } catch (interrupted_exception&) { diff --git a/cpp/tests/core/math_device.cu b/cpp/tests/core/math_device.cu index bb4ad318a6..b7d5a7f6e8 100644 --- a/cpp/tests/core/math_device.cu +++ b/cpp/tests/core/math_device.cu @@ -1,11 +1,12 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023-2024, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ #include "../test_utils.h" #include +#include #include #include @@ -31,7 +32,8 @@ auto math_eval(OpT op, Args&&... args) using OutT = cuda::std::invoke_result_t; auto stream = rmm::cuda_stream_default; rmm::device_scalar result(stream); - math_eval_kernel<<<1, 1, 0, stream>>>(result.data(), op, std::forward(args)...); + raft::launch_kernel(stream, 1, 1)( + math_eval_kernel, result.data(), op, std::forward(args)...); return result.value(stream); } diff --git a/cpp/tests/core/operators_device.cu b/cpp/tests/core/operators_device.cu index fbe0c49961..96f73c83fe 100644 --- a/cpp/tests/core/operators_device.cu +++ b/cpp/tests/core/operators_device.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -7,6 +7,7 @@ #include #include +#include #include #include @@ -28,7 +29,8 @@ auto eval_op_on_device(OpT op, Args&&... args) typedef decltype(op(args...)) OutT; auto stream = rmm::cuda_stream_default; rmm::device_scalar result(stream); - eval_op_on_device_kernel<<<1, 1, 0, stream>>>(result.data(), op, std::forward(args)...); + raft::launch_kernel(stream, 1, 1)( + eval_op_on_device_kernel, result.data(), op, std::forward(args)...); return result.value(stream); } diff --git a/cpp/tests/core/span.cu b/cpp/tests/core/span.cu index 9fdc50c101..5607b82d0d 100644 --- a/cpp/tests/core/span.cu +++ b/cpp/tests/core/span.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ #include "test_span.hpp" @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -73,20 +74,20 @@ TEST(GPUSpan, FromOther) // dynamic extent { device_span span(d_vec.data().get(), d_vec.size()); - TestFromOtherKernel<<<1, 16>>>(span); + raft::launch_kernel(cudaStream_t{0}, 1, 16)(TestFromOtherKernel, span); } { device_span span(d_vec.data().get(), d_vec.size()); - TestFromOtherKernelConst<<<1, 16>>>(span); + raft::launch_kernel(cudaStream_t{0}, 1, 16)(TestFromOtherKernelConst, span); } // static extent { device_span span(d_vec.data().get(), d_vec.data().get() + 16); - TestFromOtherKernel<<<1, 16>>>(span); + raft::launch_kernel(cudaStream_t{0}, 1, 16)(TestFromOtherKernel, span); } { device_span span(d_vec.data().get(), d_vec.data().get() + 16); - TestFromOtherKernelConst<<<1, 16>>>(span); + raft::launch_kernel(cudaStream_t{0}, 1, 16)(TestFromOtherKernelConst, span); } } @@ -185,7 +186,7 @@ TEST(GPUSpan, Modify) device_span span(d_vec.data().get(), d_vec.size()); - TestModifyKernel<<<1, 16>>>(span); + raft::launch_kernel(cudaStream_t{0}, 1, 16)(TestModifyKernel, span); for (size_t i = 0; i < d_vec.size(); ++i) { ASSERT_EQ(d_vec[i], d_vec.size() - i); diff --git a/cpp/tests/linalg/add.cuh b/cpp/tests/linalg/add.cuh index 359e0a47e0..d16ff2d3dc 100644 --- a/cpp/tests/linalg/add.cuh +++ b/cpp/tests/linalg/add.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2018-2023, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2018-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -7,6 +7,7 @@ #include #include +#include namespace raft { namespace linalg { @@ -23,8 +24,7 @@ void naiveAddElem(OutT* out, const InT* in1, const InT* in2, int len, cudaStream { static const int TPB = 64; int nblks = raft::ceildiv(len, TPB); - naiveAddElemKernel<<>>(out, in1, in2, len); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, nblks, TPB)(naiveAddElemKernel, out, in1, in2, len); } template diff --git a/cpp/tests/linalg/axpy.cu b/cpp/tests/linalg/axpy.cu index 5f0ac772b4..c1798df78d 100644 --- a/cpp/tests/linalg/axpy.cu +++ b/cpp/tests/linalg/axpy.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ #include "../test_utils.cuh" @@ -8,6 +8,7 @@ #include #include #include +#include #include @@ -79,8 +80,8 @@ class AxpyTest : public ::testing::TestWithParam> { int threads = 64; int blocks = raft::ceildiv(params.len, threads); - naiveAxpy<<>>( - params.len, params.alpha, x.data(), refy.data(), params.incx, params.incy); + raft::launch_kernel(stream, blocks, threads)( + naiveAxpy, params.len, params.alpha, x.data(), refy.data(), params.incx, params.incy); auto host_alpha_view = make_host_scalar_view(¶ms.alpha); diff --git a/cpp/tests/linalg/binary_op.cuh b/cpp/tests/linalg/binary_op.cuh index 5e18f994a5..db0e537dd1 100644 --- a/cpp/tests/linalg/binary_op.cuh +++ b/cpp/tests/linalg/binary_op.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2018-2024, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2018-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -9,6 +9,7 @@ #include #include +#include namespace raft { namespace linalg { @@ -25,8 +26,8 @@ void naiveAdd(OutType* out, const InType* in1, const InType* in2, IdxType len) { static const IdxType TPB = 64; IdxType nblks = raft::ceildiv(len, TPB); - naiveAddKernel<<>>(out, in1, in2, len); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(cudaStream_t{0}, nblks, TPB)( + naiveAddKernel, out, in1, in2, len); } template diff --git a/cpp/tests/linalg/divide.cu b/cpp/tests/linalg/divide.cu index c081698b20..559d5ac9a5 100644 --- a/cpp/tests/linalg/divide.cu +++ b/cpp/tests/linalg/divide.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2018-2024, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2018-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -10,6 +10,7 @@ #include #include #include +#include #include @@ -28,8 +29,7 @@ void naiveDivide(Type* out, const Type* in, Type scalar, int len, cudaStream_t s { static const int TPB = 64; int nblks = raft::ceildiv(len, TPB); - naiveDivideKernel<<>>(out, in, scalar, len); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, nblks, TPB)(naiveDivideKernel, out, in, scalar, len); } template diff --git a/cpp/tests/linalg/dot.cu b/cpp/tests/linalg/dot.cu index 53e6d4cb3c..d23b3b96a5 100644 --- a/cpp/tests/linalg/dot.cu +++ b/cpp/tests/linalg/dot.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ #include "../test_utils.cuh" @@ -8,6 +8,7 @@ #include #include #include +#include #include @@ -63,8 +64,8 @@ class DotTest : public ::testing::TestWithParam> { uniform(handle, r, y.data(), y_len, T(-1.0), T(1.0)); rmm::device_scalar ref(0, resource::get_cuda_stream(handle)); - naiveDot<<<256, 256, 0, stream>>>( - params.len, x.data(), params.incx, y.data(), params.incy, ref.data()); + raft::launch_kernel(stream, 256, 256)( + naiveDot, params.len, x.data(), params.incx, y.data(), params.incy, ref.data()); raft::update_host(&ref_output, ref.data(), 1, stream); // Test out both the device and host api's diff --git a/cpp/tests/linalg/eltwise.cu b/cpp/tests/linalg/eltwise.cu index fb128c7e12..bd02300e52 100644 --- a/cpp/tests/linalg/eltwise.cu +++ b/cpp/tests/linalg/eltwise.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2018-2024, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2018-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -9,6 +9,7 @@ #include #include #include +#include #include @@ -29,8 +30,7 @@ void naiveScale(Type* out, const Type* in, Type scalar, int len, cudaStream_t st { static const int TPB = 64; int nblks = raft::ceildiv(len, TPB); - naiveScaleKernel<<>>(out, in, scalar, len); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, nblks, TPB)(naiveScaleKernel, out, in, scalar, len); } template @@ -116,8 +116,7 @@ void naiveAdd(Type* out, const Type* in1, const Type* in2, int len, cudaStream_t { static const int TPB = 64; int nblks = raft::ceildiv(len, TPB); - naiveAddKernel<<>>(out, in1, in2, len); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, nblks, TPB)(naiveAddKernel, out, in1, in2, len); } template diff --git a/cpp/tests/linalg/gemm_layout.cu b/cpp/tests/linalg/gemm_layout.cu index 638b5921e3..f04ab23870 100644 --- a/cpp/tests/linalg/gemm_layout.cu +++ b/cpp/tests/linalg/gemm_layout.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -9,6 +9,7 @@ #include #include #include +#include #include @@ -83,8 +84,16 @@ class GemmLayoutTest : public ::testing::TestWithParam> { dim3 blocks(raft::ceildiv(params.M, 128), raft::ceildiv(params.N, 4), 1); dim3 threads(128, 4, 1); - naiveGemm<<>>( - refZ, X, Y, params.M, params.N, params.K, params.zLayout, params.xLayout, params.yLayout); + raft::launch_kernel(cudaStream_t{0}, blocks, threads)(naiveGemm, + refZ, + X, + Y, + params.M, + params.N, + params.K, + params.zLayout, + params.xLayout, + params.yLayout); auto x_view_row_major = raft::make_device_matrix_view(X, params.M, params.K); auto y_view_row_major = raft::make_device_matrix_view(Y, params.K, params.N); diff --git a/cpp/tests/linalg/gemv.cu b/cpp/tests/linalg/gemv.cu index be6d928c24..935a27f642 100644 --- a/cpp/tests/linalg/gemv.cu +++ b/cpp/tests/linalg/gemv.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2024, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -9,6 +9,7 @@ #include #include #include +#include #include @@ -98,8 +99,14 @@ class GemvTest : public ::testing::TestWithParam> { dim3 blocks(raft::ceildiv(yElems, 256), 1, 1); dim3 threads(256, 1, 1); - naiveGemv<<>>( - refy.data(), A.data(), x.data(), params.n_rows, params.n_cols, params.lda, params.trans_a); + raft::launch_kernel(stream, blocks, threads)(naiveGemv, + refy.data(), + A.data(), + x.data(), + params.n_rows, + params.n_cols, + params.lda, + params.trans_a); auto A_row_major = raft::make_device_matrix_view(A.data(), params.n_rows, params.n_cols); diff --git a/cpp/tests/linalg/map_then_reduce.cu b/cpp/tests/linalg/map_then_reduce.cu index c2ef0b6c6b..5a234e9946 100644 --- a/cpp/tests/linalg/map_then_reduce.cu +++ b/cpp/tests/linalg/map_then_reduce.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2018-2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2018-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -37,8 +38,8 @@ void naiveMapReduce(OutType* out, const InType* in, size_t len, MapOp map, cudaS { static const int TPB = 64; int nblks = raft::ceildiv(len, (size_t)TPB); - naiveMapReduceKernel<<>>(out, in, len, map); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, nblks, TPB)( + naiveMapReduceKernel, out, in, len, map); } template diff --git a/cpp/tests/linalg/matrix_vector_op.cuh b/cpp/tests/linalg/matrix_vector_op.cuh index cc1ef11272..67c85848af 100644 --- a/cpp/tests/linalg/matrix_vector_op.cuh +++ b/cpp/tests/linalg/matrix_vector_op.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2018-2024, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2018-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -7,6 +7,7 @@ #include #include +#include namespace raft { namespace linalg { @@ -50,8 +51,8 @@ void naiveMatVec(OutT* out, static const IdxType TPB = 64; IdxType len = N * D; IdxType nblks = raft::ceildiv(len, TPB); - naiveMatVecKernel<<>>(out, mat, vec, D, N, rowMajor, bcastAlongRows, op); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, nblks, TPB)( + naiveMatVecKernel, out, mat, vec, D, N, rowMajor, bcastAlongRows, op); } template @@ -128,9 +129,8 @@ void naiveMatVec(OutT* out, static const IdxType TPB = 64; IdxType len = N * D; IdxType nblks = raft::ceildiv(len, TPB); - naiveMatVecKernel<<>>( - out, mat, vec1, vec2, D, N, rowMajor, bcastAlongRows, op); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, nblks, TPB)( + naiveMatVecKernel, out, mat, vec1, vec2, D, N, rowMajor, bcastAlongRows, op); } template diff --git a/cpp/tests/linalg/mean_squared_error.cu b/cpp/tests/linalg/mean_squared_error.cu index 17c7105416..0f4129fa2d 100644 --- a/cpp/tests/linalg/mean_squared_error.cu +++ b/cpp/tests/linalg/mean_squared_error.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ #include "../test_utils.cuh" @@ -8,6 +8,7 @@ #include #include #include +#include #include @@ -75,8 +76,8 @@ class MeanSquaredErrorTest : public ::testing::TestWithParam(output.data()), params.weight); - naiveMeanSquaredError<<<256, 256, 0, stream>>>( - params.len, a.data(), b.data(), params.weight, refoutput.data()); + raft::launch_kernel(stream, 256, 256)( + naiveMeanSquaredError, params.len, a.data(), b.data(), params.weight, refoutput.data()); resource::sync_stream(handle); } diff --git a/cpp/tests/linalg/norm.cu b/cpp/tests/linalg/norm.cu index 6abe8aca92..b531e5b13c 100644 --- a/cpp/tests/linalg/norm.cu +++ b/cpp/tests/linalg/norm.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2018-2024, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2018-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -12,6 +12,7 @@ #include #include #include +#include #include @@ -71,9 +72,8 @@ void naiveRowNorm( { static const IdxT TPB = 64; IdxT nblks = raft::ceildiv(N, TPB); - naiveRowNormKernel - <<>>(dots, data, D, N, type, do_sqrt); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, nblks, TPB)( + naiveRowNormKernel, dots, data, D, N, type, do_sqrt); } template @@ -163,9 +163,8 @@ void naiveColNorm( { static const IdxT TPB = 64; IdxT nblks = raft::ceildiv(D, TPB); - naiveColNormKernel - <<>>(dots, data, D, N, type, do_sqrt); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, nblks, TPB)( + naiveColNormKernel, dots, data, D, N, type, do_sqrt); } template diff --git a/cpp/tests/linalg/power.cu b/cpp/tests/linalg/power.cu index c3a672f296..8cf2f30b6b 100644 --- a/cpp/tests/linalg/power.cu +++ b/cpp/tests/linalg/power.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2018-2024, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2018-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -9,6 +9,7 @@ #include #include #include +#include #include @@ -27,8 +28,7 @@ void naivePowerElem(Type* out, const Type* in1, const Type* in2, int len, cudaSt { static const int TPB = 64; int nblks = raft::ceildiv(len, TPB); - naivePowerElemKernel<<>>(out, in1, in2, len); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, nblks, TPB)(naivePowerElemKernel, out, in1, in2, len); } template @@ -43,8 +43,7 @@ void naivePowerScalar(Type* out, const Type* in1, const Type in2, int len, cudaS { static const int TPB = 64; int nblks = raft::ceildiv(len, TPB); - naivePowerScalarKernel<<>>(out, in1, in2, len); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, nblks, TPB)(naivePowerScalarKernel, out, in1, in2, len); } template diff --git a/cpp/tests/linalg/reduce.cuh b/cpp/tests/linalg/reduce.cuh index bd5ddbe2a6..99123d84cb 100644 --- a/cpp/tests/linalg/reduce.cuh +++ b/cpp/tests/linalg/reduce.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -9,6 +9,7 @@ #include #include #include +#include #include @@ -66,9 +67,8 @@ void naiveCoalescedReduction(OutType* dots, { static const IdxType TPB = 64; IdxType nblks = raft::ceildiv(N, TPB); - naiveCoalescedReductionKernel<<>>( - dots, data, D, N, init, inplace, main_op, reduce_op, fin_op); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, nblks, TPB)( + naiveCoalescedReductionKernel, dots, data, D, N, init, inplace, main_op, reduce_op, fin_op); } template >>( - dots, data, D, N, init, inplace, main_op, reduce_op, fin_op); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, nblks, TPB)( + naiveStridedReductionKernel, dots, data, D, N, init, inplace, main_op, reduce_op, fin_op); } template #include #include +#include #include @@ -55,8 +56,17 @@ void naiveReduceRowsByKey(const Type* d_A, { cudaMemset(d_sums, 0, sizeof(Type) * nkeys * ncols); - naiveReduceRowsByKeyKernel<<>>( - d_A, lda, d_keys, d_weight, d_char_keys, nrows, ncols, nkeys, d_sums); + raft::launch_kernel(stream, dim3((ncols + 31) / 32, nkeys), dim3(32, 1))( + naiveReduceRowsByKeyKernel, + d_A, + lda, + d_keys, + d_weight, + d_char_keys, + nrows, + ncols, + nkeys, + d_sums); } template diff --git a/cpp/tests/linalg/sqrt.cu b/cpp/tests/linalg/sqrt.cu index 94f85fbe81..d74ab17786 100644 --- a/cpp/tests/linalg/sqrt.cu +++ b/cpp/tests/linalg/sqrt.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2018-2024, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2018-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -9,6 +9,7 @@ #include #include #include +#include #include @@ -27,8 +28,7 @@ void naiveSqrtElem(Type* out, const Type* in1, int len) { static const int TPB = 64; int nblks = raft::ceildiv(len, TPB); - naiveSqrtElemKernel<<>>(out, in1, len); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(cudaStream_t{0}, nblks, TPB)(naiveSqrtElemKernel, out, in1, len); } template diff --git a/cpp/tests/linalg/subtract.cu b/cpp/tests/linalg/subtract.cu index caa52a33f5..402f2474da 100644 --- a/cpp/tests/linalg/subtract.cu +++ b/cpp/tests/linalg/subtract.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2018-2024, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2018-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -9,6 +9,7 @@ #include #include #include +#include #include @@ -27,8 +28,7 @@ void naiveSubtractElem(Type* out, const Type* in1, const Type* in2, int len, cud { static const int TPB = 64; int nblks = raft::ceildiv(len, TPB); - naiveSubtractElemKernel<<>>(out, in1, in2, len); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, nblks, TPB)(naiveSubtractElemKernel, out, in1, in2, len); } template @@ -43,8 +43,7 @@ void naiveSubtractScalar(Type* out, const Type* in1, const Type in2, int len, cu { static const int TPB = 64; int nblks = raft::ceildiv(len, TPB); - naiveSubtractScalarKernel<<>>(out, in1, in2, len); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, nblks, TPB)(naiveSubtractScalarKernel, out, in1, in2, len); } template diff --git a/cpp/tests/linalg/unary_op.cuh b/cpp/tests/linalg/unary_op.cuh index 70e841b509..c24f059089 100644 --- a/cpp/tests/linalg/unary_op.cuh +++ b/cpp/tests/linalg/unary_op.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2018-2024, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2018-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -9,6 +9,7 @@ #include #include +#include namespace raft { namespace linalg { @@ -32,8 +33,8 @@ void naiveScale(OutType* out, const InType* in, InType scalar, int len, cudaStre { static const int TPB = 64; int nblks = raft::ceildiv(len, TPB); - naiveScaleKernel<<>>(out, in, scalar, len); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, nblks, TPB)( + naiveScaleKernel, out, in, scalar, len); } template diff --git a/cpp/tests/matrix/math.cu b/cpp/tests/matrix/math.cu index cf35a36294..5f4e06d3f3 100644 --- a/cpp/tests/matrix/math.cu +++ b/cpp/tests/matrix/math.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2018-2024, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2018-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -15,6 +15,7 @@ #include #include #include +#include #include @@ -33,8 +34,7 @@ void naivePower(Type* in, Type* out, int len, cudaStream_t stream) { static const int TPB = 64; int nblks = raft::ceildiv(len, TPB); - naivePowerKernel<<>>(in, out, len); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, nblks, TPB)(naivePowerKernel, in, out, len); } template @@ -49,8 +49,7 @@ void naiveSqrt(Type* in, Type* out, int len, cudaStream_t stream) { static const int TPB = 64; int nblks = raft::ceildiv(len, TPB); - naiveSqrtKernel<<>>(in, out, len); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, nblks, TPB)(naiveSqrtKernel, in, out, len); } template @@ -86,8 +85,7 @@ RAFT_KERNEL naiveSignFlipKernel(Type* in, Type* out, int rowCount, int colCount) template void naiveSignFlip(Type* in, Type* out, int rowCount, int colCount, cudaStream_t stream) { - naiveSignFlipKernel<<>>(in, out, rowCount, colCount); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, colCount, 1)(naiveSignFlipKernel, in, out, rowCount, colCount); } template diff --git a/cpp/tests/random/make_blobs.cu b/cpp/tests/random/make_blobs.cu index 75060ece13..c9219cc9ec 100644 --- a/cpp/tests/random/make_blobs.cu +++ b/cpp/tests/random/make_blobs.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -11,6 +11,7 @@ #include #include #include +#include #include @@ -112,22 +113,23 @@ class MakeBlobsTest : public ::testing::TestWithParam> { bool row_major = std::is_same::value; static const int threads = 128; - meanKernel<<>>(stats.data_handle(), + raft::launch_kernel(stream, raft::ceildiv(len, threads), threads)(meanKernel, + stats.data_handle(), + lens.data_handle(), + data.data_handle(), + labels.data_handle(), + params.rows, + params.cols, + params.n_clusters, + row_major); + int len1 = params.n_clusters * params.cols; + raft::launch_kernel(stream, raft::ceildiv(len1, threads), threads)(compute_mean_var, + mean_var.data_handle(), + stats.data_handle(), lens.data_handle(), - data.data_handle(), - labels.data_handle(), - params.rows, - params.cols, params.n_clusters, + params.cols, row_major); - int len1 = params.n_clusters * params.cols; - compute_mean_var - <<>>(mean_var.data_handle(), - stats.data_handle(), - lens.data_handle(), - params.n_clusters, - params.cols, - row_major); } void check() diff --git a/cpp/tests/random/multi_variable_gaussian.cu b/cpp/tests/random/multi_variable_gaussian.cu index d58fda18b4..2d4b732494 100644 --- a/cpp/tests/random/multi_variable_gaussian.cu +++ b/cpp/tests/random/multi_variable_gaussian.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2018-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2018-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -154,16 +155,15 @@ class MVGTest : public ::testing::TestWithParam> { RAFT_CUDA_TRY(cudaMemset(Rand_mean.data(), 0, dim * sizeof(T))); dim3 block = (64); dim3 grid = (raft::ceildiv(nPoints * dim, (int)block.x)); - En_KF_accumulate<<>>(nPoints, dim, X_d.data(), Rand_mean.data()); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, grid, block)( + En_KF_accumulate, nPoints, dim, X_d.data(), Rand_mean.data()); grid = (raft::ceildiv(dim, (int)block.x)); - En_KF_normalize<<>>(nPoints, dim, Rand_mean.data()); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, grid, block)(En_KF_normalize, nPoints, dim, Rand_mean.data()); // storing the error wrt random point mean in X_d grid = (raft::ceildiv(dim * nPoints, (int)block.x)); - En_KF_dif<<>>(nPoints, dim, X_d.data(), Rand_mean.data(), X_d.data()); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, grid, block)( + En_KF_dif, nPoints, dim, X_d.data(), Rand_mean.data(), X_d.data()); // finding the cov matrix, placing in Rand_cov T alfa = 1.0 / (nPoints - 1), beta = 0.0; @@ -285,16 +285,15 @@ class MVGMdspanTest : public ::testing::TestWithParam> { RAFT_CUDA_TRY(cudaMemset(Rand_mean.data(), 0, dim * sizeof(T))); dim3 block = (64); dim3 grid = (raft::ceildiv(nPoints * dim, (int)block.x)); - En_KF_accumulate<<>>(nPoints, dim, X_d.data(), Rand_mean.data()); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, grid, block)( + En_KF_accumulate, nPoints, dim, X_d.data(), Rand_mean.data()); grid = (raft::ceildiv(dim, (int)block.x)); - En_KF_normalize<<>>(nPoints, dim, Rand_mean.data()); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, grid, block)(En_KF_normalize, nPoints, dim, Rand_mean.data()); // storing the error wrt random point mean in X_d grid = (raft::ceildiv(dim * nPoints, (int)block.x)); - En_KF_dif<<>>(nPoints, dim, X_d.data(), Rand_mean.data(), X_d.data()); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, grid, block)( + En_KF_dif, nPoints, dim, X_d.data(), Rand_mean.data(), X_d.data()); // finding the cov matrix, placing in Rand_cov T alfa = 1.0 / (nPoints - 1), beta = 0.0; diff --git a/cpp/tests/random/rmat_rectangular_generator.cu b/cpp/tests/random/rmat_rectangular_generator.cu index 2eb1b9f23b..ffc8db58e2 100644 --- a/cpp/tests/random/rmat_rectangular_generator.cu +++ b/cpp/tests/random/rmat_rectangular_generator.cu @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -124,15 +125,13 @@ void normalize(OutT* theta, { // one threadblock with 256 threads is more than enough as the 'scale' parameters // won't be that large! - normalize_kernel<<<1, 256, 0, stream>>>(theta, in_vals, max_scale, r_scale, c_scale); - RAFT_CUDA_TRY(cudaGetLastError()); + raft::launch_kernel(stream, 1, 256)( + normalize_kernel, theta, in_vals, max_scale, r_scale, c_scale); if (handle_rect) { - handle_rect_kernel<<<1, 256, 0, stream>>>(theta, max_scale, r_scale, c_scale); - RAFT_CUDA_TRY(cudaGetLastError()); + raft::launch_kernel(stream, 1, 256)(handle_rect_kernel, theta, max_scale, r_scale, c_scale); } if (!theta_array) { - theta_kernel<<<1, 256, 0, stream>>>(theta, max_scale, r_scale, c_scale); - RAFT_CUDA_TRY(cudaGetLastError()); + raft::launch_kernel(stream, 1, 256)(theta_kernel, theta, max_scale, r_scale, c_scale); } } @@ -214,9 +213,8 @@ class RmatGenTest : public ::testing::TestWithParam { { rmm::device_uvector hist{theta.size(), stream}; RAFT_CUDA_TRY(cudaMemsetAsync(hist.data(), 0, hist.size() * sizeof(int), stream)); - compute_hist<<(out.size() / 2, 256), 256, 0, stream>>>( - hist.data(), out.data(), out.size(), max_scale, params.r_scale, params.c_scale); - RAFT_CUDA_TRY(cudaGetLastError()); + raft::launch_kernel(stream, raft::ceildiv(out.size() / 2, 256), 256)( + compute_hist, hist.data(), out.data(), out.size(), max_scale, params.r_scale, params.c_scale); rmm::device_uvector computed_theta{theta.size(), stream}; normalize(computed_theta.data(), hist.data(), @@ -319,9 +317,8 @@ class RmatGenMdspanTest : public ::testing::TestWithParam { { rmm::device_uvector hist{theta.size(), stream}; RAFT_CUDA_TRY(cudaMemsetAsync(hist.data(), 0, hist.size() * sizeof(int), stream)); - compute_hist<<(out.size() / 2, 256), 256, 0, stream>>>( - hist.data(), out.data(), out.size(), max_scale, params.r_scale, params.c_scale); - RAFT_CUDA_TRY(cudaGetLastError()); + raft::launch_kernel(stream, raft::ceildiv(out.size() / 2, 256), 256)( + compute_hist, hist.data(), out.data(), out.size(), max_scale, params.r_scale, params.c_scale); rmm::device_uvector computed_theta{theta.size(), stream}; normalize(computed_theta.data(), hist.data(), diff --git a/cpp/tests/random/rng.cu b/cpp/tests/random/rng.cu index 3c09a2f0a8..a88e093b7a 100644 --- a/cpp/tests/random/rng.cu +++ b/cpp/tests/random/rng.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2018-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2018-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -12,6 +12,7 @@ #include #include #include +#include #include @@ -120,8 +121,8 @@ class RngTest : public ::testing::TestWithParam> { break; }; static const int threads = 128; - meanKernel<<>>( - stats.data(), data.data(), params.len); + raft::launch_kernel(stream, raft::ceildiv(params.len, threads), threads)( + meanKernel, stats.data(), data.data(), params.len); update_host(h_stats, stats.data(), 2, stream); RAFT_CUDA_TRY(cudaStreamSynchronize(stream)); h_stats[0] /= params.len; @@ -214,8 +215,8 @@ class RngMdspanTest : public ::testing::TestWithParam> { case RNG_Laplace: laplace(handle, r, data_view, params.start, params.end); break; }; static const int threads = 128; - meanKernel<<>>( - stats.data(), data.data(), params.len); + raft::launch_kernel(stream, raft::ceildiv(params.len, threads), threads)( + meanKernel, stats.data(), data.data(), params.len); update_host(h_stats, stats.data(), 2, stream); RAFT_CUDA_TRY(cudaStreamSynchronize(stream)); h_stats[0] /= params.len; @@ -611,8 +612,8 @@ class RngNormalTableTest : public ::testing::TestWithParam - <<>>(stats.data(), data.data(), len); + raft::launch_kernel(stream, raft::ceildiv(len, threads), threads)( + meanKernel, stats.data(), data.data(), len); update_host(h_stats, stats.data(), 2, stream); RAFT_CUDA_TRY(cudaStreamSynchronize(stream)); h_stats[0] /= len; @@ -667,8 +668,8 @@ class RngNormalTableMdspanTest : public ::testing::TestWithParam - <<>>(stats.data(), data.data(), len); + raft::launch_kernel(stream, raft::ceildiv(len, threads), threads)( + meanKernel, stats.data(), data.data(), len); update_host(h_stats, stats.data(), 2, stream); RAFT_CUDA_TRY(cudaStreamSynchronize(stream)); h_stats[0] /= len; diff --git a/cpp/tests/random/rng_int.cu b/cpp/tests/random/rng_int.cu index 87a0719e74..a206913930 100644 --- a/cpp/tests/random/rng_int.cu +++ b/cpp/tests/random/rng_int.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2018-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2018-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -10,6 +10,7 @@ #include #include #include +#include #include @@ -85,8 +86,8 @@ class RngTest : public ::testing::TestWithParam> { break; }; static const int threads = 128; - meanKernel<<>>( - stats.data(), data.data(), params.len); + raft::launch_kernel(stream, raft::ceildiv(params.len, threads), threads)( + meanKernel, stats.data(), data.data(), params.len); update_host(h_stats, stats.data(), 2, stream); resource::sync_stream(handle, stream); h_stats[0] /= params.len; @@ -138,8 +139,8 @@ class RngMdspanTest : public ::testing::TestWithParam> { case RNG_Uniform: uniformInt(handle, r, data_view, params.start, params.end); break; }; static const int threads = 128; - meanKernel<<>>( - stats.data(), data.data(), params.len); + raft::launch_kernel(stream, raft::ceildiv(params.len, threads), threads)( + meanKernel, stats.data(), data.data(), params.len); update_host(h_stats, stats.data(), 2, stream); resource::sync_stream(handle, stream); h_stats[0] /= params.len; diff --git a/cpp/tests/random/rng_pcg_host_api.cu b/cpp/tests/random/rng_pcg_host_api.cu index 713ac8c5a8..e0d0f83e6e 100644 --- a/cpp/tests/random/rng_pcg_host_api.cu +++ b/cpp/tests/random/rng_pcg_host_api.cu @@ -1,10 +1,12 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ #include "../test_utils.cuh" +#include + namespace raft { namespace random { @@ -57,8 +59,13 @@ class HostApiTest { RngState r(seed, GenPC); DeviceState d_state(r); - pcg_device_kernel<<>>( - d_buffer.data(), d_state, dist_params, total_threads, len); + raft::launch_kernel(handle, n_blocks, n_threads)( + pcg_device_kernel, + d_buffer.data(), + d_state, + dist_params, + total_threads, + len); RAFT_CUDA_TRY(cudaStreamSynchronize(stream)); for (size_t tid = 0; tid < len; tid++) { diff --git a/cpp/tests/sparse/convert_csr.cu b/cpp/tests/sparse/convert_csr.cu index a529041068..173ef6558b 100644 --- a/cpp/tests/sparse/convert_csr.cu +++ b/cpp/tests/sparse/convert_csr.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2024, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -11,6 +11,7 @@ #include #include #include +#include #include @@ -101,8 +102,8 @@ void init_adj(bool* adj, index_t num_rows, index_t num_cols, index_t divisor, cu dim3 block(32, 32); const index_t max_y_grid_dim = 65535; dim3 grid(num_cols / 32 + 1, (int)min(num_rows / 32 + 1, max_y_grid_dim)); - init_adj_kernel<<>>(adj, num_rows, num_cols, divisor); - RAFT_CHECK_CUDA(stream); + raft::launch_kernel(stream, grid, block)( + init_adj_kernel, adj, num_rows, num_cols, divisor); } template diff --git a/cpp/tests/sparse/spmm.cu b/cpp/tests/sparse/spmm.cu index d3df89aecb..80777058d0 100644 --- a/cpp/tests/sparse/spmm.cu +++ b/cpp/tests/sparse/spmm.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -10,6 +10,7 @@ #include #include #include +#include #include @@ -172,22 +173,23 @@ class SpmmTest : public ::testing::TestWithParam> { dim3 blocks(raft::ceildiv(params.M, 128), raft::ceildiv(params.N, 4), 1); dim3 threads(128, 4, 1); - naiveGemm<<>>(params.trans_x, - params.trans_y, - params.M, - params.N, - params.K, - alpha, - X, - ldx, - true, - Y, - ldy, - params.row_major, - beta, - Z_ref, - ldz, - params.row_major); + raft::launch_kernel(stream, blocks, threads)(naiveGemm, + params.trans_x, + params.trans_y, + params.M, + params.N, + params.K, + alpha, + X, + ldx, + true, + Y, + ldy, + params.row_major, + beta, + Z_ref, + ldz, + params.row_major); spmm( handle, params.trans_x, params.trans_y, &alpha, X_csr, y_stride_view, &beta, z_stride_view); diff --git a/cpp/tests/sparse/symmetrize.cu b/cpp/tests/sparse/symmetrize.cu index 17c3390f2a..5e983480c4 100644 --- a/cpp/tests/sparse/symmetrize.cu +++ b/cpp/tests/sparse/symmetrize.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2024, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -100,8 +101,8 @@ class SparseSymmetrizeTest rmm::device_scalar sum(stream); sum.set_value_to_zero_async(stream); - assert_symmetry<<>>( - out.rows(), out.cols(), out.vals(), (nnz_t)out.nnz, sum.data()); + raft::launch_kernel(stream, raft::ceildiv(out.nnz, (nnz_t)256), 256)( + assert_symmetry, out.rows(), out.cols(), out.vals(), (nnz_t)out.nnz, sum.data()); sum_h = sum.value(stream); resource::sync_stream(handle, stream); diff --git a/cpp/tests/stats/histogram.cu b/cpp/tests/stats/histogram.cu index 9c074e250c..a47aa80292 100644 --- a/cpp/tests/stats/histogram.cu +++ b/cpp/tests/stats/histogram.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2024, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -12,6 +12,7 @@ #include #include #include +#include #include @@ -41,8 +42,7 @@ void naiveHist(int* bins, int nbins, int* in, int nrows, int ncols, cudaStream_t const int TPB = 128; int nblksx = raft::ceildiv(nrows, TPB); dim3 blks(nblksx, ncols); - naiveHistKernel<<>>(bins, nbins, in, nrows); - RAFT_CUDA_TRY(cudaGetLastError()); + raft::launch_kernel(stream, blks, TPB)(naiveHistKernel, bins, nbins, in, nrows); } struct HistInputs { diff --git a/cpp/tests/stats/minmax.cu b/cpp/tests/stats/minmax.cu index 8d17b13e01..2a66a5fdab 100644 --- a/cpp/tests/stats/minmax.cu +++ b/cpp/tests/stats/minmax.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2024, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -67,11 +68,11 @@ void naiveMinMax( const int TPB = 128; int nblks = raft::ceildiv(ncols, TPB); T init_val = std::numeric_limits::max(); - naiveMinMaxInitKernel<<>>(ncols, globalmin, globalmax, init_val); - RAFT_CUDA_TRY(cudaGetLastError()); + raft::launch_kernel(stream, nblks, TPB)( + naiveMinMaxInitKernel, ncols, globalmin, globalmax, init_val); nblks = raft::ceildiv(nrows * ncols, TPB); - naiveMinMaxKernel<<>>(data, nrows, ncols, globalmin, globalmax); - RAFT_CUDA_TRY(cudaGetLastError()); + raft::launch_kernel(stream, nblks, TPB)( + naiveMinMaxKernel, data, nrows, ncols, globalmin, globalmax); } template @@ -107,9 +108,8 @@ class MinMaxTest : public ::testing::TestWithParam> { T nan_prob = 0.01; bernoulli(handle, r, mask.data(), len, nan_prob); const int TPB = 256; - nanKernel<<>>( - data.data(), mask.data(), len, std::numeric_limits::quiet_NaN()); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, raft::ceildiv(len, TPB), TPB)( + nanKernel, data.data(), mask.data(), len, std::numeric_limits::quiet_NaN()); naiveMinMax(data.data(), params.rows, params.cols, diff --git a/cpp/tests/util/bitonic_sort.cu b/cpp/tests/util/bitonic_sort.cu index 742cf8aeda..7ccbab59c6 100644 --- a/cpp/tests/util/bitonic_sort.cu +++ b/cpp/tests/util/bitonic_sort.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -8,6 +8,7 @@ #include #include #include +#include #include @@ -85,9 +86,8 @@ struct bitonic_launch { const int block_dim = n_warps * WarpSize; const int n_subwarps = block_dim / spec.warp_width; const int grid_dim = ceildiv(spec.n_inputs, n_subwarps); - bitonic_kernel - <<>>(arr, spec.ascending, spec.warp_width, spec.n_inputs); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(stream, grid_dim, block_dim)( + bitonic_kernel, arr, spec.ascending, spec.warp_width, spec.n_inputs); } }; diff --git a/cpp/tests/util/device_atomics.cu b/cpp/tests/util/device_atomics.cu index c71f917112..5fb831a53f 100644 --- a/cpp/tests/util/device_atomics.cu +++ b/cpp/tests/util/device_atomics.cu @@ -1,11 +1,12 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ #include #include #include +#include #include #include @@ -42,8 +43,8 @@ TEST(Raft, AtomicIncWarp) std::array out_host{0}; // Write all 1M thread indices to a unique location in `out_device` - test_atomic_inc_warp_kernel<<>>(counter.data(), - out_device.data()); + raft::launch_kernel(s, num_blocks, threads_per_block)( + test_atomic_inc_warp_kernel, counter.data(), out_device.data()); // Copy data to host RAFT_CUDA_TRY(cudaMemcpyAsync(out_host.data(), (const void*)out_device.data(), diff --git a/cpp/tests/util/integer_utils.cu b/cpp/tests/util/integer_utils.cu index 583e7d5df5..0122748abf 100644 --- a/cpp/tests/util/integer_utils.cu +++ b/cpp/tests/util/integer_utils.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023-2024, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -7,6 +7,7 @@ #include #include +#include #include @@ -53,12 +54,13 @@ class Multiplication64bit : public testing::TestWithParam { wmul_64bit(result_high, result_low, params.operand_1, params.operand_2); wmul_64bit(swapped_result_high, swapped_result_low, params.operand_2, params.operand_1); - mul64_test_kernel<<<1, 1, 0, stream>>>(d_result_high.data(), - d_result_low.data(), - d_swapped_result_high.data(), - d_swapped_result_low.data(), - params.operand_1, - params.operand_2); + raft::launch_kernel(stream, 1, 1)(mul64_test_kernel, + d_result_high.data(), + d_result_low.data(), + d_swapped_result_high.data(), + d_swapped_result_low.data(), + params.operand_1, + params.operand_2); } raft::resources handle; diff --git a/cpp/tests/util/kernel_launch.cu b/cpp/tests/util/kernel_launch.cu new file mode 100644 index 0000000000..de96f9f140 --- /dev/null +++ b/cpp/tests/util/kernel_launch.cu @@ -0,0 +1,81 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include +#include + +#include + +#include + +#include +#include + +namespace raft { + +namespace { + +RAFT_KERNEL noop_kernel() {} + +RAFT_KERNEL write_one_kernel(int* out) +{ + if (threadIdx.x == 0 && blockIdx.x == 0) { *out = 1; } +} + +} // namespace + +TEST(KernelLaunch, SuccessfulLaunch) +{ + raft::resources res; + rmm::device_uvector out(1, resource::get_cuda_stream(res)); + RAFT_CUDA_TRY(cudaMemsetAsync(out.data(), 0, sizeof(int), resource::get_cuda_stream(res))); + + raft::launch_kernel(res, 1, 32)(write_one_kernel, out.data()); + resource::sync_stream(res); + + int host_out = 0; + RAFT_CUDA_TRY(cudaMemcpy(&host_out, out.data(), sizeof(int), cudaMemcpyDeviceToHost)); + EXPECT_EQ(host_out, 1); +} + +TEST(KernelLaunch, StreamOverload) +{ + raft::resources res; + auto stream = resource::get_cuda_stream(res); + EXPECT_NO_THROW(raft::launch_kernel(stream, 1, 1)(noop_kernel)); + resource::sync_stream(res); +} + +TEST(KernelLaunch, ErrorReportsCallSite) +{ + raft::resources res; + + // Intentionally invalid configuration: block size exceeds hardware limit. + constexpr int k_bad_block = 2048; + std::string caught; + int launch_line = 0; + try { + launch_line = __LINE__ + 1; + raft::launch_kernel(res, 1, k_bad_block)(noop_kernel); + FAIL() << "Expected cuda_error from invalid launch configuration"; + } catch (raft::cuda_error const& e) { + caught = e.what(); + } + + // Must blame this test translation unit, not the launcher header. + EXPECT_EQ(caught.find("kernel_launch.hpp"), std::string::npos) << caught; + EXPECT_NE(caught.find("kernel_launch.cu"), std::string::npos) << caught; + + std::string re_exp{R"(CUDA error encountered at: file=.*kernel_launch\.cu line=)"}; + re_exp += std::to_string(launch_line); + re_exp += R"( function=.*ErrorReportsCallSite.*: call='cudaLaunchKernel', Reason=.*)"; + EXPECT_TRUE(std::regex_search(caught, std::regex(re_exp))) + << "message:'" << caught << "'\nexpected regex:'" << re_exp << "'"; +} + +} // namespace raft diff --git a/cpp/tests/util/reduction.cu b/cpp/tests/util/reduction.cu index 03756a1d2b..810dc91d40 100644 --- a/cpp/tests/util/reduction.cu +++ b/cpp/tests/util/reduction.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -7,6 +7,7 @@ #include #include +#include #include #include @@ -79,8 +80,8 @@ struct reduction_launch { rmm::device_scalar ref_d(stream); const int block_dim = 64; const int grid_dim = 1; - test_reduction_kernel<<>>( - arr_d.data(), ref_d.data(), reduce_op); + raft::launch_kernel(stream, grid_dim, block_dim)( + test_reduction_kernel, arr_d.data(), ref_d.data(), reduce_op); stream.synchronize(); RAFT_CUDA_TRY(cudaPeekAtLastError()); ASSERT_EQ(ref_d.value(stream), ref_val); @@ -97,8 +98,8 @@ struct reduction_launch { rmm::device_scalar rank_d(stream); const int block_dim = 64; const int grid_dim = 1; - test_ranked_reduction_kernel<<>>( - arr_d.data(), ref_d.data(), rank_d.data(), reduce_op); + raft::launch_kernel(stream, grid_dim, block_dim)( + test_ranked_reduction_kernel, arr_d.data(), ref_d.data(), rank_d.data(), reduce_op); stream.synchronize(); RAFT_CUDA_TRY(cudaPeekAtLastError()); ASSERT_EQ(ref_d.value(stream), ref_val); @@ -112,7 +113,8 @@ struct reduction_launch { rmm::device_scalar ref_d(stream); const int block_dim = 64; const int grid_dim = 1; - test_block_random_sample_kernel<<>>(arr_d.data(), ref_d.data()); + raft::launch_kernel(stream, grid_dim, block_dim)( + test_block_random_sample_kernel, arr_d.data(), ref_d.data()); stream.synchronize(); RAFT_CUDA_TRY(cudaPeekAtLastError()); ASSERT_EQ(ref_d.value(stream), ref_val); @@ -125,8 +127,8 @@ struct reduction_launch { rmm::device_scalar ref_d(stream); constexpr int block_dim = 64; const int grid_dim = 1; - test_binary_reduction_kernel - <<>>(arr_d.data(), ref_d.data()); + raft::launch_kernel(stream, grid_dim, block_dim)( + test_binary_reduction_kernel, arr_d.data(), ref_d.data()); stream.synchronize(); RAFT_CUDA_TRY(cudaPeekAtLastError()); ASSERT_EQ(ref_d.value(stream), ref_val); From ee94b00ceb367faa441c57eef73c9cec64daec20 Mon Sep 17 00:00:00 2001 From: achirkin Date: Thu, 6 Aug 2026 17:36:06 +0200 Subject: [PATCH 2/4] Replace stream with res where possible --- cpp/include/raft/core/bitset.cuh | 3 +-- cpp/include/raft/linalg/detail/transpose.cuh | 5 ++--- cpp/include/raft/matrix/detail/shift.cuh | 9 ++++----- .../random/detail/multi_variable_gaussian.cuh | 4 ++-- .../raft/sparse/convert/detail/adj_to_csr.cuh | 2 +- .../sparse/convert/detail/bitmap_to_csr.cuh | 6 ++---- .../sparse/convert/detail/bitset_to_csr.cuh | 3 +-- .../raft/sparse/linalg/detail/laplacian.cuh | 3 +-- .../raft/sparse/linalg/detail/symmetrize.cuh | 2 +- cpp/include/raft/sparse/linalg/detail/utils.cuh | 10 ++++------ cpp/include/raft/sparse/op/detail/reduce.cuh | 2 +- .../raft/sparse/solver/detail/lanczos.cuh | 12 ++++++------ .../solver/detail/svds_sign_correction.cuh | 4 +--- .../raft/spectral/detail/matrix_wrappers.hpp | 2 +- .../stats/detail/batched/silhouette_score.cuh | 2 +- .../raft/stats/detail/trustworthiness_score.cuh | 17 ++++++++--------- cpp/tests/linalg/axpy.cu | 2 +- cpp/tests/linalg/dot.cu | 2 +- cpp/tests/linalg/gemv.cu | 2 +- cpp/tests/linalg/mean_squared_error.cu | 2 +- cpp/tests/random/multi_variable_gaussian.cu | 12 ++++++------ cpp/tests/sparse/spmm.cu | 2 +- cpp/tests/stats/minmax.cu | 2 +- 23 files changed, 49 insertions(+), 61 deletions(-) diff --git a/cpp/include/raft/core/bitset.cuh b/cpp/include/raft/core/bitset.cuh index 80ce7c9a4e..7042c98a0c 100644 --- a/cpp/include/raft/core/bitset.cuh +++ b/cpp/include/raft/core/bitset.cuh @@ -148,7 +148,6 @@ void bitset_repeat(raft::resources const& handle, index_t repeat_times) { if (src_bit_len == 0 || repeat_times == 0) return; - auto stream = resource::get_cuda_stream(handle); constexpr index_t bits_per_element = sizeof(bitset_t) * 8; const index_t total_bits = src_bit_len * repeat_times; @@ -156,7 +155,7 @@ void bitset_repeat(raft::resources const& handle, int threadsPerBlock = 128; int blocksPerGrid = (output_size + threadsPerBlock - 1) / threadsPerBlock; - raft::launch_kernel(stream, blocksPerGrid, threadsPerBlock)( + raft::launch_kernel(handle, blocksPerGrid, threadsPerBlock)( bitset_repeat_kernel, d_src, d_output, src_bit_len, repeat_times); return; diff --git a/cpp/include/raft/linalg/detail/transpose.cuh b/cpp/include/raft/linalg/detail/transpose.cuh index cdc926d0e2..87c191a196 100644 --- a/cpp/include/raft/linalg/detail/transpose.cuh +++ b/cpp/include/raft/linalg/detail/transpose.cuh @@ -89,7 +89,6 @@ void transpose_half(raft::resources const& handle, const IndexType stride_out = 1) { if (n_cols == 0 || n_rows == 0) return; - auto stream = resource::get_cuda_stream(handle); int dev_id, sm_count; @@ -118,7 +117,7 @@ void transpose_half(raft::resources const& handle, dim3 grids(adjusted_grid_x, adjusted_grid_y); if (stride_in > 1 || stride_out > 1) { - raft::launch_kernel(stream, grids, blocks)( + raft::launch_kernel(handle, grids, blocks)( transpose_half_kernel, n_rows, n_cols, @@ -127,7 +126,7 @@ void transpose_half(raft::resources const& handle, stride_in, stride_out); } else { - raft::launch_kernel(stream, grids, blocks)( + raft::launch_kernel(handle, grids, blocks)( transpose_half_kernel, n_rows, n_cols, diff --git a/cpp/include/raft/matrix/detail/shift.cuh b/cpp/include/raft/matrix/detail/shift.cuh index a5d611f4ba..d59fe72837 100644 --- a/cpp/include/raft/matrix/detail/shift.cuh +++ b/cpp/include/raft/matrix/detail/shift.cuh @@ -139,12 +139,11 @@ void shift_dispatch(raft::resources const& handle, size_t n_rows = in_out.extent(0); size_t n_cols = in_out.extent(1); size_t TPB = 256; - auto stream = raft::resource::get_cuda_stream(handle); if (shift_type == ShiftType::COL) { size_t num_blocks = static_cast((n_rows + TPB) / TPB); if (shift_direction == ShiftDirection::TOWARDS_BEGINNING) { - raft::launch_kernel(stream, num_blocks, TPB)( + raft::launch_kernel(handle, num_blocks, TPB)( col_shift_towards_beginning, in_out.data_handle(), n_rows, @@ -152,7 +151,7 @@ void shift_dispatch(raft::resources const& handle, k, value); } else { // ShiftDirection::TOWARDS_END - raft::launch_kernel(stream, num_blocks, TPB)( + raft::launch_kernel(handle, num_blocks, TPB)( col_shift_towards_end, in_out.data_handle(), n_rows, @@ -163,7 +162,7 @@ void shift_dispatch(raft::resources const& handle, } else { // ShiftType::ROW size_t num_blocks = static_cast((n_cols + TPB) / TPB); if (shift_direction == ShiftDirection::TOWARDS_BEGINNING) { - raft::launch_kernel(stream, num_blocks, TPB)( + raft::launch_kernel(handle, num_blocks, TPB)( row_shift_towards_beginning, in_out.data_handle(), n_rows, @@ -171,7 +170,7 @@ void shift_dispatch(raft::resources const& handle, k, value); } else { // ShiftDirection::TOWARDS_END - raft::launch_kernel(stream, num_blocks, TPB)( + raft::launch_kernel(handle, num_blocks, TPB)( row_shift_towards_end, in_out.data_handle(), n_rows, diff --git a/cpp/include/raft/random/detail/multi_variable_gaussian.cuh b/cpp/include/raft/random/detail/multi_variable_gaussian.cuh index 4966e601f9..c6b21306d1 100644 --- a/cpp/include/raft/random/detail/multi_variable_gaussian.cuh +++ b/cpp/include/raft/random/detail/multi_variable_gaussian.cuh @@ -222,7 +222,7 @@ class multi_variable_gaussian_impl { // upper part (0) being filled with 0.0 dim3 block(32, 32); dim3 grid(raft::ceildiv(dim, (int)block.x), raft::ceildiv(dim, (int)block.y)); - raft::launch_kernel(cudaStream, grid, block)(fill_uplo, dim, UPPER, (T)0.0, P); + raft::launch_kernel(handle, grid, block)(fill_uplo, dim, UPPER, (T)0.0, P); // P is lower triangular chol decomp mtrx raft::linalg::gemm( @@ -233,7 +233,7 @@ class multi_variable_gaussian_impl { dim3 grid(raft::ceildiv(dim, (int)block.x)); RAFT_CUDA_TRY(cudaMemsetAsync(info, 0, sizeof(int), cudaStream)); grid.x = raft::ceildiv(dim * dim, (int)block.x); - raft::launch_kernel(cudaStream, grid, block)(combined_dot_product, dim, dim, eig, P, info); + raft::launch_kernel(handle, grid, block)(combined_dot_product, dim, dim, eig, P, info); // checking if any eigen vals were negative raft::update_host(&info_h, info, 1, cudaStream); diff --git a/cpp/include/raft/sparse/convert/detail/adj_to_csr.cuh b/cpp/include/raft/sparse/convert/detail/adj_to_csr.cuh index c13a809112..12ed6dfdc7 100644 --- a/cpp/include/raft/sparse/convert/detail/adj_to_csr.cuh +++ b/cpp/include/raft/sparse/convert/detail/adj_to_csr.cuh @@ -157,7 +157,7 @@ void adj_to_csr(raft::resources const& handle, dim3 block(adj_to_csr_tpb, 1); dim3 grid(blocks_per_row, grid_rows); - raft::launch_kernel(stream, grid, block)( + raft::launch_kernel(handle, grid, block)( adj_to_csr_kernel, adj, row_ind, num_rows, num_cols, tmp, out_col_ind); } diff --git a/cpp/include/raft/sparse/convert/detail/bitmap_to_csr.cuh b/cpp/include/raft/sparse/convert/detail/bitmap_to_csr.cuh index d9775c5d64..52fc355871 100644 --- a/cpp/include/raft/sparse/convert/detail/bitmap_to_csr.cuh +++ b/cpp/include/raft/sparse/convert/detail/bitmap_to_csr.cuh @@ -106,7 +106,6 @@ void calc_nnz_by_rows(raft::resources const& handle, sub_nnz_size = num_rows * ((num_cols + bits_per_sub_col - 1) / bits_per_sub_col); return; } - auto stream = resource::get_cuda_stream(handle); const size_t total = num_rows * num_cols; const size_t bitmap_num = (total + index_t(sizeof(bitmap_t) * 8) - 1) / index_t(sizeof(bitmap_t) * 8); @@ -117,7 +116,7 @@ void calc_nnz_by_rows(raft::resources const& handle, auto block = bitmap_to_csr_tpb; - raft::launch_kernel(stream, grid, block)(calc_nnz_by_rows_kernel, + raft::launch_kernel(handle, grid, block)(calc_nnz_by_rows_kernel, bitmap, num_rows, num_cols, @@ -249,14 +248,13 @@ void fill_indices_by_rows(raft::resources const& handle, index_t bits_per_sub_col, size_t sub_nnz_size) { - auto stream = resource::get_cuda_stream(handle); auto block_x = num_rows; auto block_y = sub_nnz_size / num_rows; dim3 grid(block_x, block_y, 1); auto block = bitmap_to_csr_tpb; - raft::launch_kernel(stream, grid, block)( + raft::launch_kernel(handle, grid, block)( fill_indices_by_rows_kernel, bitmap, indptr, diff --git a/cpp/include/raft/sparse/convert/detail/bitset_to_csr.cuh b/cpp/include/raft/sparse/convert/detail/bitset_to_csr.cuh index 3e471687da..8add49a9bc 100644 --- a/cpp/include/raft/sparse/convert/detail/bitset_to_csr.cuh +++ b/cpp/include/raft/sparse/convert/detail/bitset_to_csr.cuh @@ -65,11 +65,10 @@ void gpu_repeat_csr(raft::resources const& handle, { if (nnz == 0) return; - auto stream = resource::get_cuda_stream(handle); index_t repeat_csr_tpb = 256; index_t grid = (nnz + repeat_csr_tpb - 1) / (repeat_csr_tpb); - raft::launch_kernel(stream, grid, repeat_csr_tpb)(repeat_csr_kernel, + raft::launch_kernel(handle, grid, repeat_csr_tpb)(repeat_csr_kernel, d_indptr, d_indices, d_repeated_indptr, diff --git a/cpp/include/raft/sparse/linalg/detail/laplacian.cuh b/cpp/include/raft/sparse/linalg/detail/laplacian.cuh index fc356ce6a0..a905232237 100644 --- a/cpp/include/raft/sparse/linalg/detail/laplacian.cuh +++ b/cpp/include/raft/sparse/linalg/detail/laplacian.cuh @@ -105,8 +105,7 @@ auto compute_graph_laplacian( auto result_structure = result.structure_view(); auto static constexpr const threads_per_block = 256; auto blocks = std::min(int((dim + threads_per_block - 1) / threads_per_block), 65535); - auto stream = resource::get_cuda_stream(res); - raft::launch_kernel(stream, blocks, threads_per_block)( + raft::launch_kernel(res, blocks, threads_per_block)( detail::compute_graph_laplacian_kernel, std::remove_const_t, std::remove_const_t>, diff --git a/cpp/include/raft/sparse/linalg/detail/symmetrize.cuh b/cpp/include/raft/sparse/linalg/detail/symmetrize.cuh index 12f250e494..14cca3ad2c 100644 --- a/cpp/include/raft/sparse/linalg/detail/symmetrize.cuh +++ b/cpp/include/raft/sparse/linalg/detail/symmetrize.cuh @@ -208,7 +208,7 @@ void coo_symmetrize(raft::resources const& handle, handle, raft::make_device_vector_view(out_cols, out_nnz), static_cast(0)); raft::matrix::fill(handle, raft::make_device_vector_view(out_vals, out_nnz), static_cast(0.0)); - raft::launch_kernel(stream, grid, blk)(coo_symmetrize_kernel, + raft::launch_kernel(handle, grid, blk)(coo_symmetrize_kernel, in_row_ind.data(), in_rows, in_cols, diff --git a/cpp/include/raft/sparse/linalg/detail/utils.cuh b/cpp/include/raft/sparse/linalg/detail/utils.cuh index 725f4f4a59..9ed67675aa 100644 --- a/cpp/include/raft/sparse/linalg/detail/utils.cuh +++ b/cpp/include/raft/sparse/linalg/detail/utils.cuh @@ -97,8 +97,6 @@ void faster_dot_on_csr(raft::resources const& handle, { if (nnz == 0 || n_rows == 0) return; - auto stream = resource::get_cuda_stream(handle); - constexpr value_idx MAX_ROW_PER_ITER = 500; int dev_id, sm_count, blocks_per_sm; @@ -115,7 +113,7 @@ void faster_dot_on_csr(raft::resources const& handle, (std::min(value_idx(blocks_per_sm * sm_count * 16), nnz) + block_x - 1) / block_x; dim3 blocks(block_x, block_y, 1); - raft::launch_kernel(stream, blocks, tpb, smem_size)( + raft::launch_kernel(handle, blocks, tpb, smem_size)( faster_dot_on_csr_kernel, dot, indptr, @@ -135,7 +133,7 @@ void faster_dot_on_csr(raft::resources const& handle, (std::min(value_idx(blocks_per_sm * sm_count * 16), nnz) + block_x - 1) / block_x; dim3 blocks(block_x, block_y, 1); - raft::launch_kernel(stream, blocks, tpb, smem_size)( + raft::launch_kernel(handle, blocks, tpb, smem_size)( faster_dot_on_csr_kernel, dot, indptr, @@ -154,7 +152,7 @@ void faster_dot_on_csr(raft::resources const& handle, (std::min(value_idx(blocks_per_sm * sm_count * 16), nnz) + block_x - 1) / block_x; dim3 blocks(block_x, block_y, 1); - raft::launch_kernel(stream, blocks, tpb, smem_size)( + raft::launch_kernel(handle, blocks, tpb, smem_size)( faster_dot_on_csr_kernel, dot, indptr, @@ -173,7 +171,7 @@ void faster_dot_on_csr(raft::resources const& handle, (std::min(value_idx(blocks_per_sm * sm_count * 16), nnz) + block_x - 1) / block_x; dim3 blocks(block_x, block_y, 1); - raft::launch_kernel(stream, blocks, tpb, smem_size)( + raft::launch_kernel(handle, blocks, tpb, smem_size)( faster_dot_on_csr_kernel, dot, indptr, diff --git a/cpp/include/raft/sparse/op/detail/reduce.cuh b/cpp/include/raft/sparse/op/detail/reduce.cuh index 699c6238ea..ad3049cd8d 100644 --- a/cpp/include/raft/sparse/op/detail/reduce.cuh +++ b/cpp/include/raft/sparse/op/detail/reduce.cuh @@ -144,7 +144,7 @@ void max_duplicates(raft::resources const& handle, out.allocate(size, m, n, true, stream); // perform reduce - raft::launch_kernel(stream, raft::ceildiv(nnz, (nnz_t)256), 256)( + raft::launch_kernel(handle, raft::ceildiv(nnz, (nnz_t)256), 256)( max_duplicates_kernel, rows, cols, diff --git a/cpp/include/raft/sparse/solver/detail/lanczos.cuh b/cpp/include/raft/sparse/solver/detail/lanczos.cuh index c03ce32383..7fbdc45a0d 100644 --- a/cpp/include/raft/sparse/solver/detail/lanczos.cuh +++ b/cpp/include/raft/sparse/solver/detail/lanczos.cuh @@ -159,7 +159,7 @@ void lanczos_solve_ritz( int blockSize = 256; int numBlocks = (ncv + blockSize - 1) / blockSize; - raft::launch_kernel(stream, blockSize, numBlocks)(kernel_triangular_populate, + raft::launch_kernel(handle, blockSize, numBlocks)(kernel_triangular_populate, triangular_matrix.data_handle(), beta.data_handle(), ncv); @@ -167,7 +167,7 @@ void lanczos_solve_ritz( if (beta_k) { int threadsPerBlock = 256; int blocksPerGrid = (k + threadsPerBlock - 1) / threadsPerBlock; - raft::launch_kernel(stream, blocksPerGrid, threadsPerBlock)( + raft::launch_kernel(handle, blocksPerGrid, threadsPerBlock)( kernel_triangular_beta_k, triangular_matrix.data_handle(), beta_k.value().data_handle(), @@ -378,7 +378,7 @@ void lanczos_aux(raft::resources const& handle, auto uu_i = raft::make_device_scalar_view(uu.data_handle() + uu.stride(1) * i); // uu(0, i) raft::linalg::add(handle, make_const_mdspan(alpha_i), make_const_mdspan(uu_i), alpha_i); - raft::launch_kernel(stream, 1, 1)( + raft::launch_kernel(handle, 1, 1)( kernel_clamp_down, alpha_i.data_handle(), static_cast(1e-9)); auto output = raft::make_device_vector_view( @@ -390,10 +390,10 @@ void lanczos_aux(raft::resources const& handle, int blockSize = 256; int numBlocks = (n + blockSize - 1) / blockSize; - raft::launch_kernel(stream, numBlocks, blockSize)( + raft::launch_kernel(handle, numBlocks, blockSize)( kernel_clamp_down_vector, u.data_handle(), static_cast(1e-7), n); - raft::launch_kernel(stream, 1, 1)(kernel_clamp_down, + raft::launch_kernel(handle, 1, 1)(kernel_clamp_down, beta.data_handle() + beta.stride(1) * i, static_cast(1e-6)); @@ -402,7 +402,7 @@ void lanczos_aux(raft::resources const& handle, int threadsPerBlock = 256; int blocksPerGrid = (n + threadsPerBlock - 1) / threadsPerBlock; - raft::launch_kernel(stream, blocksPerGrid, threadsPerBlock)(kernel_normalize, + raft::launch_kernel(handle, blocksPerGrid, threadsPerBlock)(kernel_normalize, u.data_handle(), beta.data_handle(), i, diff --git a/cpp/include/raft/sparse/solver/detail/svds_sign_correction.cuh b/cpp/include/raft/sparse/solver/detail/svds_sign_correction.cuh index cfa7993524..c1eb2e956f 100644 --- a/cpp/include/raft/sparse/solver/detail/svds_sign_correction.cuh +++ b/cpp/include/raft/sparse/solver/detail/svds_sign_correction.cuh @@ -140,8 +140,6 @@ void svd_sign_correction( int m = U ? static_cast(U->extent(0)) : 0; int n = Vt ? static_cast(Vt->extent(1)) : 0; - auto stream = raft::resource::get_cuda_stream(handle); - // threads_per_block must be a power of 2 for the tree reduction in the kernel constexpr int threads_per_block = 256; int smem_size = threads_per_block * (sizeof(ValueTypeT) + sizeof(int)); @@ -149,7 +147,7 @@ void svd_sign_correction( ValueTypeT* U_ptr = U ? U->data_handle() : nullptr; ValueTypeT* Vt_ptr = Vt ? Vt->data_handle() : nullptr; - raft::launch_kernel(stream, k, threads_per_block, smem_size)( + raft::launch_kernel(handle, k, threads_per_block, smem_size)( svd_sign_correction_kernel, U_ptr, Vt_ptr, m, n, k); } diff --git a/cpp/include/raft/spectral/detail/matrix_wrappers.hpp b/cpp/include/raft/spectral/detail/matrix_wrappers.hpp index 16ac7e4192..b876cf497a 100644 --- a/cpp/include/raft/spectral/detail/matrix_wrappers.hpp +++ b/cpp/include/raft/spectral/detail/matrix_wrappers.hpp @@ -385,7 +385,7 @@ struct laplacian_matrix_t : sparse_matrix_t { dim3 gridDim{std::min((n + BLOCK_SIZE - 1) / BLOCK_SIZE, 65535), 1, 1}; dim3 blockDim{BLOCK_SIZE, 1, 1}; - raft::launch_kernel(stream, gridDim, blockDim)( + raft::launch_kernel(handle, gridDim, blockDim)( diagmv, n, alpha, diagonal_.raw(), x, y); // Apply adjacency matrix diff --git a/cpp/include/raft/stats/detail/batched/silhouette_score.cuh b/cpp/include/raft/stats/detail/batched/silhouette_score.cuh index 2af5fcb4f1..a3d8d46479 100644 --- a/cpp/include/raft/stats/detail/batched/silhouette_score.cuh +++ b/cpp/include/raft/stats/detail/batched/silhouette_score.cuh @@ -214,7 +214,7 @@ value_t silhouette_score( dim3 block_size(std::min(n_rows, 32), std::min(n_labels, 32)); dim3 grid_size(raft::ceildiv(n_rows, (value_idx)block_size.x), raft::ceildiv(n_labels, (label_idx)block_size.y)); - raft::launch_kernel(stream, grid_size, block_size)( + raft::launch_kernel(handle, grid_size, block_size)( detail::fill_b_kernel, b_ptr, y, n_rows, n_labels, cluster_counts.data()); resource::wait_stream_pool_on_stream(handle); diff --git a/cpp/include/raft/stats/detail/trustworthiness_score.cuh b/cpp/include/raft/stats/detail/trustworthiness_score.cuh index 9bcd8045fe..9600570fed 100644 --- a/cpp/include/raft/stats/detail/trustworthiness_score.cuh +++ b/cpp/include/raft/stats/detail/trustworthiness_score.cuh @@ -181,21 +181,20 @@ double trustworthiness_score(const raft::resources& h, int work = curBatchSize * n; int n_blocks = raft::ceildiv(work, N_THREADS); - raft::launch_kernel(stream, n_blocks, N_THREADS)( + raft::launch_kernel(h, n_blocks, N_THREADS)( build_lookup_table, lookup_table.data(), X_ind.data(), n, work); RAFT_CUDA_TRY(cudaMemsetAsync(t_dbuf.data(), 0, sizeof(double), stream)); work = curBatchSize * (n_neighbors + 1); n_blocks = raft::ceildiv(work, N_THREADS); - raft::launch_kernel(stream, n_blocks, N_THREADS)( - compute_rank, - t_dbuf.data(), - lookup_table.data(), - &emb_ind.data()[(n - toDo) * (n_neighbors + 1)], - n, - n_neighbors + 1, - work); + raft::launch_kernel(h, n_blocks, N_THREADS)(compute_rank, + t_dbuf.data(), + lookup_table.data(), + &emb_ind.data()[(n - toDo) * (n_neighbors + 1)], + n, + n_neighbors + 1, + work); t += t_dbuf.value(stream); diff --git a/cpp/tests/linalg/axpy.cu b/cpp/tests/linalg/axpy.cu index c1798df78d..aa4993ed60 100644 --- a/cpp/tests/linalg/axpy.cu +++ b/cpp/tests/linalg/axpy.cu @@ -80,7 +80,7 @@ class AxpyTest : public ::testing::TestWithParam> { int threads = 64; int blocks = raft::ceildiv(params.len, threads); - raft::launch_kernel(stream, blocks, threads)( + raft::launch_kernel(handle, blocks, threads)( naiveAxpy, params.len, params.alpha, x.data(), refy.data(), params.incx, params.incy); auto host_alpha_view = make_host_scalar_view(¶ms.alpha); diff --git a/cpp/tests/linalg/dot.cu b/cpp/tests/linalg/dot.cu index d23b3b96a5..cd1fc1b7e5 100644 --- a/cpp/tests/linalg/dot.cu +++ b/cpp/tests/linalg/dot.cu @@ -64,7 +64,7 @@ class DotTest : public ::testing::TestWithParam> { uniform(handle, r, y.data(), y_len, T(-1.0), T(1.0)); rmm::device_scalar ref(0, resource::get_cuda_stream(handle)); - raft::launch_kernel(stream, 256, 256)( + raft::launch_kernel(handle, 256, 256)( naiveDot, params.len, x.data(), params.incx, y.data(), params.incy, ref.data()); raft::update_host(&ref_output, ref.data(), 1, stream); diff --git a/cpp/tests/linalg/gemv.cu b/cpp/tests/linalg/gemv.cu index 935a27f642..d63ea8e025 100644 --- a/cpp/tests/linalg/gemv.cu +++ b/cpp/tests/linalg/gemv.cu @@ -99,7 +99,7 @@ class GemvTest : public ::testing::TestWithParam> { dim3 blocks(raft::ceildiv(yElems, 256), 1, 1); dim3 threads(256, 1, 1); - raft::launch_kernel(stream, blocks, threads)(naiveGemv, + raft::launch_kernel(handle, blocks, threads)(naiveGemv, refy.data(), A.data(), x.data(), diff --git a/cpp/tests/linalg/mean_squared_error.cu b/cpp/tests/linalg/mean_squared_error.cu index 0f4129fa2d..86a3d004ae 100644 --- a/cpp/tests/linalg/mean_squared_error.cu +++ b/cpp/tests/linalg/mean_squared_error.cu @@ -76,7 +76,7 @@ class MeanSquaredErrorTest : public ::testing::TestWithParam(output.data()), params.weight); - raft::launch_kernel(stream, 256, 256)( + raft::launch_kernel(handle, 256, 256)( naiveMeanSquaredError, params.len, a.data(), b.data(), params.weight, refoutput.data()); resource::sync_stream(handle); } diff --git a/cpp/tests/random/multi_variable_gaussian.cu b/cpp/tests/random/multi_variable_gaussian.cu index 2d4b732494..c8a3958bdc 100644 --- a/cpp/tests/random/multi_variable_gaussian.cu +++ b/cpp/tests/random/multi_variable_gaussian.cu @@ -155,14 +155,14 @@ class MVGTest : public ::testing::TestWithParam> { RAFT_CUDA_TRY(cudaMemset(Rand_mean.data(), 0, dim * sizeof(T))); dim3 block = (64); dim3 grid = (raft::ceildiv(nPoints * dim, (int)block.x)); - raft::launch_kernel(stream, grid, block)( + raft::launch_kernel(handle, grid, block)( En_KF_accumulate, nPoints, dim, X_d.data(), Rand_mean.data()); grid = (raft::ceildiv(dim, (int)block.x)); - raft::launch_kernel(stream, grid, block)(En_KF_normalize, nPoints, dim, Rand_mean.data()); + raft::launch_kernel(handle, grid, block)(En_KF_normalize, nPoints, dim, Rand_mean.data()); // storing the error wrt random point mean in X_d grid = (raft::ceildiv(dim * nPoints, (int)block.x)); - raft::launch_kernel(stream, grid, block)( + raft::launch_kernel(handle, grid, block)( En_KF_dif, nPoints, dim, X_d.data(), Rand_mean.data(), X_d.data()); // finding the cov matrix, placing in Rand_cov @@ -285,14 +285,14 @@ class MVGMdspanTest : public ::testing::TestWithParam> { RAFT_CUDA_TRY(cudaMemset(Rand_mean.data(), 0, dim * sizeof(T))); dim3 block = (64); dim3 grid = (raft::ceildiv(nPoints * dim, (int)block.x)); - raft::launch_kernel(stream, grid, block)( + raft::launch_kernel(handle, grid, block)( En_KF_accumulate, nPoints, dim, X_d.data(), Rand_mean.data()); grid = (raft::ceildiv(dim, (int)block.x)); - raft::launch_kernel(stream, grid, block)(En_KF_normalize, nPoints, dim, Rand_mean.data()); + raft::launch_kernel(handle, grid, block)(En_KF_normalize, nPoints, dim, Rand_mean.data()); // storing the error wrt random point mean in X_d grid = (raft::ceildiv(dim * nPoints, (int)block.x)); - raft::launch_kernel(stream, grid, block)( + raft::launch_kernel(handle, grid, block)( En_KF_dif, nPoints, dim, X_d.data(), Rand_mean.data(), X_d.data()); // finding the cov matrix, placing in Rand_cov diff --git a/cpp/tests/sparse/spmm.cu b/cpp/tests/sparse/spmm.cu index 80777058d0..96e19fd158 100644 --- a/cpp/tests/sparse/spmm.cu +++ b/cpp/tests/sparse/spmm.cu @@ -173,7 +173,7 @@ class SpmmTest : public ::testing::TestWithParam> { dim3 blocks(raft::ceildiv(params.M, 128), raft::ceildiv(params.N, 4), 1); dim3 threads(128, 4, 1); - raft::launch_kernel(stream, blocks, threads)(naiveGemm, + raft::launch_kernel(handle, blocks, threads)(naiveGemm, params.trans_x, params.trans_y, params.M, diff --git a/cpp/tests/stats/minmax.cu b/cpp/tests/stats/minmax.cu index 2a66a5fdab..f2c2f31f27 100644 --- a/cpp/tests/stats/minmax.cu +++ b/cpp/tests/stats/minmax.cu @@ -108,7 +108,7 @@ class MinMaxTest : public ::testing::TestWithParam> { T nan_prob = 0.01; bernoulli(handle, r, mask.data(), len, nan_prob); const int TPB = 256; - raft::launch_kernel(stream, raft::ceildiv(len, TPB), TPB)( + raft::launch_kernel(handle, raft::ceildiv(len, TPB), TPB)( nanKernel, data.data(), mask.data(), len, std::numeric_limits::quiet_NaN()); naiveMinMax(data.data(), params.rows, From ee2e96c1dec8f046a140f596f2f91538fdda37d5 Mon Sep 17 00:00:00 2001 From: achirkin Date: Mon, 10 Aug 2026 10:13:59 +0200 Subject: [PATCH 3/4] Fix the docs --- cpp/include/raft/util/kernel_launch.hpp | 41 ++++++++++++++++++++++++- docs/source/cpp_api/utils.rst | 12 ++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/cpp/include/raft/util/kernel_launch.hpp b/cpp/include/raft/util/kernel_launch.hpp index 5c845aa3ca..20be376c1b 100644 --- a/cpp/include/raft/util/kernel_launch.hpp +++ b/cpp/include/raft/util/kernel_launch.hpp @@ -76,6 +76,8 @@ inline void throw_on_cuda_launch_error(cudaError_t status, std::source_location /** * @brief Temporary object that launches a CUDA kernel with call-site error reporting. * + * @ingroup kernel_launch + * * Capture @c std::source_location on @ref launch_kernel, then launch via rvalue @c operator(). * Prefer the one-liner form so diagnostics point at the launch expression: * @code @@ -134,7 +136,7 @@ class kernel_launcher { rmm::cuda_stream_view, dim3, dim3, std::size_t, std::source_location); /** - * @brief Copy the launch arguments into parameters and pass their addresses to @ref dispatch. + * @brief Copy the launch arguments into parameters and pass their addresses to dispatch(). * * Taking the address of a copy rather than of the caller's object means passing a constant (e.g. * a @@ -183,6 +185,29 @@ class kernel_launcher { std::source_location location_{}; }; +/** + * @defgroup kernel_launch Type-checked CUDA kernel launch + * @{ + */ + +/** + * @brief Launch a CUDA kernel on the stream of @p res, reporting errors at the call site. + * + * The returned launcher is a temporary that must be called immediately, so that the diagnostics of + * a failed launch blame the launch expression rather than this header: + * @code + * raft::launch_kernel(res, grid, block)(my_kernel, arg0, arg1); + * @endcode + * The launch arguments are checked against the kernel parameters at compile time, and a failed + * launch throws @c raft::cuda_error. + * + * @param[in] res raft resources providing the stream to launch on + * @param[in] grid grid dimensions + * @param[in] block block dimensions + * @param[in] shared_mem_bytes dynamic shared memory size in bytes + * @param[in] location call site to blame for launch errors; leave at its default + * @return a launcher to invoke with the kernel and its arguments + */ inline kernel_launcher launch_kernel( resources const& res, dim3 grid, @@ -193,6 +218,18 @@ inline kernel_launcher launch_kernel( return kernel_launcher{resource::get_cuda_stream(res), grid, block, shared_mem_bytes, location}; } +/** + * @brief Launch a CUDA kernel on @p stream, reporting errors at the call site. + * + * Same as the overload taking raft resources; prefer that one where resources are available. + * + * @param[in] stream stream to launch on + * @param[in] grid grid dimensions + * @param[in] block block dimensions + * @param[in] shared_mem_bytes dynamic shared memory size in bytes + * @param[in] location call site to blame for launch errors; leave at its default + * @return a launcher to invoke with the kernel and its arguments + */ inline kernel_launcher launch_kernel( rmm::cuda_stream_view stream, dim3 grid, @@ -203,4 +240,6 @@ inline kernel_launcher launch_kernel( return kernel_launcher{stream, grid, block, shared_mem_bytes, location}; } +/** @} */ // end group kernel_launch + } // namespace raft diff --git a/docs/source/cpp_api/utils.rst b/docs/source/cpp_api/utils.rst index 4471093c8b..ec847cbd35 100644 --- a/docs/source/cpp_api/utils.rst +++ b/docs/source/cpp_api/utils.rst @@ -19,3 +19,15 @@ namespace *raft* :project: RAFT :members: :content-only: + +Kernel Launch +------------- + +``#include `` + +namespace *raft* + +.. doxygengroup:: kernel_launch + :project: RAFT + :members: + :content-only: From 811adad1cdc909ea98ee3b8684ad4650a825dd7e Mon Sep 17 00:00:00 2001 From: achirkin Date: Wed, 12 Aug 2026 15:36:21 +0200 Subject: [PATCH 4/4] Launch config as an argument to launch_kernel --- cpp/include/raft/common/detail/scatter.cuh | 4 +- cpp/include/raft/core/bitset.cuh | 10 +- cpp/include/raft/core/detail/copy.hpp | 8 +- cpp/include/raft/label/detail/classlabels.cuh | 19 +- .../raft/label/detail/merge_labels.cuh | 23 +- cpp/include/raft/linalg/detail/add.cuh | 4 +- .../linalg/detail/coalesced_reduction-inl.cuh | 105 +++++-- cpp/include/raft/linalg/detail/map.cuh | 4 +- .../raft/linalg/detail/map_then_reduce.cuh | 5 +- cpp/include/raft/linalg/detail/normalize.cuh | 30 +- .../raft/linalg/detail/reduce_cols_by_key.cuh | 16 +- .../raft/linalg/detail/reduce_rows_by_key.cuh | 50 ++- .../raft/linalg/detail/strided_reduction.cuh | 25 +- cpp/include/raft/linalg/detail/subtract.cuh | 4 +- cpp/include/raft/linalg/detail/transpose.cuh | 36 ++- .../raft/matrix/detail/columnWiseSort.cuh | 22 +- cpp/include/raft/matrix/detail/gather.cuh | 45 ++- .../raft/matrix/detail/linewise_op.cuh | 64 ++-- cpp/include/raft/matrix/detail/math.cuh | 16 +- cpp/include/raft/matrix/detail/matrix.cuh | 23 +- .../raft/matrix/detail/select_radix.cuh | 84 ++--- .../raft/matrix/detail/select_warpsort.cuh | 46 +-- cpp/include/raft/matrix/detail/shift.cuh | 64 ++-- cpp/include/raft/random/detail/make_blobs.cuh | 25 +- .../raft/random/detail/make_regression.cuh | 32 +- .../random/detail/multi_variable_gaussian.cuh | 4 +- cpp/include/raft/random/detail/permute.cuh | 26 +- .../detail/rmat_rectangular_generator.cuh | 32 +- cpp/include/raft/random/detail/rng_impl.cuh | 13 +- .../random/detail/rng_impl_deprecated.cuh | 23 +- .../raft/solver/detail/lap_functions.cuh | 294 ++++++++++-------- .../raft/sparse/convert/detail/adj_to_csr.cuh | 12 +- .../sparse/convert/detail/bitmap_to_csr.cuh | 39 ++- .../sparse/convert/detail/bitset_to_csr.cuh | 17 +- .../raft/sparse/convert/detail/coo.cuh | 4 +- .../raft/sparse/convert/detail/dense.cuh | 11 +- cpp/include/raft/sparse/detail/csr.cuh | 34 +- cpp/include/raft/sparse/detail/utils.h | 2 +- cpp/include/raft/sparse/linalg/detail/add.cuh | 54 ++-- .../raft/sparse/linalg/detail/degree.cuh | 23 +- .../raft/sparse/linalg/detail/laplacian.cuh | 24 +- .../raft/sparse/linalg/detail/norm.cuh | 8 +- .../raft/sparse/linalg/detail/symmetrize.cuh | 88 ++++-- .../raft/sparse/linalg/detail/utils.cuh | 90 +++--- cpp/include/raft/sparse/op/detail/filter.cuh | 27 +- cpp/include/raft/sparse/op/detail/reduce.cuh | 32 +- cpp/include/raft/sparse/op/detail/row_op.cuh | 4 +- .../raft/sparse/solver/detail/lanczos.cuh | 72 +++-- .../sparse/solver/detail/mst_solver_inl.cuh | 148 +++++---- .../solver/detail/svds_sign_correction.cuh | 6 +- .../raft/spectral/detail/matrix_wrappers.hpp | 4 +- .../stats/detail/batched/silhouette_score.cuh | 36 ++- .../raft/stats/detail/contingencyMatrix.cuh | 35 ++- cpp/include/raft/stats/detail/dispersion.cuh | 23 +- cpp/include/raft/stats/detail/histogram.cuh | 58 ++-- cpp/include/raft/stats/detail/meanvar.cuh | 27 +- cpp/include/raft/stats/detail/minmax.cuh | 35 ++- .../raft/stats/detail/mutual_info_score.cuh | 18 +- .../raft/stats/detail/neighborhood_recall.cuh | 5 +- cpp/include/raft/stats/detail/rand_index.cuh | 16 +- cpp/include/raft/stats/detail/scores.cuh | 15 +- .../raft/stats/detail/silhouette_score.cuh | 19 +- .../stats/detail/trustworthiness_score.cuh | 21 +- cpp/include/raft/util/cache.cuh | 69 ++-- cpp/include/raft/util/detail/scatter.cuh | 4 +- cpp/include/raft/util/kernel_launch.hpp | 234 +++++++------- cpp/tests/core/interruptible.cu | 4 +- cpp/tests/core/math_device.cu | 4 +- cpp/tests/core/operators_device.cu | 4 +- cpp/tests/core/span.cu | 10 +- cpp/tests/linalg/add.cuh | 2 +- cpp/tests/linalg/axpy.cu | 12 +- cpp/tests/linalg/binary_op.cuh | 4 +- cpp/tests/linalg/divide.cu | 2 +- cpp/tests/linalg/dot.cu | 12 +- cpp/tests/linalg/eltwise.cu | 4 +- cpp/tests/linalg/gemm_layout.cu | 23 +- cpp/tests/linalg/gemv.cu | 19 +- cpp/tests/linalg/map_then_reduce.cu | 4 +- cpp/tests/linalg/matrix_vector_op.cuh | 19 +- cpp/tests/linalg/mean_squared_error.cu | 11 +- cpp/tests/linalg/norm.cu | 8 +- cpp/tests/linalg/power.cu | 4 +- cpp/tests/linalg/reduce.cuh | 30 +- cpp/tests/linalg/reduce_rows_by_key.cu | 24 +- cpp/tests/linalg/sqrt.cu | 2 +- cpp/tests/linalg/subtract.cu | 4 +- cpp/tests/linalg/unary_op.cuh | 4 +- cpp/tests/matrix/math.cu | 6 +- cpp/tests/random/make_blobs.cu | 38 ++- cpp/tests/random/multi_variable_gaussian.cu | 20 +- .../random/rmat_rectangular_generator.cu | 32 +- cpp/tests/random/rng.cu | 36 ++- cpp/tests/random/rng_int.cu | 18 +- cpp/tests/random/rng_pcg_host_api.cu | 16 +- cpp/tests/sparse/convert_csr.cu | 4 +- cpp/tests/sparse/spmm.cu | 37 ++- cpp/tests/sparse/symmetrize.cu | 11 +- cpp/tests/stats/histogram.cu | 2 +- cpp/tests/stats/minmax.cu | 18 +- cpp/tests/util/bitonic_sort.cu | 10 +- cpp/tests/util/device_atomics.cu | 8 +- cpp/tests/util/integer_utils.cu | 17 +- cpp/tests/util/kernel_launch.cu | 69 +++- cpp/tests/util/reduction.cu | 26 +- 105 files changed, 1917 insertions(+), 1131 deletions(-) diff --git a/cpp/include/raft/common/detail/scatter.cuh b/cpp/include/raft/common/detail/scatter.cuh index 3b17e9f0a3..7307fffd5f 100644 --- a/cpp/include/raft/common/detail/scatter.cuh +++ b/cpp/include/raft/common/detail/scatter.cuh @@ -37,8 +37,8 @@ void scatterImpl( DataT* out, const DataT* in, const IdxT* idx, IdxT len, Lambda op, cudaStream_t stream) { const IdxT nblks = raft::ceildiv(VecLen ? len / VecLen : len, (IdxT)TPB); - raft::launch_kernel(stream, nblks, TPB)( - scatterKernel, out, in, idx, len, op); + raft::launch_kernel( + stream, nblks, TPB, scatterKernel, out, in, idx, len, op); } } // namespace detail diff --git a/cpp/include/raft/core/bitset.cuh b/cpp/include/raft/core/bitset.cuh index 7042c98a0c..1b98826267 100644 --- a/cpp/include/raft/core/bitset.cuh +++ b/cpp/include/raft/core/bitset.cuh @@ -155,8 +155,14 @@ void bitset_repeat(raft::resources const& handle, int threadsPerBlock = 128; int blocksPerGrid = (output_size + threadsPerBlock - 1) / threadsPerBlock; - raft::launch_kernel(handle, blocksPerGrid, threadsPerBlock)( - bitset_repeat_kernel, d_src, d_output, src_bit_len, repeat_times); + raft::launch_kernel(handle, + blocksPerGrid, + threadsPerBlock, + bitset_repeat_kernel, + d_src, + d_output, + src_bit_len, + repeat_times); return; } diff --git a/cpp/include/raft/core/detail/copy.hpp b/cpp/include/raft/core/detail/copy.hpp index c437bf835e..e00f9424c4 100644 --- a/cpp/include/raft/core/detail/copy.hpp +++ b/cpp/include/raft/core/detail/copy.hpp @@ -504,8 +504,12 @@ mdspan_copyable_t copy(resources const& res, DstType&& dst, Sr raft::ceildiv(typename config::index_type(dst.size()), typename config::index_type(mdspan_copy_tile_elems))); auto constexpr const threads = dim3{mdspan_copy_tile_dim, mdspan_copy_tile_dim, 1}; - raft::launch_kernel(res, blocks, threads)( - mdspan_copy_kernel, dst, src); + raft::launch_kernel(res, + blocks, + threads, + mdspan_copy_kernel, + dst, + src); #else // Should never actually reach this because of enable_ifs. Included for // safety. diff --git a/cpp/include/raft/label/detail/classlabels.cuh b/cpp/include/raft/label/detail/classlabels.cuh index 2a29be01f4..61c68426d1 100644 --- a/cpp/include/raft/label/detail/classlabels.cuh +++ b/cpp/include/raft/label/detail/classlabels.cuh @@ -164,14 +164,17 @@ void make_monotonic( rmm::device_uvector map_ids(0, stream); int num_clusters = getUniquelabels(map_ids, in, N, stream); - raft::launch_kernel(stream, blocks, threads)(map_label_kernel, - map_ids.data(), - num_clusters, - in, - out, - N, - filter_op, - zero_based); + raft::launch_kernel(stream, + blocks, + threads, + map_label_kernel, + map_ids.data(), + num_clusters, + in, + out, + N, + filter_op, + zero_based); } /** diff --git a/cpp/include/raft/label/detail/merge_labels.cuh b/cpp/include/raft/label/detail/merge_labels.cuh index a0e5797d57..a2506977a7 100644 --- a/cpp/include/raft/label/detail/merge_labels.cuh +++ b/cpp/include/raft/label/detail/merge_labels.cuh @@ -130,16 +130,31 @@ void merge_labels(value_idx* labels_a, do { RAFT_CUDA_TRY(cudaMemsetAsync(m, false, sizeof(bool), stream)); - raft::launch_kernel(stream, blocks, threads)( - propagate_label_kernel, labels_a, labels_b, R, mask, m, N); + raft::launch_kernel(stream, + blocks, + threads, + propagate_label_kernel, + labels_a, + labels_b, + R, + mask, + m, + N); raft::update_host(&host_m, m, 1, stream); RAFT_CUDA_TRY(cudaStreamSynchronize(stream)); } while (host_m); // Step 2: re-assign minimum equivalent label - raft::launch_kernel(stream, blocks, threads)( - reassign_label_kernel, labels_a, labels_b, R, N, MAX_LABEL); + raft::launch_kernel(stream, + blocks, + threads, + reassign_label_kernel, + labels_a, + labels_b, + R, + N, + MAX_LABEL); } } // namespace detail diff --git a/cpp/include/raft/linalg/detail/add.cuh b/cpp/include/raft/linalg/detail/add.cuh index 5fc394d497..b69085622a 100644 --- a/cpp/include/raft/linalg/detail/add.cuh +++ b/cpp/include/raft/linalg/detail/add.cuh @@ -45,8 +45,8 @@ void addDevScalar( // TODO: block dimension has not been tuned dim3 block(256); dim3 grid(raft::ceildiv(len, (IdxType)block.x)); - raft::launch_kernel(stream, grid, block)( - add_dev_scalar_kernel, outDev, inDev, singleScalarDev, len); + raft::launch_kernel( + stream, grid, block, add_dev_scalar_kernel, outDev, inDev, singleScalarDev, len); } } // namespace detail diff --git a/cpp/include/raft/linalg/detail/coalesced_reduction-inl.cuh b/cpp/include/raft/linalg/detail/coalesced_reduction-inl.cuh index 9e9b0a13ad..e921611f6e 100644 --- a/cpp/include/raft/linalg/detail/coalesced_reduction-inl.cuh +++ b/cpp/include/raft/linalg/detail/coalesced_reduction-inl.cuh @@ -253,19 +253,32 @@ void coalescedReductionThin(OutType* dots, dim3 threads(Policy::LogicalWarpSize, Policy::NumLogicalWarps, 1); dim3 blocks(ceildiv(N, Policy::RowsPerBlock), 1, 1); if constexpr (std::is_same_v) { - raft::launch_kernel(stream, blocks, threads)( - coalescedSumThinKernel, dots, data, D, N, init, main_op, final_op, inplace); + raft::launch_kernel(stream, + blocks, + threads, + coalescedSumThinKernel, + dots, + data, + D, + N, + init, + main_op, + final_op, + inplace); } else { - raft::launch_kernel(stream, blocks, threads)(coalescedReductionThinKernel, - dots, - data, - D, - N, - init, - main_op, - reduce_op, - final_op, - inplace); + raft::launch_kernel(stream, + blocks, + threads, + coalescedReductionThinKernel, + dots, + data, + D, + N, + init, + main_op, + reduce_op, + final_op, + inplace); } } @@ -406,19 +419,32 @@ void coalescedReductionMedium(OutType* dots, { common::nvtx::range fun_scope("coalescedReductionMedium<%d>", TPB); if constexpr (std::is_same_v) { - raft::launch_kernel(stream, N, TPB)( - coalescedSumMediumKernel, dots, data, D, N, init, main_op, final_op, inplace); + raft::launch_kernel(stream, + N, + TPB, + coalescedSumMediumKernel, + dots, + data, + D, + N, + init, + main_op, + final_op, + inplace); } else { - raft::launch_kernel(stream, N, TPB)(coalescedReductionMediumKernel, - dots, - data, - D, - N, - init, - main_op, - reduce_op, - final_op, - inplace); + raft::launch_kernel(stream, + N, + TPB, + coalescedReductionMediumKernel, + dots, + data, + D, + N, + init, + main_op, + reduce_op, + final_op, + inplace); } } @@ -540,17 +566,28 @@ void coalescedReductionThick(OutType* dots, * main_op but applies final_op. If in-place, the existing and new values are reduced. */ if constexpr (std::is_same_v) { - raft::launch_kernel(stream, blocks, threads)( - coalescedSumThickKernel, buffer.data(), data, D, N, init, main_op); + raft::launch_kernel(stream, + blocks, + threads, + coalescedSumThickKernel, + buffer.data(), + data, + D, + N, + init, + main_op); } else { - raft::launch_kernel(stream, blocks, threads)(coalescedReductionThickKernel, - buffer.data(), - data, - D, - N, - init, - main_op, - reduce_op); + raft::launch_kernel(stream, + blocks, + threads, + coalescedReductionThickKernel, + buffer.data(), + data, + D, + N, + init, + main_op, + reduce_op); } coalescedReductionThin(dots, diff --git a/cpp/include/raft/linalg/detail/map.cuh b/cpp/include/raft/linalg/detail/map.cuh index d91cce2ba2..7d03890c85 100644 --- a/cpp/include/raft/linalg/detail/map.cuh +++ b/cpp/include/raft/linalg/detail/map.cuh @@ -98,8 +98,8 @@ void map_call(rmm::cuda_stream_view stream, OutT* out_ptr, IdxT len, Func f, con const int threads = std::max(WarpSize, std::min(raft::bound_by_power_of_two(len_vectorized), 256)); const IdxT blocks = raft::div_rounding_up_unsafe(len_vectorized, threads); - raft::launch_kernel(stream, blocks, threads)( - map_kernel, out_ptr, len, f, in_ptrs...); + raft::launch_kernel( + stream, blocks, threads, map_kernel, out_ptr, len, f, in_ptrs...); } constexpr int kCoalescedVectorSize = 16; diff --git a/cpp/include/raft/linalg/detail/map_then_reduce.cuh b/cpp/include/raft/linalg/detail/map_then_reduce.cuh index 062b0cecdf..86bbb6af81 100644 --- a/cpp/include/raft/linalg/detail/map_then_reduce.cuh +++ b/cpp/include/raft/linalg/detail/map_then_reduce.cuh @@ -80,7 +80,10 @@ void mapThenReduceImpl(OutType* out, { raft::update_device(out, &neutral, 1, stream); const int nblks = raft::ceildiv(len, IdxType(TPB)); - raft::launch_kernel(stream, nblks, TPB)( + raft::launch_kernel( + stream, + nblks, + TPB, mapThenReduceKernel, out, len, diff --git a/cpp/include/raft/linalg/detail/normalize.cuh b/cpp/include/raft/linalg/detail/normalize.cuh index 44ed474c5c..918df6084a 100644 --- a/cpp/include/raft/linalg/detail/normalize.cuh +++ b/cpp/include/raft/linalg/detail/normalize.cuh @@ -74,8 +74,19 @@ inline void coalesced_normalize_thin(Type* out, { dim3 grid(ceildiv(N, (IdxType)Policy::RowsPerBlock), 1, 1); dim3 block(Policy::LogicalWarpSize, Policy::RowsPerBlock, 1); - raft::launch_kernel(stream, grid, block)( - coalesced_normalize_thin_kernel, out, in, D, N, init, main_op, reduce_op, fin_op, eps); + raft::launch_kernel(stream, + grid, + block, + coalesced_normalize_thin_kernel, + out, + in, + D, + N, + init, + main_op, + reduce_op, + fin_op, + eps); } template , out, in, D, N, init, main_op, reduce_op, fin_op, eps); + raft::launch_kernel(stream, + N, + TPB, + coalesced_normalize_medium_kernel, + out, + in, + D, + N, + init, + main_op, + reduce_op, + fin_op, + eps); } template (nrows * ncols, TPB); int nblks = std::min(target_nblks, max_nblks); - raft::launch_kernel(stream, nblks, TPB, cache_size)( - reduce_cols_by_key_cached_kernel, data, keys, out, nrows, ncols, nkeys); + raft::launch_kernel({stream, cache_size}, + nblks, + TPB, + reduce_cols_by_key_cached_kernel, + data, + keys, + out, + nrows, + ncols, + nkeys); } else { constexpr int TPB = 256; int nblks = raft::ceildiv(nrows * ncols, TPB); - raft::launch_kernel(stream, nblks, TPB)( - reduce_cols_by_key_direct_kernel, data, keys, out, nrows, ncols, nkeys); + raft::launch_kernel( + stream, nblks, TPB, reduce_cols_by_key_direct_kernel, data, keys, out, nrows, ncols, nkeys); } RAFT_CUDA_TRY(cudaPeekAtLastError()); } diff --git a/cpp/include/raft/linalg/detail/reduce_rows_by_key.cuh b/cpp/include/raft/linalg/detail/reduce_rows_by_key.cuh index 0604e705e1..347a6a7382 100644 --- a/cpp/include/raft/linalg/detail/reduce_rows_by_key.cuh +++ b/cpp/include/raft/linalg/detail/reduce_rows_by_key.cuh @@ -46,7 +46,7 @@ void convert_array(IteratorT1 dst, IteratorT2 src, int n, cudaStream_t st) grid.x = raft::ceildiv(n, (int)block.x); grid.x = std::min(grid.x, MAX_BLOCKS); - raft::launch_kernel(st, grid, block)(convert_array_kernel, dst, src, n); + raft::launch_kernel(st, grid, block, convert_array_kernel, dst, src, n); } template @@ -168,8 +168,18 @@ void sum_rows_by_key_small_nkeys(const DataIteratorT d_A, grid.x = std::min(grid.x, 32u); grid.y = ncols; grid.y = std::min(grid.y, MAX_BLOCKS); - raft::launch_kernel(st, grid, block)( - sum_rows_by_key_small_nkeys_kernel, d_A, lda, d_keys, d_weights, nrows, ncols, nkeys, d_sums); + raft::launch_kernel(st, + grid, + block, + sum_rows_by_key_small_nkeys_kernel, + d_A, + lda, + d_keys, + d_weights, + nrows, + ncols, + nkeys, + d_sums); } // @@ -252,15 +262,18 @@ void sum_rows_by_key_large_nkeys_colmajor(const DataIteratorT d_A, grid.x = std::min(grid.x, 32u); grid.y = ncols; grid.y = std::min(grid.y, MAX_BLOCKS); - raft::launch_kernel(st, grid, block)(sum_rows_by_key_large_nkeys_kernel_colmajor, - d_A, - lda, - d_keys, - nrows, - ncols, - key_offset, - nkeys, - d_sums); + raft::launch_kernel(st, + grid, + block, + sum_rows_by_key_large_nkeys_kernel_colmajor, + d_A, + lda, + d_keys, + nrows, + ncols, + key_offset, + nkeys, + d_sums); } template (ceildiv(nrows * ncols, (IdxT)block_dim)); - raft::launch_kernel(st, grid_dim, block_dim)( - sum_rows_by_key_large_nkeys_kernel_rowmajor, d_A, lda, d_weights, d_keys, nrows, ncols, d_sums); + raft::launch_kernel(st, + grid_dim, + block_dim, + sum_rows_by_key_large_nkeys_kernel_rowmajor, + d_A, + lda, + d_weights, + d_keys, + nrows, + ncols, + d_sums); } /** diff --git a/cpp/include/raft/linalg/detail/strided_reduction.cuh b/cpp/include/raft/linalg/detail/strided_reduction.cuh index 0c56f7ab4c..22596689c2 100644 --- a/cpp/include/raft/linalg/detail/strided_reduction.cuh +++ b/cpp/include/raft/linalg/detail/strided_reduction.cuh @@ -156,8 +156,16 @@ void stridedReduction(OutType* dots, raft::min((IdxType)MaxBlocksDimY, raft::ceildiv(N, (IdxType)MinRowsPerBlk))); const size_t shmemSize = sizeof(OutType) * Block.x * 2; - raft::launch_kernel(stream, grid, Block, shmemSize)( - stridedSummationKernel, dots, data, D, N, init, main_op); + raft::launch_kernel({stream, shmemSize}, + grid, + Block, + stridedSummationKernel, + dots, + data, + D, + N, + init, + main_op); } else { // Arbitrary numbers for now, probably need to tune const dim3 thrds(32, 16); @@ -167,8 +175,17 @@ void stridedReduction(OutType* dots, raft::ceildiv(N, (IdxType)thrds.y * elemsPerThread)); const size_t shmemSize = sizeof(OutType) * thrds.x * thrds.y; - raft::launch_kernel(stream, nblks, thrds, shmemSize)( - stridedReductionKernel, dots, data, D, N, init, main_op, reduce_op); + raft::launch_kernel({stream, shmemSize}, + nblks, + thrds, + stridedReductionKernel, + dots, + data, + D, + N, + init, + main_op, + reduce_op); } ///@todo: this complication should go away once we have eliminated the need diff --git a/cpp/include/raft/linalg/detail/subtract.cuh b/cpp/include/raft/linalg/detail/subtract.cuh index b8cc8680a3..229ca3d295 100644 --- a/cpp/include/raft/linalg/detail/subtract.cuh +++ b/cpp/include/raft/linalg/detail/subtract.cuh @@ -49,8 +49,8 @@ void subtractDevScalar(math_t* outDev, // Just for the note - there is no way to express such operation with cuBLAS in effective way // https://stackoverflow.com/questions/14051064/add-scalar-to-vector-in-blas-cublas-cuda const IdxType nblks = raft::ceildiv(len, (IdxType)TPB); - raft::launch_kernel(stream, nblks, TPB)( - subtract_dev_scalar_kernel, outDev, inDev, singleScalarDev, len); + raft::launch_kernel( + stream, nblks, TPB, subtract_dev_scalar_kernel, outDev, inDev, singleScalarDev, len); } }; // end namespace detail diff --git a/cpp/include/raft/linalg/detail/transpose.cuh b/cpp/include/raft/linalg/detail/transpose.cuh index 87c191a196..29b98ec48c 100644 --- a/cpp/include/raft/linalg/detail/transpose.cuh +++ b/cpp/include/raft/linalg/detail/transpose.cuh @@ -117,23 +117,27 @@ void transpose_half(raft::resources const& handle, dim3 grids(adjusted_grid_x, adjusted_grid_y); if (stride_in > 1 || stride_out > 1) { - raft::launch_kernel(handle, grids, blocks)( - transpose_half_kernel, - n_rows, - n_cols, - in, - out, - stride_in, - stride_out); + raft::launch_kernel(handle, + grids, + blocks, + transpose_half_kernel, + n_rows, + n_cols, + in, + out, + stride_in, + stride_out); } else { - raft::launch_kernel(handle, grids, blocks)( - transpose_half_kernel, - n_rows, - n_cols, - in, - out, - n_cols, - n_rows); + raft::launch_kernel(handle, + grids, + blocks, + transpose_half_kernel, + n_rows, + n_cols, + in, + out, + n_cols, + n_rows); } RAFT_CUDA_TRY(cudaPeekAtLastError()); diff --git a/cpp/include/raft/matrix/detail/columnWiseSort.cuh b/cpp/include/raft/matrix/detail/columnWiseSort.cuh index 11d5ccc825..bc4eebcecb 100644 --- a/cpp/include/raft/matrix/detail/columnWiseSort.cuh +++ b/cpp/include/raft/matrix/detail/columnWiseSort.cuh @@ -20,14 +20,16 @@ #include #define INST_BLOCK_SORT(keyIn, keyOut, valueInOut, rows, columns, blockSize, elemPT, stream) \ - raft::launch_kernel(stream, rows, blockSize)( \ - devKeyValSortColumnPerRow, \ - keyIn, \ - keyOut, \ - valueInOut, \ - rows, \ - columns, \ - std::numeric_limits::max()) + raft::launch_kernel(stream, \ + rows, \ + blockSize, \ + devKeyValSortColumnPerRow, \ + keyIn, \ + keyOut, \ + valueInOut, \ + rows, \ + columns, \ + std::numeric_limits::max()) namespace raft { namespace matrix { @@ -143,7 +145,7 @@ cudaError_t layoutIdx(OutType* in, int n_rows, int n_columns, cudaStream_t strea int totalElements = n_rows * n_columns; dim3 block(256); dim3 grid((totalElements + block.x - 1) / block.x); - raft::launch_kernel(stream, grid, block)(devLayoutIdx, in, n_columns, totalElements); + raft::launch_kernel(stream, grid, block, devLayoutIdx, in, n_columns, totalElements); return cudaGetLastError(); } @@ -153,7 +155,7 @@ cudaError_t layoutSortOffset(T* in, T value, int n_times, cudaStream_t stream) { dim3 block(128); dim3 grid((n_times + block.x - 1) / block.x); - raft::launch_kernel(stream, grid, block)(devOffsetKernel, in, value, n_times); + raft::launch_kernel(stream, grid, block, devOffsetKernel, in, value, n_times); return cudaGetLastError(); } diff --git a/cpp/include/raft/matrix/detail/gather.cuh b/cpp/include/raft/matrix/detail/gather.cuh index 9b88c8a8e5..9b5706098f 100644 --- a/cpp/include/raft/matrix/detail/gather.cuh +++ b/cpp/include/raft/matrix/detail/gather.cuh @@ -149,18 +149,51 @@ void gatherImpl(const InputIteratorT in, if (len < static_cast(32 * TPB * n_sm)) { using Policy = gather_policy; IndexT n_blocks = raft::ceildiv(map_length * D, static_cast(Policy::stride)); - raft::launch_kernel(stream, n_blocks, Policy::n_threads)( - gather_kernel, in, ld, D, len, map, stencil, out, pred_op, transform_op); + raft::launch_kernel(stream, + n_blocks, + Policy::n_threads, + gather_kernel, + in, + ld, + D, + len, + map, + stencil, + out, + pred_op, + transform_op); } else if (len < static_cast(32 * 4 * TPB * n_sm)) { using Policy = gather_policy; IndexT n_blocks = raft::ceildiv(map_length * D, static_cast(Policy::stride)); - raft::launch_kernel(stream, n_blocks, Policy::n_threads)( - gather_kernel, in, ld, D, len, map, stencil, out, pred_op, transform_op); + raft::launch_kernel(stream, + n_blocks, + Policy::n_threads, + gather_kernel, + in, + ld, + D, + len, + map, + stencil, + out, + pred_op, + transform_op); } else { using Policy = gather_policy; IndexT n_blocks = raft::ceildiv(map_length * D, static_cast(Policy::stride)); - raft::launch_kernel(stream, n_blocks, Policy::n_threads)( - gather_kernel, in, ld, D, len, map, stencil, out, pred_op, transform_op); + raft::launch_kernel(stream, + n_blocks, + Policy::n_threads, + gather_kernel, + in, + ld, + D, + len, + map, + stencil, + out, + pred_op, + transform_op); } RAFT_CUDA_TRY(cudaPeekAtLastError()); } diff --git a/cpp/include/raft/matrix/detail/linewise_op.cuh b/cpp/include/raft/matrix/detail/linewise_op.cuh index dbea350c23..9bdd7cc646 100644 --- a/cpp/include/raft/matrix/detail/linewise_op.cuh +++ b/cpp/include/raft/matrix/detail/linewise_op.cuh @@ -532,7 +532,10 @@ void matrixLinewiseVecCols(Type* out, // The value below determines how many scalar elements are processed by on thread in total. const IdxType elemsPerThread = raft::ceildiv(alignedLen, gs.x * VecElems * BlockSize) * VecElems; - raft::launch_kernel(stream, gs, bs)( + raft::launch_kernel( + stream, + gs, + bs, matrixLinewiseVecColsMainKernel, out, in, @@ -546,16 +549,18 @@ void matrixLinewiseVecCols(Type* out, if (alignedLen < totalLen) { // should be not smaller than the warp size for better branching constexpr std::size_t MaxOffset = std::max(std::size_t(raft::WarpSize), VecBytes); - raft::launch_kernel(stream, dim3(2, 1, 1), dim3(MaxOffset, 1, 1))( - matrixLinewiseVecColsTailKernel, - out, - in, - alignedOff, - alignedEnd, - rowLen, - totalLen, - op, - vecs...); + raft::launch_kernel(stream, + dim3(2, 1, 1), + dim3(MaxOffset, 1, 1), + matrixLinewiseVecColsTailKernel, + out, + in, + alignedOff, + alignedEnd, + rowLen, + totalLen, + op, + vecs...); } } @@ -599,7 +604,10 @@ void matrixLinewiseVecColsSpan( // The value below determines how many scalar elements are processed by on thread in total. const IdxType elemsPerThread = raft::ceildiv(alignedLen, gs.x * VecElems * BlockSize) * VecElems; - raft::launch_kernel(stream, gs, bs)( + raft::launch_kernel( + stream, + gs, + bs, matrixLinewiseVecColsMainKernel, out.data_handle(), in.data_handle(), @@ -661,7 +669,10 @@ void matrixLinewiseVecRows(Type* out, 1, 1); - raft::launch_kernel(stream, gs, bs)( + raft::launch_kernel( + stream, + gs, + bs, matrixLinewiseVecRowsMainKernel, out + alignedOff, alignedStart, @@ -674,16 +685,18 @@ void matrixLinewiseVecRows(Type* out, if (alignedLen < totalLen) { // should be not smaller than the warp size for better branching constexpr std::size_t MaxOffset = std::max(std::size_t(raft::WarpSize), VecBytes); - raft::launch_kernel(stream, dim3(2, 1, 1), dim3(MaxOffset, 1, 1))( - matrixLinewiseVecRowsTailKernel, - out, - in, - alignedOff, - alignedEnd, - rowLen, - totalLen, - op, - vecs...); + raft::launch_kernel(stream, + dim3(2, 1, 1), + dim3(MaxOffset, 1, 1), + matrixLinewiseVecRowsTailKernel, + out, + in, + alignedOff, + alignedEnd, + rowLen, + totalLen, + op, + vecs...); } } @@ -744,7 +757,10 @@ void matrixLinewiseVecRowsSpan( 1, 1); - raft::launch_kernel(stream, gs, bs)( + raft::launch_kernel( + stream, + gs, + bs, matrixLinewiseVecRowsSpanKernel, out.data_handle(), in.data_handle(), diff --git a/cpp/include/raft/matrix/detail/math.cuh b/cpp/include/raft/matrix/detail/math.cuh index 785484bf42..a7844b7c91 100644 --- a/cpp/include/raft/matrix/detail/math.cuh +++ b/cpp/include/raft/matrix/detail/math.cuh @@ -328,13 +328,13 @@ template inline void argReduce(const math_t* in, idx_t D, idx_t N, out_t* out, cudaStream_t stream) { if (D <= 32) { - raft::launch_kernel(stream, N, 32)(argReduceKernel, in, D, N, out); + raft::launch_kernel(stream, N, 32, argReduceKernel, in, D, N, out); } else if (D <= 64) { - raft::launch_kernel(stream, N, 64)(argReduceKernel, in, D, N, out); + raft::launch_kernel(stream, N, 64, argReduceKernel, in, D, N, out); } else if (D <= 128) { - raft::launch_kernel(stream, N, 128)(argReduceKernel, in, D, N, out); + raft::launch_kernel(stream, N, 128, argReduceKernel, in, D, N, out); } else { - raft::launch_kernel(stream, N, 256)(argReduceKernel, in, D, N, out); + raft::launch_kernel(stream, N, 256, argReduceKernel, in, D, N, out); } RAFT_CUDA_TRY(cudaPeekAtLastError()); } @@ -390,13 +390,13 @@ void signFlip(math_t* inout, int n_rows, int n_cols, cudaStream_t stream) int N = n_cols; auto data = inout; if (D <= 32) { - raft::launch_kernel(stream, N, 32)(signFlipKernel, data, D, N); + raft::launch_kernel(stream, N, 32, signFlipKernel, data, D, N); } else if (D <= 64) { - raft::launch_kernel(stream, N, 64)(signFlipKernel, data, D, N); + raft::launch_kernel(stream, N, 64, signFlipKernel, data, D, N); } else if (D <= 128) { - raft::launch_kernel(stream, N, 128)(signFlipKernel, data, D, N); + raft::launch_kernel(stream, N, 128, signFlipKernel, data, D, N); } else { - raft::launch_kernel(stream, N, 256)(signFlipKernel, data, D, N); + raft::launch_kernel(stream, N, 256, signFlipKernel, data, D, N); } RAFT_CUDA_TRY(cudaPeekAtLastError()); } diff --git a/cpp/include/raft/matrix/detail/matrix.cuh b/cpp/include/raft/matrix/detail/matrix.cuh index e3e43d7efe..dc03d2c575 100644 --- a/cpp/include/raft/matrix/detail/matrix.cuh +++ b/cpp/include/raft/matrix/detail/matrix.cuh @@ -41,8 +41,15 @@ void copyRows(const m_t* in, { if (rowMajor) { const idx_t TPB = 256; - raft::launch_kernel(stream, raft::ceildiv(n_rows_indices * n_cols, TPB), TPB)( - cache::get_vecs, in, n_cols, indices, n_rows_indices, out); + raft::launch_kernel(stream, + raft::ceildiv(n_rows_indices * n_cols, TPB), + TPB, + cache::get_vecs, + in, + n_cols, + indices, + n_rows_indices, + out); return; } @@ -187,9 +194,9 @@ void sliceMatrix(const m_t* in, dim3 block(64); dim3 grid(((x2 - x1) * (y2 - y1) + block.x - 1) / block.x); if (row_major) - raft::launch_kernel(stream, grid, block)(slice, in, lda, out, y1, x1, y2, x2); + raft::launch_kernel(stream, grid, block, slice, in, lda, out, y1, x1, y2, x2); else - raft::launch_kernel(stream, grid, block)(slice, in, lda, out, x1, y1, x2, y2); + raft::launch_kernel(stream, grid, block, slice, in, lda, out, x1, y1, x2, y2); } /** @@ -218,7 +225,7 @@ void copyUpperTriangular(const m_t* src, m_t* dst, idx_t n_rows, idx_t n_cols, c idx_t k = std::min(m, n); dim3 block(64); dim3 grid((m * n + block.x - 1) / block.x); - raft::launch_kernel(stream, grid, block)(getUpperTriangular, src, dst, m, n, k); + raft::launch_kernel(stream, grid, block, getUpperTriangular, src, dst, m, n, k); } /** @@ -259,7 +266,7 @@ void initializeDiagonalMatrix( idx_t lda = row_major ? n_cols : n_rows; dim3 block(64); dim3 grid((k + block.x - 1) / block.x); - raft::launch_kernel(stream, grid, block)(copyVectorToMatrixDiagonal, vec, matrix, lda, k); + raft::launch_kernel(stream, grid, block, copyVectorToMatrixDiagonal, vec, matrix, lda, k); } template @@ -270,7 +277,7 @@ void getDiagonalMatrix( idx_t lda = row_major ? n_cols : n_rows; dim3 block(64); dim3 grid((k + block.x - 1) / block.x); - raft::launch_kernel(stream, grid, block)(copyVectorFromMatrixDiagonal, vec, matrix, lda, k); + raft::launch_kernel(stream, grid, block, copyVectorFromMatrixDiagonal, vec, matrix, lda, k); } /** @@ -291,7 +298,7 @@ void getDiagonalInverseMatrix(m_t* in, idx_t len, cudaStream_t stream) { dim3 block(64); dim3 grid((len + block.x - 1) / block.x); - raft::launch_kernel(stream, grid, block)(matrixDiagonalInverse, in, len); + raft::launch_kernel(stream, grid, block, matrixDiagonalInverse, in, len); } template diff --git a/cpp/include/raft/matrix/detail/select_radix.cuh b/cpp/include/raft/matrix/detail/select_radix.cuh index 38bdf52467..c8de792513 100644 --- a/cpp/include/raft/matrix/detail/select_radix.cuh +++ b/cpp/include/raft/matrix/detail/select_radix.cuh @@ -931,36 +931,41 @@ void radix_topk(const T* in, kernel = radix_kernel; } - raft::launch_kernel(stream, blocks, BlockSize)(kernel, - in, - in_idx, - reinterpret_cast(bufs.data()), - offset, - chunk_out, - chunk_out_idx, - counters.data(), - histograms.data(), - len, - chunk_len_i, - k, - select_min, - pass); + raft::launch_kernel(stream, + blocks, + BlockSize, + kernel, + in, + in_idx, + reinterpret_cast(bufs.data()), + offset, + chunk_out, + chunk_out_idx, + counters.data(), + histograms.data(), + len, + chunk_len_i, + k, + select_min, + pass); } if (!fused_last_filter) { - raft::launch_kernel(stream, blocks, BlockSize)( - last_filter_kernel, - in, - in_idx, - reinterpret_cast(bufs.data()), - offset, - chunk_out, - chunk_out_idx, - len, - chunk_len_i, - k, - counters.data(), - select_min); + raft::launch_kernel(stream, + blocks, + BlockSize, + last_filter_kernel, + in, + in_idx, + reinterpret_cast(bufs.data()), + offset, + chunk_out, + chunk_out_idx, + len, + chunk_len_i, + k, + counters.data(), + select_min); } } } @@ -1178,17 +1183,20 @@ void radix_topk_one_block(const T* in, for (size_t offset = 0; offset < static_cast(batch_size); offset += max_chunk_size) { int chunk_size = std::min(max_chunk_size, batch_size - offset); const IdxT* chunk_len_i = len_i ? (len_i + offset) : nullptr; - raft::launch_kernel(stream, chunk_size, BlockSize)(kernel, - in, - in_idx, - len, - chunk_len_i, - k, - out + offset * k, - out_idx + offset * k, - select_min, - reinterpret_cast(bufs.data()), - offset); + raft::launch_kernel(stream, + chunk_size, + BlockSize, + kernel, + in, + in_idx, + len, + chunk_len_i, + k, + out + offset * k, + out_idx + offset * k, + select_min, + reinterpret_cast(bufs.data()), + offset); } } diff --git a/cpp/include/raft/matrix/detail/select_warpsort.cuh b/cpp/include/raft/matrix/detail/select_warpsort.cuh index b6027e240a..b2595db644 100644 --- a/cpp/include/raft/matrix/detail/select_warpsort.cuh +++ b/cpp/include/raft/matrix/detail/select_warpsort.cuh @@ -858,7 +858,7 @@ struct launch_setup { size_t len, int num_blocks, int block_dim, - int smem_size, + size_t smem_size, const T* in_key, const IdxT* in_idx, const IdxT* in_indptr, @@ -894,27 +894,31 @@ struct launch_setup { size_t batch_chunk = std::min(kMaxGridDimY, batch_size - offset); dim3 gs(num_blocks, batch_chunk, 1); if (select_min) { - raft::launch_kernel(stream, gs, block_dim, smem_size)( - block_kernel, - in_key, - in_idx, - in_indptr, - g_offset, - IdxT(len), - k, - out_key, - out_idx); + raft::launch_kernel({stream, smem_size}, + gs, + block_dim, + block_kernel, + in_key, + in_idx, + in_indptr, + g_offset, + IdxT(len), + k, + out_key, + out_idx); } else { - raft::launch_kernel(stream, gs, block_dim, smem_size)( - block_kernel, - in_key, - in_idx, - in_indptr, - g_offset, - IdxT(len), - k, - out_key, - out_idx); + raft::launch_kernel({stream, smem_size}, + gs, + block_dim, + block_kernel, + in_key, + in_idx, + in_indptr, + g_offset, + IdxT(len), + k, + out_key, + out_idx); } RAFT_CUDA_TRY(cudaPeekAtLastError()); out_key += batch_chunk * num_blocks * k; diff --git a/cpp/include/raft/matrix/detail/shift.cuh b/cpp/include/raft/matrix/detail/shift.cuh index d59fe72837..8be9516db7 100644 --- a/cpp/include/raft/matrix/detail/shift.cuh +++ b/cpp/include/raft/matrix/detail/shift.cuh @@ -143,40 +143,48 @@ void shift_dispatch(raft::resources const& handle, if (shift_type == ShiftType::COL) { size_t num_blocks = static_cast((n_rows + TPB) / TPB); if (shift_direction == ShiftDirection::TOWARDS_BEGINNING) { - raft::launch_kernel(handle, num_blocks, TPB)( - col_shift_towards_beginning, - in_out.data_handle(), - n_rows, - n_cols, - k, - value); + raft::launch_kernel(handle, + num_blocks, + TPB, + col_shift_towards_beginning, + in_out.data_handle(), + n_rows, + n_cols, + k, + value); } else { // ShiftDirection::TOWARDS_END - raft::launch_kernel(handle, num_blocks, TPB)( - col_shift_towards_end, - in_out.data_handle(), - n_rows, - n_cols, - k, - value); + raft::launch_kernel(handle, + num_blocks, + TPB, + col_shift_towards_end, + in_out.data_handle(), + n_rows, + n_cols, + k, + value); } } else { // ShiftType::ROW size_t num_blocks = static_cast((n_cols + TPB) / TPB); if (shift_direction == ShiftDirection::TOWARDS_BEGINNING) { - raft::launch_kernel(handle, num_blocks, TPB)( - row_shift_towards_beginning, - in_out.data_handle(), - n_rows, - n_cols, - k, - value); + raft::launch_kernel(handle, + num_blocks, + TPB, + row_shift_towards_beginning, + in_out.data_handle(), + n_rows, + n_cols, + k, + value); } else { // ShiftDirection::TOWARDS_END - raft::launch_kernel(handle, num_blocks, TPB)( - row_shift_towards_end, - in_out.data_handle(), - n_rows, - n_cols, - k, - value); + raft::launch_kernel(handle, + num_blocks, + TPB, + row_shift_towards_end, + in_out.data_handle(), + n_rows, + n_cols, + k, + value); } } raft::resource::sync_stream(handle); diff --git a/cpp/include/raft/random/detail/make_blobs.cuh b/cpp/include/raft/random/detail/make_blobs.cuh index 9d1747bc52..5a80415ef3 100644 --- a/cpp/include/raft/random/detail/make_blobs.cuh +++ b/cpp/include/raft/random/detail/make_blobs.cuh @@ -152,17 +152,20 @@ void call_generate_data_kernel(raft::random::DeviceState const& rng_sta int64_t n_blocks, cudaStream_t stream) { - raft::launch_kernel(stream, n_blocks, 128)(generate_data_kernel, - rng_state, - out, - labels, - n_rows, - n_cols, - n_clusters, - row_major, - centers, - cluster_std, - cluster_std_scalar); + raft::launch_kernel(stream, + n_blocks, + 128, + generate_data_kernel, + rng_state, + out, + labels, + n_rows, + n_cols, + n_clusters, + row_major, + centers, + cluster_std, + cluster_std_scalar); } template diff --git a/cpp/include/raft/random/detail/make_regression.cuh b/cpp/include/raft/random/detail/make_regression.cuh index d445ef995b..aa25db929a 100644 --- a/cpp/include/raft/random/detail/make_regression.cuh +++ b/cpp/include/raft/random/detail/make_regression.cuh @@ -68,8 +68,14 @@ static void _make_low_rank_matrix(raft::resources const& handle, // Build the singular profile by assembling signal and noise components rmm::device_uvector singular_vec(n, stream); - raft::launch_kernel(stream, raft::ceildiv(n, 256), 256)( - _singular_profile_kernel, singular_vec.data(), n, tail_strength, effective_rank); + raft::launch_kernel(stream, + raft::ceildiv(n, 256), + 256, + _singular_profile_kernel, + singular_vec.data(), + n, + tail_strength, + effective_rank); rmm::device_uvector singular_mat(n * n, stream); RAFT_CUDA_TRY(cudaMemsetAsync(singular_mat.data(), 0, n * n * sizeof(DataT), stream)); @@ -250,8 +256,15 @@ void make_regression_caller(raft::resources const& handle, raft::random::permute( perms_samples.data(), tmp_out.data(), out, n_cols, n_rows, true, stream); IdxT nblks_rows = raft::ceildiv(n_rows, Nthreads); - raft::launch_kernel(stream, nblks_rows, Nthreads)( - _gather2d_kernel, values, _values, perms_samples.data(), n_rows, n_targets); + raft::launch_kernel(stream, + nblks_rows, + Nthreads, + _gather2d_kernel, + values, + _values, + perms_samples.data(), + n_rows, + n_targets); // Shuffle the features from tmp_out to out raft::random::permute( @@ -260,8 +273,15 @@ void make_regression_caller(raft::resources const& handle, // Shuffle the coefficients accordingly if (coef != nullptr) { IdxT nblks_cols = raft::ceildiv(n_cols, Nthreads); - raft::launch_kernel(stream, nblks_cols, Nthreads)( - _gather2d_kernel, coef, _coef, perms_features.data(), n_cols, n_targets); + raft::launch_kernel(stream, + nblks_cols, + Nthreads, + _gather2d_kernel, + coef, + _coef, + perms_features.data(), + n_cols, + n_targets); } } } diff --git a/cpp/include/raft/random/detail/multi_variable_gaussian.cuh b/cpp/include/raft/random/detail/multi_variable_gaussian.cuh index c6b21306d1..8e26e6b60b 100644 --- a/cpp/include/raft/random/detail/multi_variable_gaussian.cuh +++ b/cpp/include/raft/random/detail/multi_variable_gaussian.cuh @@ -222,7 +222,7 @@ class multi_variable_gaussian_impl { // upper part (0) being filled with 0.0 dim3 block(32, 32); dim3 grid(raft::ceildiv(dim, (int)block.x), raft::ceildiv(dim, (int)block.y)); - raft::launch_kernel(handle, grid, block)(fill_uplo, dim, UPPER, (T)0.0, P); + raft::launch_kernel(handle, grid, block, fill_uplo, dim, UPPER, (T)0.0, P); // P is lower triangular chol decomp mtrx raft::linalg::gemm( @@ -233,7 +233,7 @@ class multi_variable_gaussian_impl { dim3 grid(raft::ceildiv(dim, (int)block.x)); RAFT_CUDA_TRY(cudaMemsetAsync(info, 0, sizeof(int), cudaStream)); grid.x = raft::ceildiv(dim * dim, (int)block.x); - raft::launch_kernel(handle, grid, block)(combined_dot_product, dim, dim, eig, P, info); + raft::launch_kernel(handle, grid, block, combined_dot_product, dim, dim, eig, P, info); // checking if any eigen vals were negative raft::update_host(&info_h, info, 1, cudaStream); diff --git a/cpp/include/raft/random/detail/permute.cuh b/cpp/include/raft/random/detail/permute.cuh index edf265d805..cab212d000 100644 --- a/cpp/include/raft/random/detail/permute.cuh +++ b/cpp/include/raft/random/detail/permute.cuh @@ -85,8 +85,17 @@ struct permute_impl_t { // check if we can execute at this vector length if (D % VLen == 0 && raft::is_aligned(vout, sizeof(VType)) && raft::is_aligned(vin, sizeof(VType))) { - raft::launch_kernel(stream, nblks, TPB)( - permuteKernel, perms, vout, vin, a, b, N, D / VLen); + raft::launch_kernel(stream, + nblks, + TPB, + permuteKernel, + perms, + vout, + vin, + a, + b, + N, + D / VLen); } else { // otherwise try the next lower vector length permute_impl_t::permuteImpl( perms, out, in, N, D, nblks, a, b, stream); @@ -107,8 +116,17 @@ struct permute_impl_t { IdxType b, cudaStream_t stream) { - raft::launch_kernel(stream, nblks, TPB)( - permuteKernel, perms, out, in, a, b, N, D); + raft::launch_kernel(stream, + nblks, + TPB, + permuteKernel, + perms, + out, + in, + a, + b, + N, + D); } }; diff --git a/cpp/include/raft/random/detail/rmat_rectangular_generator.cuh b/cpp/include/raft/random/detail/rmat_rectangular_generator.cuh index 3bff053b8a..afbc4db9f3 100644 --- a/cpp/include/raft/random/detail/rmat_rectangular_generator.cuh +++ b/cpp/include/raft/random/detail/rmat_rectangular_generator.cuh @@ -118,8 +118,19 @@ void rmat_rectangular_gen_caller(IdxT* out, auto max_scale = max(r_scale, c_scale); size_t smem_size = sizeof(ProbT) * max_scale * 2 * 2; auto n_blks = raft::ceildiv(n_edges, N_THREADS); - raft::launch_kernel(stream, n_blks, N_THREADS, smem_size)( - rmat_gen_kernel, out, out_src, out_dst, theta, r_scale, c_scale, n_edges, max_scale, r); + raft::launch_kernel({stream, smem_size}, + n_blks, + N_THREADS, + rmat_gen_kernel, + out, + out_src, + out_dst, + theta, + r_scale, + c_scale, + n_edges, + max_scale, + r); r.advance(n_edges, max_scale); } @@ -171,8 +182,21 @@ void rmat_rectangular_gen_caller(IdxT* out, static constexpr int N_THREADS = 512; auto max_scale = max(r_scale, c_scale); auto n_blks = raft::ceildiv(n_edges, N_THREADS); - raft::launch_kernel(stream, n_blks, N_THREADS)( - rmat_gen_kernel, out, out_src, out_dst, a, b, c, r_scale, c_scale, n_edges, max_scale, r); + raft::launch_kernel(stream, + n_blks, + N_THREADS, + rmat_gen_kernel, + out, + out_src, + out_dst, + a, + b, + c, + r_scale, + c_scale, + n_edges, + max_scale, + r); r.advance(n_edges, max_scale); } diff --git a/cpp/include/raft/random/detail/rng_impl.cuh b/cpp/include/raft/random/detail/rng_impl.cuh index a5d8a59c2a..0eb0718536 100644 --- a/cpp/include/raft/random/detail/rng_impl.cuh +++ b/cpp/include/raft/random/detail/rng_impl.cuh @@ -70,7 +70,7 @@ void call_rng_kernel(DeviceState const& dev_state, { auto n_threads = 256; auto n_blocks = 4 * getMultiProcessorCount(); - raft::launch_kernel(stream, n_blocks, n_threads)(rngKernel, dev_state, args...); + raft::launch_kernel(stream, n_blocks, n_threads, rngKernel, dev_state, args...); rng_state.advance(uint64_t(n_blocks) * n_threads, 16); } @@ -246,8 +246,15 @@ void call_sample_with_replacement_kernel(DeviceState const& dev_state, { IdxType n_threads = 256; IdxType n_blocks = raft::ceildiv(sampledLen, n_threads); - raft::launch_kernel(stream, n_blocks, n_threads)( - sample_with_replacement_kernel, dev_state, out, weights_csum, sampledLen, len); + raft::launch_kernel(stream, + n_blocks, + n_threads, + sample_with_replacement_kernel, + dev_state, + out, + weights_csum, + sampledLen, + len); rng_state.advance(uint64_t(n_blocks) * n_threads, 1); } diff --git a/cpp/include/raft/random/detail/rng_impl_deprecated.cuh b/cpp/include/raft/random/detail/rng_impl_deprecated.cuh index 782905b3ab..1d641fe11a 100644 --- a/cpp/include/raft/random/detail/rng_impl_deprecated.cuh +++ b/cpp/include/raft/random/detail/rng_impl_deprecated.cuh @@ -236,7 +236,10 @@ class RngImpl { { switch (state.type) { case GenPhilox: - raft::launch_kernel(stream, nBlocks, nThreads)( + raft::launch_kernel( + stream, + nBlocks, + nThreads, fillKernel, state.seed, state.base_subsequence, @@ -246,14 +249,16 @@ class RngImpl { params); break; case GenPC: - raft::launch_kernel(stream, nBlocks, nThreads)( - fillKernel, - state.seed, - state.base_subsequence, - 0, - ptr, - len, - params); + raft::launch_kernel(stream, + nBlocks, + nThreads, + fillKernel, + state.seed, + state.base_subsequence, + 0, + ptr, + len, + params); break; default: break; } diff --git a/cpp/include/raft/solver/detail/lap_functions.cuh b/cpp/include/raft/solver/detail/lap_functions.cuh index ba39d5d17c..e1cb35bff3 100644 --- a/cpp/include/raft/solver/detail/lap_functions.cuh +++ b/cpp/include/raft/solver/detail/lap_functions.cuh @@ -117,16 +117,24 @@ inline void initialReduction(raft::resources const& handle, detail::calculateRectangularDims(blocks_per_grid, threads_per_block, total_blocks, N, SP); - raft::launch_kernel(handle, blocks_per_grid, threads_per_block)( - kernel_rowReduction, d_costs, d_vertices_dev.row_duals, SP, N); - - raft::launch_kernel(handle, blocks_per_grid, threads_per_block)( - kernel_columnReduction, - d_costs, - d_vertices_dev.row_duals, - d_vertices_dev.col_duals, - SP, - N); + raft::launch_kernel(handle, + blocks_per_grid, + threads_per_block, + kernel_rowReduction, + d_costs, + d_vertices_dev.row_duals, + SP, + N); + + raft::launch_kernel(handle, + blocks_per_grid, + threads_per_block, + kernel_columnReduction, + d_costs, + d_vertices_dev.row_duals, + d_vertices_dev.col_duals, + SP, + N); } template @@ -153,18 +161,20 @@ inline void computeInitialAssignments(raft::resources const& handle, detail::calculateRectangularDims(blocks_per_grid, threads_per_block, total_blocks, N, SP); - raft::launch_kernel(handle, blocks_per_grid, threads_per_block)( - kernel_computeInitialAssignments, - d_costs, - d_vertices.row_duals, - d_vertices.col_duals, - d_vertices.row_assignments, - d_vertices.col_assignments, - row_lock_v.data(), - col_lock_v.data(), - SP, - N, - epsilon); + raft::launch_kernel(handle, + blocks_per_grid, + threads_per_block, + kernel_computeInitialAssignments, + d_costs, + d_vertices.row_duals, + d_vertices.col_duals, + d_vertices.row_assignments, + d_vertices.col_assignments, + row_lock_v.data(), + col_lock_v.data(), + SP, + N, + epsilon); } // Function for finding row cover on individual devices. @@ -193,12 +203,15 @@ inline int computeRowCovers(raft::resources const& handle, thrust::fill_n(thrust::device, d_col_data.children, size, vertex_t{-1}); detail::calculateRectangularDims(blocks_per_grid, threads_per_block, total_blocks, N, SP); - raft::launch_kernel(handle, blocks_per_grid, threads_per_block)(kernel_computeRowCovers, - d_vertices.row_assignments, - d_vertices.row_covers, - d_row_data.is_visited, - SP, - N); + raft::launch_kernel(handle, + blocks_per_grid, + threads_per_block, + kernel_computeRowCovers, + d_vertices.row_assignments, + d_vertices.row_covers, + d_row_data.is_visited, + SP, + N); return thrust::reduce(thrust::device, d_vertices.row_covers, d_vertices.row_covers + size); } @@ -223,18 +236,20 @@ inline void coverZeroAndExpand(raft::resources const& handle, detail::calculateRectangularDims(blocks_per_grid, threads_per_block, total_blocks, N, SP); - raft::launch_kernel(handle, blocks_per_grid, threads_per_block)( - kernel_coverAndExpand, - d_flag, - d_rows_csr_ptrs, - d_rows_csr_neighbors, - d_costs_dev, - d_vertices_dev, - d_row_data_dev, - d_col_data_dev, - SP, - N, - epsilon); + raft::launch_kernel(handle, + blocks_per_grid, + threads_per_block, + kernel_coverAndExpand, + d_flag, + d_rows_csr_ptrs, + d_rows_csr_neighbors, + d_costs_dev, + d_vertices_dev, + d_row_data_dev, + d_col_data_dev, + SP, + N, + epsilon); } template @@ -271,13 +286,15 @@ inline vertex_t zeroCoverIteration(raft::resources const& handle, detail::calculateRectangularDims(blocks_per_grid, threads_per_block, total_blocks, N, SP); // construct predicate matrix for edges. - raft::launch_kernel(handle, blocks_per_grid, threads_per_block)( - kernel_rowPredicateConstructionCSR, - predicates_v.data(), - addresses_v.data(), - d_row_data_dev.is_visited, - SP, - N); + raft::launch_kernel(handle, + blocks_per_grid, + threads_per_block, + kernel_rowPredicateConstructionCSR, + predicates_v.data(), + addresses_v.data(), + d_row_data_dev.is_visited, + SP, + N); M = thrust::reduce(thrust::device, addresses_v.begin(), addresses_v.end()); thrust::exclusive_scan( @@ -286,15 +303,17 @@ inline vertex_t zeroCoverIteration(raft::resources const& handle, if (M > 0) { csr_neighbors_v.resize(M, resource::get_cuda_stream(handle)); - raft::launch_kernel(handle, blocks_per_grid, threads_per_block)( - kernel_rowScatterCSR, - predicates_v.data(), - addresses_v.data(), - csr_neighbors_v.data(), - csr_ptrs_v.data(), - M, - SP, - N); + raft::launch_kernel(handle, + blocks_per_grid, + threads_per_block, + kernel_rowScatterCSR, + predicates_v.data(), + addresses_v.data(), + csr_neighbors_v.data(), + csr_ptrs_v.data(), + M, + SP, + N); } } @@ -358,12 +377,14 @@ inline void reversePass(raft::resources const& handle, thrust::fill_n(thrust::device, addresses_v.data(), size, vertex_t{0}); // compact the reverse pass row vertices. - raft::launch_kernel(handle, blocks_per_grid, threads_per_block)( - kernel_augmentPredicateConstruction, - predicates_v.data(), - addresses_v.data(), - d_col_data_dev.is_visited, - size); + raft::launch_kernel(handle, + blocks_per_grid, + threads_per_block, + kernel_augmentPredicateConstruction, + predicates_v.data(), + addresses_v.data(), + d_col_data_dev.is_visited, + size); // calculate total number of vertices. std::size_t csr_size = thrust::reduce(thrust::device, addresses_v.begin(), addresses_v.end()); @@ -379,18 +400,23 @@ inline void reversePass(raft::resources const& handle, rmm::device_uvector elements_v(csr_size, resource::get_cuda_stream(handle)); - raft::launch_kernel(handle, blocks_per_grid, threads_per_block)(kernel_augmentScatter, - elements_v.data(), - predicates_v.data(), - addresses_v.data(), - size); - - raft::launch_kernel(handle, blocks_per_grid_1, threads_per_block_1)( - kernel_reverseTraversal, - elements_v.data(), - d_row_data_dev, - d_col_data_dev, - csr_size); + raft::launch_kernel(handle, + blocks_per_grid, + threads_per_block, + kernel_augmentScatter, + elements_v.data(), + predicates_v.data(), + addresses_v.data(), + size); + + raft::launch_kernel(handle, + blocks_per_grid_1, + threads_per_block_1, + kernel_reverseTraversal, + elements_v.data(), + d_row_data_dev, + d_col_data_dev, + csr_size); } } @@ -415,12 +441,14 @@ inline void augmentationPass(raft::resources const& handle, thrust::fill_n(thrust::device, addresses_v.data(), SP * N, vertex_t{0}); // compact the reverse pass row vertices. - raft::launch_kernel(handle, blocks_per_grid, threads_per_block)( - kernel_augmentPredicateConstruction, - predicates_v.data(), - addresses_v.data(), - d_row_data_dev.is_visited, - SP * N); + raft::launch_kernel(handle, + blocks_per_grid, + threads_per_block, + kernel_augmentPredicateConstruction, + predicates_v.data(), + addresses_v.data(), + d_row_data_dev.is_visited, + SP * N); // calculate total number of vertices. // TODO: should be vertex_t @@ -439,21 +467,26 @@ inline void augmentationPass(raft::resources const& handle, rmm::device_uvector elements_v(row_ids_csr_size, resource::get_cuda_stream(handle)); - raft::launch_kernel(handle, blocks_per_grid, threads_per_block)(kernel_augmentScatter, - elements_v.data(), - predicates_v.data(), - addresses_v.data(), - vertex_t{SP * N}); - - raft::launch_kernel(handle, blocks_per_grid_1, threads_per_block_1)( - kernel_augmentation, - d_vertices_dev.row_assignments, - d_vertices_dev.col_assignments, - elements_v.data(), - d_row_data_dev, - d_col_data_dev, - vertex_t{N}, - row_ids_csr_size); + raft::launch_kernel(handle, + blocks_per_grid, + threads_per_block, + kernel_augmentScatter, + elements_v.data(), + predicates_v.data(), + addresses_v.data(), + vertex_t{SP * N}); + + raft::launch_kernel(handle, + blocks_per_grid_1, + threads_per_block_1, + kernel_augmentation, + d_vertices_dev.row_assignments, + d_vertices_dev.col_assignments, + elements_v.data(), + d_row_data_dev, + d_col_data_dev, + vertex_t{N}, + row_ids_csr_size); } } @@ -473,28 +506,32 @@ inline void dualUpdate(raft::resources const& handle, rmm::device_uvector sp_min_v(SP, resource::get_cuda_stream(handle)); detail::calculateLinearDims(blocks_per_grid, threads_per_block, total_blocks, SP); - raft::launch_kernel(handle, blocks_per_grid, threads_per_block)( - kernel_dualUpdate_1, - sp_min_v.data(), - d_vertices_dev.col_slacks, - d_vertices_dev.col_covers, - SP, - N); + raft::launch_kernel(handle, + blocks_per_grid, + threads_per_block, + kernel_dualUpdate_1, + sp_min_v.data(), + d_vertices_dev.col_slacks, + d_vertices_dev.col_covers, + SP, + N); detail::calculateRectangularDims(blocks_per_grid, threads_per_block, total_blocks, N, SP); - raft::launch_kernel(handle, blocks_per_grid, threads_per_block)( - kernel_dualUpdate_2, - sp_min_v.data(), - d_vertices_dev.row_duals, - d_vertices_dev.col_duals, - d_vertices_dev.col_slacks, - d_vertices_dev.row_covers, - d_vertices_dev.col_covers, - d_row_data_dev.is_visited, - d_col_data_dev.parents, - SP, - N, - epsilon); + raft::launch_kernel(handle, + blocks_per_grid, + threads_per_block, + kernel_dualUpdate_2, + sp_min_v.data(), + d_vertices_dev.row_duals, + d_vertices_dev.col_duals, + d_vertices_dev.col_slacks, + d_vertices_dev.row_covers, + d_vertices_dev.col_covers, + d_row_data_dev.is_visited, + d_col_data_dev.parents, + SP, + N, + epsilon); } // Function for calculating optimal objective function value using dual variables. @@ -511,13 +548,15 @@ inline void calcObjValDual(raft::resources const& handle, detail::calculateLinearDims(blocks_per_grid, threads_per_block, total_blocks, SP); - raft::launch_kernel(handle, blocks_per_grid, threads_per_block)( - kernel_calcObjValDual, - d_obj_val, - d_vertices_dev.row_duals, - d_vertices_dev.col_duals, - SP, - N); + raft::launch_kernel(handle, + blocks_per_grid, + threads_per_block, + kernel_calcObjValDual, + d_obj_val, + d_vertices_dev.row_duals, + d_vertices_dev.col_duals, + SP, + N); } // Function for calculating optimal objective function value using dual variables. @@ -535,8 +574,15 @@ inline void calcObjValPrimal(raft::resources const& handle, detail::calculateLinearDims(blocks_per_grid, threads_per_block, total_blocks, SP); - raft::launch_kernel(handle, blocks_per_grid, threads_per_block)( - kernel_calcObjValPrimal, d_obj_val, d_costs, d_row_assignments, SP, N); + raft::launch_kernel(handle, + blocks_per_grid, + threads_per_block, + kernel_calcObjValPrimal, + d_obj_val, + d_costs, + d_row_assignments, + SP, + N); } } // namespace solver::detail diff --git a/cpp/include/raft/sparse/convert/detail/adj_to_csr.cuh b/cpp/include/raft/sparse/convert/detail/adj_to_csr.cuh index 12ed6dfdc7..d545c116b8 100644 --- a/cpp/include/raft/sparse/convert/detail/adj_to_csr.cuh +++ b/cpp/include/raft/sparse/convert/detail/adj_to_csr.cuh @@ -157,8 +157,16 @@ void adj_to_csr(raft::resources const& handle, dim3 block(adj_to_csr_tpb, 1); dim3 grid(blocks_per_row, grid_rows); - raft::launch_kernel(handle, grid, block)( - adj_to_csr_kernel, adj, row_ind, num_rows, num_cols, tmp, out_col_ind); + raft::launch_kernel(handle, + grid, + block, + adj_to_csr_kernel, + adj, + row_ind, + num_rows, + num_cols, + tmp, + out_col_ind); } }; // end NAMESPACE detail diff --git a/cpp/include/raft/sparse/convert/detail/bitmap_to_csr.cuh b/cpp/include/raft/sparse/convert/detail/bitmap_to_csr.cuh index 52fc355871..0b42b07b1f 100644 --- a/cpp/include/raft/sparse/convert/detail/bitmap_to_csr.cuh +++ b/cpp/include/raft/sparse/convert/detail/bitmap_to_csr.cuh @@ -116,13 +116,16 @@ void calc_nnz_by_rows(raft::resources const& handle, auto block = bitmap_to_csr_tpb; - raft::launch_kernel(handle, grid, block)(calc_nnz_by_rows_kernel, - bitmap, - num_rows, - num_cols, - bitmap_num, - sub_col_nnz, - bits_per_sub_col); + raft::launch_kernel(handle, + grid, + block, + calc_nnz_by_rows_kernel, + bitmap, + num_rows, + num_cols, + bitmap_num, + sub_col_nnz, + bits_per_sub_col); } template @@ -254,16 +257,18 @@ void fill_indices_by_rows(raft::resources const& handle, auto block = bitmap_to_csr_tpb; - raft::launch_kernel(handle, grid, block)( - fill_indices_by_rows_kernel, - bitmap, - indptr, - num_rows, - num_cols, - nnz, - indices, - sub_col_nnz, - bits_per_sub_col); + raft::launch_kernel(handle, + grid, + block, + fill_indices_by_rows_kernel, + bitmap, + indptr, + num_rows, + num_cols, + nnz, + indices, + sub_col_nnz, + bits_per_sub_col); } template , row_ind, m, coo_rows, nnz); + raft::launch_kernel( + stream, grid, blk, csr_to_coo_kernel, row_ind, m, coo_rows, nnz); } }; // end NAMESPACE detail diff --git a/cpp/include/raft/sparse/convert/detail/dense.cuh b/cpp/include/raft/sparse/convert/detail/dense.cuh index 57107c7a15..fa22902e89 100644 --- a/cpp/include/raft/sparse/convert/detail/dense.cuh +++ b/cpp/include/raft/sparse/convert/detail/dense.cuh @@ -124,8 +124,15 @@ void csr_to_dense(cusparseHandle_t handle, } else { int blockdim = block_dim(ncols); RAFT_CUDA_TRY(cudaMemsetAsync(out, 0, nrows * ncols * sizeof(value_t), stream)); - raft::launch_kernel(stream, nrows, blockdim)( - csr_to_dense_warp_per_row_kernel, ncols, csr_data, csr_indptr, csr_indices, out); + raft::launch_kernel(stream, + nrows, + blockdim, + csr_to_dense_warp_per_row_kernel, + ncols, + csr_data, + csr_indptr, + csr_indices, + out); } } diff --git a/cpp/include/raft/sparse/detail/csr.cuh b/cpp/include/raft/sparse/detail/csr.cuh index 152649af1c..18b34569ce 100644 --- a/cpp/include/raft/sparse/detail/csr.cuh +++ b/cpp/include/raft/sparse/detail/csr.cuh @@ -137,24 +137,32 @@ void weak_cc_batched(Index_* labels, bool host_m; Index_ MAX_LABEL = std::numeric_limits::max(); - raft::launch_kernel(stream, raft::ceildiv(N, Index_(TPB_X)), TPB_X)( - weak_cc_init_all_kernel, labels, N, MAX_LABEL, filter_op); + raft::launch_kernel(stream, + raft::ceildiv(N, Index_(TPB_X)), + TPB_X, + weak_cc_init_all_kernel, + labels, + N, + MAX_LABEL, + filter_op); int n_iters = 0; do { RAFT_CUDA_TRY(cudaMemsetAsync(state->m, false, sizeof(bool), stream)); - raft::launch_kernel(stream, raft::ceildiv(batch_size, Index_(TPB_X)), TPB_X)( - weak_cc_label_device, - labels, - row_ind, - row_ind_ptr, - nnz, - state->m, - start_vertex_id, - batch_size, - N, - filter_op); + raft::launch_kernel(stream, + raft::ceildiv(batch_size, Index_(TPB_X)), + TPB_X, + weak_cc_label_device, + labels, + row_ind, + row_ind_ptr, + nnz, + state->m, + start_vertex_id, + batch_size, + N, + filter_op); //** Updating m * raft::update_host(&host_m, state->m, 1, stream); diff --git a/cpp/include/raft/sparse/detail/utils.h b/cpp/include/raft/sparse/detail/utils.h index c51c9c6164..1f0145bb21 100644 --- a/cpp/include/raft/sparse/detail/utils.h +++ b/cpp/include/raft/sparse/detail/utils.h @@ -91,7 +91,7 @@ void iota_fill(value_idx* indices, value_idx nrows, value_idx ncols, cudaStream_ { int blockdim = block_dim(ncols); - raft::launch_kernel(stream, nrows, blockdim)(iota_fill_block_kernel, indices, ncols); + raft::launch_kernel(stream, nrows, blockdim, iota_fill_block_kernel, indices, ncols); } template diff --git a/cpp/include/raft/sparse/linalg/detail/add.cuh b/cpp/include/raft/sparse/linalg/detail/add.cuh index 700c492d02..2d1aff1b1a 100644 --- a/cpp/include/raft/sparse/linalg/detail/add.cuh +++ b/cpp/include/raft/sparse/linalg/detail/add.cuh @@ -181,17 +181,20 @@ size_t csr_add_calc_inds(const int* a_ind, rmm::device_uvector row_counts(m + 1, stream); RAFT_CUDA_TRY(cudaMemsetAsync(row_counts.data(), 0, (m + 1) * sizeof(int), stream)); - raft::launch_kernel(stream, grid, blk)(csr_add_calc_row_counts_kernel, - a_ind, - a_indptr, - a_val, - nnz1, - b_ind, - b_indptr, - b_val, - nnz2, - m, - row_counts.data()); + raft::launch_kernel(stream, + grid, + blk, + csr_add_calc_row_counts_kernel, + a_ind, + a_indptr, + a_val, + nnz1, + b_ind, + b_indptr, + b_val, + nnz2, + m, + row_counts.data()); int cnnz = 0; raft::update_host(&cnnz, row_counts.data() + m, 1, stream); @@ -240,19 +243,22 @@ void csr_add_finalize(const int* a_ind, dim3 grid(raft::ceildiv(m, TPB_X), 1, 1); dim3 blk(TPB_X, 1, 1); - raft::launch_kernel(stream, grid, blk)(csr_add_kernel, - a_ind, - a_indptr, - a_val, - nnz1, - b_ind, - b_indptr, - b_val, - nnz2, - m, - c_ind, - c_indptr, - c_val); + raft::launch_kernel(stream, + grid, + blk, + csr_add_kernel, + a_ind, + a_indptr, + a_val, + nnz1, + b_ind, + b_indptr, + b_val, + nnz2, + m, + c_ind, + c_indptr, + c_val); } }; // end NAMESPACE detail diff --git a/cpp/include/raft/sparse/linalg/detail/degree.cuh b/cpp/include/raft/sparse/linalg/detail/degree.cuh index 8679814f5e..61d6fe9baf 100644 --- a/cpp/include/raft/sparse/linalg/detail/degree.cuh +++ b/cpp/include/raft/sparse/linalg/detail/degree.cuh @@ -51,7 +51,7 @@ void coo_degree(const T* rows, nnz_t nnz, outT* results, cudaStream_t stream) dim3 grid_rc(raft::ceildiv((nnz_t)nnz, (nnz_t)TPB_X), 1, 1); dim3 blk_rc(TPB_X, 1, 1); - raft::launch_kernel(stream, grid_rc, blk_rc)(coo_degree_kernel, rows, nnz, results); + raft::launch_kernel(stream, grid_rc, blk_rc, coo_degree_kernel, rows, nnz, results); } template @@ -86,8 +86,15 @@ void coo_degree_scalar( { dim3 grid_rc(raft::ceildiv(nnz, static_cast(TPB_X)), 1, 1); dim3 blk_rc(TPB_X, 1, 1); - raft::launch_kernel(stream, grid_rc, blk_rc)( - coo_degree_scalar_kernel, rows, vals, nnz, scalar, results); + raft::launch_kernel(stream, + grid_rc, + blk_rc, + coo_degree_scalar_kernel, + rows, + vals, + nnz, + scalar, + results); } /** @@ -105,8 +112,14 @@ void coo_degree_nz(const idx_t* rows, const T* vals, nnz_t nnz, idx_t* results, { dim3 grid_rc(raft::ceildiv(nnz, TPB_X), 1, 1); dim3 blk_rc(TPB_X, 1, 1); - raft::launch_kernel(stream, grid_rc, blk_rc)( - coo_degree_nz_kernel, rows, vals, nnz, results); + raft::launch_kernel(stream, + grid_rc, + blk_rc, + coo_degree_nz_kernel, + rows, + vals, + nnz, + results); } }; // end NAMESPACE detail diff --git a/cpp/include/raft/sparse/linalg/detail/laplacian.cuh b/cpp/include/raft/sparse/linalg/detail/laplacian.cuh index a905232237..72232e9a77 100644 --- a/cpp/include/raft/sparse/linalg/detail/laplacian.cuh +++ b/cpp/include/raft/sparse/linalg/detail/laplacian.cuh @@ -105,17 +105,19 @@ auto compute_graph_laplacian( auto result_structure = result.structure_view(); auto static constexpr const threads_per_block = 256; auto blocks = std::min(int((dim + threads_per_block - 1) / threads_per_block), 65535); - raft::launch_kernel(res, blocks, threads_per_block)( - detail::compute_graph_laplacian_kernel, - std::remove_const_t, - std::remove_const_t>, - result.get_elements().data(), - result_structure.get_indices().data(), - result_structure.get_indptr().data(), - dim, - input.get_elements().data(), - input_structure.get_indices().data(), - input_structure.get_indptr().data()); + raft::launch_kernel(res, + blocks, + threads_per_block, + detail::compute_graph_laplacian_kernel, + std::remove_const_t, + std::remove_const_t>, + result.get_elements().data(), + result_structure.get_indices().data(), + result_structure.get_indptr().data(), + dim, + input.get_elements().data(), + input_structure.get_indices().data(), + input_structure.get_indptr().data()); return result; } diff --git a/cpp/include/raft/sparse/linalg/detail/norm.cuh b/cpp/include/raft/sparse/linalg/detail/norm.cuh index c57767ee45..929ce4ea87 100644 --- a/cpp/include/raft/sparse/linalg/detail/norm.cuh +++ b/cpp/include/raft/sparse/linalg/detail/norm.cuh @@ -93,8 +93,8 @@ void csr_row_normalize_l1(const indT* ia, // csr row ex_scan (sorted by row) dim3 grid(raft::ceildiv(m, TPB_X), 1, 1); dim3 blk(TPB_X, 1, 1); - raft::launch_kernel(stream, grid, blk)( - csr_row_normalize_l1_kernel, ia, vals, nnz, m, result); + raft::launch_kernel( + stream, grid, blk, csr_row_normalize_l1_kernel, ia, vals, nnz, m, result); } template @@ -160,8 +160,8 @@ void csr_row_normalize_max(const int* ia, // csr row ind array (sorted by row) dim3 grid(raft::ceildiv(m, TPB_X), 1, 1); dim3 blk(TPB_X, 1, 1); - raft::launch_kernel(stream, grid, blk)( - csr_row_normalize_max_kernel, ia, vals, nnz, m, result); + raft::launch_kernel( + stream, grid, blk, csr_row_normalize_max_kernel, ia, vals, nnz, m, result); } template * in, out->allocate(in->nnz * 2, in->n_rows, in->n_cols, true, stream); - raft::launch_kernel(stream, grid, blk)(coo_symmetrize_kernel, - in_row_ind.data(), - in->rows(), - in->cols(), - in->vals(), - out->rows(), - out->cols(), - out->vals(), - in->n_rows, - in->nnz, - reduction_op); + raft::launch_kernel(stream, + grid, + blk, + coo_symmetrize_kernel, + in_row_ind.data(), + in->rows(), + in->cols(), + in->vals(), + out->rows(), + out->cols(), + out->vals(), + in->n_rows, + in->nnz, + reduction_op); } /** @@ -208,17 +211,20 @@ void coo_symmetrize(raft::resources const& handle, handle, raft::make_device_vector_view(out_cols, out_nnz), static_cast(0)); raft::matrix::fill(handle, raft::make_device_vector_view(out_vals, out_nnz), static_cast(0.0)); - raft::launch_kernel(handle, grid, blk)(coo_symmetrize_kernel, - in_row_ind.data(), - in_rows, - in_cols, - in_vals, - out_rows, - out_cols, - out_vals, - in_n_rows, - in_nnz, - reduction_op); + raft::launch_kernel(handle, + grid, + blk, + coo_symmetrize_kernel, + in_row_ind.data(), + in_rows, + in_cols, + in_vals, + out_rows, + out_cols, + out_vals, + in_n_rows, + in_nnz, + reduction_op); } /** @@ -349,11 +355,25 @@ void from_knn_symmetrize_matrix(const value_idx* __restrict__ knn_indices, rmm::device_uvector row_sizes2(n, stream); RAFT_CUDA_TRY(cudaMemsetAsync(row_sizes2.data(), 0, sizeof(value_idx) * n, stream)); - raft::launch_kernel(stream, numBlocks, threadsPerBlock)( - symmetric_find_size, knn_dists, knn_indices, n, k, row_sizes.data(), row_sizes2.data()); - - raft::launch_kernel(stream, raft::ceildiv(n, (value_idx)1024), 1024)( - reduce_find_size, n, k, row_sizes.data(), row_sizes2.data()); + raft::launch_kernel(stream, + numBlocks, + threadsPerBlock, + symmetric_find_size, + knn_dists, + knn_indices, + n, + k, + row_sizes.data(), + row_sizes2.data()); + + raft::launch_kernel(stream, + raft::ceildiv(n, (value_idx)1024), + 1024, + reduce_find_size, + n, + k, + row_sizes.data(), + row_sizes2.data()); // (2) Compute final space needed (n*k + sum(row_sizes)) == 2*n*k // Notice we don't do any merging and leave the result as 2*NNZ @@ -374,8 +394,18 @@ void from_knn_symmetrize_matrix(const value_idx* __restrict__ knn_indices, thrust::exclusive_scan(rmm::exec_policy(stream), __row_sizes, __row_sizes + n, __edges); // (5) Perform final data + data.T operation in tandem with memcpying - raft::launch_kernel(stream, numBlocks, threadsPerBlock)( - symmetric_sum, edges, knn_dists, knn_indices, out->vals(), out->cols(), out->rows(), n, k); + raft::launch_kernel(stream, + numBlocks, + threadsPerBlock, + symmetric_sum, + edges, + knn_dists, + knn_indices, + out->vals(), + out->cols(), + out->rows(), + n, + k); } /** diff --git a/cpp/include/raft/sparse/linalg/detail/utils.cuh b/cpp/include/raft/sparse/linalg/detail/utils.cuh index 9ed67675aa..98a0604275 100644 --- a/cpp/include/raft/sparse/linalg/detail/utils.cuh +++ b/cpp/include/raft/sparse/linalg/detail/utils.cuh @@ -100,7 +100,7 @@ void faster_dot_on_csr(raft::resources const& handle, constexpr value_idx MAX_ROW_PER_ITER = 500; int dev_id, sm_count, blocks_per_sm; - const int smem_size = dim * sizeof(value_t); + const size_t smem_size = dim * sizeof(value_t); cudaGetDevice(&dev_id); cudaDeviceGetAttribute(&sm_count, cudaDevAttrMultiProcessorCount, dev_id); @@ -113,16 +113,18 @@ void faster_dot_on_csr(raft::resources const& handle, (std::min(value_idx(blocks_per_sm * sm_count * 16), nnz) + block_x - 1) / block_x; dim3 blocks(block_x, block_y, 1); - raft::launch_kernel(handle, blocks, tpb, smem_size)( - faster_dot_on_csr_kernel, - dot, - indptr, - cols, - A, - B, - nnz, - n_rows, - dim); + raft::launch_kernel({handle, smem_size}, + blocks, + tpb, + faster_dot_on_csr_kernel, + dot, + indptr, + cols, + A, + B, + nnz, + n_rows, + dim); } else if (dim < 256) { constexpr int tpb = 128; @@ -133,16 +135,18 @@ void faster_dot_on_csr(raft::resources const& handle, (std::min(value_idx(blocks_per_sm * sm_count * 16), nnz) + block_x - 1) / block_x; dim3 blocks(block_x, block_y, 1); - raft::launch_kernel(handle, blocks, tpb, smem_size)( - faster_dot_on_csr_kernel, - dot, - indptr, - cols, - A, - B, - nnz, - n_rows, - dim); + raft::launch_kernel({handle, smem_size}, + blocks, + tpb, + faster_dot_on_csr_kernel, + dot, + indptr, + cols, + A, + B, + nnz, + n_rows, + dim); } else if (dim < 512) { constexpr int tpb = 256; cudaOccupancyMaxActiveBlocksPerMultiprocessor( @@ -152,16 +156,18 @@ void faster_dot_on_csr(raft::resources const& handle, (std::min(value_idx(blocks_per_sm * sm_count * 16), nnz) + block_x - 1) / block_x; dim3 blocks(block_x, block_y, 1); - raft::launch_kernel(handle, blocks, tpb, smem_size)( - faster_dot_on_csr_kernel, - dot, - indptr, - cols, - A, - B, - nnz, - n_rows, - dim); + raft::launch_kernel({handle, smem_size}, + blocks, + tpb, + faster_dot_on_csr_kernel, + dot, + indptr, + cols, + A, + B, + nnz, + n_rows, + dim); } else { constexpr int tpb = 512; cudaOccupancyMaxActiveBlocksPerMultiprocessor( @@ -171,16 +177,18 @@ void faster_dot_on_csr(raft::resources const& handle, (std::min(value_idx(blocks_per_sm * sm_count * 16), nnz) + block_x - 1) / block_x; dim3 blocks(block_x, block_y, 1); - raft::launch_kernel(handle, blocks, tpb, smem_size)( - faster_dot_on_csr_kernel, - dot, - indptr, - cols, - A, - B, - nnz, - n_rows, - dim); + raft::launch_kernel({handle, smem_size}, + blocks, + tpb, + faster_dot_on_csr_kernel, + dot, + indptr, + cols, + A, + B, + nnz, + n_rows, + dim); } RAFT_CUDA_TRY(cudaPeekAtLastError()); diff --git a/cpp/include/raft/sparse/op/detail/filter.cuh b/cpp/include/raft/sparse/op/detail/filter.cuh index f246cfb04a..e7810b9404 100644 --- a/cpp/include/raft/sparse/op/detail/filter.cuh +++ b/cpp/include/raft/sparse/op/detail/filter.cuh @@ -117,18 +117,21 @@ void coo_remove_scalar(const idx_t* rows, dim3 grid(raft::ceildiv(n, static_cast(TPB_X)), 1, 1); dim3 blk(TPB_X, 1, 1); - raft::launch_kernel(stream, grid, blk)(coo_remove_scalar_kernel, - rows, - cols, - vals, - nnz, - crows, - ccols, - cvals, - dev_ex_scan.get(), - dev_cur_ex_scan.get(), - n, - scalar); + raft::launch_kernel(stream, + grid, + blk, + coo_remove_scalar_kernel, + rows, + cols, + vals, + nnz, + crows, + ccols, + cvals, + dev_ex_scan.get(), + dev_cur_ex_scan.get(), + n, + scalar); } /** diff --git a/cpp/include/raft/sparse/op/detail/reduce.cuh b/cpp/include/raft/sparse/op/detail/reduce.cuh index ad3049cd8d..f2a9c5af51 100644 --- a/cpp/include/raft/sparse/op/detail/reduce.cuh +++ b/cpp/include/raft/sparse/op/detail/reduce.cuh @@ -95,8 +95,14 @@ void compute_duplicates_mask( { RAFT_CUDA_TRY(cudaMemsetAsync(mask, 0, nnz * sizeof(value_idx), stream)); - raft::launch_kernel(stream, raft::ceildiv(nnz, (nnz_t)256), 256)( - compute_duplicates_diffs_kernel, rows, cols, mask, nnz); + raft::launch_kernel(stream, + raft::ceildiv(nnz, (nnz_t)256), + 256, + compute_duplicates_diffs_kernel, + rows, + cols, + mask, + nnz); } /** @@ -144,16 +150,18 @@ void max_duplicates(raft::resources const& handle, out.allocate(size, m, n, true, stream); // perform reduce - raft::launch_kernel(handle, raft::ceildiv(nnz, (nnz_t)256), 256)( - max_duplicates_kernel, - rows, - cols, - vals, - diff.data() + 1, - out.rows(), - out.cols(), - out.vals(), - nnz); + raft::launch_kernel(handle, + raft::ceildiv(nnz, (nnz_t)256), + 256, + max_duplicates_kernel, + rows, + cols, + vals, + diff.data() + 1, + out.rows(), + out.cols(), + out.vals(), + nnz); } }; // END namespace detail diff --git a/cpp/include/raft/sparse/op/detail/row_op.cuh b/cpp/include/raft/sparse/op/detail/row_op.cuh index b81c385434..08bd9d0524 100644 --- a/cpp/include/raft/sparse/op/detail/row_op.cuh +++ b/cpp/include/raft/sparse/op/detail/row_op.cuh @@ -54,8 +54,8 @@ void csr_row_op(const Index_* row_ind, Index_ n_rows, Index_ nnz, Lambda op, cud { dim3 grid(raft::ceildiv(n_rows, Index_(TPB_X)), 1, 1); dim3 blk(TPB_X, 1, 1); - raft::launch_kernel(stream, grid, blk)( - csr_row_op_kernel, row_ind, n_rows, nnz, op); + raft::launch_kernel( + stream, grid, blk, csr_row_op_kernel, row_ind, n_rows, nnz, op); } }; // namespace detail diff --git a/cpp/include/raft/sparse/solver/detail/lanczos.cuh b/cpp/include/raft/sparse/solver/detail/lanczos.cuh index e89b49c083..b7c24124f7 100644 --- a/cpp/include/raft/sparse/solver/detail/lanczos.cuh +++ b/cpp/include/raft/sparse/solver/detail/lanczos.cuh @@ -159,20 +159,25 @@ void lanczos_solve_ritz( int blockSize = 256; int numBlocks = raft::div_rounding_up_safe(ncv, blockSize); - raft::launch_kernel(handle, numBlocks, blockSize)(kernel_triangular_populate, - triangular_matrix.data_handle(), - beta.data_handle(), - ncv); + raft::launch_kernel(handle, + numBlocks, + blockSize, + kernel_triangular_populate, + triangular_matrix.data_handle(), + beta.data_handle(), + ncv); if (beta_k) { int threadsPerBlock = 256; int blocksPerGrid = raft::div_rounding_up_safe(k, threadsPerBlock); - raft::launch_kernel(handle, blocksPerGrid, threadsPerBlock)( - kernel_triangular_beta_k, - triangular_matrix.data_handle(), - beta_k.value().data_handle(), - k, - ncv); + raft::launch_kernel(handle, + blocksPerGrid, + threadsPerBlock, + kernel_triangular_beta_k, + triangular_matrix.data_handle(), + beta_k.value().data_handle(), + k, + ncv); } auto triangular_matrix_view = @@ -378,8 +383,12 @@ void lanczos_aux(raft::resources const& handle, auto uu_i = raft::make_device_scalar_view(uu.data_handle() + uu.stride(1) * i); // uu(0, i) raft::linalg::add(handle, make_const_mdspan(alpha_i), make_const_mdspan(uu_i), alpha_i); - raft::launch_kernel(handle, 1, 1)( - kernel_clamp_down, alpha_i.data_handle(), static_cast(1e-9)); + raft::launch_kernel(handle, + 1, + 1, + kernel_clamp_down, + alpha_i.data_handle(), + static_cast(1e-9)); auto output = raft::make_device_vector_view( beta.data_handle() + beta.stride(1) * i, 1); @@ -390,26 +399,37 @@ void lanczos_aux(raft::resources const& handle, int blockSize = 256; int numBlocks = (n + blockSize - 1) / blockSize; - raft::launch_kernel(handle, numBlocks, blockSize)( - kernel_clamp_down_vector, u.data_handle(), static_cast(1e-7), n); - - raft::launch_kernel(handle, 1, 1)(kernel_clamp_down, - beta.data_handle() + beta.stride(1) * i, - static_cast(1e-6)); + raft::launch_kernel(handle, + numBlocks, + blockSize, + kernel_clamp_down_vector, + u.data_handle(), + static_cast(1e-7), + n); + + raft::launch_kernel(handle, + 1, + 1, + kernel_clamp_down, + beta.data_handle() + beta.stride(1) * i, + static_cast(1e-6)); if (i >= end_idx - 1) { break; } int threadsPerBlock = 256; int blocksPerGrid = (n + threadsPerBlock - 1) / threadsPerBlock; - raft::launch_kernel(handle, blocksPerGrid, threadsPerBlock)(kernel_normalize, - u.data_handle(), - beta.data_handle(), - i, - n, - v.data_handle(), - V.data_handle(), - n); + raft::launch_kernel(handle, + blocksPerGrid, + threadsPerBlock, + kernel_normalize, + u.data_handle(), + beta.data_handle(), + i, + n, + v.data_handle(), + V.data_handle(), + n); } } diff --git a/cpp/include/raft/sparse/solver/detail/mst_solver_inl.cuh b/cpp/include/raft/sparse/solver/detail/mst_solver_inl.cuh index e867c1f31a..9d7332d706 100644 --- a/cpp/include/raft/sparse/solver/detail/mst_solver_inl.cuh +++ b/cpp/include/raft/sparse/solver/detail/mst_solver_inl.cuh @@ -234,15 +234,18 @@ void MST_solver::alteration() RAFT_EXPECTS(curand_status == CURAND_STATUS_SUCCESS, "MST: CURAND cleanup failed"); // Alterate the weights, make all undirected edge weight unique while keeping Wuv == Wvu - raft::launch_kernel(stream, nblocks, nthreads)(detail::alteration_kernel, - v, - e, - offsets, - indices, - weights, - max, - rand_values.data(), - altered_weights.data()); + raft::launch_kernel(stream, + nblocks, + nthreads, + detail::alteration_kernel, + v, + e, + offsets, + indices, + weights, + max, + rand_values.data(), + altered_weights.data()); } // updates colors of vertices by propagating the lower color to the higher @@ -270,22 +273,36 @@ void MST_solver::label_prop(vertex_t* while (!done.value(stream)) { done.set_value_async(true_val, stream); - raft::launch_kernel(stream, min_pair_nblocks, min_pair_nthreads)( - detail::min_pair_colors, - v, - indices, - new_mst_edge_ptr, - color_ptr, - color_index, - next_color_ptr); - - raft::launch_kernel(stream, min_pair_nblocks, min_pair_nthreads)( - detail::update_colors, v, color_ptr, color_index, next_color_ptr, done_ptr); + raft::launch_kernel(stream, + min_pair_nblocks, + min_pair_nthreads, + detail::min_pair_colors, + v, + indices, + new_mst_edge_ptr, + color_ptr, + color_index, + next_color_ptr); + + raft::launch_kernel(stream, + min_pair_nblocks, + min_pair_nthreads, + detail::update_colors, + v, + color_ptr, + color_index, + next_color_ptr, + done_ptr); i++; } - raft::launch_kernel(stream, min_pair_nblocks, min_pair_nthreads)( - detail::final_color_indices, v, color_ptr, color_index); + raft::launch_kernel(stream, + min_pair_nblocks, + min_pair_nthreads, + detail::final_color_indices, + v, + color_ptr, + color_index); } // Finds the minimum edge from each vertex to the lowest color @@ -306,17 +323,19 @@ void MST_solver::min_edge_per_vertex() alteration_t* min_edge_color_ptr = min_edge_color.data(); alteration_t* altered_weights_ptr = altered_weights.data(); - raft::launch_kernel(stream, v, n_threads)( - detail::kernel_min_edge_per_vertex, - offsets, - indices, - altered_weights_ptr, - color_ptr, - color_index, - new_mst_edge_ptr, - mst_edge_ptr, - min_edge_color_ptr, - v); + raft::launch_kernel(stream, + v, + n_threads, + detail::kernel_min_edge_per_vertex, + offsets, + indices, + altered_weights_ptr, + color_ptr, + color_index, + new_mst_edge_ptr, + mst_edge_ptr, + min_edge_color_ptr, + v); } // Finds the minimum edge from each supervertex to the lowest color @@ -338,36 +357,40 @@ void MST_solver::min_edge_per_superver vertex_t* temp_dst_ptr = temp_dst.data(); weight_t* temp_weights_ptr = temp_weights.data(); - raft::launch_kernel(stream, nblocks, nthreads)( - detail::min_edge_per_supervertex, - color_ptr, - color_index, - new_mst_edge_ptr, - mst_edge_ptr, - indices, - weights, - altered_weights_ptr, - temp_src_ptr, - temp_dst_ptr, - temp_weights_ptr, - min_edge_color_ptr, - v, - symmetrize_output); + raft::launch_kernel(stream, + nblocks, + nthreads, + detail::min_edge_per_supervertex, + color_ptr, + color_index, + new_mst_edge_ptr, + mst_edge_ptr, + indices, + weights, + altered_weights_ptr, + temp_src_ptr, + temp_dst_ptr, + temp_weights_ptr, + min_edge_color_ptr, + v, + symmetrize_output); // the above kernel only adds directed mst edges in the case where // a pair of vertices don't pick the same min edge between them // so, now we add the reverse edge to make it undirected if (symmetrize_output) { - raft::launch_kernel(stream, nblocks, nthreads)( - detail::add_reverse_edge, - new_mst_edge_ptr, - indices, - weights, - temp_src_ptr, - temp_dst_ptr, - temp_weights_ptr, - v, - symmetrize_output); + raft::launch_kernel(stream, + nblocks, + nthreads, + detail::add_reverse_edge, + new_mst_edge_ptr, + indices, + weights, + temp_src_ptr, + temp_dst_ptr, + temp_weights_ptr, + v, + symmetrize_output); } } @@ -381,8 +404,13 @@ void MST_solver::check_termination() edge_t* mst_edge_count_ptr = mst_edge_count.data(); vertex_t* temp_src_ptr = temp_src.data(); - raft::launch_kernel(stream, nblocks, nthreads)( - detail::kernel_count_new_mst_edges, temp_src_ptr, mst_edge_count_ptr, 2 * v); + raft::launch_kernel(stream, + nblocks, + nthreads, + detail::kernel_count_new_mst_edges, + temp_src_ptr, + mst_edge_count_ptr, + 2 * v); } template diff --git a/cpp/include/raft/sparse/solver/detail/svds_sign_correction.cuh b/cpp/include/raft/sparse/solver/detail/svds_sign_correction.cuh index c1eb2e956f..7401af0a07 100644 --- a/cpp/include/raft/sparse/solver/detail/svds_sign_correction.cuh +++ b/cpp/include/raft/sparse/solver/detail/svds_sign_correction.cuh @@ -142,13 +142,13 @@ void svd_sign_correction( // threads_per_block must be a power of 2 for the tree reduction in the kernel constexpr int threads_per_block = 256; - int smem_size = threads_per_block * (sizeof(ValueTypeT) + sizeof(int)); + size_t smem_size = threads_per_block * (sizeof(ValueTypeT) + sizeof(int)); ValueTypeT* U_ptr = U ? U->data_handle() : nullptr; ValueTypeT* Vt_ptr = Vt ? Vt->data_handle() : nullptr; - raft::launch_kernel(handle, k, threads_per_block, smem_size)( - svd_sign_correction_kernel, U_ptr, Vt_ptr, m, n, k); + raft::launch_kernel( + {handle, smem_size}, k, threads_per_block, svd_sign_correction_kernel, U_ptr, Vt_ptr, m, n, k); } } // namespace raft::sparse::solver::detail diff --git a/cpp/include/raft/spectral/detail/matrix_wrappers.hpp b/cpp/include/raft/spectral/detail/matrix_wrappers.hpp index b876cf497a..0cd3849545 100644 --- a/cpp/include/raft/spectral/detail/matrix_wrappers.hpp +++ b/cpp/include/raft/spectral/detail/matrix_wrappers.hpp @@ -385,8 +385,8 @@ struct laplacian_matrix_t : sparse_matrix_t { dim3 gridDim{std::min((n + BLOCK_SIZE - 1) / BLOCK_SIZE, 65535), 1, 1}; dim3 blockDim{BLOCK_SIZE, 1, 1}; - raft::launch_kernel(handle, gridDim, blockDim)( - diagmv, n, alpha, diagonal_.raw(), x, y); + raft::launch_kernel( + handle, gridDim, blockDim, diagmv, n, alpha, diagonal_.raw(), x, y); // Apply adjacency matrix // diff --git a/cpp/include/raft/stats/detail/batched/silhouette_score.cuh b/cpp/include/raft/stats/detail/batched/silhouette_score.cuh index a3d8d46479..cedc5282b7 100644 --- a/cpp/include/raft/stats/detail/batched/silhouette_score.cuh +++ b/cpp/include/raft/stats/detail/batched/silhouette_score.cuh @@ -160,17 +160,20 @@ void compute_chunked_a_b(raft::resources const& handle, dim3 grid_size(raft::ceildiv(dist_rows, (value_idx)block_size.x), raft::ceildiv(dist_cols, (value_idx)block_size.y)); - raft::launch_kernel(stream, grid_size, block_size)(detail::compute_chunked_a_b_kernel, - a, - b, - row_offset, - col_offset, - y, - n_labels, - cluster_counts, - distances, - dist_rows, - dist_cols); + raft::launch_kernel(stream, + grid_size, + block_size, + detail::compute_chunked_a_b_kernel, + a, + b, + row_offset, + col_offset, + y, + n_labels, + cluster_counts, + distances, + dist_rows, + dist_cols); } template @@ -214,8 +217,15 @@ value_t silhouette_score( dim3 block_size(std::min(n_rows, 32), std::min(n_labels, 32)); dim3 grid_size(raft::ceildiv(n_rows, (value_idx)block_size.x), raft::ceildiv(n_labels, (label_idx)block_size.y)); - raft::launch_kernel(handle, grid_size, block_size)( - detail::fill_b_kernel, b_ptr, y, n_rows, n_labels, cluster_counts.data()); + raft::launch_kernel(handle, + grid_size, + block_size, + detail::fill_b_kernel, + b_ptr, + y, + n_rows, + n_labels, + cluster_counts.data()); resource::wait_stream_pool_on_stream(handle); diff --git a/cpp/include/raft/stats/detail/contingencyMatrix.cuh b/cpp/include/raft/stats/detail/contingencyMatrix.cuh index bb54aec935..a12378ef64 100644 --- a/cpp/include/raft/stats/detail/contingencyMatrix.cuh +++ b/cpp/include/raft/stats/detail/contingencyMatrix.cuh @@ -59,13 +59,16 @@ void computeCMatWAtomics(const T* groundTruth, cudaFuncSetCacheConfig(devConstructContingencyMatrix, cudaFuncCachePreferL1)); static const int block = 128; auto grid = raft::ceildiv(nSamples, block); - raft::launch_kernel(stream, grid, block)(devConstructContingencyMatrix, - groundTruth, - predictedLabel, - nSamples, - outMat, - outIdxOffset, - outDimN); + raft::launch_kernel(stream, + grid, + block, + devConstructContingencyMatrix, + groundTruth, + predictedLabel, + nSamples, + outMat, + outIdxOffset, + outDimN); } template @@ -107,14 +110,16 @@ void computeCMatWSmemAtomics(const T* groundTruth, static const int block = 128; auto grid = raft::ceildiv(nSamples, block); size_t smemSizePerBlock = outDimN * outDimN * sizeof(OutT); - raft::launch_kernel(stream, grid, block, smemSizePerBlock)( - devConstructContingencyMatrixSmem, - groundTruth, - predictedLabel, - nSamples, - outMat, - outIdxOffset, - outDimN); + raft::launch_kernel({stream, smemSizePerBlock}, + grid, + block, + devConstructContingencyMatrixSmem, + groundTruth, + predictedLabel, + nSamples, + outMat, + outIdxOffset, + outDimN); } template diff --git a/cpp/include/raft/stats/detail/dispersion.cuh b/cpp/include/raft/stats/detail/dispersion.cuh index 3fc21de6a3..ae8acf6d2e 100644 --- a/cpp/include/raft/stats/detail/dispersion.cuh +++ b/cpp/include/raft/stats/detail/dispersion.cuh @@ -110,15 +110,30 @@ DataT dispersion(const DataT* centroids, } RAFT_CUDA_TRY(cudaMemsetAsync(mu, 0, sizeof(DataT) * dim, stream)); RAFT_CUDA_TRY(cudaMemsetAsync(result.data(), 0, sizeof(DataT), stream)); - raft::launch_kernel(stream, grid, TPB)( - weightedMeanKernel, mu, centroids, clusterSizes, dim, nClusters); + raft::launch_kernel(stream, + grid, + TPB, + weightedMeanKernel, + mu, + centroids, + clusterSizes, + dim, + nClusters); DataT ratio = DataT(1) / DataT(nPoints); raft::linalg::scalarMultiply(mu, mu, ratio, dim, stream); // finally, compute the dispersion constexpr int ItemsPerThread = 4; int nblks = raft::ceildiv(dim * nClusters, TPB * ItemsPerThread); - raft::launch_kernel(stream, nblks, TPB)( - dispersionKernel, result.data(), centroids, clusterSizes, mu, dim, nClusters); + raft::launch_kernel(stream, + nblks, + TPB, + dispersionKernel, + result.data(), + centroids, + clusterSizes, + mu, + dim, + nClusters); DataT h_result; raft::update_host(&h_result, result.data(), 1, stream); raft::interruptible::synchronize(stream); diff --git a/cpp/include/raft/stats/detail/histogram.cuh b/cpp/include/raft/stats/detail/histogram.cuh index 253d2d324f..62f97ae0a1 100644 --- a/cpp/include/raft/stats/detail/histogram.cuh +++ b/cpp/include/raft/stats/detail/histogram.cuh @@ -95,8 +95,15 @@ void gmemHist(int* bins, { auto blks = computeGridDim( nrows, ncols, (const void*)gmemHistKernel); - raft::launch_kernel(stream, blks, ThreadsPerBlock)( - gmemHistKernel, bins, data, nrows, nbins, binner); + raft::launch_kernel(stream, + blks, + ThreadsPerBlock, + gmemHistKernel, + bins, + data, + nrows, + nbins, + binner); } template @@ -146,8 +153,15 @@ void smemHist(int* bins, auto blks = computeGridDim( nrows, ncols, (const void*)smemHistKernel); size_t smemSize = nbins * sizeof(unsigned); - raft::launch_kernel(stream, blks, ThreadsPerBlock, smemSize)( - smemHistKernel, bins, data, nrows, nbins, binner); + raft::launch_kernel({stream, smemSize}, + blks, + ThreadsPerBlock, + smemHistKernel, + bins, + data, + nrows, + nbins, + binner); } template @@ -232,13 +246,15 @@ void smemBitsHist(int* bins, auto blks = computeGridDim( nrows, ncols, (const void*)smemBitsHistKernel); size_t smemSize = raft::ceildiv(nbins, Bits::WORD_BITS / Bits::BIN_BITS) * sizeof(int); - raft::launch_kernel(stream, blks, ThreadsPerBlock, smemSize)( - smemBitsHistKernel, - bins, - data, - nrows, - nbins, - binner); + raft::launch_kernel({stream, smemSize}, + blks, + ThreadsPerBlock, + smemBitsHistKernel, + bins, + data, + nrows, + nbins, + binner); } #define INVALID_KEY -1 @@ -355,15 +371,17 @@ void smemHashHist(int* bins, nrows, ncols, (const void*)smemHashHistKernel); int hashSize = computeHashTableSize(); size_t smemSize = hashSize * sizeof(int2) + sizeof(int); - raft::launch_kernel(stream, blks, ThreadsPerBlock, smemSize)( - smemHashHistKernel, - bins, - data, - nrows, - nbins, - binner, - hashSize, - flushThreshold); + raft::launch_kernel({stream, smemSize}, + blks, + ThreadsPerBlock, + smemHashHistKernel, + bins, + data, + nrows, + nbins, + binner, + hashSize, + flushThreshold); } template diff --git a/cpp/include/raft/stats/detail/meanvar.cuh b/cpp/include/raft/stats/detail/meanvar.cuh index 19e3ba421f..d542bfdc25 100644 --- a/cpp/include/raft/stats/detail/meanvar.cuh +++ b/cpp/include/raft/stats/detail/meanvar.cuh @@ -209,13 +209,28 @@ void meanvar( const uint64_t len = uint64_t(D) * uint64_t(N); ASSERT(len <= uint64_t(std::numeric_limits::max()), "N * D does not fit the indexing type"); - raft::launch_kernel(stream, gs, bs)( - meanvar_kernel_rowmajor, data, mvs, locks, len, D); - raft::launch_kernel(stream, raft::ceildiv(D, BlockSize), BlockSize)( - meanvar_kernel_fill, mean, var, mvs, D, sample); + raft::launch_kernel( + stream, gs, bs, meanvar_kernel_rowmajor, data, mvs, locks, len, D); + raft::launch_kernel(stream, + raft::ceildiv(D, BlockSize), + BlockSize, + meanvar_kernel_fill, + mean, + var, + mvs, + D, + sample); } else { - raft::launch_kernel(stream, D, BlockSize)( - meanvar_kernel_colmajor, mean, var, data, D, N, sample); + raft::launch_kernel(stream, + D, + BlockSize, + meanvar_kernel_colmajor, + mean, + var, + data, + D, + N, + sample); } RAFT_CHECK_CUDA(stream); } diff --git a/cpp/include/raft/stats/detail/minmax.cuh b/cpp/include/raft/stats/detail/minmax.cuh index 0e53c4a3e8..19e610c664 100644 --- a/cpp/include/raft/stats/detail/minmax.cuh +++ b/cpp/include/raft/stats/detail/minmax.cuh @@ -187,8 +187,8 @@ void minmax(const T* data, using E = typename encode_traits::E; int nblks = raft::ceildiv(ncols, TPB); T init_val = std::numeric_limits::max(); - raft::launch_kernel(stream, nblks, TPB)( - minmaxInitKernel, ncols, globalmin, globalmax, init_val); + raft::launch_kernel( + stream, nblks, TPB, minmaxInitKernel, ncols, globalmin, globalmax, init_val); nblks = raft::ceildiv(nrows * ncols, TPB); nblks = min(nblks, 65536); size_t smemSize = sizeof(T) * 2 * ncols; @@ -200,20 +200,23 @@ void minmax(const T* data, int num_batches = raft::ceildiv(ncols, batch_ncols); smemSize = sizeof(T) * 2 * batch_ncols; - raft::launch_kernel(stream, nblks, TPB, smemSize)(minmaxKernel, - data, - rowids, - colids, - nrows, - ncols, - row_stride, - globalmin, - globalmax, - sampledcols, - init_val, - batch_ncols, - num_batches); - raft::launch_kernel(stream, nblks, TPB)(decodeKernel, globalmin, globalmax, ncols); + raft::launch_kernel({stream, smemSize}, + nblks, + TPB, + minmaxKernel, + data, + rowids, + colids, + nrows, + ncols, + row_stride, + globalmin, + globalmax, + sampledcols, + init_val, + batch_ncols, + num_batches); + raft::launch_kernel(stream, nblks, TPB, decodeKernel, globalmin, globalmax, ncols); } }; // end namespace detail diff --git a/cpp/include/raft/stats/detail/mutual_info_score.cuh b/cpp/include/raft/stats/detail/mutual_info_score.cuh index 0ec6600207..8eff81f9bf 100644 --- a/cpp/include/raft/stats/detail/mutual_info_score.cuh +++ b/cpp/include/raft/stats/detail/mutual_info_score.cuh @@ -151,14 +151,16 @@ double mutual_info_score(const T* firstClusterArray, raft::ceildiv(numUniqueClasses, numThreadsPerBlock.y)); // calling the kernel - raft::launch_kernel(stream, numBlocks, numThreadsPerBlock)( - mutual_info_kernel, - dContingencyMatrix.data(), - a.data(), - b.data(), - numUniqueClasses, - size, - d_MI.data()); + raft::launch_kernel(stream, + numBlocks, + numThreadsPerBlock, + mutual_info_kernel, + dContingencyMatrix.data(), + a.data(), + b.data(), + numUniqueClasses, + size, + d_MI.data()); // updating in the host memory h_MI = d_MI.value(stream); diff --git a/cpp/include/raft/stats/detail/neighborhood_recall.cuh b/cpp/include/raft/stats/detail/neighborhood_recall.cuh index f856261a3c..80ae616990 100644 --- a/cpp/include/raft/stats/detail/neighborhood_recall.cuh +++ b/cpp/include/raft/stats/detail/neighborhood_recall.cuh @@ -100,7 +100,10 @@ void neighborhood_recall( auto constexpr kThreadsPerBlock = 32; auto const num_blocks = indices.extent(0); - raft::launch_kernel(res, num_blocks, kThreadsPerBlock)( + raft::launch_kernel( + res, + num_blocks, + kThreadsPerBlock, neighborhood_recall, indices, ref_indices, diff --git a/cpp/include/raft/stats/detail/rand_index.cuh b/cpp/include/raft/stats/detail/rand_index.cuh index 0a3fc6b43c..99a731aca2 100644 --- a/cpp/include/raft/stats/detail/rand_index.cuh +++ b/cpp/include/raft/stats/detail/rand_index.cuh @@ -140,13 +140,15 @@ double compute_rand_index(const T* firstClusterArray, raft::ceildiv(size, numThreadsPerBlock.y)); // calling the kernel - raft::launch_kernel(stream, numBlocks, numThreadsPerBlock)( - computeTheNumerator, - firstClusterArray, - secondClusterArray, - size, - arr_buf.data(), - arr_buf.data() + 1); + raft::launch_kernel(stream, + numBlocks, + numThreadsPerBlock, + computeTheNumerator, + firstClusterArray, + secondClusterArray, + size, + arr_buf.data(), + arr_buf.data() + 1); // synchronizing and updating the calculated values of a and b from device to host uint64_t ab_host[2] = {0}; diff --git a/cpp/include/raft/stats/detail/scores.cuh b/cpp/include/raft/stats/detail/scores.cuh index 8a81deb969..b288d47ef1 100644 --- a/cpp/include/raft/stats/detail/scores.cuh +++ b/cpp/include/raft/stats/detail/scores.cuh @@ -164,12 +164,15 @@ void regression_metrics(const T* predictions, rmm::device_uvector tmp_sums(2 * sizeof(double), stream); RAFT_CUDA_TRY(cudaMemsetAsync(tmp_sums.data(), 0, 2 * sizeof(double), stream)); - raft::launch_kernel(stream, block_cnt, thread_cnt)(reg_metrics_kernel, - predictions, - ref_predictions, - n, - abs_diffs_array.data(), - tmp_sums.data()); + raft::launch_kernel(stream, + block_cnt, + thread_cnt, + reg_metrics_kernel, + predictions, + ref_predictions, + n, + abs_diffs_array.data(), + tmp_sums.data()); raft::update_host(&mean_errors[0], tmp_sums.data(), 2, stream); raft::interruptible::synchronize(stream); diff --git a/cpp/include/raft/stats/detail/silhouette_score.cuh b/cpp/include/raft/stats/detail/silhouette_score.cuh index 3d11cf4710..929a442f96 100644 --- a/cpp/include/raft/stats/detail/silhouette_score.cuh +++ b/cpp/include/raft/stats/detail/silhouette_score.cuh @@ -247,14 +247,17 @@ DataT silhouette_score( dim3 numBlocks(raft::ceildiv(nRows, numThreadsPerBlock.x), 1, 1); // calling the kernel - raft::launch_kernel(stream, numBlocks, numThreadsPerBlock)(populateAKernel, - sampleToClusterSumOfDistances.data(), - binCountArray.data(), - d_aArray.data(), - labels, - nRows, - nLabels, - std::numeric_limits::max()); + raft::launch_kernel(stream, + numBlocks, + numThreadsPerBlock, + populateAKernel, + sampleToClusterSumOfDistances.data(), + binCountArray.data(), + d_aArray.data(), + labels, + nRows, + nLabels, + std::numeric_limits::max()); // elementwise dividing by bincounts rmm::device_uvector averageDistanceBetweenSampleAndCluster(nRows * nLabels, stream); diff --git a/cpp/include/raft/stats/detail/trustworthiness_score.cuh b/cpp/include/raft/stats/detail/trustworthiness_score.cuh index 9600570fed..7f68840a78 100644 --- a/cpp/include/raft/stats/detail/trustworthiness_score.cuh +++ b/cpp/include/raft/stats/detail/trustworthiness_score.cuh @@ -181,20 +181,23 @@ double trustworthiness_score(const raft::resources& h, int work = curBatchSize * n; int n_blocks = raft::ceildiv(work, N_THREADS); - raft::launch_kernel(h, n_blocks, N_THREADS)( - build_lookup_table, lookup_table.data(), X_ind.data(), n, work); + raft::launch_kernel( + h, n_blocks, N_THREADS, build_lookup_table, lookup_table.data(), X_ind.data(), n, work); RAFT_CUDA_TRY(cudaMemsetAsync(t_dbuf.data(), 0, sizeof(double), stream)); work = curBatchSize * (n_neighbors + 1); n_blocks = raft::ceildiv(work, N_THREADS); - raft::launch_kernel(h, n_blocks, N_THREADS)(compute_rank, - t_dbuf.data(), - lookup_table.data(), - &emb_ind.data()[(n - toDo) * (n_neighbors + 1)], - n, - n_neighbors + 1, - work); + raft::launch_kernel(h, + n_blocks, + N_THREADS, + compute_rank, + t_dbuf.data(), + lookup_table.data(), + &emb_ind.data()[(n - toDo) * (n_neighbors + 1)], + n, + n_neighbors + 1, + work); t += t_dbuf.value(stream); diff --git a/cpp/include/raft/util/cache.cuh b/cpp/include/raft/util/cache.cuh index af75372339..e4774258d6 100644 --- a/cpp/include/raft/util/cache.cuh +++ b/cpp/include/raft/util/cache.cuh @@ -181,8 +181,8 @@ class Cache { void GetVecs(const int* idx, int n, math_t* out, cudaStream_t stream) { if (n > 0) { - raft::launch_kernel(stream, raft::ceildiv(n * n_vec, TPB), TPB)( - get_vecs, cache.data(), n_vec, idx, n, out); + raft::launch_kernel( + stream, raft::ceildiv(n * n_vec, TPB), TPB, get_vecs, cache.data(), n_vec, idx, n, out); } } @@ -216,15 +216,18 @@ class Cache { const int* tile_idx = nullptr) { if (n > 0) { - raft::launch_kernel(stream, raft::ceildiv(n * n_vec, TPB), TPB)(store_vecs, - tile, - n_tile, - n_vec, - tile_idx, - n, - cache_idx, - cache.data(), - cache.size() / n_vec); + raft::launch_kernel(stream, + raft::ceildiv(n * n_vec, TPB), + TPB, + store_vecs, + tile, + n_tile, + n_vec, + tile_idx, + n, + cache_idx, + cache.data(), + cache.size() / n_vec); } } @@ -254,16 +257,19 @@ class Cache { { n_iter++; // we increase the iteration counter, that is used to time stamp // accessing entries from the cache - raft::launch_kernel(stream, raft::ceildiv(n, TPB), TPB)(get_cache_idx, - keys, - n, - cached_keys.data(), - n_cache_sets, - associativity, - cache_time.data(), - cache_idx, - is_cached, - n_iter); + raft::launch_kernel(stream, + raft::ceildiv(n, TPB), + TPB, + get_cache_idx, + keys, + n, + cached_keys.data(), + n_cache_sets, + associativity, + cache_time.data(), + cache_idx, + is_cached, + n_iter); } /** @brief Map a set of keys to cache indices. @@ -346,15 +352,18 @@ class Cache { RAFT_CUDA_TRY(cudaMemsetAsync(cidx, 255, n * sizeof(int), stream)); const int nthreads = associativity <= 32 ? associativity : 32; - raft::launch_kernel(stream, n_cache_sets, nthreads)(assign_cache_idx, - keys, - n, - ws_tmp.data(), - cached_keys.data(), - n_cache_sets, - cache_time.data(), - n_iter, - cidx); + raft::launch_kernel(stream, + n_cache_sets, + nthreads, + assign_cache_idx, + keys, + n, + ws_tmp.data(), + cached_keys.data(), + n_cache_sets, + cache_time.data(), + n_iter, + cidx); if (debug_mode) RAFT_CUDA_TRY(cudaDeviceSynchronize()); } diff --git a/cpp/include/raft/util/detail/scatter.cuh b/cpp/include/raft/util/detail/scatter.cuh index 3b17e9f0a3..7307fffd5f 100644 --- a/cpp/include/raft/util/detail/scatter.cuh +++ b/cpp/include/raft/util/detail/scatter.cuh @@ -37,8 +37,8 @@ void scatterImpl( DataT* out, const DataT* in, const IdxT* idx, IdxT len, Lambda op, cudaStream_t stream) { const IdxT nblks = raft::ceildiv(VecLen ? len / VecLen : len, (IdxT)TPB); - raft::launch_kernel(stream, nblks, TPB)( - scatterKernel, out, in, idx, len, op); + raft::launch_kernel( + stream, nblks, TPB, scatterKernel, out, in, idx, len, op); } } // namespace detail diff --git a/cpp/include/raft/util/kernel_launch.hpp b/cpp/include/raft/util/kernel_launch.hpp index 20be376c1b..834d1a5982 100644 --- a/cpp/include/raft/util/kernel_launch.hpp +++ b/cpp/include/raft/util/kernel_launch.hpp @@ -20,6 +20,7 @@ #include #include #include +#include #include namespace raft { @@ -38,7 +39,7 @@ inline std::string format_cuda_launch_error(cudaError_t status, std::source_loca char const* location_prefix = "CUDA error encountered at: "; char const* location_fmt = "file=%s line=%d function=%s: "; char const* fmt = "call='%s', Reason=%s:%s"; - char const* call = "cudaLaunchKernel"; + char const* call = "cudaLaunchKernelExC"; char const* file = location.file_name(); auto line = static_cast(location.line()); char const* function = location.function_name(); @@ -64,8 +65,21 @@ inline std::string format_cuda_launch_error(cudaError_t status, std::source_loca return std::string(buf.data(), buf.data() + size - 1); } -inline void throw_on_cuda_launch_error(cudaError_t status, std::source_location location) +/** + * @brief Launch a kernel, copying the launch arguments into parameters first. + * + * Taking the address of a copy rather than of the caller's object means passing a constant (e.g. a + * @c static @c const data member) does not odr-use it, matching the @c <<<>>> launch syntax. + */ +template +void dispatch(cudaLaunchConfig_t const& config, + void* kernel, + std::source_location location, + Params... params) { + std::array arg_ptrs{ + {const_cast(static_cast(std::addressof(params)))...}}; + cudaError_t status = cudaLaunchKernelExC(&config, kernel, arg_ptrs.data()); if (status == cudaSuccess) { return; } cudaGetLastError(); // clear sticky error throw raft::cuda_error(format_cuda_launch_error(status, location)); @@ -74,170 +88,142 @@ inline void throw_on_cuda_launch_error(cudaError_t status, std::source_location } // namespace detail /** - * @brief Temporary object that launches a CUDA kernel with call-site error reporting. - * - * @ingroup kernel_launch + * @defgroup kernel_launch Type-checked CUDA kernel launch + * @{ + */ + +/** + * @brief Where a kernel is launched: the stream, the dynamic shared memory size, and the call site + * to blame for launch errors. * - * Capture @c std::source_location on @ref launch_kernel, then launch via rvalue @c operator(). - * Prefer the one-liner form so diagnostics point at the launch expression: + * Converts implicitly from raft resources or from a stream, so that a launch reads as a single + * call and the diagnostics of a failed launch point at the launch expression: * @code - * raft::launch_kernel(res, grid, block)(kernel, args...); - * raft::launch_kernel(stream, grid, block, smem)(kernel, args...); + * raft::launch_kernel(res, grid, block, my_kernel, arg0, arg1); + * raft::launch_kernel({stream, smem}, grid, block, my_kernel, arg0, arg1); * @endcode * - * The kernel must resolve to a unique @c __global__ function pointer. Partially-specified function - * templates are OK when the remaining parameters can be deduced from the launch argument types. - * Overload sets that remain ambiguous after that conversion are not supported. + * Copy and move are deleted and @c launch_kernel takes this by value, so the parameter can only be + * initialized from a prvalue: an instance stored in a variable can never be launched, and the + * captured location is therefore always the one of the launch expression. */ -class kernel_launcher { +struct launch_on { public: - kernel_launcher(kernel_launcher const&) = delete; - kernel_launcher& operator=(kernel_launcher const&) = delete; - kernel_launcher(kernel_launcher&&) = default; - kernel_launcher& operator=(kernel_launcher&&) = delete; - /** - * @brief Launch @p kernel with @p args, which already have the kernel parameter types. - * - * The function-pointer parameter type is a non-deduced context derived from @p args, so a - * partially specified function template (e.g. @c map_kernel) can still convert to - * a unique @c __global__ pointer by deducing its remaining template parameters from that type. + * @param[in] res raft resources providing the stream to launch on + * @param[in] smem dynamic shared memory size in bytes + * @param[in] loc call site to blame for launch errors; leave at its default */ - template - void operator()(std::type_identity_t...)> kernel, - Args&&... args) && + launch_on( // NOLINT(google-explicit-constructor) + resources const& res, + std::size_t smem = 0, + std::source_location loc = std::source_location::current()) + : launch_on{resource::get_cuda_stream(res), smem, loc} { - dispatch_by_value...>(reinterpret_cast(kernel), - std::forward(args)...); } /** - * @brief Launch @p kernel, converting @p args to the kernel parameter types. - * - * Handles call sites where an argument merely converts to its parameter (e.g. @c T* to - * @c const T*), so they do not need casts. @p kernel must name a single specialization here, - * because its parameter types are what the arguments are converted to. + * @param[in] stream stream to launch on + * @param[in] smem dynamic shared memory size in bytes + * @param[in] loc call site to blame for launch errors; leave at its default */ - template - requires(sizeof...(Params) == sizeof...(Args) && - !(std::is_same_v, Params> && ...)) void - operator()(void (*kernel)(Params...), Args&&... args) && + launch_on( // NOLINT(google-explicit-constructor) + rmm::cuda_stream_view stream, + std::size_t smem = 0, + std::source_location loc = std::source_location::current()) + : launch_on{stream.value(), smem, loc} { - static_assert((std::is_convertible_v && ...), - "Each launch argument must be convertible to the corresponding kernel parameter"); - - dispatch_by_value(reinterpret_cast(kernel), std::forward(args)...); } - private: - friend kernel_launcher launch_kernel( - resources const&, dim3, dim3, std::size_t, std::source_location); - friend kernel_launcher launch_kernel( - rmm::cuda_stream_view, dim3, dim3, std::size_t, std::source_location); - /** - * @brief Copy the launch arguments into parameters and pass their addresses to dispatch(). - * - * Taking the address of a copy rather than of the caller's object means passing a constant (e.g. - * a - * @c static @c const data member) does not odr-use it, matching the @c <<<>>> launch syntax. + * @param[in] stream stream to launch on + * @param[in] smem dynamic shared memory size in bytes + * @param[in] loc call site to blame for launch errors; leave at its default */ - template - void dispatch_by_value(void* kernel, Params... params) const - { - std::array arg_ptrs{ - {const_cast(static_cast(std::addressof(params)))...}}; - dispatch(kernel, arg_ptrs.data()); - } - - void dispatch(void* kernel, void** arg_ptrs) const + launch_on( // NOLINT(google-explicit-constructor) + cudaStream_t stream, + std::size_t smem = 0, + std::source_location loc = std::source_location::current()) + : location{loc} { - cudaError_t status = - cudaLaunchKernel(kernel, grid_, block_, arg_ptrs, shared_mem_bytes_, stream_.value()); - - if (status == cudaSuccess) { -#ifndef NDEBUG - status = cudaStreamSynchronize(stream_.value()); -#else - status = cudaPeekAtLastError(); -#endif - } - detail::throw_on_cuda_launch_error(status, location_); + config.stream = stream; + config.dynamicSmemBytes = smem; } - kernel_launcher(rmm::cuda_stream_view stream, - dim3 grid, - dim3 block, - std::size_t shared_mem_bytes, - std::source_location location) - : stream_{stream}, - grid_{grid}, - block_{block}, - shared_mem_bytes_{shared_mem_bytes}, - location_{location} - { - } + launch_on(launch_on const&) = delete; + launch_on& operator=(launch_on const&) = delete; + launch_on(launch_on&&) = delete; + launch_on& operator=(launch_on&&) = delete; + ~launch_on() = default; - rmm::cuda_stream_view stream_; - dim3 grid_{}; - dim3 block_{}; - std::size_t shared_mem_bytes_{0}; - std::source_location location_{}; + /** Call site to blame for launch errors. */ + std::source_location location; + /** Launch configuration; the grid and block dimensions are filled in by the launch. */ + cudaLaunchConfig_t config{}; }; /** - * @defgroup kernel_launch Type-checked CUDA kernel launch - * @{ - */ - -/** - * @brief Launch a CUDA kernel on the stream of @p res, reporting errors at the call site. + * @brief Launch @p kernel with @p args, which already have the kernel parameter types. * - * The returned launcher is a temporary that must be called immediately, so that the diagnostics of - * a failed launch blame the launch expression rather than this header: + * The launch arguments are checked against the kernel parameters at compile time, and a failed + * launch throws @c raft::cuda_error blaming the call site: * @code - * raft::launch_kernel(res, grid, block)(my_kernel, arg0, arg1); + * raft::launch_kernel(res, grid, block, my_kernel, arg0, arg1); * @endcode - * The launch arguments are checked against the kernel parameters at compile time, and a failed - * launch throws @c raft::cuda_error. * - * @param[in] res raft resources providing the stream to launch on + * The function-pointer parameter type is a non-deduced context derived from @p args, so a + * partially specified function template (e.g. @c map_kernel) can still convert to a + * unique @c __global__ pointer by deducing its remaining template parameters from that type. + * Overload sets that remain ambiguous after that conversion are not supported. + * + * @param[in] where stream to launch on, dynamic shared memory size, and the call site * @param[in] grid grid dimensions * @param[in] block block dimensions - * @param[in] shared_mem_bytes dynamic shared memory size in bytes - * @param[in] location call site to blame for launch errors; leave at its default - * @return a launcher to invoke with the kernel and its arguments + * @param[in] kernel the @c __global__ function to launch + * @param[in] args arguments to pass to @p kernel */ -inline kernel_launcher launch_kernel( - resources const& res, - dim3 grid, - dim3 block, - std::size_t shared_mem_bytes = 0, - std::source_location location = std::source_location::current()) +template +void launch_kernel(launch_on where, + dim3 grid, + dim3 block, + std::type_identity_t...)> kernel, + Args&&... args) { - return kernel_launcher{resource::get_cuda_stream(res), grid, block, shared_mem_bytes, location}; + where.config.gridDim = grid; + where.config.blockDim = block; + detail::dispatch...>( + where.config, reinterpret_cast(kernel), where.location, std::forward(args)...); } /** - * @brief Launch a CUDA kernel on @p stream, reporting errors at the call site. + * @brief Launch @p kernel, converting @p args to the kernel parameter types. * - * Same as the overload taking raft resources; prefer that one where resources are available. + * Handles call sites where an argument merely converts to its parameter (e.g. @c T* to + * @c const T*), so they do not need casts. @p kernel must name a single specialization here, + * because its parameter types are what the arguments are converted to. * - * @param[in] stream stream to launch on + * @param[in] where stream to launch on, dynamic shared memory size, and the call site * @param[in] grid grid dimensions * @param[in] block block dimensions - * @param[in] shared_mem_bytes dynamic shared memory size in bytes - * @param[in] location call site to blame for launch errors; leave at its default - * @return a launcher to invoke with the kernel and its arguments + * @param[in] kernel the @c __global__ function to launch + * @param[in] args arguments to convert and pass to @p kernel */ -inline kernel_launcher launch_kernel( - rmm::cuda_stream_view stream, - dim3 grid, - dim3 block, - std::size_t shared_mem_bytes = 0, - std::source_location location = std::source_location::current()) +template +requires(sizeof...(Params) == sizeof...(Args) && + !(std::is_same_v, Params> && + ...)) void launch_kernel(launch_on where, + dim3 grid, + dim3 block, + void (*kernel)(Params...), + Args&&... args) { - return kernel_launcher{stream, grid, block, shared_mem_bytes, location}; + static_assert((std::is_convertible_v && ...), + "Each launch argument must be convertible to the corresponding kernel parameter"); + + where.config.gridDim = grid; + where.config.blockDim = block; + detail::dispatch( + where.config, reinterpret_cast(kernel), where.location, std::forward(args)...); } /** @} */ // end group kernel_launch diff --git a/cpp/tests/core/interruptible.cu b/cpp/tests/core/interruptible.cu index 2ee97e9310..cdcdd67d96 100644 --- a/cpp/tests/core/interruptible.cu +++ b/cpp/tests/core/interruptible.cu @@ -109,14 +109,14 @@ TEST(Raft, InterruptibleOpenMP) auto i = omp_get_thread_num(); common::nvtx::range omp_scope("interruptible::thread-%d", i); rmm::cuda_stream stream; - raft::launch_kernel(stream.value(), 1, 1)(gpu_wait, 1); + raft::launch_kernel(stream.value(), 1, 1, gpu_wait, 1); interruptible::synchronize(stream); thread_tokens[i] = interruptible::get_token(); #pragma omp barrier try { common::nvtx::range wait_scope("interruptible::wait-%d", i); - raft::launch_kernel(stream.value(), 1, 1)(gpu_wait, (1 + i) * thread_delay_millis); + raft::launch_kernel(stream.value(), 1, 1, gpu_wait, (1 + i) * thread_delay_millis); interruptible::synchronize(stream); n_finished = 1; } catch (interrupted_exception&) { diff --git a/cpp/tests/core/math_device.cu b/cpp/tests/core/math_device.cu index b7d5a7f6e8..1b3249d538 100644 --- a/cpp/tests/core/math_device.cu +++ b/cpp/tests/core/math_device.cu @@ -32,8 +32,8 @@ auto math_eval(OpT op, Args&&... args) using OutT = cuda::std::invoke_result_t; auto stream = rmm::cuda_stream_default; rmm::device_scalar result(stream); - raft::launch_kernel(stream, 1, 1)( - math_eval_kernel, result.data(), op, std::forward(args)...); + raft::launch_kernel( + stream, 1, 1, math_eval_kernel, result.data(), op, std::forward(args)...); return result.value(stream); } diff --git a/cpp/tests/core/operators_device.cu b/cpp/tests/core/operators_device.cu index 96f73c83fe..f46a66ef30 100644 --- a/cpp/tests/core/operators_device.cu +++ b/cpp/tests/core/operators_device.cu @@ -29,8 +29,8 @@ auto eval_op_on_device(OpT op, Args&&... args) typedef decltype(op(args...)) OutT; auto stream = rmm::cuda_stream_default; rmm::device_scalar result(stream); - raft::launch_kernel(stream, 1, 1)( - eval_op_on_device_kernel, result.data(), op, std::forward(args)...); + raft::launch_kernel( + stream, 1, 1, eval_op_on_device_kernel, result.data(), op, std::forward(args)...); return result.value(stream); } diff --git a/cpp/tests/core/span.cu b/cpp/tests/core/span.cu index 5607b82d0d..920f5d06d6 100644 --- a/cpp/tests/core/span.cu +++ b/cpp/tests/core/span.cu @@ -74,20 +74,20 @@ TEST(GPUSpan, FromOther) // dynamic extent { device_span span(d_vec.data().get(), d_vec.size()); - raft::launch_kernel(cudaStream_t{0}, 1, 16)(TestFromOtherKernel, span); + raft::launch_kernel(cudaStream_t{0}, 1, 16, TestFromOtherKernel, span); } { device_span span(d_vec.data().get(), d_vec.size()); - raft::launch_kernel(cudaStream_t{0}, 1, 16)(TestFromOtherKernelConst, span); + raft::launch_kernel(cudaStream_t{0}, 1, 16, TestFromOtherKernelConst, span); } // static extent { device_span span(d_vec.data().get(), d_vec.data().get() + 16); - raft::launch_kernel(cudaStream_t{0}, 1, 16)(TestFromOtherKernel, span); + raft::launch_kernel(cudaStream_t{0}, 1, 16, TestFromOtherKernel, span); } { device_span span(d_vec.data().get(), d_vec.data().get() + 16); - raft::launch_kernel(cudaStream_t{0}, 1, 16)(TestFromOtherKernelConst, span); + raft::launch_kernel(cudaStream_t{0}, 1, 16, TestFromOtherKernelConst, span); } } @@ -186,7 +186,7 @@ TEST(GPUSpan, Modify) device_span span(d_vec.data().get(), d_vec.size()); - raft::launch_kernel(cudaStream_t{0}, 1, 16)(TestModifyKernel, span); + raft::launch_kernel(cudaStream_t{0}, 1, 16, TestModifyKernel, span); for (size_t i = 0; i < d_vec.size(); ++i) { ASSERT_EQ(d_vec[i], d_vec.size() - i); diff --git a/cpp/tests/linalg/add.cuh b/cpp/tests/linalg/add.cuh index d16ff2d3dc..c467ce8fe6 100644 --- a/cpp/tests/linalg/add.cuh +++ b/cpp/tests/linalg/add.cuh @@ -24,7 +24,7 @@ void naiveAddElem(OutT* out, const InT* in1, const InT* in2, int len, cudaStream { static const int TPB = 64; int nblks = raft::ceildiv(len, TPB); - raft::launch_kernel(stream, nblks, TPB)(naiveAddElemKernel, out, in1, in2, len); + raft::launch_kernel(stream, nblks, TPB, naiveAddElemKernel, out, in1, in2, len); } template diff --git a/cpp/tests/linalg/axpy.cu b/cpp/tests/linalg/axpy.cu index aa4993ed60..0e65ac79d5 100644 --- a/cpp/tests/linalg/axpy.cu +++ b/cpp/tests/linalg/axpy.cu @@ -80,8 +80,16 @@ class AxpyTest : public ::testing::TestWithParam> { int threads = 64; int blocks = raft::ceildiv(params.len, threads); - raft::launch_kernel(handle, blocks, threads)( - naiveAxpy, params.len, params.alpha, x.data(), refy.data(), params.incx, params.incy); + raft::launch_kernel(handle, + blocks, + threads, + naiveAxpy, + params.len, + params.alpha, + x.data(), + refy.data(), + params.incx, + params.incy); auto host_alpha_view = make_host_scalar_view(¶ms.alpha); diff --git a/cpp/tests/linalg/binary_op.cuh b/cpp/tests/linalg/binary_op.cuh index db0e537dd1..316d3002d8 100644 --- a/cpp/tests/linalg/binary_op.cuh +++ b/cpp/tests/linalg/binary_op.cuh @@ -26,8 +26,8 @@ void naiveAdd(OutType* out, const InType* in1, const InType* in2, IdxType len) { static const IdxType TPB = 64; IdxType nblks = raft::ceildiv(len, TPB); - raft::launch_kernel(cudaStream_t{0}, nblks, TPB)( - naiveAddKernel, out, in1, in2, len); + raft::launch_kernel( + cudaStream_t{0}, nblks, TPB, naiveAddKernel, out, in1, in2, len); } template diff --git a/cpp/tests/linalg/divide.cu b/cpp/tests/linalg/divide.cu index 559d5ac9a5..f9e1be6c34 100644 --- a/cpp/tests/linalg/divide.cu +++ b/cpp/tests/linalg/divide.cu @@ -29,7 +29,7 @@ void naiveDivide(Type* out, const Type* in, Type scalar, int len, cudaStream_t s { static const int TPB = 64; int nblks = raft::ceildiv(len, TPB); - raft::launch_kernel(stream, nblks, TPB)(naiveDivideKernel, out, in, scalar, len); + raft::launch_kernel(stream, nblks, TPB, naiveDivideKernel, out, in, scalar, len); } template diff --git a/cpp/tests/linalg/dot.cu b/cpp/tests/linalg/dot.cu index cd1fc1b7e5..1ef5d3d948 100644 --- a/cpp/tests/linalg/dot.cu +++ b/cpp/tests/linalg/dot.cu @@ -64,8 +64,16 @@ class DotTest : public ::testing::TestWithParam> { uniform(handle, r, y.data(), y_len, T(-1.0), T(1.0)); rmm::device_scalar ref(0, resource::get_cuda_stream(handle)); - raft::launch_kernel(handle, 256, 256)( - naiveDot, params.len, x.data(), params.incx, y.data(), params.incy, ref.data()); + raft::launch_kernel(handle, + 256, + 256, + naiveDot, + params.len, + x.data(), + params.incx, + y.data(), + params.incy, + ref.data()); raft::update_host(&ref_output, ref.data(), 1, stream); // Test out both the device and host api's diff --git a/cpp/tests/linalg/eltwise.cu b/cpp/tests/linalg/eltwise.cu index bd02300e52..cbbcce746e 100644 --- a/cpp/tests/linalg/eltwise.cu +++ b/cpp/tests/linalg/eltwise.cu @@ -30,7 +30,7 @@ void naiveScale(Type* out, const Type* in, Type scalar, int len, cudaStream_t st { static const int TPB = 64; int nblks = raft::ceildiv(len, TPB); - raft::launch_kernel(stream, nblks, TPB)(naiveScaleKernel, out, in, scalar, len); + raft::launch_kernel(stream, nblks, TPB, naiveScaleKernel, out, in, scalar, len); } template @@ -116,7 +116,7 @@ void naiveAdd(Type* out, const Type* in1, const Type* in2, int len, cudaStream_t { static const int TPB = 64; int nblks = raft::ceildiv(len, TPB); - raft::launch_kernel(stream, nblks, TPB)(naiveAddKernel, out, in1, in2, len); + raft::launch_kernel(stream, nblks, TPB, naiveAddKernel, out, in1, in2, len); } template diff --git a/cpp/tests/linalg/gemm_layout.cu b/cpp/tests/linalg/gemm_layout.cu index f04ab23870..2708f3f671 100644 --- a/cpp/tests/linalg/gemm_layout.cu +++ b/cpp/tests/linalg/gemm_layout.cu @@ -84,16 +84,19 @@ class GemmLayoutTest : public ::testing::TestWithParam> { dim3 blocks(raft::ceildiv(params.M, 128), raft::ceildiv(params.N, 4), 1); dim3 threads(128, 4, 1); - raft::launch_kernel(cudaStream_t{0}, blocks, threads)(naiveGemm, - refZ, - X, - Y, - params.M, - params.N, - params.K, - params.zLayout, - params.xLayout, - params.yLayout); + raft::launch_kernel(cudaStream_t{0}, + blocks, + threads, + naiveGemm, + refZ, + X, + Y, + params.M, + params.N, + params.K, + params.zLayout, + params.xLayout, + params.yLayout); auto x_view_row_major = raft::make_device_matrix_view(X, params.M, params.K); auto y_view_row_major = raft::make_device_matrix_view(Y, params.K, params.N); diff --git a/cpp/tests/linalg/gemv.cu b/cpp/tests/linalg/gemv.cu index d63ea8e025..3f2a0187db 100644 --- a/cpp/tests/linalg/gemv.cu +++ b/cpp/tests/linalg/gemv.cu @@ -99,14 +99,17 @@ class GemvTest : public ::testing::TestWithParam> { dim3 blocks(raft::ceildiv(yElems, 256), 1, 1); dim3 threads(256, 1, 1); - raft::launch_kernel(handle, blocks, threads)(naiveGemv, - refy.data(), - A.data(), - x.data(), - params.n_rows, - params.n_cols, - params.lda, - params.trans_a); + raft::launch_kernel(handle, + blocks, + threads, + naiveGemv, + refy.data(), + A.data(), + x.data(), + params.n_rows, + params.n_cols, + params.lda, + params.trans_a); auto A_row_major = raft::make_device_matrix_view(A.data(), params.n_rows, params.n_cols); diff --git a/cpp/tests/linalg/map_then_reduce.cu b/cpp/tests/linalg/map_then_reduce.cu index 5a234e9946..be61674ff1 100644 --- a/cpp/tests/linalg/map_then_reduce.cu +++ b/cpp/tests/linalg/map_then_reduce.cu @@ -38,8 +38,8 @@ void naiveMapReduce(OutType* out, const InType* in, size_t len, MapOp map, cudaS { static const int TPB = 64; int nblks = raft::ceildiv(len, (size_t)TPB); - raft::launch_kernel(stream, nblks, TPB)( - naiveMapReduceKernel, out, in, len, map); + raft::launch_kernel( + stream, nblks, TPB, naiveMapReduceKernel, out, in, len, map); } template diff --git a/cpp/tests/linalg/matrix_vector_op.cuh b/cpp/tests/linalg/matrix_vector_op.cuh index 67c85848af..29b356642b 100644 --- a/cpp/tests/linalg/matrix_vector_op.cuh +++ b/cpp/tests/linalg/matrix_vector_op.cuh @@ -51,8 +51,8 @@ void naiveMatVec(OutT* out, static const IdxType TPB = 64; IdxType len = N * D; IdxType nblks = raft::ceildiv(len, TPB); - raft::launch_kernel(stream, nblks, TPB)( - naiveMatVecKernel, out, mat, vec, D, N, rowMajor, bcastAlongRows, op); + raft::launch_kernel( + stream, nblks, TPB, naiveMatVecKernel, out, mat, vec, D, N, rowMajor, bcastAlongRows, op); } template @@ -129,8 +129,19 @@ void naiveMatVec(OutT* out, static const IdxType TPB = 64; IdxType len = N * D; IdxType nblks = raft::ceildiv(len, TPB); - raft::launch_kernel(stream, nblks, TPB)( - naiveMatVecKernel, out, mat, vec1, vec2, D, N, rowMajor, bcastAlongRows, op); + raft::launch_kernel(stream, + nblks, + TPB, + naiveMatVecKernel, + out, + mat, + vec1, + vec2, + D, + N, + rowMajor, + bcastAlongRows, + op); } template diff --git a/cpp/tests/linalg/mean_squared_error.cu b/cpp/tests/linalg/mean_squared_error.cu index 86a3d004ae..5b137b139e 100644 --- a/cpp/tests/linalg/mean_squared_error.cu +++ b/cpp/tests/linalg/mean_squared_error.cu @@ -76,8 +76,15 @@ class MeanSquaredErrorTest : public ::testing::TestWithParam(output.data()), params.weight); - raft::launch_kernel(handle, 256, 256)( - naiveMeanSquaredError, params.len, a.data(), b.data(), params.weight, refoutput.data()); + raft::launch_kernel(handle, + 256, + 256, + naiveMeanSquaredError, + params.len, + a.data(), + b.data(), + params.weight, + refoutput.data()); resource::sync_stream(handle); } diff --git a/cpp/tests/linalg/norm.cu b/cpp/tests/linalg/norm.cu index b531e5b13c..0071c1f36e 100644 --- a/cpp/tests/linalg/norm.cu +++ b/cpp/tests/linalg/norm.cu @@ -72,8 +72,8 @@ void naiveRowNorm( { static const IdxT TPB = 64; IdxT nblks = raft::ceildiv(N, TPB); - raft::launch_kernel(stream, nblks, TPB)( - naiveRowNormKernel, dots, data, D, N, type, do_sqrt); + raft::launch_kernel( + stream, nblks, TPB, naiveRowNormKernel, dots, data, D, N, type, do_sqrt); } template @@ -163,8 +163,8 @@ void naiveColNorm( { static const IdxT TPB = 64; IdxT nblks = raft::ceildiv(D, TPB); - raft::launch_kernel(stream, nblks, TPB)( - naiveColNormKernel, dots, data, D, N, type, do_sqrt); + raft::launch_kernel( + stream, nblks, TPB, naiveColNormKernel, dots, data, D, N, type, do_sqrt); } template diff --git a/cpp/tests/linalg/power.cu b/cpp/tests/linalg/power.cu index 8cf2f30b6b..07996278dd 100644 --- a/cpp/tests/linalg/power.cu +++ b/cpp/tests/linalg/power.cu @@ -28,7 +28,7 @@ void naivePowerElem(Type* out, const Type* in1, const Type* in2, int len, cudaSt { static const int TPB = 64; int nblks = raft::ceildiv(len, TPB); - raft::launch_kernel(stream, nblks, TPB)(naivePowerElemKernel, out, in1, in2, len); + raft::launch_kernel(stream, nblks, TPB, naivePowerElemKernel, out, in1, in2, len); } template @@ -43,7 +43,7 @@ void naivePowerScalar(Type* out, const Type* in1, const Type in2, int len, cudaS { static const int TPB = 64; int nblks = raft::ceildiv(len, TPB); - raft::launch_kernel(stream, nblks, TPB)(naivePowerScalarKernel, out, in1, in2, len); + raft::launch_kernel(stream, nblks, TPB, naivePowerScalarKernel, out, in1, in2, len); } template diff --git a/cpp/tests/linalg/reduce.cuh b/cpp/tests/linalg/reduce.cuh index 99123d84cb..e7135b00cf 100644 --- a/cpp/tests/linalg/reduce.cuh +++ b/cpp/tests/linalg/reduce.cuh @@ -67,8 +67,19 @@ void naiveCoalescedReduction(OutType* dots, { static const IdxType TPB = 64; IdxType nblks = raft::ceildiv(N, TPB); - raft::launch_kernel(stream, nblks, TPB)( - naiveCoalescedReductionKernel, dots, data, D, N, init, inplace, main_op, reduce_op, fin_op); + raft::launch_kernel(stream, + nblks, + TPB, + naiveCoalescedReductionKernel, + dots, + data, + D, + N, + init, + inplace, + main_op, + reduce_op, + fin_op); } template diff --git a/cpp/tests/linalg/sqrt.cu b/cpp/tests/linalg/sqrt.cu index d74ab17786..15d464d91d 100644 --- a/cpp/tests/linalg/sqrt.cu +++ b/cpp/tests/linalg/sqrt.cu @@ -28,7 +28,7 @@ void naiveSqrtElem(Type* out, const Type* in1, int len) { static const int TPB = 64; int nblks = raft::ceildiv(len, TPB); - raft::launch_kernel(cudaStream_t{0}, nblks, TPB)(naiveSqrtElemKernel, out, in1, len); + raft::launch_kernel(cudaStream_t{0}, nblks, TPB, naiveSqrtElemKernel, out, in1, len); } template diff --git a/cpp/tests/linalg/subtract.cu b/cpp/tests/linalg/subtract.cu index 402f2474da..614e4a36d4 100644 --- a/cpp/tests/linalg/subtract.cu +++ b/cpp/tests/linalg/subtract.cu @@ -28,7 +28,7 @@ void naiveSubtractElem(Type* out, const Type* in1, const Type* in2, int len, cud { static const int TPB = 64; int nblks = raft::ceildiv(len, TPB); - raft::launch_kernel(stream, nblks, TPB)(naiveSubtractElemKernel, out, in1, in2, len); + raft::launch_kernel(stream, nblks, TPB, naiveSubtractElemKernel, out, in1, in2, len); } template @@ -43,7 +43,7 @@ void naiveSubtractScalar(Type* out, const Type* in1, const Type in2, int len, cu { static const int TPB = 64; int nblks = raft::ceildiv(len, TPB); - raft::launch_kernel(stream, nblks, TPB)(naiveSubtractScalarKernel, out, in1, in2, len); + raft::launch_kernel(stream, nblks, TPB, naiveSubtractScalarKernel, out, in1, in2, len); } template diff --git a/cpp/tests/linalg/unary_op.cuh b/cpp/tests/linalg/unary_op.cuh index c24f059089..ac33876aac 100644 --- a/cpp/tests/linalg/unary_op.cuh +++ b/cpp/tests/linalg/unary_op.cuh @@ -33,8 +33,8 @@ void naiveScale(OutType* out, const InType* in, InType scalar, int len, cudaStre { static const int TPB = 64; int nblks = raft::ceildiv(len, TPB); - raft::launch_kernel(stream, nblks, TPB)( - naiveScaleKernel, out, in, scalar, len); + raft::launch_kernel( + stream, nblks, TPB, naiveScaleKernel, out, in, scalar, len); } template diff --git a/cpp/tests/matrix/math.cu b/cpp/tests/matrix/math.cu index 5f4e06d3f3..f885382d32 100644 --- a/cpp/tests/matrix/math.cu +++ b/cpp/tests/matrix/math.cu @@ -34,7 +34,7 @@ void naivePower(Type* in, Type* out, int len, cudaStream_t stream) { static const int TPB = 64; int nblks = raft::ceildiv(len, TPB); - raft::launch_kernel(stream, nblks, TPB)(naivePowerKernel, in, out, len); + raft::launch_kernel(stream, nblks, TPB, naivePowerKernel, in, out, len); } template @@ -49,7 +49,7 @@ void naiveSqrt(Type* in, Type* out, int len, cudaStream_t stream) { static const int TPB = 64; int nblks = raft::ceildiv(len, TPB); - raft::launch_kernel(stream, nblks, TPB)(naiveSqrtKernel, in, out, len); + raft::launch_kernel(stream, nblks, TPB, naiveSqrtKernel, in, out, len); } template @@ -85,7 +85,7 @@ RAFT_KERNEL naiveSignFlipKernel(Type* in, Type* out, int rowCount, int colCount) template void naiveSignFlip(Type* in, Type* out, int rowCount, int colCount, cudaStream_t stream) { - raft::launch_kernel(stream, colCount, 1)(naiveSignFlipKernel, in, out, rowCount, colCount); + raft::launch_kernel(stream, colCount, 1, naiveSignFlipKernel, in, out, rowCount, colCount); } template diff --git a/cpp/tests/random/make_blobs.cu b/cpp/tests/random/make_blobs.cu index c9219cc9ec..fc2d36258a 100644 --- a/cpp/tests/random/make_blobs.cu +++ b/cpp/tests/random/make_blobs.cu @@ -113,23 +113,29 @@ class MakeBlobsTest : public ::testing::TestWithParam> { bool row_major = std::is_same::value; static const int threads = 128; - raft::launch_kernel(stream, raft::ceildiv(len, threads), threads)(meanKernel, - stats.data_handle(), - lens.data_handle(), - data.data_handle(), - labels.data_handle(), - params.rows, - params.cols, - params.n_clusters, - row_major); + raft::launch_kernel(stream, + raft::ceildiv(len, threads), + threads, + meanKernel, + stats.data_handle(), + lens.data_handle(), + data.data_handle(), + labels.data_handle(), + params.rows, + params.cols, + params.n_clusters, + row_major); int len1 = params.n_clusters * params.cols; - raft::launch_kernel(stream, raft::ceildiv(len1, threads), threads)(compute_mean_var, - mean_var.data_handle(), - stats.data_handle(), - lens.data_handle(), - params.n_clusters, - params.cols, - row_major); + raft::launch_kernel(stream, + raft::ceildiv(len1, threads), + threads, + compute_mean_var, + mean_var.data_handle(), + stats.data_handle(), + lens.data_handle(), + params.n_clusters, + params.cols, + row_major); } void check() diff --git a/cpp/tests/random/multi_variable_gaussian.cu b/cpp/tests/random/multi_variable_gaussian.cu index c8a3958bdc..c3f350d86b 100644 --- a/cpp/tests/random/multi_variable_gaussian.cu +++ b/cpp/tests/random/multi_variable_gaussian.cu @@ -155,15 +155,15 @@ class MVGTest : public ::testing::TestWithParam> { RAFT_CUDA_TRY(cudaMemset(Rand_mean.data(), 0, dim * sizeof(T))); dim3 block = (64); dim3 grid = (raft::ceildiv(nPoints * dim, (int)block.x)); - raft::launch_kernel(handle, grid, block)( - En_KF_accumulate, nPoints, dim, X_d.data(), Rand_mean.data()); + raft::launch_kernel( + handle, grid, block, En_KF_accumulate, nPoints, dim, X_d.data(), Rand_mean.data()); grid = (raft::ceildiv(dim, (int)block.x)); - raft::launch_kernel(handle, grid, block)(En_KF_normalize, nPoints, dim, Rand_mean.data()); + raft::launch_kernel(handle, grid, block, En_KF_normalize, nPoints, dim, Rand_mean.data()); // storing the error wrt random point mean in X_d grid = (raft::ceildiv(dim * nPoints, (int)block.x)); - raft::launch_kernel(handle, grid, block)( - En_KF_dif, nPoints, dim, X_d.data(), Rand_mean.data(), X_d.data()); + raft::launch_kernel( + handle, grid, block, En_KF_dif, nPoints, dim, X_d.data(), Rand_mean.data(), X_d.data()); // finding the cov matrix, placing in Rand_cov T alfa = 1.0 / (nPoints - 1), beta = 0.0; @@ -285,15 +285,15 @@ class MVGMdspanTest : public ::testing::TestWithParam> { RAFT_CUDA_TRY(cudaMemset(Rand_mean.data(), 0, dim * sizeof(T))); dim3 block = (64); dim3 grid = (raft::ceildiv(nPoints * dim, (int)block.x)); - raft::launch_kernel(handle, grid, block)( - En_KF_accumulate, nPoints, dim, X_d.data(), Rand_mean.data()); + raft::launch_kernel( + handle, grid, block, En_KF_accumulate, nPoints, dim, X_d.data(), Rand_mean.data()); grid = (raft::ceildiv(dim, (int)block.x)); - raft::launch_kernel(handle, grid, block)(En_KF_normalize, nPoints, dim, Rand_mean.data()); + raft::launch_kernel(handle, grid, block, En_KF_normalize, nPoints, dim, Rand_mean.data()); // storing the error wrt random point mean in X_d grid = (raft::ceildiv(dim * nPoints, (int)block.x)); - raft::launch_kernel(handle, grid, block)( - En_KF_dif, nPoints, dim, X_d.data(), Rand_mean.data(), X_d.data()); + raft::launch_kernel( + handle, grid, block, En_KF_dif, nPoints, dim, X_d.data(), Rand_mean.data(), X_d.data()); // finding the cov matrix, placing in Rand_cov T alfa = 1.0 / (nPoints - 1), beta = 0.0; diff --git a/cpp/tests/random/rmat_rectangular_generator.cu b/cpp/tests/random/rmat_rectangular_generator.cu index ffc8db58e2..3d752567b4 100644 --- a/cpp/tests/random/rmat_rectangular_generator.cu +++ b/cpp/tests/random/rmat_rectangular_generator.cu @@ -125,13 +125,13 @@ void normalize(OutT* theta, { // one threadblock with 256 threads is more than enough as the 'scale' parameters // won't be that large! - raft::launch_kernel(stream, 1, 256)( - normalize_kernel, theta, in_vals, max_scale, r_scale, c_scale); + raft::launch_kernel( + stream, 1, 256, normalize_kernel, theta, in_vals, max_scale, r_scale, c_scale); if (handle_rect) { - raft::launch_kernel(stream, 1, 256)(handle_rect_kernel, theta, max_scale, r_scale, c_scale); + raft::launch_kernel(stream, 1, 256, handle_rect_kernel, theta, max_scale, r_scale, c_scale); } if (!theta_array) { - raft::launch_kernel(stream, 1, 256)(theta_kernel, theta, max_scale, r_scale, c_scale); + raft::launch_kernel(stream, 1, 256, theta_kernel, theta, max_scale, r_scale, c_scale); } } @@ -213,8 +213,16 @@ class RmatGenTest : public ::testing::TestWithParam { { rmm::device_uvector hist{theta.size(), stream}; RAFT_CUDA_TRY(cudaMemsetAsync(hist.data(), 0, hist.size() * sizeof(int), stream)); - raft::launch_kernel(stream, raft::ceildiv(out.size() / 2, 256), 256)( - compute_hist, hist.data(), out.data(), out.size(), max_scale, params.r_scale, params.c_scale); + raft::launch_kernel(stream, + raft::ceildiv(out.size() / 2, 256), + 256, + compute_hist, + hist.data(), + out.data(), + out.size(), + max_scale, + params.r_scale, + params.c_scale); rmm::device_uvector computed_theta{theta.size(), stream}; normalize(computed_theta.data(), hist.data(), @@ -317,8 +325,16 @@ class RmatGenMdspanTest : public ::testing::TestWithParam { { rmm::device_uvector hist{theta.size(), stream}; RAFT_CUDA_TRY(cudaMemsetAsync(hist.data(), 0, hist.size() * sizeof(int), stream)); - raft::launch_kernel(stream, raft::ceildiv(out.size() / 2, 256), 256)( - compute_hist, hist.data(), out.data(), out.size(), max_scale, params.r_scale, params.c_scale); + raft::launch_kernel(stream, + raft::ceildiv(out.size() / 2, 256), + 256, + compute_hist, + hist.data(), + out.data(), + out.size(), + max_scale, + params.r_scale, + params.c_scale); rmm::device_uvector computed_theta{theta.size(), stream}; normalize(computed_theta.data(), hist.data(), diff --git a/cpp/tests/random/rng.cu b/cpp/tests/random/rng.cu index a88e093b7a..5486a51fbc 100644 --- a/cpp/tests/random/rng.cu +++ b/cpp/tests/random/rng.cu @@ -121,8 +121,13 @@ class RngTest : public ::testing::TestWithParam> { break; }; static const int threads = 128; - raft::launch_kernel(stream, raft::ceildiv(params.len, threads), threads)( - meanKernel, stats.data(), data.data(), params.len); + raft::launch_kernel(stream, + raft::ceildiv(params.len, threads), + threads, + meanKernel, + stats.data(), + data.data(), + params.len); update_host(h_stats, stats.data(), 2, stream); RAFT_CUDA_TRY(cudaStreamSynchronize(stream)); h_stats[0] /= params.len; @@ -215,8 +220,13 @@ class RngMdspanTest : public ::testing::TestWithParam> { case RNG_Laplace: laplace(handle, r, data_view, params.start, params.end); break; }; static const int threads = 128; - raft::launch_kernel(stream, raft::ceildiv(params.len, threads), threads)( - meanKernel, stats.data(), data.data(), params.len); + raft::launch_kernel(stream, + raft::ceildiv(params.len, threads), + threads, + meanKernel, + stats.data(), + data.data(), + params.len); update_host(h_stats, stats.data(), 2, stream); RAFT_CUDA_TRY(cudaStreamSynchronize(stream)); h_stats[0] /= params.len; @@ -612,8 +622,13 @@ class RngNormalTableTest : public ::testing::TestWithParam, stats.data(), data.data(), len); + raft::launch_kernel(stream, + raft::ceildiv(len, threads), + threads, + meanKernel, + stats.data(), + data.data(), + len); update_host(h_stats, stats.data(), 2, stream); RAFT_CUDA_TRY(cudaStreamSynchronize(stream)); h_stats[0] /= len; @@ -668,8 +683,13 @@ class RngNormalTableMdspanTest : public ::testing::TestWithParam, stats.data(), data.data(), len); + raft::launch_kernel(stream, + raft::ceildiv(len, threads), + threads, + meanKernel, + stats.data(), + data.data(), + len); update_host(h_stats, stats.data(), 2, stream); RAFT_CUDA_TRY(cudaStreamSynchronize(stream)); h_stats[0] /= len; diff --git a/cpp/tests/random/rng_int.cu b/cpp/tests/random/rng_int.cu index a206913930..96d531dea2 100644 --- a/cpp/tests/random/rng_int.cu +++ b/cpp/tests/random/rng_int.cu @@ -86,8 +86,13 @@ class RngTest : public ::testing::TestWithParam> { break; }; static const int threads = 128; - raft::launch_kernel(stream, raft::ceildiv(params.len, threads), threads)( - meanKernel, stats.data(), data.data(), params.len); + raft::launch_kernel(stream, + raft::ceildiv(params.len, threads), + threads, + meanKernel, + stats.data(), + data.data(), + params.len); update_host(h_stats, stats.data(), 2, stream); resource::sync_stream(handle, stream); h_stats[0] /= params.len; @@ -139,8 +144,13 @@ class RngMdspanTest : public ::testing::TestWithParam> { case RNG_Uniform: uniformInt(handle, r, data_view, params.start, params.end); break; }; static const int threads = 128; - raft::launch_kernel(stream, raft::ceildiv(params.len, threads), threads)( - meanKernel, stats.data(), data.data(), params.len); + raft::launch_kernel(stream, + raft::ceildiv(params.len, threads), + threads, + meanKernel, + stats.data(), + data.data(), + params.len); update_host(h_stats, stats.data(), 2, stream); resource::sync_stream(handle, stream); h_stats[0] /= params.len; diff --git a/cpp/tests/random/rng_pcg_host_api.cu b/cpp/tests/random/rng_pcg_host_api.cu index e0d0f83e6e..dba311aba8 100644 --- a/cpp/tests/random/rng_pcg_host_api.cu +++ b/cpp/tests/random/rng_pcg_host_api.cu @@ -59,13 +59,15 @@ class HostApiTest { RngState r(seed, GenPC); DeviceState d_state(r); - raft::launch_kernel(handle, n_blocks, n_threads)( - pcg_device_kernel, - d_buffer.data(), - d_state, - dist_params, - total_threads, - len); + raft::launch_kernel(handle, + n_blocks, + n_threads, + pcg_device_kernel, + d_buffer.data(), + d_state, + dist_params, + total_threads, + len); RAFT_CUDA_TRY(cudaStreamSynchronize(stream)); for (size_t tid = 0; tid < len; tid++) { diff --git a/cpp/tests/sparse/convert_csr.cu b/cpp/tests/sparse/convert_csr.cu index 173ef6558b..699a1172af 100644 --- a/cpp/tests/sparse/convert_csr.cu +++ b/cpp/tests/sparse/convert_csr.cu @@ -102,8 +102,8 @@ void init_adj(bool* adj, index_t num_rows, index_t num_cols, index_t divisor, cu dim3 block(32, 32); const index_t max_y_grid_dim = 65535; dim3 grid(num_cols / 32 + 1, (int)min(num_rows / 32 + 1, max_y_grid_dim)); - raft::launch_kernel(stream, grid, block)( - init_adj_kernel, adj, num_rows, num_cols, divisor); + raft::launch_kernel( + stream, grid, block, init_adj_kernel, adj, num_rows, num_cols, divisor); } template diff --git a/cpp/tests/sparse/spmm.cu b/cpp/tests/sparse/spmm.cu index 96e19fd158..f97b0eba23 100644 --- a/cpp/tests/sparse/spmm.cu +++ b/cpp/tests/sparse/spmm.cu @@ -173,23 +173,26 @@ class SpmmTest : public ::testing::TestWithParam> { dim3 blocks(raft::ceildiv(params.M, 128), raft::ceildiv(params.N, 4), 1); dim3 threads(128, 4, 1); - raft::launch_kernel(handle, blocks, threads)(naiveGemm, - params.trans_x, - params.trans_y, - params.M, - params.N, - params.K, - alpha, - X, - ldx, - true, - Y, - ldy, - params.row_major, - beta, - Z_ref, - ldz, - params.row_major); + raft::launch_kernel(handle, + blocks, + threads, + naiveGemm, + params.trans_x, + params.trans_y, + params.M, + params.N, + params.K, + alpha, + X, + ldx, + true, + Y, + ldy, + params.row_major, + beta, + Z_ref, + ldz, + params.row_major); spmm( handle, params.trans_x, params.trans_y, &alpha, X_csr, y_stride_view, &beta, z_stride_view); diff --git a/cpp/tests/sparse/symmetrize.cu b/cpp/tests/sparse/symmetrize.cu index 5e983480c4..47132ff8eb 100644 --- a/cpp/tests/sparse/symmetrize.cu +++ b/cpp/tests/sparse/symmetrize.cu @@ -101,8 +101,15 @@ class SparseSymmetrizeTest rmm::device_scalar sum(stream); sum.set_value_to_zero_async(stream); - raft::launch_kernel(stream, raft::ceildiv(out.nnz, (nnz_t)256), 256)( - assert_symmetry, out.rows(), out.cols(), out.vals(), (nnz_t)out.nnz, sum.data()); + raft::launch_kernel(stream, + raft::ceildiv(out.nnz, (nnz_t)256), + 256, + assert_symmetry, + out.rows(), + out.cols(), + out.vals(), + (nnz_t)out.nnz, + sum.data()); sum_h = sum.value(stream); resource::sync_stream(handle, stream); diff --git a/cpp/tests/stats/histogram.cu b/cpp/tests/stats/histogram.cu index a47aa80292..06cd38deff 100644 --- a/cpp/tests/stats/histogram.cu +++ b/cpp/tests/stats/histogram.cu @@ -42,7 +42,7 @@ void naiveHist(int* bins, int nbins, int* in, int nrows, int ncols, cudaStream_t const int TPB = 128; int nblksx = raft::ceildiv(nrows, TPB); dim3 blks(nblksx, ncols); - raft::launch_kernel(stream, blks, TPB)(naiveHistKernel, bins, nbins, in, nrows); + raft::launch_kernel(stream, blks, TPB, naiveHistKernel, bins, nbins, in, nrows); } struct HistInputs { diff --git a/cpp/tests/stats/minmax.cu b/cpp/tests/stats/minmax.cu index f2c2f31f27..3ad1937dc0 100644 --- a/cpp/tests/stats/minmax.cu +++ b/cpp/tests/stats/minmax.cu @@ -68,11 +68,11 @@ void naiveMinMax( const int TPB = 128; int nblks = raft::ceildiv(ncols, TPB); T init_val = std::numeric_limits::max(); - raft::launch_kernel(stream, nblks, TPB)( - naiveMinMaxInitKernel, ncols, globalmin, globalmax, init_val); + raft::launch_kernel( + stream, nblks, TPB, naiveMinMaxInitKernel, ncols, globalmin, globalmax, init_val); nblks = raft::ceildiv(nrows * ncols, TPB); - raft::launch_kernel(stream, nblks, TPB)( - naiveMinMaxKernel, data, nrows, ncols, globalmin, globalmax); + raft::launch_kernel( + stream, nblks, TPB, naiveMinMaxKernel, data, nrows, ncols, globalmin, globalmax); } template @@ -108,8 +108,14 @@ class MinMaxTest : public ::testing::TestWithParam> { T nan_prob = 0.01; bernoulli(handle, r, mask.data(), len, nan_prob); const int TPB = 256; - raft::launch_kernel(handle, raft::ceildiv(len, TPB), TPB)( - nanKernel, data.data(), mask.data(), len, std::numeric_limits::quiet_NaN()); + raft::launch_kernel(handle, + raft::ceildiv(len, TPB), + TPB, + nanKernel, + data.data(), + mask.data(), + len, + std::numeric_limits::quiet_NaN()); naiveMinMax(data.data(), params.rows, params.cols, diff --git a/cpp/tests/util/bitonic_sort.cu b/cpp/tests/util/bitonic_sort.cu index 7ccbab59c6..61be78c9c8 100644 --- a/cpp/tests/util/bitonic_sort.cu +++ b/cpp/tests/util/bitonic_sort.cu @@ -86,8 +86,14 @@ struct bitonic_launch { const int block_dim = n_warps * WarpSize; const int n_subwarps = block_dim / spec.warp_width; const int grid_dim = ceildiv(spec.n_inputs, n_subwarps); - raft::launch_kernel(stream, grid_dim, block_dim)( - bitonic_kernel, arr, spec.ascending, spec.warp_width, spec.n_inputs); + raft::launch_kernel(stream, + grid_dim, + block_dim, + bitonic_kernel, + arr, + spec.ascending, + spec.warp_width, + spec.n_inputs); } }; diff --git a/cpp/tests/util/device_atomics.cu b/cpp/tests/util/device_atomics.cu index 5fb831a53f..fe700e3e7d 100644 --- a/cpp/tests/util/device_atomics.cu +++ b/cpp/tests/util/device_atomics.cu @@ -43,8 +43,12 @@ TEST(Raft, AtomicIncWarp) std::array out_host{0}; // Write all 1M thread indices to a unique location in `out_device` - raft::launch_kernel(s, num_blocks, threads_per_block)( - test_atomic_inc_warp_kernel, counter.data(), out_device.data()); + raft::launch_kernel(s, + num_blocks, + threads_per_block, + test_atomic_inc_warp_kernel, + counter.data(), + out_device.data()); // Copy data to host RAFT_CUDA_TRY(cudaMemcpyAsync(out_host.data(), (const void*)out_device.data(), diff --git a/cpp/tests/util/integer_utils.cu b/cpp/tests/util/integer_utils.cu index 0122748abf..ef23fd4373 100644 --- a/cpp/tests/util/integer_utils.cu +++ b/cpp/tests/util/integer_utils.cu @@ -54,13 +54,16 @@ class Multiplication64bit : public testing::TestWithParam { wmul_64bit(result_high, result_low, params.operand_1, params.operand_2); wmul_64bit(swapped_result_high, swapped_result_low, params.operand_2, params.operand_1); - raft::launch_kernel(stream, 1, 1)(mul64_test_kernel, - d_result_high.data(), - d_result_low.data(), - d_swapped_result_high.data(), - d_swapped_result_low.data(), - params.operand_1, - params.operand_2); + raft::launch_kernel(stream, + 1, + 1, + mul64_test_kernel, + d_result_high.data(), + d_result_low.data(), + d_swapped_result_high.data(), + d_swapped_result_low.data(), + params.operand_1, + params.operand_2); } raft::resources handle; diff --git a/cpp/tests/util/kernel_launch.cu b/cpp/tests/util/kernel_launch.cu index de96f9f140..733c986737 100644 --- a/cpp/tests/util/kernel_launch.cu +++ b/cpp/tests/util/kernel_launch.cu @@ -15,6 +15,7 @@ #include #include +#include namespace raft { @@ -27,6 +28,43 @@ RAFT_KERNEL write_one_kernel(int* out) if (threadIdx.x == 0 && blockIdx.x == 0) { *out = 1; } } +RAFT_KERNEL smem_kernel(int* out) +{ + extern __shared__ int shared[]; // NOLINT(modernize-avoid-c-arrays) + if (threadIdx.x == 0 && blockIdx.x == 0) { + shared[0] = 1; + __threadfence_block(); + *out = shared[0]; + } +} + +/** Whether `w` can be launched as it is named, i.e. as an lvalue. */ +template +concept launchable_as_named = requires(W w) +{ + raft::launch_kernel(w, dim3{}, dim3{}, noop_kernel); +}; + +/** Whether `w` can be launched after being moved from. */ +template +concept launchable_when_moved = requires(W w) +{ + raft::launch_kernel(std::move(w), dim3{}, dim3{}, noop_kernel); +}; + +// Only a prvalue built inside the launch expression may be launched, so that the reported location +// is always the one of the launch. Everything else must fail to compile. +static_assert(launchable_as_named, + "resources must convert to a launch_on prvalue"); +static_assert(launchable_as_named, + "a stream view must convert to a launch_on prvalue"); +static_assert(launchable_as_named, + "a raw stream handle must convert to a launch_on prvalue"); +static_assert(!launchable_as_named, "a stored launch_on must not be launchable"); +static_assert(!launchable_as_named, "an lvalue launch_on must not be launchable"); +static_assert(!launchable_when_moved, + "a moved-from launch_on must not be launchable"); + } // namespace TEST(KernelLaunch, SuccessfulLaunch) @@ -35,7 +73,7 @@ TEST(KernelLaunch, SuccessfulLaunch) rmm::device_uvector out(1, resource::get_cuda_stream(res)); RAFT_CUDA_TRY(cudaMemsetAsync(out.data(), 0, sizeof(int), resource::get_cuda_stream(res))); - raft::launch_kernel(res, 1, 32)(write_one_kernel, out.data()); + raft::launch_kernel(res, 1, 32, write_one_kernel, out.data()); resource::sync_stream(res); int host_out = 0; @@ -47,8 +85,31 @@ TEST(KernelLaunch, StreamOverload) { raft::resources res; auto stream = resource::get_cuda_stream(res); - EXPECT_NO_THROW(raft::launch_kernel(stream, 1, 1)(noop_kernel)); + EXPECT_NO_THROW(raft::launch_kernel(stream, 1, 1, noop_kernel)); + resource::sync_stream(res); +} + +TEST(KernelLaunch, RawStreamHandleOverload) +{ + raft::resources res; + cudaStream_t stream = resource::get_cuda_stream(res).value(); + EXPECT_NO_THROW(raft::launch_kernel(stream, 1, 1, noop_kernel)); + resource::sync_stream(res); +} + +TEST(KernelLaunch, SharedMemory) +{ + raft::resources res; + auto stream = resource::get_cuda_stream(res); + rmm::device_uvector out(1, stream); + RAFT_CUDA_TRY(cudaMemsetAsync(out.data(), 0, sizeof(int), stream)); + + raft::launch_kernel({stream, sizeof(int)}, 1, 32, smem_kernel, out.data()); resource::sync_stream(res); + + int host_out = 0; + RAFT_CUDA_TRY(cudaMemcpy(&host_out, out.data(), sizeof(int), cudaMemcpyDeviceToHost)); + EXPECT_EQ(host_out, 1); } TEST(KernelLaunch, ErrorReportsCallSite) @@ -61,7 +122,7 @@ TEST(KernelLaunch, ErrorReportsCallSite) int launch_line = 0; try { launch_line = __LINE__ + 1; - raft::launch_kernel(res, 1, k_bad_block)(noop_kernel); + raft::launch_kernel(res, 1, k_bad_block, noop_kernel); FAIL() << "Expected cuda_error from invalid launch configuration"; } catch (raft::cuda_error const& e) { caught = e.what(); @@ -73,7 +134,7 @@ TEST(KernelLaunch, ErrorReportsCallSite) std::string re_exp{R"(CUDA error encountered at: file=.*kernel_launch\.cu line=)"}; re_exp += std::to_string(launch_line); - re_exp += R"( function=.*ErrorReportsCallSite.*: call='cudaLaunchKernel', Reason=.*)"; + re_exp += R"( function=.*ErrorReportsCallSite.*: call='cudaLaunchKernelExC', Reason=.*)"; EXPECT_TRUE(std::regex_search(caught, std::regex(re_exp))) << "message:'" << caught << "'\nexpected regex:'" << re_exp << "'"; } diff --git a/cpp/tests/util/reduction.cu b/cpp/tests/util/reduction.cu index 810dc91d40..bf970e1ef2 100644 --- a/cpp/tests/util/reduction.cu +++ b/cpp/tests/util/reduction.cu @@ -80,8 +80,8 @@ struct reduction_launch { rmm::device_scalar ref_d(stream); const int block_dim = 64; const int grid_dim = 1; - raft::launch_kernel(stream, grid_dim, block_dim)( - test_reduction_kernel, arr_d.data(), ref_d.data(), reduce_op); + raft::launch_kernel( + stream, grid_dim, block_dim, test_reduction_kernel, arr_d.data(), ref_d.data(), reduce_op); stream.synchronize(); RAFT_CUDA_TRY(cudaPeekAtLastError()); ASSERT_EQ(ref_d.value(stream), ref_val); @@ -98,8 +98,14 @@ struct reduction_launch { rmm::device_scalar rank_d(stream); const int block_dim = 64; const int grid_dim = 1; - raft::launch_kernel(stream, grid_dim, block_dim)( - test_ranked_reduction_kernel, arr_d.data(), ref_d.data(), rank_d.data(), reduce_op); + raft::launch_kernel(stream, + grid_dim, + block_dim, + test_ranked_reduction_kernel, + arr_d.data(), + ref_d.data(), + rank_d.data(), + reduce_op); stream.synchronize(); RAFT_CUDA_TRY(cudaPeekAtLastError()); ASSERT_EQ(ref_d.value(stream), ref_val); @@ -113,8 +119,8 @@ struct reduction_launch { rmm::device_scalar ref_d(stream); const int block_dim = 64; const int grid_dim = 1; - raft::launch_kernel(stream, grid_dim, block_dim)( - test_block_random_sample_kernel, arr_d.data(), ref_d.data()); + raft::launch_kernel( + stream, grid_dim, block_dim, test_block_random_sample_kernel, arr_d.data(), ref_d.data()); stream.synchronize(); RAFT_CUDA_TRY(cudaPeekAtLastError()); ASSERT_EQ(ref_d.value(stream), ref_val); @@ -127,8 +133,12 @@ struct reduction_launch { rmm::device_scalar ref_d(stream); constexpr int block_dim = 64; const int grid_dim = 1; - raft::launch_kernel(stream, grid_dim, block_dim)( - test_binary_reduction_kernel, arr_d.data(), ref_d.data()); + raft::launch_kernel(stream, + grid_dim, + block_dim, + test_binary_reduction_kernel, + arr_d.data(), + ref_d.data()); stream.synchronize(); RAFT_CUDA_TRY(cudaPeekAtLastError()); ASSERT_EQ(ref_d.value(stream), ref_val);