From 4dfc295f0a4b751cab125cff3d4814045895f08a Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Mon, 1 Jun 2026 11:10:20 -0700 Subject: [PATCH 01/16] PERF/API: Specify datasource size to avoid HEAD requests for S3 endpoints This updates libcudf and its usage of kvikio to avoid unnecessary HEAD requests in KvikIO to "open" a remote S3 file. --- cpp/include/cudf/io/datasource.hpp | 9 ++- cpp/include/cudf/io/types.hpp | 49 ++++++++++++- cpp/src/io/functions.cpp | 18 +++-- cpp/src/io/utilities/datasource.cpp | 41 +++++++++-- cpp/tests/CMakeLists.txt | 1 + cpp/tests/io/filepath_source_test.cpp | 73 +++++++++++++++++++ docs/cudf/source/cudf/io/io.md | 23 ++++++ python/pylibcudf/pylibcudf/io/__init__.py | 3 +- python/pylibcudf/pylibcudf/io/types.pxd | 5 ++ python/pylibcudf/pylibcudf/io/types.pyi | 10 ++- python/pylibcudf/pylibcudf/io/types.pyx | 49 ++++++++++++- .../pylibcudf/pylibcudf/libcudf/io/types.pxd | 14 +++- .../tests/io/test_source_sink_info.py | 42 ++++++++++- 13 files changed, 312 insertions(+), 25 deletions(-) create mode 100644 cpp/tests/io/filepath_source_test.cpp diff --git a/cpp/include/cudf/io/datasource.hpp b/cpp/include/cudf/io/datasource.hpp index 6753c2638e39..da0404da0970 100644 --- a/cpp/include/cudf/io/datasource.hpp +++ b/cpp/include/cudf/io/datasource.hpp @@ -13,7 +13,7 @@ #include #include -#include +#include namespace CUDF_EXPORT cudf { //! IO interfaces @@ -98,11 +98,14 @@ class datasource { * @param[in] offset Starting byte offset from which data will be read (the default is zero) * @param[in] max_size_estimate Upper estimate of the data range that will be read (the default is * zero, which means the whole file after `offset`) + * @param[in] known_size Optional known file size in bytes. When set for remote URLs, KvikIO skips + * the size query HEAD request at open time. * @return Constructed datasource object */ static std::unique_ptr create(std::string const& filepath, - size_t offset = 0, - size_t max_size_estimate = 0); + size_t offset = 0, + size_t max_size_estimate = 0, + std::optional known_size = std::nullopt); /** * @brief Creates a source from a host memory buffer. diff --git a/cpp/include/cudf/io/types.hpp b/cpp/include/cudf/io/types.hpp index 4978960b7c30..6eed76c06c8a 100644 --- a/cpp/include/cudf/io/types.hpp +++ b/cpp/include/cudf/io/types.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ @@ -310,6 +310,17 @@ constexpr inline auto is_byte_like_type() std::is_same_v; } +/** + * @brief A file path with an optional known size in bytes. + * + * When `size` is set for a remote URL, libcudf passes it to KvikIO at open time so the remote + * server is not queried for file size. + */ +struct filepath_source { + std::string path; ///< Path or URL of the input file + std::optional size{}; ///< Known file size; omit to query via KvikIO +}; + /** * @brief Source information for read interfaces */ @@ -325,8 +336,13 @@ struct source_info { * @param file_paths Input files paths */ explicit source_info(std::vector file_paths) - : _type(io_type::FILEPATH), _num_sources(file_paths.size()), _filepaths(std::move(file_paths)) + : _type(io_type::FILEPATH), _num_sources(file_paths.size()) { + _filepath_sources.reserve(file_paths.size()); + for (auto& path : file_paths) { + _filepath_sources.push_back({std::move(path), std::nullopt}); + } + rebuild_filepaths(); } /** @@ -335,10 +351,21 @@ struct source_info { * @param file_path Single input file */ explicit source_info(std::string file_path) - : _type(io_type::FILEPATH), _num_sources(1), _filepaths({std::move(file_path)}) + : source_info(std::vector{std::move(file_path)}) { } + /** + * @brief Construct a new source info object from filepath sources with optional known sizes + * + * @param sources Input filepath sources + */ + explicit source_info(std::vector sources) + : _type(io_type::FILEPATH), _num_sources(sources.size()), _filepath_sources(std::move(sources)) + { + rebuild_filepaths(); + } + /** * @brief Construct a new source info object for multiple buffers in host memory * @@ -424,6 +451,12 @@ struct source_info { * @return The type of the input */ [[nodiscard]] auto type() const { return _type; } + /** + * @brief Get the filepath sources of the input + * + * @return The filepath sources of the input + */ + [[nodiscard]] auto const& filepath_sources() const { return _filepath_sources; } /** * @brief Get the filepaths of the input * @@ -457,8 +490,18 @@ struct source_info { [[nodiscard]] auto num_sources() const { return _num_sources; } private: + void rebuild_filepaths() + { + _filepaths.clear(); + _filepaths.reserve(_filepath_sources.size()); + for (auto const& source : _filepath_sources) { + _filepaths.push_back(source.path); + } + } + io_type _type = io_type::VOID; size_t _num_sources = 0; + std::vector _filepath_sources; std::vector _filepaths; std::vector> _host_buffers; std::vector> _device_buffers; diff --git a/cpp/src/io/functions.cpp b/cpp/src/io/functions.cpp index 96dce2c1e2cb..eed9e71a90d1 100644 --- a/cpp/src/io/functions.cpp +++ b/cpp/src/io/functions.cpp @@ -162,24 +162,26 @@ std::vector> make_datasources(source_info switch (info.type()) { case io_type::FILEPATH: { std::vector> sources; - sources.reserve(info.filepaths().size()); + sources.reserve(info.filepath_sources().size()); // Creating sources in a single thread is faster for a small number of sources auto const pool_use_threshold = cudf::detail::getenv_or("LIBCUDF_DATASOURCE_PARALLEL_CREATION_THRESHOLD", 8ul); - if (info.filepaths().size() >= pool_use_threshold) { + if (info.filepath_sources().size() >= pool_use_threshold) { std::vector>> source_tasks; - source_tasks.reserve(info.filepaths().size()); - for (auto const& path : info.filepaths()) { - source_tasks.emplace_back(cudf::detail::host_worker_pool().submit_task( - [=] { return cudf::io::datasource::create(path, offset, max_size_estimate); })); + source_tasks.reserve(info.filepath_sources().size()); + for (auto const& fs : info.filepath_sources()) { + source_tasks.emplace_back(cudf::detail::host_worker_pool().submit_task([=] { + return cudf::io::datasource::create(fs.path, offset, max_size_estimate, fs.size); + })); } std::transform( source_tasks.begin(), source_tasks.end(), std::back_inserter(sources), [](auto& task) { return task.get(); }); } else { - for (auto const& filepath : info.filepaths()) { - sources.emplace_back(cudf::io::datasource::create(filepath, offset, max_size_estimate)); + for (auto const& fs : info.filepath_sources()) { + sources.emplace_back( + cudf::io::datasource::create(fs.path, offset, max_size_estimate, fs.size)); } } return sources; diff --git a/cpp/src/io/utilities/datasource.cpp b/cpp/src/io/utilities/datasource.cpp index aee5b7abfe52..07e01e6fecbb 100644 --- a/cpp/src/io/utilities/datasource.cpp +++ b/cpp/src/io/utilities/datasource.cpp @@ -27,6 +27,7 @@ #include #ifdef CUDF_KVIKIO_REMOTE_IO +#include #include #endif @@ -353,14 +354,43 @@ class user_datasource_wrapper : public datasource { datasource* const source; ///< A non-owning pointer to the user-implemented datasource }; +#ifdef CUDF_KVIKIO_REMOTE_IO +/** + * @brief Infer the KvikIO remote endpoint type from a URL (no network I/O). + * + * Mirrors the order used by `kvikio::RemoteHandle::open()` in AUTO mode. + */ +kvikio::RemoteEndpointType infer_remote_endpoint_type(std::string const& url) +{ + if (kvikio::S3Endpoint::is_url_valid(url)) { return kvikio::RemoteEndpointType::S3; } + if (kvikio::S3PublicEndpoint::is_url_valid(url)) { return kvikio::RemoteEndpointType::S3_PUBLIC; } + if (kvikio::S3EndpointWithPresignedUrl::is_url_valid(url)) { + return kvikio::RemoteEndpointType::S3_PRESIGNED_URL; + } + if (kvikio::WebHdfsEndpoint::is_url_valid(url)) { return kvikio::RemoteEndpointType::WEBHDFS; } + if (kvikio::HttpEndpoint::is_url_valid(url)) { return kvikio::RemoteEndpointType::HTTP; } + return kvikio::RemoteEndpointType::HTTP; +} + +kvikio::RemoteHandle open_remote_handle(char const* filepath, std::optional known_size) +{ + if (known_size.has_value()) { + auto const endpoint_type = infer_remote_endpoint_type(filepath); + return kvikio::RemoteHandle::open(filepath, endpoint_type, std::nullopt, *known_size); + } + return kvikio::RemoteHandle::open(filepath); +} +#endif + #ifdef CUDF_KVIKIO_REMOTE_IO /** * @brief Remote file source backed by KvikIO, which handles S3 filepaths seamlessly. */ class remote_file_source : public kvikio_source { public: - explicit remote_file_source(char const* filepath) - : kvikio_source{kvikio::RemoteHandle::open(filepath)} + explicit remote_file_source(char const* filepath, + std::optional known_size = std::nullopt) + : kvikio_source{open_remote_handle(filepath, known_size)} { } @@ -397,7 +427,8 @@ class remote_file_source : public file_source { std::unique_ptr datasource::create(std::string const& filepath, size_t offset, - size_t max_size_estimate) + size_t max_size_estimate, + std::optional known_size) { auto const use_memory_mapping = [] { auto const policy = cudf::detail::getenv_or("LIBCUDF_MMAP_ENABLED", std::string{"OFF"}); @@ -410,7 +441,7 @@ std::unique_ptr datasource::create(std::string const& filepath, if (remote_file_source::could_be_remote_url(filepath)) { try { - return std::make_unique(filepath.c_str()); + return std::make_unique(filepath.c_str(), known_size); } catch (std::exception const& ex) { std::string redacted_msg; try { @@ -450,7 +481,7 @@ std::unique_ptr datasource::create(std::string const& filepath, // Create a remote file resource only when the pattern is found and replaced; otherwise, still // create a local file resource if (filepath != remote_file_path) { - return std::make_unique(remote_file_path.c_str()); + return std::make_unique(remote_file_path.c_str(), known_size); } } diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index ce7bfdbed0f0..466f9283b017 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -320,6 +320,7 @@ ConfigureTest( # * io tests -------------------------------------------------------------------------------------- ConfigureTest(COMPRESSION_TEST io/comp/comp_test.cpp) ConfigureTest(ROW_SELECTION_TEST io/row_selection_test.cpp) +ConfigureTest(FILEPATH_SOURCE_TEST io/filepath_source_test.cpp) ConfigureTest( CSV_TEST io/csv_test.cpp diff --git a/cpp/tests/io/filepath_source_test.cpp b/cpp/tests/io/filepath_source_test.cpp new file mode 100644 index 000000000000..a6ad01adb1dd --- /dev/null +++ b/cpp/tests/io/filepath_source_test.cpp @@ -0,0 +1,73 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include + +#include +#include +#include + +#include + +struct FilepathSourceTest : public cudf::test::BaseFixture {}; + +TEST_F(FilepathSourceTest, StringConstructorsPopulateFilepathSources) +{ + auto const single = cudf::io::source_info{"test.parquet"}; + ASSERT_EQ(single.filepath_sources().size(), 1); + EXPECT_EQ(single.filepath_sources().front().path, "test.parquet"); + EXPECT_FALSE(single.filepath_sources().front().size.has_value()); + EXPECT_EQ(single.filepaths(), std::vector{"test.parquet"}); + + auto const multi = cudf::io::source_info{std::vector{"a.parquet", "b.parquet"}}; + ASSERT_EQ(multi.filepath_sources().size(), 2); + EXPECT_EQ(multi.filepaths().size(), 2); + EXPECT_FALSE(multi.filepath_sources()[1].size.has_value()); +} + +TEST_F(FilepathSourceTest, FilepathSourceConstructorPreservesSize) +{ + std::vector sources{ + {"s3://bucket/object.parquet", 12345}, + {"https://example.com/data.parquet", std::nullopt}, + }; + + auto const info = cudf::io::source_info{std::move(sources)}; + ASSERT_EQ(info.filepath_sources().size(), 2); + EXPECT_EQ(info.filepath_sources()[0].path, "s3://bucket/object.parquet"); + ASSERT_TRUE(info.filepath_sources()[0].size.has_value()); + EXPECT_EQ(info.filepath_sources()[0].size.value(), 12345); + EXPECT_FALSE(info.filepath_sources()[1].size.has_value()); + EXPECT_EQ(info.filepaths()[0], "s3://bucket/object.parquet"); + EXPECT_EQ(info.filepaths()[1], "https://example.com/data.parquet"); +} + +TEST_F(FilepathSourceTest, KnownSizePlumbsThroughMakeDatasources) +{ + auto const filepath = + (std::filesystem::temp_directory_path() / "filepath_source_test.parquet").string(); + + auto col = cudf::test::fixed_width_column_wrapper{1, 2, 3}; + cudf::table_view const table{{col}}; + + cudf::io::parquet_writer_options write_opts = + cudf::io::parquet_writer_options::builder(cudf::io::sink_info{filepath}, table); + cudf::io::write_parquet(write_opts); + + auto const file_size = std::filesystem::file_size(filepath); + std::vector sources{{filepath, file_size}}; + auto const source_info = cudf::io::source_info{std::move(sources)}; + + auto datasources = cudf::io::make_datasources(source_info); + ASSERT_EQ(datasources.size(), 1); + EXPECT_EQ(datasources.front()->size(), file_size); + + auto const read_opts = + cudf::io::parquet_reader_options::builder(source_info).column_names({"0"}).build(); + auto const result = cudf::io::read_parquet(read_opts); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(col, result.tbl->get_column(0).view()); +} diff --git a/docs/cudf/source/cudf/io/io.md b/docs/cudf/source/cudf/io/io.md index e46f5d09a973..688ef87fde0e 100644 --- a/docs/cudf/source/cudf/io/io.md +++ b/docs/cudf/source/cudf/io/io.md @@ -117,6 +117,29 @@ Note that: For more information about error handling, compatibility mode, and tuning parameters in KvikIO see: +### Remote file sizes and HEAD requests + +When reading remote files (for example `s3://...` URLs) via `pylibcudf.io.SourceInfo`, KvikIO +may send HEAD requests at open time to probe connectivity and query file size. To skip those +requests when the file size is already known (for example from object-store metadata), pass a +`pylibcudf.io.FilepathSource` with the `size` argument set: + +```python +import pylibcudf as plc + +content_length = ... # from external metadata +sources = plc.io.SourceInfo([ + plc.io.FilepathSource("s3://bucket/object.parquet", size=content_length), +]) +table = plc.io.parquet.read_parquet( + plc.io.parquet.ParquetReaderOptions.builder(sources).build() +) +``` + +Providing an incorrect size avoids the extra HEAD requests but will break footer reads and other +operations that depend on the true file length. Plain string paths in `SourceInfo` preserve the +previous behavior (size queried via KvikIO). + Operations that support the use of GPUDirect Storage: - {py:func}`cudf.read_avro` diff --git a/python/pylibcudf/pylibcudf/io/__init__.py b/python/pylibcudf/pylibcudf/io/__init__.py index 2162b50e963c..a256857920c9 100644 --- a/python/pylibcudf/pylibcudf/io/__init__.py +++ b/python/pylibcudf/pylibcudf/io/__init__.py @@ -15,10 +15,11 @@ types, ) from .parquet_metadata import FileMetaData -from .types import SinkInfo, SourceInfo, TableWithMetadata +from .types import FilepathSource, SinkInfo, SourceInfo, TableWithMetadata __all__ = [ "FileMetaData", + "FilepathSource", "SinkInfo", "SourceInfo", "TableWithMetadata", diff --git a/python/pylibcudf/pylibcudf/io/types.pxd b/python/pylibcudf/pylibcudf/io/types.pxd index 1e52f4faa058..5b476d066627 100644 --- a/python/pylibcudf/pylibcudf/io/types.pxd +++ b/python/pylibcudf/pylibcudf/io/types.pxd @@ -14,6 +14,7 @@ from pylibcudf.libcudf.io.types cimport ( column_name_info, compression_type, dictionary_policy, + filepath_source, io_type, partition_info, quote_style, @@ -88,6 +89,10 @@ cdef class TableWithMetadata: table_with_metadata& tbl, object stream, DeviceMemoryResource mr ) +cdef class FilepathSource: + cdef public object path + cdef public object size + cdef class SourceInfo: cdef source_info c_obj # Keep the bytes converted from stringio alive diff --git a/python/pylibcudf/pylibcudf/io/types.pyi b/python/pylibcudf/pylibcudf/io/types.pyi index f2050a5b1f91..0d491524b5c0 100644 --- a/python/pylibcudf/pylibcudf/io/types.pyi +++ b/python/pylibcudf/pylibcudf/io/types.pyi @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 import io import os @@ -111,11 +111,19 @@ class TableWithMetadata: @property def num_row_groups_after_bloom_filter(self) -> int | None: ... +class FilepathSource: + def __init__( + self, path: str | os.PathLike[Any], size: int | None = None + ): ... + path: str + size: int | None + class SourceInfo: def __init__( self, sources: Sequence[str] | Sequence[os.PathLike[Any]] + | Sequence[FilepathSource] | Sequence[Datasource], ) -> None: ... @staticmethod diff --git a/python/pylibcudf/pylibcudf/io/types.pyx b/python/pylibcudf/pylibcudf/io/types.pyx index 27c3bb47caf0..6805f635f9ce 100644 --- a/python/pylibcudf/pylibcudf/io/types.pyx +++ b/python/pylibcudf/pylibcudf/io/types.pyx @@ -22,6 +22,7 @@ from pylibcudf.libcudf.io.types cimport ( column_encoding, column_in_metadata, column_name_info, + filepath_source, partition_info, source_info, table_input_metadata, @@ -55,6 +56,7 @@ __all__ = [ "ColumnInMetadata", "CompressionType", "DictionaryPolicy", + "FilepathSource", "JSONRecoveryMode", "PartitionInfo", "QuoteStyle", @@ -453,6 +455,27 @@ cdef class TableWithMetadata: return None +cdef class FilepathSource: + """ + A file path or URL with an optional known size in bytes. + + When ``size`` is set for a remote URL, libcudf passes it to KvikIO at open + time so the remote server is not queried for file size (avoiding HEAD + requests). An incorrect size will cause read failures. + + Parameters + ---------- + path : str or os.PathLike + Path or URL of the input file. + size : int, optional + Known file size in bytes. Omit to query size via KvikIO (HEAD for remote URLs). + """ + + def __init__(self, path, size=None): + self.path = os.fspath(path) + self.size = size + + cdef class SourceInfo: """ A class containing details on a source to read from. @@ -464,6 +487,7 @@ cdef class SourceInfo: sources : List[Union[ str, os.PathLike, + FilepathSource, bytes, io.BytesIO, DataSource, @@ -480,9 +504,30 @@ cdef class SourceInfo: return cdef vector[string] c_files + cdef vector[filepath_source] c_filepath_sources cdef vector[datasource*] c_datasources + cdef filepath_source fs + + if isinstance(sources[0], FilepathSource): + c_filepath_sources.reserve(len(sources)) + + for src in sources: + if not isinstance(src, FilepathSource): + raise ValueError("All sources must be of the same type!") + if not ( + os.path.isfile(src.path) or SourceInfo._is_remote_uri(src.path) + ): + raise FileNotFoundError( + errno.ENOENT, os.strerror(errno.ENOENT), src.path + ) + fs = filepath_source( str(src.path).encode()) + if src.size is not None: + fs.size = src.size + c_filepath_sources.push_back(fs) - if isinstance(sources[0], (os.PathLike, str)): + self.c_obj = move(source_info(c_filepath_sources)) + return + elif isinstance(sources[0], (os.PathLike, str)): c_files.reserve(len(sources)) for src in sources: @@ -537,7 +582,7 @@ cdef class SourceInfo: self.c_obj = move(source_info(host_span[device_span[const_byte]](d_spans))) return else: - raise ValueError("Sources must be a list of str/paths, " + raise ValueError("Sources must be a list of str/paths, FilepathSource, " "bytes, io.BytesIO, io.StringIO, or a Datasource") self.c_obj = source_info(host_span[host_span[const_byte]](self._hspans)) diff --git a/python/pylibcudf/pylibcudf/libcudf/io/types.pxd b/python/pylibcudf/pylibcudf/libcudf/io/types.pxd index 6a6d6356801e..98d07b06c4f8 100644 --- a/python/pylibcudf/pylibcudf/libcudf/io/types.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/io/types.pxd @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2020-2025, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 cimport pylibcudf.libcudf.io.data_sink as cudf_io_data_sink cimport pylibcudf.libcudf.io.datasource as cudf_io_datasource @@ -125,13 +125,25 @@ cdef extern from "cudf/io/types.hpp" \ size_type start_row, size_type num_rows ) except +libcudf_exception_handler + cdef cppclass filepath_source: + string path + optional[size_t] size + + filepath_source() except +libcudf_exception_handler + filepath_source(string path) except +libcudf_exception_handler + cdef cppclass source_info: const vector[string]& filepaths() except +libcudf_exception_handler + const vector[filepath_source]& filepath_sources() \ + except +libcudf_exception_handler source_info() except +libcudf_exception_handler source_info( const vector[string] &filepaths ) except +libcudf_exception_handler + source_info( + const vector[filepath_source] &sources + ) except +libcudf_exception_handler source_info( cudf_io_datasource.datasource *source ) except +libcudf_exception_handler diff --git a/python/pylibcudf/tests/io/test_source_sink_info.py b/python/pylibcudf/tests/io/test_source_sink_info.py index 5a2bc95bd109..220b4edbceb9 100644 --- a/python/pylibcudf/tests/io/test_source_sink_info.py +++ b/python/pylibcudf/tests/io/test_source_sink_info.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 import io @@ -93,3 +93,43 @@ def test_source_info_ctor_mixing_invalid(io_class, sources, tmp_path): def test_source_info_invalid(): with pytest.raises(ValueError): plc.io.SourceInfo([123]) + + +def test_filepath_source_local_parquet(tmp_path): + path = tmp_path / "data.parquet" + table = plc.Table({"a": plc.Column([1, 2, 3])}) + plc.io.parquet.write_parquet( + plc.io.parquet.ParquetWriterOptions.builder( + plc.io.SinkInfo([str(path)]), table + ).build() + ) + file_size = path.stat().st_size + + source = plc.io.FilepathSource(str(path), size=file_size) + assert source.path == str(path) + assert source.size == file_size + + source_info = plc.io.SourceInfo([source]) + read_opts = plc.io.parquet.ParquetReaderOptions.builder( + source_info + ).build() + result = plc.io.parquet.read_parquet(read_opts) + assert result.columns[0].to_arrow().to_pylist() == [1, 2, 3] + + +def test_filepath_source_remote_uri_without_size(): + source = plc.io.FilepathSource("s3://bucket/object.parquet") + assert source.size is None + plc.io.SourceInfo([source]) + + +def test_filepath_source_mixed_sources_invalid(): + with pytest.raises( + ValueError, match="All sources must be of the same type" + ): + plc.io.SourceInfo( + [ + plc.io.FilepathSource("s3://bucket/object.parquet", size=100), + "s3://bucket/other.parquet", + ] + ) From ed52e1dcbd276cc68895a3dc6c71c32454d60931 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Mon, 1 Jun 2026 12:36:59 -0700 Subject: [PATCH 02/16] Fixups --- cpp/include/cudf/io/datasource.hpp | 4 ++-- cpp/include/cudf/io/types.hpp | 6 +++--- cpp/src/io/utilities/datasource.cpp | 2 -- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/cpp/include/cudf/io/datasource.hpp b/cpp/include/cudf/io/datasource.hpp index da0404da0970..a4a87bba180d 100644 --- a/cpp/include/cudf/io/datasource.hpp +++ b/cpp/include/cudf/io/datasource.hpp @@ -98,8 +98,8 @@ class datasource { * @param[in] offset Starting byte offset from which data will be read (the default is zero) * @param[in] max_size_estimate Upper estimate of the data range that will be read (the default is * zero, which means the whole file after `offset`) - * @param[in] known_size Optional known file size in bytes. When set for remote URLs, KvikIO skips - * the size query HEAD request at open time. + * @param[in] known_size Optional known file size in bytes. When set for remote URLs, the IO + * backend may skip querying the remote server for file size at open time. * @return Constructed datasource object */ static std::unique_ptr create(std::string const& filepath, diff --git a/cpp/include/cudf/io/types.hpp b/cpp/include/cudf/io/types.hpp index 6eed76c06c8a..d912740257ef 100644 --- a/cpp/include/cudf/io/types.hpp +++ b/cpp/include/cudf/io/types.hpp @@ -313,12 +313,12 @@ constexpr inline auto is_byte_like_type() /** * @brief A file path with an optional known size in bytes. * - * When `size` is set for a remote URL, libcudf passes it to KvikIO at open time so the remote - * server is not queried for file size. + * When `size` is set for a remote URL, the IO backend may skip querying the remote server for file + * size at open time. */ struct filepath_source { std::string path; ///< Path or URL of the input file - std::optional size{}; ///< Known file size; omit to query via KvikIO + std::optional size{}; ///< Known file size; omit to query size at open time }; /** diff --git a/cpp/src/io/utilities/datasource.cpp b/cpp/src/io/utilities/datasource.cpp index 07e01e6fecbb..fb6c49c81d8e 100644 --- a/cpp/src/io/utilities/datasource.cpp +++ b/cpp/src/io/utilities/datasource.cpp @@ -380,9 +380,7 @@ kvikio::RemoteHandle open_remote_handle(char const* filepath, std::optional Date: Mon, 1 Jun 2026 13:01:29 -0700 Subject: [PATCH 03/16] const type --- python/pylibcudf/pylibcudf/libcudf/io/types.pxd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pylibcudf/pylibcudf/libcudf/io/types.pxd b/python/pylibcudf/pylibcudf/libcudf/io/types.pxd index 98d07b06c4f8..938154f8be0f 100644 --- a/python/pylibcudf/pylibcudf/libcudf/io/types.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/io/types.pxd @@ -142,7 +142,7 @@ cdef extern from "cudf/io/types.hpp" \ const vector[string] &filepaths ) except +libcudf_exception_handler source_info( - const vector[filepath_source] &sources + vector[filepath_source] sources ) except +libcudf_exception_handler source_info( cudf_io_datasource.datasource *source From a65f65d247d69afde878c139b0eaf301c1df782f Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Mon, 1 Jun 2026 13:08:47 -0700 Subject: [PATCH 04/16] test style --- cpp/tests/io/filepath_source_test.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cpp/tests/io/filepath_source_test.cpp b/cpp/tests/io/filepath_source_test.cpp index a6ad01adb1dd..08eda203773a 100644 --- a/cpp/tests/io/filepath_source_test.cpp +++ b/cpp/tests/io/filepath_source_test.cpp @@ -13,6 +13,9 @@ #include +auto const temp_env = static_cast( + ::testing::AddGlobalTestEnvironment(new cudf::test::TempDirTestEnvironment)); + struct FilepathSourceTest : public cudf::test::BaseFixture {}; TEST_F(FilepathSourceTest, StringConstructorsPopulateFilepathSources) @@ -48,8 +51,7 @@ TEST_F(FilepathSourceTest, FilepathSourceConstructorPreservesSize) TEST_F(FilepathSourceTest, KnownSizePlumbsThroughMakeDatasources) { - auto const filepath = - (std::filesystem::temp_directory_path() / "filepath_source_test.parquet").string(); + auto const filepath = temp_env->get_temp_filepath("KnownSize.parquet"); auto col = cudf::test::fixed_width_column_wrapper{1, 2, 3}; cudf::table_view const table{{col}}; From 315220517140ab03ea1fb0c4771b38a610805adf Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Tue, 2 Jun 2026 04:43:16 -0700 Subject: [PATCH 05/16] Test fixup --- cpp/tests/io/filepath_source_test.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cpp/tests/io/filepath_source_test.cpp b/cpp/tests/io/filepath_source_test.cpp index 08eda203773a..5eb11489615f 100644 --- a/cpp/tests/io/filepath_source_test.cpp +++ b/cpp/tests/io/filepath_source_test.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -68,8 +69,7 @@ TEST_F(FilepathSourceTest, KnownSizePlumbsThroughMakeDatasources) ASSERT_EQ(datasources.size(), 1); EXPECT_EQ(datasources.front()->size(), file_size); - auto const read_opts = - cudf::io::parquet_reader_options::builder(source_info).column_names({"0"}).build(); - auto const result = cudf::io::read_parquet(read_opts); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(col, result.tbl->get_column(0).view()); + auto const read_opts = cudf::io::parquet_reader_options::builder(source_info).build(); + auto const result = cudf::io::read_parquet(read_opts); + CUDF_TEST_EXPECT_TABLES_EQUAL(table, result.tbl->view()); } From b0876272e8c2276e92fc016c33c1ce550ce02d42 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Tue, 2 Jun 2026 06:02:35 -0700 Subject: [PATCH 06/16] test fix --- python/pylibcudf/tests/io/test_source_sink_info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pylibcudf/tests/io/test_source_sink_info.py b/python/pylibcudf/tests/io/test_source_sink_info.py index 220b4edbceb9..0bf2091be514 100644 --- a/python/pylibcudf/tests/io/test_source_sink_info.py +++ b/python/pylibcudf/tests/io/test_source_sink_info.py @@ -97,7 +97,7 @@ def test_source_info_invalid(): def test_filepath_source_local_parquet(tmp_path): path = tmp_path / "data.parquet" - table = plc.Table({"a": plc.Column([1, 2, 3])}) + table = plc.Table([plc.Column.from_iterable_of_py([1, 2, 3])]) plc.io.parquet.write_parquet( plc.io.parquet.ParquetWriterOptions.builder( plc.io.SinkInfo([str(path)]), table From 62e82215bd446e787202ac2f75bdd49985a6155d Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Tue, 9 Jun 2026 06:04:04 -0700 Subject: [PATCH 07/16] Build cudf_streaming against CI run's pylibcudf This updates our cudf_streaming build-wheel script build against the pylibcudf wheel built in this same CI run. This is motivated by an error observed in CI runs for https://github.com/rapidsai/cudf/pull/22739, which changes the `SourceInfo` class. This led to an ABI mismatch between the `SourceInfo` class in the runtime pylibcudf (from the CI run) and the `SourceInfo` class cudf-streaming was built against (from `main` / nightly). This matches how we build cudf in build_wheel_cudf. --- ci/build_wheel_cudf_streaming.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/build_wheel_cudf_streaming.sh b/ci/build_wheel_cudf_streaming.sh index d485f4741ac6..b8a19d3cdf79 100755 --- a/ci/build_wheel_cudf_streaming.sh +++ b/ci/build_wheel_cudf_streaming.sh @@ -15,7 +15,9 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}")" # Downloads libcudf_streaming wheel from this current build, # then ensures 'cudf_streaming' wheel builds always use the 'libcudf_streaming' just built in the same CI run. LIBCUDF_STREAMING_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="libcudf_streaming_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-github cpp) +PYLIBCUDF_WHEELHOUSE=$(rapids-download-from-github "$(rapids-package-name "wheel_python" pylibcudf --stable --cuda "$RAPIDS_CUDA_VERSION")") echo "libcudf-streaming-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${LIBCUDF_STREAMING_WHEELHOUSE}/libcudf_streaming_*.whl)" >> "${PIP_CONSTRAINT}" +echo "pylibcudf-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${PYLIBCUDF_WHEELHOUSE}/pylibcudf_*.whl)" >> "${PIP_CONSTRAINT}" rapids-logger "Generating build requirements" From e0c459626e8c95a085256c31f65418d54247225a Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Tue, 9 Jun 2026 06:11:59 -0700 Subject: [PATCH 08/16] fix quoting from https://github.com/rapidsai/cudf/pull/22760/changes#r3358910212 --- ci/build_wheel_cudf_streaming.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/build_wheel_cudf_streaming.sh b/ci/build_wheel_cudf_streaming.sh index b8a19d3cdf79..c2619fe55798 100755 --- a/ci/build_wheel_cudf_streaming.sh +++ b/ci/build_wheel_cudf_streaming.sh @@ -16,8 +16,8 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}")" # then ensures 'cudf_streaming' wheel builds always use the 'libcudf_streaming' just built in the same CI run. LIBCUDF_STREAMING_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="libcudf_streaming_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-github cpp) PYLIBCUDF_WHEELHOUSE=$(rapids-download-from-github "$(rapids-package-name "wheel_python" pylibcudf --stable --cuda "$RAPIDS_CUDA_VERSION")") -echo "libcudf-streaming-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${LIBCUDF_STREAMING_WHEELHOUSE}/libcudf_streaming_*.whl)" >> "${PIP_CONSTRAINT}" -echo "pylibcudf-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${PYLIBCUDF_WHEELHOUSE}/pylibcudf_*.whl)" >> "${PIP_CONSTRAINT}" +echo "libcudf-streaming-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo "${LIBCUDF_STREAMING_WHEELHOUSE}"/libcudf_streaming_*.whl)" >> "${PIP_CONSTRAINT}" +echo "pylibcudf-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo "${PYLIBCUDF_WHEELHOUSE}"/pylibcudf_*.whl)" >> "${PIP_CONSTRAINT}" rapids-logger "Generating build requirements" From e76a02b1b7f2f2da775f5e7b6a27c8b888087636 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Tue, 9 Jun 2026 11:58:57 -0700 Subject: [PATCH 09/16] libcudf too --- ci/build_wheel_cudf_streaming.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ci/build_wheel_cudf_streaming.sh b/ci/build_wheel_cudf_streaming.sh index c2619fe55798..c6c08919476b 100755 --- a/ci/build_wheel_cudf_streaming.sh +++ b/ci/build_wheel_cudf_streaming.sh @@ -12,11 +12,13 @@ dependency_file_key_suffix="cudf_streaming" RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}")" -# Downloads libcudf_streaming wheel from this current build, -# then ensures 'cudf_streaming' wheel builds always use the 'libcudf_streaming' just built in the same CI run. +# Downloads libcudf, pylibcudf, and libcudf_streaming wheels from the current build. +# Then ensures 'cudf_streaming' wheel builds always use wheels built in the same CI run. LIBCUDF_STREAMING_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="libcudf_streaming_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-github cpp) +LIBCUDF_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-github cpp) PYLIBCUDF_WHEELHOUSE=$(rapids-download-from-github "$(rapids-package-name "wheel_python" pylibcudf --stable --cuda "$RAPIDS_CUDA_VERSION")") echo "libcudf-streaming-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo "${LIBCUDF_STREAMING_WHEELHOUSE}"/libcudf_streaming_*.whl)" >> "${PIP_CONSTRAINT}" +echo "libcudf-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo "${LIBCUDF_WHEELHOUSE}"/libcudf_*.whl)" >> "${PIP_CONSTRAINT}" echo "pylibcudf-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo "${PYLIBCUDF_WHEELHOUSE}"/pylibcudf_*.whl)" >> "${PIP_CONSTRAINT}" rapids-logger "Generating build requirements" From 5b504da5ebcc694a3a9e3a8fed4c9e84d866d5e0 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Mon, 22 Jun 2026 08:44:50 -0700 Subject: [PATCH 10/16] CI --- ci/build_wheel_cudf_streaming.sh | 2 +- cpp/include/cudf/io/datasource.hpp | 2 +- cpp/include/cudf/io/types.hpp | 2 +- cpp/src/io/functions.cpp | 2 +- cpp/src/io/utilities/datasource.cpp | 2 +- cpp/tests/CMakeLists.txt | 2 +- cpp/tests/io/filepath_source_test.cpp | 2 +- python/cudf/cudf/pandas/_wrappers/numpy.py | 4 ++-- python/pylibcudf/pylibcudf/io/__init__.py | 2 +- python/pylibcudf/pylibcudf/io/types.pxd | 2 +- python/pylibcudf/pylibcudf/io/types.pyi | 2 +- python/pylibcudf/pylibcudf/io/types.pyx | 2 +- python/pylibcudf/pylibcudf/libcudf/io/types.pxd | 2 +- python/pylibcudf/tests/io/test_source_sink_info.py | 2 +- 14 files changed, 15 insertions(+), 15 deletions(-) diff --git a/ci/build_wheel_cudf_streaming.sh b/ci/build_wheel_cudf_streaming.sh index c0fb43711661..f572dcbb84e3 100755 --- a/ci/build_wheel_cudf_streaming.sh +++ b/ci/build_wheel_cudf_streaming.sh @@ -1,5 +1,5 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 set -euo pipefail diff --git a/cpp/include/cudf/io/datasource.hpp b/cpp/include/cudf/io/datasource.hpp index a4a87bba180d..6731c0d64c24 100644 --- a/cpp/include/cudf/io/datasource.hpp +++ b/cpp/include/cudf/io/datasource.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/include/cudf/io/types.hpp b/cpp/include/cudf/io/types.hpp index d912740257ef..6d7ca5e80c05 100644 --- a/cpp/include/cudf/io/types.hpp +++ b/cpp/include/cudf/io/types.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/io/functions.cpp b/cpp/src/io/functions.cpp index eed9e71a90d1..6df5bfe5a9e7 100644 --- a/cpp/src/io/functions.cpp +++ b/cpp/src/io/functions.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/io/utilities/datasource.cpp b/cpp/src/io/utilities/datasource.cpp index fb6c49c81d8e..2aaf881fd7bc 100644 --- a/cpp/src/io/utilities/datasource.cpp +++ b/cpp/src/io/utilities/datasource.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 21074652664c..b4af7d296e4a 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -1,6 +1,6 @@ # ============================================================================= # cmake-format: off -# SPDX-FileCopyrightText: Copyright (c) 2018-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2018-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # cmake-format: on # ============================================================================= diff --git a/cpp/tests/io/filepath_source_test.cpp b/cpp/tests/io/filepath_source_test.cpp index 5eb11489615f..b6def9770fd2 100644 --- a/cpp/tests/io/filepath_source_test.cpp +++ b/cpp/tests/io/filepath_source_test.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/python/cudf/cudf/pandas/_wrappers/numpy.py b/python/cudf/cudf/pandas/_wrappers/numpy.py index 6036add94c0a..bcddfd5e3dba 100644 --- a/python/cudf/cudf/pandas/_wrappers/numpy.py +++ b/python/cudf/cudf/pandas/_wrappers/numpy.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations @@ -328,7 +328,7 @@ def _ndarray_fsproxy_fast_to_slow(self): # NumPy 2 introduced `_core` and gives warnings for access to `core`. from numpy._core.multiarray import flagsobj as _numpy_flagsobj else: - from numpy.core.multiarray import ( # type: ignore[no-redef] + from numpy.core.multiarray import ( flagsobj as _numpy_flagsobj, ) diff --git a/python/pylibcudf/pylibcudf/io/__init__.py b/python/pylibcudf/pylibcudf/io/__init__.py index a256857920c9..a6a0ebad3a1e 100644 --- a/python/pylibcudf/pylibcudf/io/__init__.py +++ b/python/pylibcudf/pylibcudf/io/__init__.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from . import ( diff --git a/python/pylibcudf/pylibcudf/io/types.pxd b/python/pylibcudf/pylibcudf/io/types.pxd index 5b476d066627..72e056041d08 100644 --- a/python/pylibcudf/pylibcudf/io/types.pxd +++ b/python/pylibcudf/pylibcudf/io/types.pxd @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from libc.stdint cimport uint8_t, int32_t diff --git a/python/pylibcudf/pylibcudf/io/types.pyi b/python/pylibcudf/pylibcudf/io/types.pyi index 0d491524b5c0..e6b17f169f2d 100644 --- a/python/pylibcudf/pylibcudf/io/types.pyi +++ b/python/pylibcudf/pylibcudf/io/types.pyi @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 import io import os diff --git a/python/pylibcudf/pylibcudf/io/types.pyx b/python/pylibcudf/pylibcudf/io/types.pyx index 6805f635f9ce..af07f31d02e2 100644 --- a/python/pylibcudf/pylibcudf/io/types.pyx +++ b/python/pylibcudf/pylibcudf/io/types.pyx @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from cpython.buffer cimport PyBUF_READ from cpython.memoryview cimport PyMemoryView_FromMemory diff --git a/python/pylibcudf/pylibcudf/libcudf/io/types.pxd b/python/pylibcudf/pylibcudf/libcudf/io/types.pxd index 938154f8be0f..0c35c07ca4ee 100644 --- a/python/pylibcudf/pylibcudf/libcudf/io/types.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/io/types.pxd @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 cimport pylibcudf.libcudf.io.data_sink as cudf_io_data_sink cimport pylibcudf.libcudf.io.datasource as cudf_io_datasource diff --git a/python/pylibcudf/tests/io/test_source_sink_info.py b/python/pylibcudf/tests/io/test_source_sink_info.py index 0bf2091be514..4409702db659 100644 --- a/python/pylibcudf/tests/io/test_source_sink_info.py +++ b/python/pylibcudf/tests/io/test_source_sink_info.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 import io From dcda6e1b66efa559788e90e761aa3b332aaf544a Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Mon, 22 Jun 2026 11:56:45 -0700 Subject: [PATCH 11/16] Match the siganture with remote_io off --- cpp/src/io/utilities/datasource.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cpp/src/io/utilities/datasource.cpp b/cpp/src/io/utilities/datasource.cpp index 2aaf881fd7bc..d07583c0e4a8 100644 --- a/cpp/src/io/utilities/datasource.cpp +++ b/cpp/src/io/utilities/datasource.cpp @@ -417,7 +417,10 @@ class remote_file_source : public kvikio_source { */ class remote_file_source : public file_source { public: - explicit remote_file_source(char const* filepath) : file_source(filepath) {} + explicit remote_file_source(char const* filepath, std::optional = std::nullopt) + : file_source(filepath) + { + } static constexpr bool could_be_remote_url(std::string const&) { return false; } }; #endif From 60a5ecddf9259238676f10068ffe2580521338a5 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Mon, 22 Jun 2026 13:01:36 -0700 Subject: [PATCH 12/16] reinclude memory --- cpp/include/cudf/io/datasource.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/include/cudf/io/datasource.hpp b/cpp/include/cudf/io/datasource.hpp index 6731c0d64c24..336e44134a4d 100644 --- a/cpp/include/cudf/io/datasource.hpp +++ b/cpp/include/cudf/io/datasource.hpp @@ -13,6 +13,7 @@ #include #include +#include #include namespace CUDF_EXPORT cudf { From b460be2319dd72cab1b0ad5c90b27bf806c2f271 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Tue, 23 Jun 2026 05:37:12 -0700 Subject: [PATCH 13/16] Remove unreachable public s3 endpoint type --- cpp/src/io/utilities/datasource.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/cpp/src/io/utilities/datasource.cpp b/cpp/src/io/utilities/datasource.cpp index d07583c0e4a8..46eeeea4be4b 100644 --- a/cpp/src/io/utilities/datasource.cpp +++ b/cpp/src/io/utilities/datasource.cpp @@ -363,7 +363,6 @@ class user_datasource_wrapper : public datasource { kvikio::RemoteEndpointType infer_remote_endpoint_type(std::string const& url) { if (kvikio::S3Endpoint::is_url_valid(url)) { return kvikio::RemoteEndpointType::S3; } - if (kvikio::S3PublicEndpoint::is_url_valid(url)) { return kvikio::RemoteEndpointType::S3_PUBLIC; } if (kvikio::S3EndpointWithPresignedUrl::is_url_valid(url)) { return kvikio::RemoteEndpointType::S3_PRESIGNED_URL; } From e65fb89d4adb3046923a0286289350fe729b6ac1 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Wed, 24 Jun 2026 07:09:16 -0700 Subject: [PATCH 14/16] Use kvikio API for inferring the remote endpoint type --- cpp/src/io/utilities/datasource.cpp | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/cpp/src/io/utilities/datasource.cpp b/cpp/src/io/utilities/datasource.cpp index 46eeeea4be4b..025b294489d2 100644 --- a/cpp/src/io/utilities/datasource.cpp +++ b/cpp/src/io/utilities/datasource.cpp @@ -355,26 +355,10 @@ class user_datasource_wrapper : public datasource { }; #ifdef CUDF_KVIKIO_REMOTE_IO -/** - * @brief Infer the KvikIO remote endpoint type from a URL (no network I/O). - * - * Mirrors the order used by `kvikio::RemoteHandle::open()` in AUTO mode. - */ -kvikio::RemoteEndpointType infer_remote_endpoint_type(std::string const& url) -{ - if (kvikio::S3Endpoint::is_url_valid(url)) { return kvikio::RemoteEndpointType::S3; } - if (kvikio::S3EndpointWithPresignedUrl::is_url_valid(url)) { - return kvikio::RemoteEndpointType::S3_PRESIGNED_URL; - } - if (kvikio::WebHdfsEndpoint::is_url_valid(url)) { return kvikio::RemoteEndpointType::WEBHDFS; } - if (kvikio::HttpEndpoint::is_url_valid(url)) { return kvikio::RemoteEndpointType::HTTP; } - return kvikio::RemoteEndpointType::HTTP; -} - kvikio::RemoteHandle open_remote_handle(char const* filepath, std::optional known_size) { if (known_size.has_value()) { - auto const endpoint_type = infer_remote_endpoint_type(filepath); + auto const endpoint_type = kvikio::infer_remote_endpoint_type(filepath); return kvikio::RemoteHandle::open(filepath, endpoint_type, std::nullopt, *known_size); } return kvikio::RemoteHandle::open(filepath); From b375237d57fe01b61111374a7784b0c86f0ac859 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Thu, 25 Jun 2026 14:33:54 -0700 Subject: [PATCH 15/16] Also pin libcudf in build_wheel_libcudf_streaming.sh --- ci/build_wheel_libcudf_streaming.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/ci/build_wheel_libcudf_streaming.sh b/ci/build_wheel_libcudf_streaming.sh index 39b432153da5..2810119f587a 100755 --- a/ci/build_wheel_libcudf_streaming.sh +++ b/ci/build_wheel_libcudf_streaming.sh @@ -1,5 +1,5 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 set -euo pipefail @@ -9,6 +9,14 @@ source rapids-init-pip package_name="libcudf_streaming" package_dir="python/libcudf_streaming" +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}")" + +# Downloads libcudf wheel from this current build, +# then ensures 'libcudf_streaming' wheel builds always use the 'libcudf' +# just built in the same CI run. +LIBCUDF_WHEELHOUSE=$(rapids-download-from-github "$(rapids-artifact-name wheel_cpp libcudf cudf --cuda "$RAPIDS_CUDA_VERSION")") +echo "libcudf-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo "${LIBCUDF_WHEELHOUSE}"/libcudf_*.whl)" >> "${PIP_CONSTRAINT}" + rapids-logger "Generating build requirements" rapids-dependency-file-generator \ From 7dbe50a23fea49952fa47ed332c0bbe9a121f75c Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Fri, 26 Jun 2026 14:25:33 -0700 Subject: [PATCH 16/16] Added note about s3 --- cpp/src/io/utilities/datasource.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cpp/src/io/utilities/datasource.cpp b/cpp/src/io/utilities/datasource.cpp index 025b294489d2..f918402d3ae1 100644 --- a/cpp/src/io/utilities/datasource.cpp +++ b/cpp/src/io/utilities/datasource.cpp @@ -366,6 +366,10 @@ kvikio::RemoteHandle open_remote_handle(char const* filepath, std::optional { public: