diff --git a/ci/build_wheel_cudf_streaming.sh b/ci/build_wheel_cudf_streaming.sh index f265528848c4..cc961508b253 100755 --- a/ci/build_wheel_cudf_streaming.sh +++ b/ci/build_wheel_cudf_streaming.sh @@ -15,8 +15,10 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}")" # Downloads libcudf_streaming wheel from this current build, # then ensures 'cudf_streaming' wheel builds always use the 'libcudf_streaming' just built in the same CI run. LIBCUDF_STREAMING_WHEELHOUSE=$(rapids-download-from-github "$(rapids-artifact-name wheel_cpp libcudf-streaming cudf --cuda "$RAPIDS_CUDA_VERSION")") +LIBCUDF_WHEELHOUSE=$(rapids-download-from-github "$(rapids-artifact-name wheel_cpp libcudf cudf --cuda "$RAPIDS_CUDA_VERSION")") PYLIBCUDF_WHEELHOUSE=$(rapids-download-from-github "$(rapids-artifact-name wheel_python pylibcudf cudf --stable --cuda "$RAPIDS_CUDA_VERSION")") echo "libcudf-streaming-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo "${LIBCUDF_STREAMING_WHEELHOUSE}"/libcudf_streaming_*.whl)" >> "${PIP_CONSTRAINT}" +echo "libcudf-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo "${LIBCUDF_WHEELHOUSE}"/libcudf_*.whl)" >> "${PIP_CONSTRAINT}" echo "pylibcudf-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo "${PYLIBCUDF_WHEELHOUSE}"/pylibcudf_*.whl)" >> "${PIP_CONSTRAINT}" rapids-logger "Generating build requirements" diff --git a/ci/build_wheel_libcudf_streaming.sh b/ci/build_wheel_libcudf_streaming.sh index 39b432153da5..2810119f587a 100755 --- a/ci/build_wheel_libcudf_streaming.sh +++ b/ci/build_wheel_libcudf_streaming.sh @@ -1,5 +1,5 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 set -euo pipefail @@ -9,6 +9,14 @@ source rapids-init-pip package_name="libcudf_streaming" package_dir="python/libcudf_streaming" +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}")" + +# Downloads libcudf wheel from this current build, +# then ensures 'libcudf_streaming' wheel builds always use the 'libcudf' +# just built in the same CI run. +LIBCUDF_WHEELHOUSE=$(rapids-download-from-github "$(rapids-artifact-name wheel_cpp libcudf cudf --cuda "$RAPIDS_CUDA_VERSION")") +echo "libcudf-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo "${LIBCUDF_WHEELHOUSE}"/libcudf_*.whl)" >> "${PIP_CONSTRAINT}" + rapids-logger "Generating build requirements" rapids-dependency-file-generator \ diff --git a/cpp/include/cudf/io/datasource.hpp b/cpp/include/cudf/io/datasource.hpp index 6753c2638e39..336e44134a4d 100644 --- a/cpp/include/cudf/io/datasource.hpp +++ b/cpp/include/cudf/io/datasource.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -14,6 +14,7 @@ #include #include +#include namespace CUDF_EXPORT cudf { //! IO interfaces @@ -98,11 +99,14 @@ class datasource { * @param[in] offset Starting byte offset from which data will be read (the default is zero) * @param[in] max_size_estimate Upper estimate of the data range that will be read (the default is * zero, which means the whole file after `offset`) + * @param[in] known_size Optional known file size in bytes. When set for remote URLs, the IO + * backend may skip querying the remote server for file size at open time. * @return Constructed datasource object */ static std::unique_ptr create(std::string const& filepath, - size_t offset = 0, - size_t max_size_estimate = 0); + size_t offset = 0, + size_t max_size_estimate = 0, + std::optional known_size = std::nullopt); /** * @brief Creates a source from a host memory buffer. diff --git a/cpp/include/cudf/io/types.hpp b/cpp/include/cudf/io/types.hpp index 4978960b7c30..6d7ca5e80c05 100644 --- a/cpp/include/cudf/io/types.hpp +++ b/cpp/include/cudf/io/types.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -310,6 +310,17 @@ constexpr inline auto is_byte_like_type() std::is_same_v; } +/** + * @brief A file path with an optional known size in bytes. + * + * When `size` is set for a remote URL, the IO backend may skip querying the remote server for file + * size at open time. + */ +struct filepath_source { + std::string path; ///< Path or URL of the input file + std::optional size{}; ///< Known file size; omit to query size at open time +}; + /** * @brief Source information for read interfaces */ @@ -325,8 +336,13 @@ struct source_info { * @param file_paths Input files paths */ explicit source_info(std::vector file_paths) - : _type(io_type::FILEPATH), _num_sources(file_paths.size()), _filepaths(std::move(file_paths)) + : _type(io_type::FILEPATH), _num_sources(file_paths.size()) { + _filepath_sources.reserve(file_paths.size()); + for (auto& path : file_paths) { + _filepath_sources.push_back({std::move(path), std::nullopt}); + } + rebuild_filepaths(); } /** @@ -335,10 +351,21 @@ struct source_info { * @param file_path Single input file */ explicit source_info(std::string file_path) - : _type(io_type::FILEPATH), _num_sources(1), _filepaths({std::move(file_path)}) + : source_info(std::vector{std::move(file_path)}) { } + /** + * @brief Construct a new source info object from filepath sources with optional known sizes + * + * @param sources Input filepath sources + */ + explicit source_info(std::vector sources) + : _type(io_type::FILEPATH), _num_sources(sources.size()), _filepath_sources(std::move(sources)) + { + rebuild_filepaths(); + } + /** * @brief Construct a new source info object for multiple buffers in host memory * @@ -424,6 +451,12 @@ struct source_info { * @return The type of the input */ [[nodiscard]] auto type() const { return _type; } + /** + * @brief Get the filepath sources of the input + * + * @return The filepath sources of the input + */ + [[nodiscard]] auto const& filepath_sources() const { return _filepath_sources; } /** * @brief Get the filepaths of the input * @@ -457,8 +490,18 @@ struct source_info { [[nodiscard]] auto num_sources() const { return _num_sources; } private: + void rebuild_filepaths() + { + _filepaths.clear(); + _filepaths.reserve(_filepath_sources.size()); + for (auto const& source : _filepath_sources) { + _filepaths.push_back(source.path); + } + } + io_type _type = io_type::VOID; size_t _num_sources = 0; + std::vector _filepath_sources; std::vector _filepaths; std::vector> _host_buffers; std::vector> _device_buffers; diff --git a/cpp/src/io/functions.cpp b/cpp/src/io/functions.cpp index 55d1a750bc06..dec3cd34488e 100644 --- a/cpp/src/io/functions.cpp +++ b/cpp/src/io/functions.cpp @@ -163,24 +163,26 @@ std::vector> make_datasources(source_info switch (info.type()) { case io_type::FILEPATH: { std::vector> sources; - sources.reserve(info.filepaths().size()); + sources.reserve(info.filepath_sources().size()); // Creating sources in a single thread is faster for a small number of sources auto const pool_use_threshold = cudf::detail::getenv_or("LIBCUDF_DATASOURCE_PARALLEL_CREATION_THRESHOLD", 8ul); - if (info.filepaths().size() >= pool_use_threshold) { + if (info.filepath_sources().size() >= pool_use_threshold) { std::vector>> source_tasks; - source_tasks.reserve(info.filepaths().size()); - for (auto const& path : info.filepaths()) { - source_tasks.emplace_back(cudf::detail::host_worker_pool().submit_task( - [=] { return cudf::io::datasource::create(path, offset, max_size_estimate); })); + source_tasks.reserve(info.filepath_sources().size()); + for (auto const& fs : info.filepath_sources()) { + source_tasks.emplace_back(cudf::detail::host_worker_pool().submit_task([=] { + return cudf::io::datasource::create(fs.path, offset, max_size_estimate, fs.size); + })); } std::transform( source_tasks.begin(), source_tasks.end(), std::back_inserter(sources), [](auto& task) { return task.get(); }); } else { - for (auto const& filepath : info.filepaths()) { - sources.emplace_back(cudf::io::datasource::create(filepath, offset, max_size_estimate)); + for (auto const& fs : info.filepath_sources()) { + sources.emplace_back( + cudf::io::datasource::create(fs.path, offset, max_size_estimate, fs.size)); } } return sources; diff --git a/cpp/src/io/utilities/datasource.cpp b/cpp/src/io/utilities/datasource.cpp index aee5b7abfe52..f918402d3ae1 100644 --- a/cpp/src/io/utilities/datasource.cpp +++ b/cpp/src/io/utilities/datasource.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -27,6 +27,7 @@ #include #ifdef CUDF_KVIKIO_REMOTE_IO +#include #include #endif @@ -354,13 +355,27 @@ class user_datasource_wrapper : public datasource { }; #ifdef CUDF_KVIKIO_REMOTE_IO +kvikio::RemoteHandle open_remote_handle(char const* filepath, std::optional known_size) +{ + if (known_size.has_value()) { + auto const endpoint_type = kvikio::infer_remote_endpoint_type(filepath); + return kvikio::RemoteHandle::open(filepath, endpoint_type, std::nullopt, *known_size); + } + return kvikio::RemoteHandle::open(filepath); +} + /** * @brief Remote file source backed by KvikIO, which handles S3 filepaths seamlessly. + * + * Note that this datasource does not currently support anonymously reading a public + * 's3://'-style URL when 'known_size' is provided. + * */ class remote_file_source : public kvikio_source { public: - explicit remote_file_source(char const* filepath) - : kvikio_source{kvikio::RemoteHandle::open(filepath)} + explicit remote_file_source(char const* filepath, + std::optional known_size = std::nullopt) + : kvikio_source{open_remote_handle(filepath, known_size)} { } @@ -389,7 +404,10 @@ class remote_file_source : public kvikio_source { */ class remote_file_source : public file_source { public: - explicit remote_file_source(char const* filepath) : file_source(filepath) {} + explicit remote_file_source(char const* filepath, std::optional = std::nullopt) + : file_source(filepath) + { + } static constexpr bool could_be_remote_url(std::string const&) { return false; } }; #endif @@ -397,7 +415,8 @@ class remote_file_source : public file_source { std::unique_ptr datasource::create(std::string const& filepath, size_t offset, - size_t max_size_estimate) + size_t max_size_estimate, + std::optional known_size) { auto const use_memory_mapping = [] { auto const policy = cudf::detail::getenv_or("LIBCUDF_MMAP_ENABLED", std::string{"OFF"}); @@ -410,7 +429,7 @@ std::unique_ptr datasource::create(std::string const& filepath, if (remote_file_source::could_be_remote_url(filepath)) { try { - return std::make_unique(filepath.c_str()); + return std::make_unique(filepath.c_str(), known_size); } catch (std::exception const& ex) { std::string redacted_msg; try { @@ -450,7 +469,7 @@ std::unique_ptr datasource::create(std::string const& filepath, // Create a remote file resource only when the pattern is found and replaced; otherwise, still // create a local file resource if (filepath != remote_file_path) { - return std::make_unique(remote_file_path.c_str()); + return std::make_unique(remote_file_path.c_str(), known_size); } } diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index a2655216f7cb..b4af7d296e4a 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -1,6 +1,6 @@ # ============================================================================= # cmake-format: off -# SPDX-FileCopyrightText: Copyright (c) 2018-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2018-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # cmake-format: on # ============================================================================= @@ -320,6 +320,7 @@ ConfigureTest( # * io tests -------------------------------------------------------------------------------------- ConfigureTest(COMPRESSION_TEST io/comp/comp_test.cpp) ConfigureTest(ROW_SELECTION_TEST io/row_selection_test.cpp) +ConfigureTest(FILEPATH_SOURCE_TEST io/filepath_source_test.cpp) ConfigureTest( CSV_TEST io/csv_test.cpp diff --git a/cpp/tests/io/filepath_source_test.cpp b/cpp/tests/io/filepath_source_test.cpp new file mode 100644 index 000000000000..b6def9770fd2 --- /dev/null +++ b/cpp/tests/io/filepath_source_test.cpp @@ -0,0 +1,75 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include + +#include +#include +#include + +#include + +auto const temp_env = static_cast( + ::testing::AddGlobalTestEnvironment(new cudf::test::TempDirTestEnvironment)); + +struct FilepathSourceTest : public cudf::test::BaseFixture {}; + +TEST_F(FilepathSourceTest, StringConstructorsPopulateFilepathSources) +{ + auto const single = cudf::io::source_info{"test.parquet"}; + ASSERT_EQ(single.filepath_sources().size(), 1); + EXPECT_EQ(single.filepath_sources().front().path, "test.parquet"); + EXPECT_FALSE(single.filepath_sources().front().size.has_value()); + EXPECT_EQ(single.filepaths(), std::vector{"test.parquet"}); + + auto const multi = cudf::io::source_info{std::vector{"a.parquet", "b.parquet"}}; + ASSERT_EQ(multi.filepath_sources().size(), 2); + EXPECT_EQ(multi.filepaths().size(), 2); + EXPECT_FALSE(multi.filepath_sources()[1].size.has_value()); +} + +TEST_F(FilepathSourceTest, FilepathSourceConstructorPreservesSize) +{ + std::vector sources{ + {"s3://bucket/object.parquet", 12345}, + {"https://example.com/data.parquet", std::nullopt}, + }; + + auto const info = cudf::io::source_info{std::move(sources)}; + ASSERT_EQ(info.filepath_sources().size(), 2); + EXPECT_EQ(info.filepath_sources()[0].path, "s3://bucket/object.parquet"); + ASSERT_TRUE(info.filepath_sources()[0].size.has_value()); + EXPECT_EQ(info.filepath_sources()[0].size.value(), 12345); + EXPECT_FALSE(info.filepath_sources()[1].size.has_value()); + EXPECT_EQ(info.filepaths()[0], "s3://bucket/object.parquet"); + EXPECT_EQ(info.filepaths()[1], "https://example.com/data.parquet"); +} + +TEST_F(FilepathSourceTest, KnownSizePlumbsThroughMakeDatasources) +{ + auto const filepath = temp_env->get_temp_filepath("KnownSize.parquet"); + + auto col = cudf::test::fixed_width_column_wrapper{1, 2, 3}; + cudf::table_view const table{{col}}; + + cudf::io::parquet_writer_options write_opts = + cudf::io::parquet_writer_options::builder(cudf::io::sink_info{filepath}, table); + cudf::io::write_parquet(write_opts); + + auto const file_size = std::filesystem::file_size(filepath); + std::vector sources{{filepath, file_size}}; + auto const source_info = cudf::io::source_info{std::move(sources)}; + + auto datasources = cudf::io::make_datasources(source_info); + ASSERT_EQ(datasources.size(), 1); + EXPECT_EQ(datasources.front()->size(), file_size); + + auto const read_opts = cudf::io::parquet_reader_options::builder(source_info).build(); + auto const result = cudf::io::read_parquet(read_opts); + CUDF_TEST_EXPECT_TABLES_EQUAL(table, result.tbl->view()); +} diff --git a/docs/cudf/source/cudf/io/io.md b/docs/cudf/source/cudf/io/io.md index e46f5d09a973..688ef87fde0e 100644 --- a/docs/cudf/source/cudf/io/io.md +++ b/docs/cudf/source/cudf/io/io.md @@ -117,6 +117,29 @@ Note that: For more information about error handling, compatibility mode, and tuning parameters in KvikIO see: +### Remote file sizes and HEAD requests + +When reading remote files (for example `s3://...` URLs) via `pylibcudf.io.SourceInfo`, KvikIO +may send HEAD requests at open time to probe connectivity and query file size. To skip those +requests when the file size is already known (for example from object-store metadata), pass a +`pylibcudf.io.FilepathSource` with the `size` argument set: + +```python +import pylibcudf as plc + +content_length = ... # from external metadata +sources = plc.io.SourceInfo([ + plc.io.FilepathSource("s3://bucket/object.parquet", size=content_length), +]) +table = plc.io.parquet.read_parquet( + plc.io.parquet.ParquetReaderOptions.builder(sources).build() +) +``` + +Providing an incorrect size avoids the extra HEAD requests but will break footer reads and other +operations that depend on the true file length. Plain string paths in `SourceInfo` preserve the +previous behavior (size queried via KvikIO). + Operations that support the use of GPUDirect Storage: - {py:func}`cudf.read_avro` diff --git a/python/pylibcudf/pylibcudf/io/__init__.py b/python/pylibcudf/pylibcudf/io/__init__.py index 2162b50e963c..a6a0ebad3a1e 100644 --- a/python/pylibcudf/pylibcudf/io/__init__.py +++ b/python/pylibcudf/pylibcudf/io/__init__.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from . import ( @@ -15,10 +15,11 @@ types, ) from .parquet_metadata import FileMetaData -from .types import SinkInfo, SourceInfo, TableWithMetadata +from .types import FilepathSource, SinkInfo, SourceInfo, TableWithMetadata __all__ = [ "FileMetaData", + "FilepathSource", "SinkInfo", "SourceInfo", "TableWithMetadata", diff --git a/python/pylibcudf/pylibcudf/io/types.pxd b/python/pylibcudf/pylibcudf/io/types.pxd index 1e52f4faa058..72e056041d08 100644 --- a/python/pylibcudf/pylibcudf/io/types.pxd +++ b/python/pylibcudf/pylibcudf/io/types.pxd @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from libc.stdint cimport uint8_t, int32_t @@ -14,6 +14,7 @@ from pylibcudf.libcudf.io.types cimport ( column_name_info, compression_type, dictionary_policy, + filepath_source, io_type, partition_info, quote_style, @@ -88,6 +89,10 @@ cdef class TableWithMetadata: table_with_metadata& tbl, object stream, DeviceMemoryResource mr ) +cdef class FilepathSource: + cdef public object path + cdef public object size + cdef class SourceInfo: cdef source_info c_obj # Keep the bytes converted from stringio alive diff --git a/python/pylibcudf/pylibcudf/io/types.pyi b/python/pylibcudf/pylibcudf/io/types.pyi index f2050a5b1f91..e6b17f169f2d 100644 --- a/python/pylibcudf/pylibcudf/io/types.pyi +++ b/python/pylibcudf/pylibcudf/io/types.pyi @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 import io import os @@ -111,11 +111,19 @@ class TableWithMetadata: @property def num_row_groups_after_bloom_filter(self) -> int | None: ... +class FilepathSource: + def __init__( + self, path: str | os.PathLike[Any], size: int | None = None + ): ... + path: str + size: int | None + class SourceInfo: def __init__( self, sources: Sequence[str] | Sequence[os.PathLike[Any]] + | Sequence[FilepathSource] | Sequence[Datasource], ) -> None: ... @staticmethod diff --git a/python/pylibcudf/pylibcudf/io/types.pyx b/python/pylibcudf/pylibcudf/io/types.pyx index 27c3bb47caf0..af07f31d02e2 100644 --- a/python/pylibcudf/pylibcudf/io/types.pyx +++ b/python/pylibcudf/pylibcudf/io/types.pyx @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from cpython.buffer cimport PyBUF_READ from cpython.memoryview cimport PyMemoryView_FromMemory @@ -22,6 +22,7 @@ from pylibcudf.libcudf.io.types cimport ( column_encoding, column_in_metadata, column_name_info, + filepath_source, partition_info, source_info, table_input_metadata, @@ -55,6 +56,7 @@ __all__ = [ "ColumnInMetadata", "CompressionType", "DictionaryPolicy", + "FilepathSource", "JSONRecoveryMode", "PartitionInfo", "QuoteStyle", @@ -453,6 +455,27 @@ cdef class TableWithMetadata: return None +cdef class FilepathSource: + """ + A file path or URL with an optional known size in bytes. + + When ``size`` is set for a remote URL, libcudf passes it to KvikIO at open + time so the remote server is not queried for file size (avoiding HEAD + requests). An incorrect size will cause read failures. + + Parameters + ---------- + path : str or os.PathLike + Path or URL of the input file. + size : int, optional + Known file size in bytes. Omit to query size via KvikIO (HEAD for remote URLs). + """ + + def __init__(self, path, size=None): + self.path = os.fspath(path) + self.size = size + + cdef class SourceInfo: """ A class containing details on a source to read from. @@ -464,6 +487,7 @@ cdef class SourceInfo: sources : List[Union[ str, os.PathLike, + FilepathSource, bytes, io.BytesIO, DataSource, @@ -480,9 +504,30 @@ cdef class SourceInfo: return cdef vector[string] c_files + cdef vector[filepath_source] c_filepath_sources cdef vector[datasource*] c_datasources + cdef filepath_source fs + + if isinstance(sources[0], FilepathSource): + c_filepath_sources.reserve(len(sources)) + + for src in sources: + if not isinstance(src, FilepathSource): + raise ValueError("All sources must be of the same type!") + if not ( + os.path.isfile(src.path) or SourceInfo._is_remote_uri(src.path) + ): + raise FileNotFoundError( + errno.ENOENT, os.strerror(errno.ENOENT), src.path + ) + fs = filepath_source( str(src.path).encode()) + if src.size is not None: + fs.size = src.size + c_filepath_sources.push_back(fs) - if isinstance(sources[0], (os.PathLike, str)): + self.c_obj = move(source_info(c_filepath_sources)) + return + elif isinstance(sources[0], (os.PathLike, str)): c_files.reserve(len(sources)) for src in sources: @@ -537,7 +582,7 @@ cdef class SourceInfo: self.c_obj = move(source_info(host_span[device_span[const_byte]](d_spans))) return else: - raise ValueError("Sources must be a list of str/paths, " + raise ValueError("Sources must be a list of str/paths, FilepathSource, " "bytes, io.BytesIO, io.StringIO, or a Datasource") self.c_obj = source_info(host_span[host_span[const_byte]](self._hspans)) diff --git a/python/pylibcudf/pylibcudf/libcudf/io/types.pxd b/python/pylibcudf/pylibcudf/libcudf/io/types.pxd index 6a6d6356801e..0c35c07ca4ee 100644 --- a/python/pylibcudf/pylibcudf/libcudf/io/types.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/io/types.pxd @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2020-2025, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 cimport pylibcudf.libcudf.io.data_sink as cudf_io_data_sink cimport pylibcudf.libcudf.io.datasource as cudf_io_datasource @@ -125,13 +125,25 @@ cdef extern from "cudf/io/types.hpp" \ size_type start_row, size_type num_rows ) except +libcudf_exception_handler + cdef cppclass filepath_source: + string path + optional[size_t] size + + filepath_source() except +libcudf_exception_handler + filepath_source(string path) except +libcudf_exception_handler + cdef cppclass source_info: const vector[string]& filepaths() except +libcudf_exception_handler + const vector[filepath_source]& filepath_sources() \ + except +libcudf_exception_handler source_info() except +libcudf_exception_handler source_info( const vector[string] &filepaths ) except +libcudf_exception_handler + source_info( + vector[filepath_source] sources + ) except +libcudf_exception_handler source_info( cudf_io_datasource.datasource *source ) except +libcudf_exception_handler diff --git a/python/pylibcudf/tests/io/test_source_sink_info.py b/python/pylibcudf/tests/io/test_source_sink_info.py index 5a2bc95bd109..4409702db659 100644 --- a/python/pylibcudf/tests/io/test_source_sink_info.py +++ b/python/pylibcudf/tests/io/test_source_sink_info.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 import io @@ -93,3 +93,43 @@ def test_source_info_ctor_mixing_invalid(io_class, sources, tmp_path): def test_source_info_invalid(): with pytest.raises(ValueError): plc.io.SourceInfo([123]) + + +def test_filepath_source_local_parquet(tmp_path): + path = tmp_path / "data.parquet" + table = plc.Table([plc.Column.from_iterable_of_py([1, 2, 3])]) + plc.io.parquet.write_parquet( + plc.io.parquet.ParquetWriterOptions.builder( + plc.io.SinkInfo([str(path)]), table + ).build() + ) + file_size = path.stat().st_size + + source = plc.io.FilepathSource(str(path), size=file_size) + assert source.path == str(path) + assert source.size == file_size + + source_info = plc.io.SourceInfo([source]) + read_opts = plc.io.parquet.ParquetReaderOptions.builder( + source_info + ).build() + result = plc.io.parquet.read_parquet(read_opts) + assert result.columns[0].to_arrow().to_pylist() == [1, 2, 3] + + +def test_filepath_source_remote_uri_without_size(): + source = plc.io.FilepathSource("s3://bucket/object.parquet") + assert source.size is None + plc.io.SourceInfo([source]) + + +def test_filepath_source_mixed_sources_invalid(): + with pytest.raises( + ValueError, match="All sources must be of the same type" + ): + plc.io.SourceInfo( + [ + plc.io.FilepathSource("s3://bucket/object.parquet", size=100), + "s3://bucket/other.parquet", + ] + )